[ { "path": ".dockerignore", "content": "bin/*\n" }, { "path": ".github/FUNDING.yml", "content": "github: mikefarah\n" }, { "path": ".github/ISSUE_TEMPLATE/bug_report_v4.md", "content": "---\nname: Bug report - V4\nabout: Create a report to help us improve\ntitle: ''\nlabels: bug, v4\nassignees: ''\n\n---\n\n**Describe the bug**\nA clear and concise description of what the bug is.\n\nNote that any how to questions should be posted in the discussion board and not raised as an issue.\n\nVersion of yq: 4.X.X\nOperating system: mac/linux/windows/....\nInstalled via: docker/binary release/homebrew/snap/...\n\n**Input Yaml**\nConcise yaml document(s) (as simple as possible to show the bug, please keep it to 10 lines or less)\ndata1.yml:\n```yaml\nthis: should really work\n```\n\ndata2.yml:\n```yaml\nbut: it strangely didn't\n```\n\n**Command**\nThe command you ran:\n```\nyq eval-all 'select(fileIndex==0) | .a.b.c' data1.yml data2.yml\n```\n\n**Actual behaviour**\n\n```yaml\ncat: meow\n```\n\n**Expected behaviour**\n\n```yaml\nthis: should really work\nbut: it strangely didn't\n```\n\n**Additional context**\nAdd any other context about the problem here.\n" }, { "path": ".github/ISSUE_TEMPLATE/feature_request.md", "content": "---\nname: Feature request - V4\nabout: Suggest an idea for this project\ntitle: ''\nlabels: enhancement, v4\nassignees: ''\n\n---\n\n**Please describe your feature request.**\nA clear and concise description of what the request is and what it would solve. \nEg. I wish I could use yq to [...]\n\nNote:\n- how to questions should be posted in the discussion board and not raised as an issue.\n- V3 will no longer have any enhancements.\n\n**Describe the solution you'd like**\nIf we have data1.yml like:\n(please keep to around 10 lines )\n\n```yaml\ncountry: Australia\n```\n\nAnd we run a command:\n\n```bash\nyq 'predictWeatherOf(.country)'\n```\n\nit could output\n\n```yaml\ntemp: 32\n```\n\n**Describe alternatives you've considered**\nA clear and concise description of any alternative solutions or features you've considered.\n\n**Additional context**\nAdd any other context or screenshots about the feature request here.\n" }, { "path": ".github/dependabot.yml", "content": "# To get started with Dependabot version updates, you'll need to specify which\n# package ecosystems to update and where the package manifests are located.\n# Please see the documentation for all configuration options:\n# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates\n\nversion: 2\nupdates:\n - package-ecosystem: docker\n directory: /\n schedule:\n day: thursday\n interval: weekly\n - package-ecosystem: github-actions\n directory: /\n schedule:\n day: thursday\n interval: weekly\n - package-ecosystem: gomod\n directory: /\n schedule:\n day: thursday\n interval: weekly\n" }, { "path": ".github/instructions/instructions.md", "content": "When you find a bug - make sure to include a new test that exposes the bug, as well as the fix for the bug itself.\n" }, { "path": ".github/workflows/codeql.yml", "content": "# For most projects, this workflow file will not need changing; you simply need\n# to commit it to your repository.\n#\n# You may wish to alter this file to override the set of languages analyzed,\n# or to provide custom queries or build logic.\n#\n# ******** NOTE ********\n# We have attempted to detect the languages in your repository. Please check\n# the `language` matrix defined below to confirm you have the correct set of\n# supported CodeQL languages.\n#\nname: \"CodeQL\"\n\non:\n push:\n branches: [ master ]\n pull_request:\n # The branches below must be a subset of the branches above\n branches: [ master ]\n schedule:\n - cron: '24 3 * * 1'\n\njobs:\n analyze:\n name: Analyze\n runs-on: ubuntu-latest\n permissions:\n actions: read\n contents: read\n security-events: write\n\n strategy:\n fail-fast: false\n matrix:\n language: [ 'go' ]\n # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]\n # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support\n\n steps:\n - name: Checkout repository\n uses: actions/checkout@v6\n\n # Initializes the CodeQL tools for scanning.\n - name: Initialize CodeQL\n uses: github/codeql-action/init@v4\n with:\n languages: ${{ matrix.language }}\n # If you wish to specify custom queries, you can do so here or in a config file.\n # By default, queries listed here will override any specified in a config file.\n # Prefix the list here with \"+\" to use these queries and those in the config file.\n # queries: ./path/to/local/query, your-org/your-repo/queries@main\n\n # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).\n # If this step fails, then you should remove it and run the build manually (see below)\n - name: Autobuild\n uses: github/codeql-action/autobuild@v4\n\n # ℹ️ Command-line programs to run using the OS shell.\n # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun\n\n # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines\n # and modify them (or add more) to build your code if your project\n # uses a compiled language\n\n #- run: |\n # make bootstrap\n # make release\n\n - name: Perform CodeQL Analysis\n uses: github/codeql-action/analyze@v4\n" }, { "path": ".github/workflows/docker-release.yml", "content": "name: Release Docker\n\non:\n release:\n types: [released]\n\n # Allows you to run this workflow manually from the Actions tab\n workflow_dispatch:\n\njobs:\n publishDocker:\n environment: dockerhub\n env:\n IMAGE_NAME: mikefarah/yq\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v6\n\n - name: Set up QEMU\n uses: docker/setup-qemu-action@v4\n with:\n platforms: all\n\n - name: Set up Docker Buildx\n id: buildx\n uses: docker/setup-buildx-action@v3\n with:\n version: latest\n\n - name: Available platforms\n run: echo ${{ steps.buildx.outputs.platforms }} && docker version\n\n - name: Login to Docker Hub\n uses: docker/login-action@v4\n with:\n username: ${{ secrets.DOCKER_USERNAME }}\n password: ${{ secrets.DOCKER_PASSWORD }}\n\n - name: Login to GitHub Container Registry\n uses: docker/login-action@v4\n with:\n registry: ghcr.io\n username: ${{ github.actor }}\n password: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Build and push image\n run: |\n echo \"GithubRef: ${GITHUB_REF}\"\n VERSION=${GITHUB_REF##*/}\n echo \"VERSION: ${VERSION}\"\n IMAGE_VERSION=${VERSION:1}\n echo \"IMAGE_VERSION: ${IMAGE_VERSION}\"\n\n PLATFORMS=\"linux/amd64,linux/ppc64le,linux/arm64,linux/arm/v7,linux/s390x\"\n\n echo \"Building and pushing version ${IMAGE_VERSION} of image ${IMAGE_NAME}\"\n docker buildx build \\\n --label \"org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors\" \\\n --label \"org.opencontainers.image.created=$(date --rfc-3339=seconds)\" \\\n --label \"org.opencontainers.image.description=yq is a portable command-line data file processor\" \\\n --label \"org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/\" \\\n --label \"org.opencontainers.image.licenses=MIT\" \\\n --label \"org.opencontainers.image.revision=$(git rev-parse HEAD)\" \\\n --label \"org.opencontainers.image.source=https://github.com/mikefarah/yq\" \\\n --label \"org.opencontainers.image.title=yq\" \\\n --label \"org.opencontainers.image.url=https://mikefarah.gitbook.io/yq/\" \\\n --label \"org.opencontainers.image.version=${IMAGE_VERSION}\" \\\n --platform \"${PLATFORMS}\" \\\n --pull \\\n --push \\\n -t \"${IMAGE_NAME}:${IMAGE_VERSION}\" \\\n -t \"${IMAGE_NAME}:4\" \\\n -t \"${IMAGE_NAME}:latest\" \\\n -t \"ghcr.io/${IMAGE_NAME}:${IMAGE_VERSION}\" \\\n -t \"ghcr.io/${IMAGE_NAME}:4\" \\\n -t \"ghcr.io/${IMAGE_NAME}:latest\" \\\n .\n\n cd github-action\n docker buildx build \\\n --label \"org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors\" \\\n --label \"org.opencontainers.image.created=$(date --rfc-3339=seconds)\" \\\n --label \"org.opencontainers.image.description=yq is a portable command-line data file processor\" \\\n --label \"org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/\" \\\n --label \"org.opencontainers.image.licenses=MIT\" \\\n --label \"org.opencontainers.image.revision=$(git rev-parse HEAD)\" \\\n --label \"org.opencontainers.image.source=https://github.com/mikefarah/yq\" \\\n --label \"org.opencontainers.image.title=yq\" \\\n --label \"org.opencontainers.image.url=https://mikefarah.gitbook.io/yq/\" \\\n --label \"org.opencontainers.image.version=${IMAGE_VERSION}\" \\\n --platform \"${PLATFORMS}\" \\\n --pull \\\n --push \\\n -t \"${IMAGE_NAME}:${IMAGE_VERSION}-githubaction\" \\\n -t \"${IMAGE_NAME}:4-githubaction\" \\\n -t \"${IMAGE_NAME}:latest-githubaction\" \\\n -t \"ghcr.io/${IMAGE_NAME}:${IMAGE_VERSION}-githubaction\" \\\n -t \"ghcr.io/${IMAGE_NAME}:4-githubaction\" \\\n -t \"ghcr.io/${IMAGE_NAME}:latest-githubaction\" \\\n .\n" }, { "path": ".github/workflows/go.yml", "content": "name: Build\non: [push, pull_request]\npermissions:\n contents: read\n\njobs:\n\n build:\n name: Build\n runs-on: ubuntu-latest\n steps:\n\n - name: Set up Go\n uses: actions/setup-go@v6\n with:\n go-version: '^1.20'\n id: go\n\n - name: Check out code into the Go module directory\n uses: actions/checkout@v6\n\n - name: Get dependencies\n run: |\n go get -v -t -d ./...\n if [ -f Gopkg.toml ]; then\n curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh\n dep ensure\n fi\n\n - name: Check the build\n shell: bash -l {0}\n run: |\n export PATH=${PATH}:`go env GOPATH`/bin\n scripts/devtools.sh\n make local build\n" }, { "path": ".github/workflows/release.yml", "content": "name: Release YQ\non:\n push:\n tags:\n - 'v4.*'\n - 'draft-*'\n\njobs:\n publishGitRelease:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v6\n - uses: actions/setup-go@v6\n with:\n go-version: '^1.20'\n check-latest: true\n - name: Compile man page markup\n id: gen-man-page-md\n run: |\n ./scripts/generate-man-page-md.sh\n\n - name: Get the version\n id: get_version\n run: echo \"VERSION=${GITHUB_REF##*/}\" >> \"${GITHUB_OUTPUT}\"\n\n - name: Generate man page\n uses: docker://pandoc/core:2.14.2\n id: gen-man-page\n with:\n args: >-\n --standalone\n --to man\n --variable=title:\"YQ\"\n --variable=section:\"1\"\n --variable=header:\"yq (https://github.com/mikefarah/yq/) version ${{ steps.get_version.outputs.VERSION }}\"\n --variable=author:\"Mike Farah\"\n --output=yq.1\n man.md\n\n - name: Cross compile\n run: |\n sudo apt-get install rhash -y\n go install github.com/goreleaser/goreleaser/v2@latest\n ./scripts/xcompile.sh\n\n - name: Release\n uses: softprops/action-gh-release@v1\n with:\n files: build/*\n draft: true\n fail_on_unmatched_files: true\n" }, { "path": ".github/workflows/snap-release.yml", "content": "name: Release Snap\n\non:\n release:\n types: [released]\n\n # Allows you to run this workflow manually from the Actions tab\n workflow_dispatch:\n\njobs:\n buildSnap:\n environment: snap\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v6\n - uses: snapcore/action-build@v1\n id: build\n env:\n SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.STORE_LOGIN }}\n with:\n snapcraft-args: \"remote-build --launchpad-accept-public-upload\"\n - uses: snapcore/action-publish@v1\n env:\n SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.STORE_LOGIN }}\n with:\n snap: ${{ steps.build.outputs.snap }}\n release: stable\n" }, { "path": ".gitignore", "content": "# Compiled Object files, Static and Dynamic libs (Shared Objects)\n*.o\n*.a\n*.so\n\n# Folders\n_obj\n_test\nbin\nbuild\nbuild-done\n.DS_Store\n\n# Architecture specific extensions/prefixes\n*.[568vq]\n[568vq].out\n\n*.cgo1.go\n*.cgo2.c\n_cgo_defun.c\n_cgo_gotypes.go\n_cgo_export.*\n\n_testmain.go\ncover.out\ncoverage.out\ncoverage.html\ncoverage_sorted.txt\n*.exe\n*.test\n*.prof\nyaml\nvendor/\ntmp/\ncover/\nyq\n\n# snapcraft\nparts/\nprime/\n.snapcraft/\nyq*.snap\n\ntest.yml\ntest*.yml\ntest*.tf\ntest*.xml\ntest*.toml\ntest*.yaml\n*.kyaml\ntest_dir1/\ntest_dir2/\n0.yml\n1.yml\n2.yml\n\n\n# man page\nman.md\nyq.1\n\n# debian pkg\n_build\ndebian/files\n\n# intellij\n/.idea\n\n# vscode\n.vscode\n\nyq3\n\n# Golang\n.gomodcache/\n.gocache/\n" }, { "path": ".golangci.bck.yml", "content": "run:\n timeout: 5m\nlinters:\n enable:\n - asciicheck\n - depguard\n - errorlint\n - gci\n - gochecknoinits\n - gofmt\n - goimports\n - gosec\n - gosimple\n - staticcheck\n - unused\n - misspell\n - nakedret\n - nolintlint\n - predeclared\n - revive\n - unconvert\n - unparam\nlinters-settings:\n depguard:\n rules:\n prevent_unmaintained_packages:\n list-mode: lax\n files:\n - $all\n - \"!$test\"\n deny:\n - pkg: io/ioutil\n desc: \"replaced by io and os packages since Go 1.16: https://tip.golang.org/doc/go1.16#ioutil\"\nissues:\n exclude-rules:\n - linters:\n - revive\n text: \"var-naming\"\n" }, { "path": ".golangci.yml", "content": "version: \"2\"\nlinters:\n enable:\n - asciicheck\n - depguard\n - errorlint\n - gochecknoinits\n - gosec\n - misspell\n - nakedret\n - nolintlint\n - predeclared\n - revive\n - unconvert\n - unparam\n settings:\n misspell:\n locale: UK\n ignore-rules:\n - color\n - colors\n depguard:\n rules:\n prevent_unmaintained_packages:\n list-mode: lax\n files:\n - $all\n - '!$test'\n deny:\n - pkg: io/ioutil\n desc: 'replaced by io and os packages since Go 1.16: https://tip.golang.org/doc/go1.16#ioutil'\n exclusions:\n generated: lax\n presets:\n - comments\n - common-false-positives\n - legacy\n - std-error-handling\n rules:\n - linters:\n - revive\n text: var-naming\n paths:\n - third_party$\n - builtin$\n - examples$\nformatters:\n enable:\n - gci\n - gofmt\n - goimports\n exclusions:\n generated: lax\n paths:\n - third_party$\n - builtin$\n - examples$\n" }, { "path": ".goreleaser.yaml", "content": "version: 2\n\ndist: build\n\nbuilds:\n - id: yq\n\n binary: yq_{{ .Os }}_{{ .Arch }}\n\n ldflags:\n - -s -w\n\n env:\n - CGO_ENABLED=0\n\n targets:\n - darwin_amd64\n - darwin_arm64\n - freebsd_386\n - freebsd_amd64\n - freebsd_arm\n - linux_386\n - linux_amd64\n - linux_arm\n - linux_arm64\n - linux_loong64\n - linux_mips\n - linux_mips64\n - linux_mips64le\n - linux_mipsle\n - linux_ppc64\n - linux_ppc64le\n - linux_riscv64\n - linux_s390x\n - netbsd_386\n - netbsd_amd64\n - netbsd_arm\n - openbsd_386\n - openbsd_amd64\n - windows_386\n - windows_amd64\n - windows_arm64\n\n no_unique_dist_dir: true\n\nrelease:\n disable: true\n skip_upload: true\n" }, { "path": "CODE_OF_CONDUCT.md", "content": "# Contributor Covenant Code of Conduct\n\n## Our Pledge\n\nIn the interest of fostering an open and welcoming environment, we as\ncontributors and maintainers pledge to making participation in our project and\nour community a harassment-free experience for everyone, regardless of age, body\nsize, disability, ethnicity, sex characteristics, gender identity and expression,\nlevel of experience, education, socio-economic status, nationality, personal\nappearance, race, religion, or sexual identity and orientation.\n\n## Our Standards\n\nExamples of behaviour that contributes to creating a positive environment\ninclude:\n\n* Using welcoming and inclusive language\n* Being respectful of differing viewpoints and experiences\n* Gracefully accepting constructive criticism\n* Focusing on what is best for the community\n* Showing empathy towards other community members\n\nExamples of unacceptable behaviour by participants include:\n\n* The use of sexualized language or imagery and unwelcome sexual attention or\n advances\n* Trolling, insulting/derogatory comments, and personal or political attacks\n* Public or private harassment\n* Publishing others' private information, such as a physical or electronic\n address, without explicit permission\n* Other conduct which could reasonably be considered inappropriate in a\n professional setting\n\n## Our Responsibilities\n\nProject maintainers are responsible for clarifying the standards of acceptable\nbehaviour and are expected to take appropriate and fair corrective action in\nresponse to any instances of unacceptable behaviour.\n\nProject maintainers have the right and responsibility to remove, edit, or\nreject comments, commits, code, wiki edits, issues, and other contributions\nthat are not aligned to this Code of Conduct, or to ban temporarily or\npermanently any contributor for other behaviours that they deem inappropriate,\nthreatening, offensive, or harmful.\n\n## Scope\n\nThis Code of Conduct applies both within project spaces and in public spaces\nwhen an individual is representing the project or its community. Examples of\nrepresenting a project or community include using an official project e-mail\naddress, posting via an official social media account, or acting as an appointed\nrepresentative at an online or offline event. Representation of a project may be\nfurther defined and clarified by project maintainers.\n\n## Enforcement\n\nInstances of abusive, harassing, or otherwise unacceptable behaviour may be\nreported by contacting the project team at mikefarah@gmail.com. All\ncomplaints will be reviewed and investigated and will result in a response that\nis deemed necessary and appropriate to the circumstances. The project team is\nobligated to maintain confidentiality with regard to the reporter of an incident.\nFurther details of specific enforcement policies may be posted separately.\n\nProject maintainers who do not follow or enforce the Code of Conduct in good\nfaith may face temporary or permanent repercussions as determined by other\nmembers of the project's leadership.\n\n## Attribution\n\nThis Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,\navailable at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html\n\n[homepage]: https://www.contributor-covenant.org\n\nFor answers to common questions about this code of conduct, see\nhttps://www.contributor-covenant.org/faq\n" }, { "path": "CONTRIBUTING.md", "content": "# Before you begin \nNot all new PRs will be merged in \n\nIt's recommended to check with the owner first (e.g. raise an issue) to discuss a new feature before developing, to ensure your hard efforts don't go to waste.\n\nPRs to fix bugs and issues are almost always welcome :pray: please ensure you write tests as well.\n\nThe following types of PRs will _not_ be accepted:\n- **Significant refactors** take a lot of time to understand and can have all sorts of unintended side effects. If you think there's a better way to do things (that requires significant changes) raise an issue for discussion first :)\n- **Release pipeline PRs** are a security risk - it's too easy for a serious vulnerability to sneak in (either intended or not). If there is a new cool way of releasing things, raise an issue for discussion first - it will need to be gone over with a fine tooth comb.\n- **Version bumps** are handled by dependabot, the bot will auto-raise PRs and they will be regularly merged in. \n- **New release platforms** At this stage, yq is not going to maintain any other release platforms other than GitHub and Docker - that said, I'm more than happy to put in other community maintained methods in the README for visibility :heart:\n\n\n# Development\n\n## Initial Setup\n\n1. Install [Golang](https://golang.org/) (version 1.24.0 or later)\n2. Run `scripts/devtools.sh` to install required development tools:\n - golangci-lint for code linting\n - gosec for security analysis\n3. Run `make [local] vendor` to install vendor dependencies\n4. Run `make [local] test` to ensure you can run the existing tests\n\n## Development Workflow\n\n1. **Write unit tests first** - Changes will not be accepted without corresponding unit tests (see Testing section below)\n2. **Make your code changes**\n3. **Run tests and linting**: `make [local] test` (this runs formatting, linting, security checks, and tests)\n4. **Create your PR** and get kudos! :)\n\n## Make Commands\n\n- Use `make [local] ` for local development (runs in Docker container)\n- Use `make ` for CI/CD environments\n- Common commands:\n - `make [local] vendor` - Install dependencies\n - `make [local] test` - Run all checks and tests\n - `make [local] build` - Build the yq binary\n - `make [local] format` - Format code\n - `make [local] check` - Run linting and security checks\n\n# Code Quality\n\n## Linting and Formatting\n\nThe project uses strict linting rules defined in `.golangci.yml`. All code must pass:\n\n- **Code formatting**: gofmt, goimports, gci\n- **Linting**: revive, errorlint, gosec, misspell, and others\n- **Security checks**: gosec security analysis\n- **Spelling checks**: misspell detection\n\nRun `make [local] check` to verify your code meets all quality standards.\n\n## Code Style Guidelines\n\n- Follow standard Go conventions\n- Use meaningful variable names\n- Add comments for public functions and complex logic\n- Keep functions focused and reasonably sized\n- Use the project's existing patterns and conventions\n\n# Testing\n\n## Test Structure\n\nTests in yq use the `expressionScenario` pattern. Each test scenario includes:\n- `expression`: The yq expression to test\n- `document`: Input YAML/JSON (optional)\n- `expected`: Expected output\n- `skipDoc`: Whether to skip documentation generation\n\n## Writing Tests\n\n1. **Find the appropriate test file** (e.g., `operator_add_test.go` for addition operations)\n2. **Add your test scenario** to the `*OperatorScenarios` slice\n3. **Run the specific test**: `go test -run TestAddOperatorScenarios` (replace with appropriate test name)\n4. **Verify documentation generation** (see Documentation section)\n\n## Test Examples\n\n```go\nvar addOperatorScenarios = []expressionScenario{\n {\n skipDoc: true,\n expression: `\"foo\" + \"bar\"`,\n expected: []string{\n \"D0, P[], (!!str)::foobar\\n\",\n },\n },\n {\n document: \"apples: 3\",\n expression: `.apples + 3`,\n expected: []string{\n \"D0, P[apples], (!!int)::6\\n\",\n },\n },\n}\n```\n\n## Running Tests\n\n- **All tests**: `make [local] test`\n- **Specific test**: `go test -run TestName`\n- **With coverage**: `make [local] cover`\n\n# Documentation\n\n## Documentation Generation\n\nThe project uses a documentation system that combines static headers with dynamically generated content from tests.\n\n### How It Works\n\n1. **Static headers** are defined in `pkg/yqlib/doc/operators/headers/*.md`\n2. **Dynamic content** is generated from test scenarios in `*_test.go` files\n3. **Generated docs** are created in `pkg/yqlib/doc/*.md` by concatenating headers with test-generated content\n4. **Documentation is synced** to the gitbook branch for the website\n\n### Updating Operator Documentation\n\n#### For Test-Generated Documentation\n\nMost operator documentation is generated from tests. To update:\n\n1. **Find the test file** (e.g., `operator_add_test.go`)\n2. **Update test scenarios** - each `expressionScenario` with `skipDoc: false` becomes documentation\n3. **Run the test** to regenerate docs:\n ```bash\n cd pkg/yqlib\n go test -run TestAddOperatorScenarios\n ```\n4. **Verify the generated documentation** in `pkg/yqlib/doc/add.md`\n5. **Create a PR** with your changes\n\n#### For Header-Only Documentation\n\nIf documentation exists only in `headers/*.md` files:\n1. **Update the header file directly** (e.g., `pkg/yqlib/doc/operators/headers/add.md`)\n2. **Create a PR** with your changes\n\n### Updating Static Documentation\n\nFor documentation not in the master branch:\n\n1. **Check the gitbook branch** for additional pages\n2. **Update the `*.md` files** directly\n3. **Create a PR** to the gitbook branch\n\n### Documentation Best Practices\n\n- **Write clear, concise examples** in test scenarios\n- **Use meaningful variable names** in examples\n- **Include edge cases** and error conditions\n- **Test your documentation changes** by running the specific test\n- **Verify generated output** matches expectations\n\nNote: PRs with small changes (e.g. minor typos) may not be merged (see https://joel.net/how-one-guy-ruined-hacktoberfest2020-drama).\n\n# Troubleshooting\n\n## Common Setup Issues\n\n### Docker/Podman Issues\n- **Problem**: `make` commands fail with Docker errors\n- **Solution**: Ensure Docker or Podman is running and accessible\n- **Alternative**: Use `make local ` to run in containers\n\n### Go Version Issues\n- **Problem**: Build fails with Go version errors\n- **Solution**: Ensure you have Go 1.24.0 or later installed\n- **Check**: Run `go version` to verify\n\n### Vendor Dependencies\n- **Problem**: `make vendor` fails or dependencies are outdated\n- **Solution**: \n ```bash\n go mod tidy\n make [local] vendor\n ```\n\n### Linting Failures\n- **Problem**: `make check` fails with linting errors\n- **Solution**: \n ```bash\n make [local] format # Auto-fix formatting\n # Manually fix remaining linting issues\n make [local] check # Verify fixes\n ```\n\n### Test Failures\n- **Problem**: Tests fail locally but pass in CI\n- **Solution**: \n ```bash\n make [local] test # Run in Docker container\n ```\n\n- **Problem**: Tests fail with a VCS error:\n ```bash\n error obtaining VCS status: exit status 128\n Use -buildvcs=false to disable VCS stamping.\n ```\n- **Solution**:\n Git security mechanisms prevent Golang from detecting the Git details inside\n the container; either build with the `local` option, or pass GOFLAGS to\n disable Golang buildvcs behaviour.\n ```bash\n make local test\n # OR\n make test GOFLAGS='-buildvcs=true'\n ```\n\n### Documentation Generation Issues\n- **Problem**: Generated docs don't update after test changes\n- **Solution**: \n ```bash\n cd pkg/yqlib\n go test -run TestSpecificOperatorScenarios\n # Check if generated file updated in pkg/yqlib/doc/\n ```\n\n## Getting Help\n\n- **Check existing issues**: Search GitHub issues for similar problems\n- **Create an issue**: If you can't find a solution, create a detailed issue\n- **Ask questions**: Use GitHub Discussions for general questions\n- **Join the community**: Check the project's community channels" }, { "path": "Dockerfile", "content": "FROM golang:1.26.1 AS builder\r\n\r\nWORKDIR /go/src/mikefarah/yq\r\n\r\nCOPY . .\r\n\r\nRUN CGO_ENABLED=0 go build -ldflags \"-s -w\" .\r\n# RUN ./scripts/test.sh -- this too often times out in the github pipeline.\r\nRUN ./scripts/acceptance.sh\r\n\r\n# Choose alpine as a base image to make this useful for CI, as many\r\n# CI tools expect an interactive shell inside the container\r\nFROM alpine:3 AS production\r\nLABEL maintainer=\"Mike Farah \"\r\n\r\nCOPY --from=builder /go/src/mikefarah/yq/yq /usr/bin/yq\r\n\r\nWORKDIR /workdir\r\n\r\nRUN set -eux; \\\r\n addgroup -g 1000 yq; \\\r\n adduser -u 1000 -G yq -s /bin/sh -h /home/yq -D yq\r\n\r\nRUN chown -R yq:yq /workdir\r\n\r\nUSER yq\r\n\r\nENTRYPOINT [\"/usr/bin/yq\"]\r\n" }, { "path": "Dockerfile.dev", "content": "FROM golang:1.26.1\n\nRUN apt-get update && \\\n apt-get install -y npm && \\\n npm install -g npx cspell@latest\n\nCOPY scripts/devtools.sh /opt/devtools.sh\n\nRUN set -e -x && \\\n /opt/devtools.sh\n\nENV PATH=/go/bin:$PATH\n\nENV CGO_ENABLED 0\nENV GOPATH /go:/yq\n" }, { "path": "LICENSE", "content": "The MIT License (MIT)\n\nCopyright (c) 2017 Mike Farah\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "Makefile", "content": "MAKEFLAGS += --warn-undefined-variables\nSHELL := /bin/bash\n.SHELLFLAGS := -o pipefail -euc\n.DEFAULT_GOAL := install\nENGINE := $(shell { (podman version > /dev/null 2>&1 && command -v podman) || command -v docker; } 2>/dev/null)\n\ninclude Makefile.variables\n\n.PHONY: help\nhelp:\n\t@echo 'Management commands for cicdtest:'\n\t@echo\n\t@echo 'Usage:'\n\t@echo ' ## Develop / Test Commands'\n\t@echo ' make build Build yq binary.'\n\t@echo ' make install Install yq.'\n\t@echo ' make xcompile Build cross-compiled binaries of yq.'\n\t@echo ' make vendor Install dependencies to vendor directory.'\n\t@echo ' make format Run code formatter.'\n\t@echo ' make check Run static code analysis (lint).'\n\t@echo ' make secure Run gosec.'\n\t@echo ' make test Run tests on project.'\n\t@echo ' make cover Run tests and capture code coverage metrics on project.'\n\t@echo ' make clean Clean the directory tree of produced artifacts.'\n\t@echo\n\t@echo ' ## Utility Commands'\n\t@echo ' make setup Configures Minishfit/Docker directory mounts.'\n\t@echo\n\n\n.PHONY: clean\nclean:\n\t@rm -rf bin build cover *.out\n\n## prefix before other make targets to run in your local dev environment\nlocal: | quiet\n\t@$(eval ENGINERUN= )\n\t@$(eval GOFLAGS=\"$(GOFLAGS)\" )\n\t@mkdir -p tmp\n\t@touch tmp/dev_image_id\nquiet: # this is silly but shuts up 'Nothing to be done for `local`'\n\t@:\n\nprepare: tmp/dev_image_id\ntmp/dev_image_id: Dockerfile.dev scripts/devtools.sh\n\t@mkdir -p tmp\n\t@${ENGINE} rmi -f ${DEV_IMAGE} > /dev/null 2>&1 || true\n\t@${ENGINE} build -t ${DEV_IMAGE} -f Dockerfile.dev .\n\t@${ENGINE} inspect -f \"{{ .ID }}\" ${DEV_IMAGE} > tmp/dev_image_id\n\n# ----------------------------------------------\n# build\n.PHONY: build\nbuild: build/dev\n\n.PHONY: build/dev\nbuild/dev: test *.go\n\t@mkdir -p bin/\n\t${ENGINERUN} go build --ldflags \"$(LDFLAGS)\"\n\t${ENGINERUN} bash ./scripts/acceptance.sh\n\n## Compile the project for multiple OS and Architectures.\nxcompile: check\n\t@rm -rf build/\n\t@mkdir -p build\n\t${ENGINERUN} bash ./scripts/xcompile.sh\n\t@find build -type d -exec chmod 755 {} \\; || :\n\t@find build -type f -exec chmod 755 {} \\; || :\n\n.PHONY: install\ninstall: build\n\t${ENGINERUN} go install\n\n# Each of the fetch should be an entry within vendor.json; not currently included within project\n.PHONY: vendor\nvendor: tmp/dev_image_id\n\t@mkdir -p vendor\n\t${ENGINERUN} go mod vendor\n\n# ----------------------------------------------\n# develop and test\n\n.PHONY: format\nformat: vendor\n\t${ENGINERUN} bash ./scripts/format.sh\n\n\n.PHONY: spelling\nspelling: format\n\t${ENGINERUN} bash ./scripts/spelling.sh\n\n.PHONY: secure\nsecure: spelling\n\t${ENGINERUN} bash ./scripts/secure.sh\n\n.PHONY: check\ncheck: secure\n\t${ENGINERUN} bash ./scripts/check.sh\n\n\n\n.PHONY: test\ntest: check\n\t${ENGINERUN} bash ./scripts/test.sh\n\n.PHONY: cover\ncover: check\n\t@rm -rf cover/\n\t@mkdir -p cover\n\t${ENGINERUN} bash ./scripts/coverage.sh\n\t@find cover -type d -exec chmod 755 {} \\; || :\n\t@find cover -type f -exec chmod 644 {} \\; || :\n\n\n.PHONY: release\nrelease: xcompile\n\t${ENGINERUN} bash ./scripts/publish.sh\n\n# ----------------------------------------------\n# utilities\n\n.PHONY: setup\nsetup:\n\t@bash ./scripts/setup.sh\n" }, { "path": "Makefile.variables", "content": "export PROJECT = yq\nIMPORT_PATH := github.com/mikefarah/${PROJECT}\n\nexport GIT_COMMIT = $(shell git rev-parse --short HEAD)\nexport GIT_DIRTY = $(shell test -n \"$$(git status --porcelain)\" && echo \"+CHANGES\" || true)\nexport GIT_DESCRIBE = $(shell git describe --tags --always)\nGOFLAGS :=\nLDFLAGS :=\nLDFLAGS += -X main.GitCommit=${GIT_COMMIT}${GIT_DIRTY}\nLDFLAGS += -X main.GitDescribe=${GIT_DESCRIBE}\nLDFLAGS += -w -s\n\nGITHUB_TOKEN ?=\n\n# Windows environment?\nCYG_CHECK := $(shell hash cygpath 2>/dev/null && echo 1)\nifeq ($(CYG_CHECK),1)\n\tVBOX_CHECK := $(shell hash VBoxManage 2>/dev/null && echo 1)\n\n\t# Docker Toolbox (pre-Windows 10)\n\tifeq ($(VBOX_CHECK),1)\n\t\tROOT := /${PROJECT}\n\telse\n\t\t# Docker Windows\n\t\tROOT := $(shell cygpath -m -a \"$(shell pwd)\")\n\tendif\nelse\n\t# all non-windows environments\n\tROOT := $(shell pwd)\n\t# Deliberately use `command -v` instead of `which` to be POSIX compliant\n\tSELINUX := $(shell command -v getenforce >/dev/null 2>&1 && echo :z)\nendif\n\nDEV_IMAGE := ${PROJECT}_dev\n\nENGINERUN := ${ENGINE} run --rm \\\n\t-e LDFLAGS=\"${LDFLAGS}\" \\\n\t-e GOFLAGS=\"${GOFLAGS}\" \\\n\t-e GITHUB_TOKEN=\"${GITHUB_TOKEN}\" \\\n\t-v ${ROOT}/vendor:/go/src${SELINUX} \\\n\t-v ${ROOT}:/${PROJECT}/src/${IMPORT_PATH}${SELINUX} \\\n\t-w /${PROJECT}/src/${IMPORT_PATH} \\\n\t${DEV_IMAGE}\n" }, { "path": "README.md", "content": "# yq\n\n![Build](https://github.com/mikefarah/yq/workflows/Build/badge.svg) ![Docker Pulls](https://img.shields.io/docker/pulls/mikefarah/yq.svg) ![Github Releases (by Release)](https://img.shields.io/github/downloads/mikefarah/yq/total.svg) ![Go Report](https://goreportcard.com/badge/github.com/mikefarah/yq) ![CodeQL](https://github.com/mikefarah/yq/workflows/CodeQL/badge.svg)\n\n\nA lightweight and portable command-line YAML, JSON, INI and XML processor. `yq` uses [jq](https://github.com/stedolan/jq) (a popular JSON processor) like syntax but works with yaml files as well as json, kyaml, xml, ini, properties, csv and tsv. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.\n\nyq is written in Go - so you can download a dependency free binary for your platform and you are good to go! If you prefer there are a variety of package managers that can be used as well as Docker and Podman, all listed below.\n\n## Quick Usage Guide\n\n### Basic Operations\n\n**Read a value:**\n```bash\nyq '.a.b[0].c' file.yaml\n```\n\n**Pipe from STDIN:**\n```bash\nyq '.a.b[0].c' < file.yaml\n```\n\n**Update a yaml file in place:**\n```bash\nyq -i '.a.b[0].c = \"cool\"' file.yaml\n```\n\n**Update using environment variables:**\n```bash\nNAME=mike yq -i '.a.b[0].c = strenv(NAME)' file.yaml\n```\n\n### Advanced Operations\n\n**Merge multiple files:**\n```bash\n# merge two files\nyq -n 'load(\"file1.yaml\") * load(\"file2.yaml\")'\n\n# merge using globs (note: `ea` evaluates all files at once instead of in sequence)\nyq ea '. as $item ireduce ({}; . * $item )' path/to/*.yml\n```\n\n**Multiple updates to a yaml file:**\n```bash\nyq -i '\n .a.b[0].c = \"cool\" |\n .x.y.z = \"foobar\" |\n .person.name = strenv(NAME)\n' file.yaml\n```\n\n**Find and update an item in an array:**\n```bash\n# Note: requires input file - add your file at the end\nyq -i '(.[] | select(.name == \"foo\") | .address) = \"12 cat st\"' data.yaml\n```\n\n**Convert between formats:**\n```bash\n# Convert JSON to YAML (pretty print)\nyq -Poy sample.json\n\n# Convert YAML to JSON\nyq -o json file.yaml\n\n# Convert XML to YAML\nyq -o yaml file.xml\n```\n\nSee [recipes](https://mikefarah.gitbook.io/yq/recipes) for more examples and the [documentation](https://mikefarah.gitbook.io/yq/) for more information.\n\nTake a look at the discussions for [common questions](https://github.com/mikefarah/yq/discussions/categories/q-a), and [cool ideas](https://github.com/mikefarah/yq/discussions/categories/show-and-tell)\n\n## Install\n\n### [Download the latest binary](https://github.com/mikefarah/yq/releases/latest)\n\n### wget\nUse wget to download pre-compiled binaries. Choose your platform and architecture:\n\n**For Linux (example):**\n```bash\n# Set your platform variables (adjust as needed)\nVERSION=v4.2.0\nPLATFORM=linux_amd64\n\n# Download compressed binary\nwget https://github.com/mikefarah/yq/releases/download/${VERSION}/yq_${PLATFORM}.tar.gz -O - |\\\n tar xz && sudo mv yq_${PLATFORM} /usr/local/bin/yq\n\n# Or download plain binary\nwget https://github.com/mikefarah/yq/releases/download/${VERSION}/yq_${PLATFORM} -O /usr/local/bin/yq &&\\\n chmod +x /usr/local/bin/yq\n```\n\n**Latest version (Linux AMD64):**\n```bash\nwget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/local/bin/yq &&\\\n chmod +x /usr/local/bin/yq\n```\n\n**Available platforms:** `linux_amd64`, `linux_arm64`, `linux_arm`, `linux_386`, `darwin_amd64`, `darwin_arm64`, `windows_amd64`, `windows_386`, etc.\n\n### MacOS / Linux via Homebrew:\nUsing [Homebrew](https://brew.sh/)\n```\nbrew install yq\n```\n\n### Linux via snap:\n```\nsnap install yq\n```\n\n#### Snap notes\n`yq` installs with [_strict confinement_](https://docs.snapcraft.io/snap-confinement/6233) in snap, this means it doesn't have direct access to root files. To read root files you can:\n\n```\nsudo cat /etc/myfile | yq '.a.path'\n```\n\nAnd to write to a root file you can either use [sponge](https://linux.die.net/man/1/sponge):\n```\nsudo cat /etc/myfile | yq '.a.path = \"value\"' | sudo sponge /etc/myfile\n```\nor write to a temporary file:\n```\nsudo cat /etc/myfile | yq '.a.path = \"value\"' | sudo tee /etc/myfile.tmp\nsudo mv /etc/myfile.tmp /etc/myfile\nrm /etc/myfile.tmp\n```\n\n### Run with Docker or Podman\n\n#### One-time use:\n```bash\n# Docker - process files in current directory\ndocker run --rm -v \"${PWD}\":/workdir mikefarah/yq '.a.b[0].c' file.yaml\n\n# Podman - same usage as Docker\npodman run --rm -v \"${PWD}\":/workdir mikefarah/yq '.a.b[0].c' file.yaml\n```\n\n**Security note:** You can run `yq` in Docker with restricted privileges:\n```bash\ndocker run --rm --security-opt=no-new-privileges --cap-drop all --network none \\\n -v \"${PWD}\":/workdir mikefarah/yq '.a.b[0].c' file.yaml\n```\n\n#### Pipe data via STDIN:\n\nYou'll need to pass the `-i --interactive` flag to Docker/Podman:\n\n```bash\n# Process piped data\ndocker run -i --rm mikefarah/yq '.this.thing' < myfile.yml\n\n# Same with Podman\npodman run -i --rm mikefarah/yq '.this.thing' < myfile.yml\n```\n\n#### Run commands interactively:\n\n```bash\ndocker run --rm -it -v \"${PWD}\":/workdir --entrypoint sh mikefarah/yq\n```\n\n```bash\npodman run --rm -it -v \"${PWD}\":/workdir --entrypoint sh mikefarah/yq\n```\n\nIt can be useful to have a bash function to avoid typing the whole docker command:\n\n```bash\nyq() {\n docker run --rm -i -v \"${PWD}\":/workdir mikefarah/yq \"$@\"\n}\n```\n\n```bash\nyq() {\n podman run --rm -i -v \"${PWD}\":/workdir mikefarah/yq \"$@\"\n}\n```\n#### Running as root:\n\n`yq`'s container image no longer runs under root (https://github.com/mikefarah/yq/pull/860). If you'd like to install more things in the container image, or you're having permissions issues when attempting to read/write files you'll need to either:\n\n\n```\ndocker run --user=\"root\" -it --entrypoint sh mikefarah/yq\n```\n\n```\npodman run --user=\"root\" -it --entrypoint sh mikefarah/yq\n```\n\nOr, in your Dockerfile:\n\n```\nFROM mikefarah/yq\n\nUSER root\nRUN apk add --no-cache bash\nUSER yq\n```\n\n#### Missing timezone data\nBy default, the alpine image yq uses does not include timezone data. If you'd like to use the `tz` operator, you'll need to include this data:\n\n```\nFROM mikefarah/yq\n\nUSER root\nRUN apk add --no-cache tzdata\nUSER yq\n```\n\n#### Podman with SELinux\n\nIf you are using podman with SELinux, you will need to set the shared volume flag `:z` on the volume mount:\n\n```\n-v \"${PWD}\":/workdir:z\n```\n\n### GitHub Action\n```\n - name: Set foobar to cool\n uses: mikefarah/yq@master\n with:\n cmd: yq -i '.foo.bar = \"cool\"' 'config.yml'\n - name: Get an entry with a variable that might contain dots or spaces\n id: get_username\n uses: mikefarah/yq@master\n with:\n cmd: yq '.all.children.[\"${{ matrix.ip_address }}\"].username' ops/inventories/production.yml\n - name: Reuse a variable obtained in another step\n run: echo ${{ steps.get_username.outputs.result }}\n```\n\nSee https://mikefarah.gitbook.io/yq/usage/github-action for more.\n\n### Go Install:\n```\ngo install github.com/mikefarah/yq/v4@latest\n```\n\n## Community Supported Installation methods\nAs these are supported by the community :heart: - however, they may be out of date with the officially supported releases.\n\n_Please note that the Debian package (previously supported by @rmescandon) is no longer maintained. Please use an alternative installation method._\n\n\n### X-CMD\nCheckout `yq` on x-cmd: https://x-cmd.com/mod/yq\n\n- Instant Results: See the output of your yq filter in real-time.\n- Error Handling: Encounter a syntax error? It will display the error message and the results of the closest valid filter\n\nThanks @edwinjhlee!\n\n### Nix\n\n```\nnix profile install nixpkgs#yq-go\n```\n\nSee [here](https://search.nixos.org/packages?channel=unstable&show=yq-go&from=0&size=50&sort=relevance&type=packages&query=yq-go)\n\n\n### Webi\n\n```\nwebi yq\n```\n\nSee [webi](https://webinstall.dev/)\nSupported by @adithyasunil26 (https://github.com/webinstall/webi-installers/tree/master/yq)\n\n### Arch Linux\n\n```\npacman -S go-yq\n```\n\n### Windows:\n\nUsing [Chocolatey](https://chocolatey.org)\n\n[![Chocolatey](https://img.shields.io/chocolatey/v/yq.svg)](https://chocolatey.org/packages/yq)\n[![Chocolatey](https://img.shields.io/chocolatey/dt/yq.svg)](https://chocolatey.org/packages/yq)\n```\nchoco install yq\n```\nSupported by @chillum (https://chocolatey.org/packages/yq)\n\nUsing [scoop](https://scoop.sh/)\n```\nscoop install main/yq\n```\n\nUsing [winget](https://learn.microsoft.com/en-us/windows/package-manager/)\n```\nwinget install --id MikeFarah.yq\n```\n\n### MacPorts:\nUsing [MacPorts](https://www.macports.org/)\n```\nsudo port selfupdate\nsudo port install yq\n```\nSupported by @herbygillot (https://ports.macports.org/maintainer/github/herbygillot)\n\n### Alpine Linux\n\nAlpine Linux v3.20+ (and Edge):\n```\napk add yq-go\n```\n\nAlpine Linux up to v3.19:\n```\napk add yq\n```\n\nSupported by Tuan Hoang (https://pkgs.alpinelinux.org/packages?name=yq-go)\n\n### Flox:\n\nFlox can be used to install yq on Linux, MacOS, and Windows through WSL.\n\n```\nflox install yq\n```\n\n\n### MacOS / Linux via gah:\nUsing [gah](https://github.com/marverix/gah)\n\n```\ngah install yq\n```\n\n## Features\n- [Detailed documentation with many examples](https://mikefarah.gitbook.io/yq/)\n- Written in portable go, so you can download a lovely dependency free binary\n- Uses similar syntax as `jq` but works with YAML, INI, [JSON](https://mikefarah.gitbook.io/yq/usage/convert) and [XML](https://mikefarah.gitbook.io/yq/usage/xml) files\n- Fully supports multi document yaml files\n- Supports yaml [front matter](https://mikefarah.gitbook.io/yq/usage/front-matter) blocks (e.g. jekyll/assemble)\n- Colorized yaml output\n- [Date/Time manipulation and formatting with TZ](https://mikefarah.gitbook.io/yq/operators/datetime)\n- [Deep data structures](https://mikefarah.gitbook.io/yq/operators/traverse-read)\n- [Sort keys](https://mikefarah.gitbook.io/yq/operators/sort-keys)\n- Manipulate yaml [comments](https://mikefarah.gitbook.io/yq/operators/comment-operators), [styling](https://mikefarah.gitbook.io/yq/operators/style), [tags](https://mikefarah.gitbook.io/yq/operators/tag) and [anchors and aliases](https://mikefarah.gitbook.io/yq/operators/anchor-and-alias-operators).\n- [Update in place](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags)\n- [Complex expressions to select and update](https://mikefarah.gitbook.io/yq/operators/select#select-and-update-matching-values-in-map)\n- Keeps yaml formatting and comments when updating (though there are issues with whitespace)\n- [Decode/Encode base64 data](https://mikefarah.gitbook.io/yq/operators/encode-decode)\n- [Load content from other files](https://mikefarah.gitbook.io/yq/operators/load)\n- [Convert to/from json/ndjson](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert)\n- [Convert to/from xml](https://mikefarah.gitbook.io/yq/v/v4.x/usage/xml)\n- [Convert to/from hcl (terraform)](https://mikefarah.gitbook.io/yq/v/v4.x/usage/hcl)\n- [Convert to/from toml](https://mikefarah.gitbook.io/yq/v/v4.x/usage/toml)\n- [Convert to/from properties](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties)\n- [Convert to/from csv/tsv](https://mikefarah.gitbook.io/yq/usage/csv-tsv)\n- [General shell completion scripts (bash/zsh/fish/powershell)](https://mikefarah.gitbook.io/yq/v/v4.x/commands/shell-completion)\n- [Reduce](https://mikefarah.gitbook.io/yq/operators/reduce) to merge multiple files or sum an array or other fancy things.\n- [Github Action](https://mikefarah.gitbook.io/yq/usage/github-action) to use in your automated pipeline (thanks @devorbitus)\n\n## [Usage](https://mikefarah.gitbook.io/yq/)\n\nCheck out the [documentation](https://mikefarah.gitbook.io/yq/) for more detailed and advanced usage.\n\n```\nUsage:\n yq [flags]\n yq [command]\n\nExamples:\n\n# yq tries to auto-detect the file format based off the extension, and defaults to YAML if it's unknown (or piping through STDIN)\n# Use the '-p/--input-format' flag to specify a format type.\ncat file.xml | yq -p xml\n\n# read the \"stuff\" node from \"myfile.yml\"\nyq '.stuff' < myfile.yml\n\n# update myfile.yml in place\nyq -i '.stuff = \"foo\"' myfile.yml\n\n# print contents of sample.json as idiomatic YAML\nyq -P -oy sample.json\n\n\nAvailable Commands:\n completion Generate the autocompletion script for the specified shell\n eval (default) Apply the expression to each document in each yaml file in sequence\n eval-all Loads _all_ yaml documents of _all_ yaml files and runs expression once\n help Help about any command\n\nFlags:\n -C, --colors force print with colors\n --csv-auto-parse parse CSV YAML/JSON values (default true)\n --csv-separator char CSV Separator character (default ,)\n --debug-node-info debug node info\n -e, --exit-status set exit status if there are no matches or null or false is returned\n --expression string forcibly set the expression argument. Useful when yq argument detection thinks your expression is a file.\n --from-file string Load expression from specified file.\n -f, --front-matter string (extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact\n --header-preprocess Slurp any header comments and separators before processing expression. (default true)\n -h, --help help for yq\n -I, --indent int sets indent level for output (default 2)\n -i, --inplace update the file in place of first file given.\n -p, --input-format string [auto|a|yaml|y|json|j|kyaml|ky|props|p|csv|c|tsv|t|xml|x|base64|uri|toml|hcl|h|lua|l|ini|i] parse format for input. (default \"auto\")\n --lua-globals output keys as top-level global variables\n --lua-prefix string prefix (default \"return \")\n --lua-suffix string suffix (default \";\\n\")\n --lua-unquoted output unquoted string keys (e.g. {foo=\"bar\"})\n -M, --no-colors force print with no colors\n -N, --no-doc Don't print document separators (---)\n -0, --nul-output Use NUL char to separate values. If unwrap scalar is also set, fail if unwrapped scalar contains NUL char.\n -n, --null-input Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.\n -o, --output-format string [auto|a|yaml|y|json|j|kyaml|ky|props|p|csv|c|tsv|t|xml|x|base64|uri|toml|hcl|h|shell|s|lua|l|ini|i] output format type. (default \"auto\")\n -P, --prettyPrint pretty print, shorthand for '... style = \"\"'\n --properties-array-brackets use [x] in array paths (e.g. for SpringBoot)\n --properties-separator string separator to use between keys and values (default \" = \")\n --security-disable-env-ops Disable env related operations.\n --security-disable-file-ops Disable file related operations (e.g. load)\n --shell-key-separator string separator for shell variable key paths (default \"_\")\n -s, --split-exp string print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter. The necessary directories will be created.\n --split-exp-file string Use a file to specify the split-exp expression.\n --string-interpolation Toggles strings interpolation of \\(exp) (default true)\n --tsv-auto-parse parse TSV YAML/JSON values (default true)\n -r, --unwrapScalar unwrap scalar, print the value with no quotes, colors or comments. Defaults to true for yaml (default true)\n -v, --verbose verbose mode\n -V, --version Print version information and quit\n --xml-attribute-prefix string prefix for xml attributes (default \"+@\")\n --xml-content-name string name for xml content (if no attribute name is present). (default \"+content\")\n --xml-directive-name string name for xml directives (e.g. ) (default \"+directive\")\n --xml-keep-namespace enables keeping namespace after parsing attributes (default true)\n --xml-proc-inst-prefix string prefix for xml processing instructions (e.g. ) (default \"+p_\")\n --xml-raw-token enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details. (default true)\n --xml-skip-directives skip over directives (e.g. )\n --xml-skip-proc-inst skip over process instructions (e.g. )\n --xml-strict-mode enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.\n --yaml-fix-merge-anchor-to-spec Fix merge anchor to match YAML spec. Will default to true in late 2025\n\nUse \"yq [command] --help\" for more information about a command.\n```\n\n## Troubleshooting\n\n### Common Issues\n\n**PowerShell quoting issues:**\n```powershell\n# Use single quotes for expressions\nyq '.a.b[0].c' file.yaml\n\n# Or escape double quotes\nyq \".a.b[0].c = \\\"value\\\"\" file.yaml\n```\n\n### Getting Help\n\n- **Check existing issues**: [GitHub Issues](https://github.com/mikefarah/yq/issues)\n- **Ask questions**: [GitHub Discussions](https://github.com/mikefarah/yq/discussions)\n- **Documentation**: [Complete documentation](https://mikefarah.gitbook.io/yq/)\n- **Examples**: [Recipes and examples](https://mikefarah.gitbook.io/yq/recipes)\n\n## Known Issues / Missing Features\n- `yq` attempts to preserve comment positions and whitespace as much as possible, but it does not handle all scenarios (see https://github.com/go-yaml/yaml/tree/v3 for details)\n- Powershell has its own...[opinions on quoting yq](https://mikefarah.gitbook.io/yq/usage/tips-and-tricks#quotes-in-windows-powershell)\n- \"yes\", \"no\" were dropped as boolean values in the yaml 1.2 standard - which is the standard yq assumes.\n\nSee [tips and tricks](https://mikefarah.gitbook.io/yq/usage/tips-and-tricks) for more common problems and solutions.\n" }, { "path": "acceptance_tests/bad_args.sh", "content": "#!/bin/bash\n\ntestWriteInPlacePipeIn() {\n result=$(./yq e -i -n '.a' 2>&1)\n assertEquals 1 $?\n assertEquals \"Error: write in place flag only applicable when giving an expression and at least one file\" \"$result\"\n}\n\ntestWriteInPlacePipeInEvalall() {\n result=$(./yq ea -i -n '.a' 2>&1)\n assertEquals 1 $?\n assertEquals \"Error: write in place flag only applicable when giving an expression and at least one file\" \"$result\"\n}\n\ntestWriteInPlaceWithSplit() {\n result=$(./yq e -s \"cat\" -i '.a = \"thing\"' test.yml 2>&1)\n assertEquals 1 $?\n assertEquals \"Error: write in place cannot be used with split file\" \"$result\"\n}\n\ntestWriteInPlaceWithSplitEvalAll() {\n result=$(./yq ea -s \"cat\" -i '.a = \"thing\"' test.yml 2>&1)\n assertEquals 1 $?\n assertEquals \"Error: write in place cannot be used with split file\" \"$result\"\n}\n\ntestNullWithFiles() {\n result=$(./yq e -n '.a = \"thing\"' test.yml 2>&1)\n assertEquals 1 $?\n assertEquals \"Error: cannot pass files in when using null-input flag\" \"$result\"\n}\n\ntestNullWithFilesEvalAll() {\n result=$(./yq ea -n '.a = \"thing\"' test.yml 2>&1)\n assertEquals 1 $?\n assertEquals \"Error: cannot pass files in when using null-input flag\" \"$result\"\n}\n\n\n\nsource ./scripts/shunit2" }, { "path": "acceptance_tests/basic.sh", "content": "#!/bin/bash\n\nsetUp() {\n rm test*.yml 2>/dev/null || true\n rm .xyz 2>/dev/null || true\n rm instructions.txt 2>/dev/null || true\n}\n\ntestBasicEvalRoundTrip() {\n ./yq -n \".a = 123\" > test.yml\n X=$(./yq '.a' test.yml)\n assertEquals 123 \"$X\"\n}\n\ntestBasicTrailingContent() {\n cat >test-trailing.yml <test-trailing.yml <test-trailing.yml <test-trailing.yml < test.yml\n X=$(cat test.yml | ./yq '.')\n assertEquals \"a: 123\" \"$X\"\n}\n\ntestBasicExpressionMatchesFileName() {\n ./yq -n \".xyz = 123\" > test.yml\n touch .xyz\n \n X=$(./yq --expression '.xyz' test.yml)\n assertEquals \"123\" \"$X\"\n\n X=$(./yq ea --expression '.xyz' test.yml)\n assertEquals \"123\" \"$X\"\n}\n\ntestBasicExpressionFromFile() {\n ./yq -n \".xyz = 123\" > test.yml\n echo '.xyz = \"meow\" | .cool = \"frog\"' > instructions.txt\n \n X=$(./yq --from-file instructions.txt test.yml -o=j -I=0)\n assertEquals '{\"xyz\":\"meow\",\"cool\":\"frog\"}' \"$X\"\n\n X=$(./yq ea --from-file instructions.txt test.yml -o=j -I=0)\n assertEquals '{\"xyz\":\"meow\",\"cool\":\"frog\"}' \"$X\"\n}\n\ntestBasicExpressionFromFileDos() {\n ./yq -n \".xyz = 123\" > test.yml\n echo '.xyz = \"meow\" | .cool = \"frog\"' | sed 's/$'\"/`echo \\\\\\r`/\" > instructions.txt\n \n X=$(./yq --from-file instructions.txt test.yml -o=j -I=0)\n assertEquals '{\"xyz\":\"meow\",\"cool\":\"frog\"}' \"$X\"\n\n X=$(./yq ea --from-file instructions.txt test.yml -o=j -I=0)\n assertEquals '{\"xyz\":\"meow\",\"cool\":\"frog\"}' \"$X\"\n}\n\ntestBasicGitHubAction() {\n ./yq -n \".a = 123\" > test.yml\n X=$(cat /dev/null | ./yq test.yml)\n assertEquals \"a: 123\" \"$X\"\n\n X=$(cat /dev/null | ./yq e test.yml)\n assertEquals \"a: 123\" \"$X\"\n\n X=$(cat /dev/null | ./yq ea test.yml)\n assertEquals \"a: 123\" \"$X\"\n}\n\ntestBasicGitHubActionWithExpression() {\n ./yq -n \".a = 123\" > test.yml\n X=$(cat /dev/null | ./yq '.a' test.yml)\n assertEquals \"123\" \"$X\"\n\n X=$(cat /dev/null | ./yq e '.a' test.yml)\n assertEquals \"123\" \"$X\"\n\n X=$(cat /dev/null | ./yq ea '.a' test.yml)\n assertEquals \"123\" \"$X\"\n}\n\n\ntestBasicEvalAllAllFiles() {\n ./yq -n \".a = 123\" > test.yml\n ./yq -n \".a = 124\" > test2.yml\n X=$(./yq ea test.yml test2.yml)\n Y=$(./yq e '.' test.yml test2.yml)\n assertEquals \"$Y\" \"$X\"\n}\n\n# when given a file, don't read STDIN\n# otherwise strange things start happening\n# in scripts\n# https://github.com/mikefarah/yq/issues/1115\n\ntestBasicCatWithFilesNoDash() {\n ./yq -n \".a = 123\" > test.yml\n ./yq -n \".a = 124\" > test2.yml\n X=$(cat test.yml | ./yq test2.yml)\n Y=$(./yq e '.' test2.yml)\n assertEquals \"$Y\" \"$X\"\n}\n\n# when the nullinput flag is used\n# don't automatically read STDIN (this breaks github actions)\ntestBasicCreateFileGithubAction() {\n cat /dev/null | ./yq -n \".a = 123\" > test.yml\n}\n\ntestBasicEvalAllCatWithFilesNoDash() {\n ./yq -n \".a = 123\" > test.yml\n ./yq -n \".a = 124\" > test2.yml\n X=$(cat test.yml | ./yq ea test2.yml)\n Y=$(./yq e '.' test2.yml)\n assertEquals \"$Y\" \"$X\"\n}\n\ntestBasicCatWithFilesNoDashWithExp() {\n ./yq -n \".a = 123\" > test.yml\n ./yq -n \".a = 124\" > test2.yml\n X=$(cat test.yml | ./yq '.a' test2.yml)\n Y=$(./yq e '.a' test2.yml)\n assertEquals \"$Y\" \"$X\"\n}\n\ntestBasicEvalAllCatWithFilesNoDashWithExp() {\n ./yq -n \".a = 123\" > test.yml\n ./yq -n \".a = 124\" > test2.yml\n X=$(cat test.yml | ./yq ea '.a' test2.yml)\n Y=$(./yq e '.a' test2.yml)\n assertEquals \"$Y\" \"$X\"\n}\n\n\ntestBasicStdInWithFiles() {\n ./yq -n \".a = 123\" > test.yml\n ./yq -n \".a = 124\" > test2.yml\n X=$(cat test.yml | ./yq - test2.yml)\n Y=$(./yq e '.' test.yml test2.yml)\n assertEquals \"$Y\" \"$X\"\n}\n\ntestBasicEvalAllStdInWithFiles() {\n ./yq -n \".a = 123\" > test.yml\n ./yq -n \".a = 124\" > test2.yml\n X=$(cat test.yml | ./yq ea - test2.yml)\n Y=$(./yq e '.' test.yml test2.yml)\n assertEquals \"$Y\" \"$X\"\n}\n\ntestBasicStdInWithFilesReverse() {\n ./yq -n \".a = 123\" > test.yml\n ./yq -n \".a = 124\" > test2.yml\n X=$(cat test.yml | ./yq test2.yml -)\n Y=$(./yq e '.' test2.yml test.yml)\n assertEquals \"$Y\" \"$X\"\n}\n\ntestBasicEvalAllStdInWithFilesReverse() {\n ./yq -n \".a = 123\" > test.yml\n ./yq -n \".a = 124\" > test2.yml\n X=$(cat test.yml | ./yq ea test2.yml -)\n Y=$(./yq e '.' test2.yml test.yml)\n assertEquals \"$Y\" \"$X\"\n}\n\ntestBasicEvalRoundTripNoEval() {\n ./yq -n \".a = 123\" > test.yml\n X=$(./yq '.a' test.yml)\n assertEquals 123 \"$X\"\n}\n\ntestBasicStdInWithOneArg() {\n ./yq e -n \".a = 123\" > test.yml\n X=$(cat test.yml | ./yq e \".a\")\n assertEquals 123 \"$X\"\n\n X=$(cat test.yml | ./yq ea \".a\")\n assertEquals 123 \"$X\"\n\n X=$(cat test.yml | ./yq \".a\")\n assertEquals 123 \"$X\"\n}\n\ntestBasicUpdateInPlaceSequence() {\n cat >test.yml <test.yml <test.yml <test.yml <test2.yml <test.yml <test2.yml < test.yml\n X=$(./yq e '.z' test.yml)\n assertEquals \"null\" \"$X\"\n}\n\ntestBasicExitStatus() {\n echo \"a: cat\" > test.yml\n X=$(./yq e -e '.z' test.yml 2&>/dev/null)\n assertEquals 1 \"$?\"\n}\n\ntestBasicExitStatusNoEval() {\n echo \"a: cat\" > test.yml\n X=$(./yq -e '.z' test.yml 2&>/dev/null)\n assertEquals 1 \"$?\"\n}\n\ntestBasicExtractFieldWithSeparator() {\n cat >test.yml <test.yml <test.yml <test.yml <&1)\n assertEquals 0 $?\n assertContains \"$result\" \"Completion ended with directive:\"\n}\n\nsource ./scripts/shunit2\n" }, { "path": "acceptance_tests/empty.sh", "content": "#!/bin/bash\n\nsetUp() {\n rm test*.yml || true\n cat >test.yml <test.yml\n X=$(./yq e test.yml)\n expected=$(cat test.yml)\n assertEquals 0 $?\n assertEquals \"$expected\" \"$X\"\n}\n\ntestEmptyEvalNoNewLineWithExpression() {\n echo -n \"# comment\" >test.yml\n X=$(./yq e '.apple = \"tree\"' test.yml)\n read -r -d '' expected << EOM\n# comment\napple: tree\nEOM\n assertEquals \"$expected\" \"$X\"\n}\n\ntestEmptyEvalPipe() {\n X=$(./yq e - < test.yml)\n assertEquals 0 $?\n}\n\ntestEmptyCommentsWithExpressionEval() {\n read -r -d '' expected << EOM\n# comment\napple: tree\nEOM\n\n X=$(./yq e '.apple=\"tree\"' test.yml)\n\n assertEquals \"$expected\" \"$X\"\n}\n\ntestEmptyCommentsWithExpressionEvalAll() {\n read -r -d '' expected << EOM\n# comment\napple: tree\nEOM\n\n X=$(./yq ea '.apple=\"tree\"' test.yml)\n\n assertEquals \"$expected\" \"$X\"\n}\n\ntestEmptyWithExpressionEval() {\n rm test.yml\n touch test.yml\n expected=\"apple: tree\"\n\n X=$(./yq e '.apple=\"tree\"' test.yml)\n\n assertEquals \"$expected\" \"$X\"\n}\n\ntestEmptyWithExpressionEvalAll() {\n rm test.yml\n touch test.yml\n expected=\"apple: tree\"\n\n X=$(./yq ea '.apple=\"tree\"' test.yml)\n\n assertEquals \"$expected\" \"$X\"\n}\n\n\nsource ./scripts/shunit2" }, { "path": "acceptance_tests/flags.sh", "content": "#!/bin/bash\n\nsetUp() {\n rm test*.yml || true\n cat >test.yml <test.yml <test.yml <test.yml <test.yml <test.yml </dev/null || true\n rm test*.toml 2>/dev/null || true\n rm test*.tfstate 2>/dev/null || true\n rm test*.json 2>/dev/null || true\n rm test*.properties 2>/dev/null || true\n rm test*.csv 2>/dev/null || true\n rm test*.tsv 2>/dev/null || true\n rm test*.xml 2>/dev/null || true\n}\n\ntestInputJson() {\n cat >test.json <test.toml <test.tfstate <test.json <test.properties <test.properties <test.csv <test.tsv <test.xml <BiBi\nEOL\n\n read -r -d '' expected << EOM\nBiBi\nEOM\n\n X=$(./yq e test.xml)\n assertEquals \"$expected\" \"$X\"\n\n X=$(./yq ea test.xml)\n assertEquals \"$expected\" \"$X\"\n}\n\ntestInputXmlNamespaces() {\n cat >test.xml <\n\n\nEOL\n\n read -r -d '' expected << EOM\n\n\nEOM\n\n X=$(./yq e test.xml)\n assertEquals \"$expected\" \"$X\"\n\n X=$(./yq ea test.xml)\n assertEquals \"$expected\" \"$X\"\n}\n\n\n\ntestInputXmlStrict() {\n cat >test.xml <\n\n\n]>\n\n &writer;©right;\n\nEOL\n\n X=$(./yq --xml-strict-mode test.xml 2>&1)\n assertEquals 1 $?\n assertEquals \"Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;\" \"$X\"\n\n X=$(./yq ea --xml-strict-mode test.xml 2>&1)\n assertEquals \"Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;\" \"$X\"\n}\n\ntestInputXmlGithubAction() {\n cat >test.xml <BiBi\nEOL\n\n read -r -d '' expected << EOM\nBiBi\nEOM\n\n X=$(cat /dev/null | ./yq e test.xml)\n assertEquals \"$expected\" \"$X\"\n\n X=$(cat /dev/null | ./yq ea test.xml)\n assertEquals \"$expected\" \"$X\"\n}\n\nsource ./scripts/shunit2\n" }, { "path": "acceptance_tests/inputs-format.sh", "content": "#!/bin/bash\n\nsetUp() {\n rm test*.yml 2>/dev/null || true\n rm test*.properties 2>/dev/null || true\n rm test*.csv 2>/dev/null || true\n rm test*.tsv 2>/dev/null || true\n rm test*.xml 2>/dev/null || true\n rm test*.tf 2>/dev/null || true\n}\n\ntestInputProperties() {\n cat >test.properties <test.properties <test.csv <test.csv <test.csv <test.tsv <test.tsv <test.kyaml <<'EOL'\n# leading\n{\n a: 1, # a line\n # head b\n b: 2,\n c: [\n # head d\n \"d\", # d line\n ],\n}\nEOL\n\n read -r -d '' expected <<'EOM'\n# leading\na: 1 # a line\n# head b\nb: 2\nc:\n # head d\n - d # d line\nEOM\n\n X=$(./yq e -p=kyaml -P test.kyaml)\n assertEquals \"$expected\" \"$X\"\n\n X=$(./yq ea -p=kyaml -P test.kyaml)\n assertEquals \"$expected\" \"$X\"\n}\n\n\n\n\ntestInputXml() {\n cat >test.yml <BiBi\nEOL\n\n read -r -d '' expected << EOM\ncat:\n +content: BiBi\n +@legs: \"4\"\nEOM\n\n X=$(./yq e -p=xml test.yml)\n assertEquals \"$expected\" \"$X\"\n\n X=$(./yq ea -p=xml test.yml)\n assertEquals \"$expected\" \"$X\"\n}\n\ntestInputXmlNamespaces() {\n cat >test.xml <\n\n\nEOL\n\n read -r -d '' expected << EOM\n+p_xml: version=\"1.0\"\nmap:\n +@xmlns: some-namespace\n +@xmlns:xsi: some-instance\n +@xsi:schemaLocation: some-url\nEOM\n\n X=$(./yq e -p=xml test.xml)\n assertEquals \"$expected\" \"$X\"\n\n X=$(./yq ea -p=xml test.xml)\n assertEquals \"$expected\" \"$X\"\n}\n\ntestInputXmlRoundtrip() {\n cat >test.yml <\n\nMeow\nEOL\n\n read -r -d '' expected << EOM\n\n\nMeow\nEOM\n\n X=$(./yq -p=xml -o=xml test.yml)\n assertEquals \"$expected\" \"$X\"\n\n X=$(./yq ea -p=xml -o=xml test.yml)\n assertEquals \"$expected\" \"$X\"\n}\n\n\ntestInputXmlStrict() {\n cat >test.yml <\n\n\n]>\n\n &writer;©right;\n\nEOL\n\n X=$(./yq -p=xml --xml-strict-mode test.yml -o=xml 2>&1)\n assertEquals 1 $?\n assertEquals \"Error: bad file 'test.yml': XML syntax error on line 7: invalid character entity &writer;\" \"$X\"\n\n X=$(./yq ea -p=xml --xml-strict-mode test.yml -o=xml 2>&1)\n assertEquals \"Error: bad file 'test.yml': XML syntax error on line 7: invalid character entity &writer;\" \"$X\"\n}\n\ntestInputXmlGithubAction() {\n cat >test.yml <BiBi\nEOL\n\n read -r -d '' expected << EOM\ncat:\n +content: BiBi\n +@legs: \"4\"\nEOM\n\n X=$(cat /dev/null | ./yq e -p=xml test.yml)\n assertEquals \"$expected\" \"$X\"\n\n X=$(cat /dev/null | ./yq ea -p=xml test.yml)\n assertEquals \"$expected\" \"$X\"\n}\n\ntestInputTerraform() {\n cat >test.tf <test.tf <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test2.yml <test.yml <test2.yml <test.yml <test2.yml <test.yml <test2.yml <test.yml <test2.yml <test.yml <test.yml <test.yml <test2.yml <test.yml <test2.yml <&1)\n assertEquals 1 $?\n assertEquals \"Error: failed to load cat.yml: open cat.yml: no such file or directory\" \"$result\"\n}\n\ntestLoadFileExpNotExist() {\n result=$(./yq e -n 'load(.a)' 2>&1)\n assertEquals 1 $?\n assertEquals \"Error: filename expression returned nil\" \"$result\"\n}\n\ntestStrLoadFileNotExist() {\n result=$(./yq e -n 'strload(\"cat.yml\")' 2>&1)\n assertEquals 1 $?\n assertEquals \"Error: failed to load cat.yml: open cat.yml: no such file or directory\" \"$result\"\n}\n\ntestStrLoadFileExpNotExist() {\n result=$(./yq e -n 'strload(.a)' 2>&1)\n assertEquals 1 $?\n assertEquals \"Error: filename expression returned nil\" \"$result\"\n}\n\nsource ./scripts/shunit2" }, { "path": "acceptance_tests/nul-separator.sh", "content": "#!/bin/bash\n\nsetUp() {\n rm test*.yml || true\n}\n\n## Convenient bash shortcut to read records of NUL separated values\n## from stdin the safe way. See example usage in the next tests.\nread-0() {\n local eof=\"\" IFS=''\n while [ \"$1\" ]; do\n ## - The `-r` avoids bad surprise with '\\n' and other interpreted\n ## sequences that can be read.\n ## - The `-d ''` is the (strange?) way to refer to NUL delimiter.\n ## - The `--` is how to avoid unpleasant surprises if your\n ## \"$1\" starts with \"-\" (minus) sign. This protection also\n ## will produce a readable error if you want to try to start\n ## your variable names with a \"-\".\n read -r -d '' -- \"$1\" || eof=1\n shift\n done\n [ -z \"$eof\" ] ## fail on EOF\n}\n\n## Convenient bash shortcut to be used with the next function `p-err`\n## to read NUL separated values the safe way AND catch any errors from\n## the process creating the stream of NUL separated data. See example\n## usage in the tests.\nread-0-err() {\n local ret=\"$1\" eof=\"\" idx=0 last=\n read -r -- \"${ret?}\" <<<\"0\"\n shift\n while [ \"$1\" ]; do\n last=$idx\n read -r -d '' -- \"$1\" || {\n ## Put this last value in ${!ret}\n eof=\"$1\"\n read -r -- \"$ret\" <<<\"${!eof}\"\n break\n }\n ((idx++))\n shift\n done\n [ -z \"$eof\" ] || {\n if [ \"$last\" != 0 ]; then\n ## Uhoh, we have no idea if the errorlevel of the internal\n ## command was properly delimited with a NUL char, and\n ## anyway something went really wrong at least about the\n ## number of fields separated by NUL char and the one\n ## expected.\n echo \"Error: read-0-err couldn't fill all value $ret = '${!ret}', '$eof', '${!eof}'\" >&2\n read -r -- \"$ret\" <<<\"not-enough-values\"\n else\n if ! [[ \"${!ret}\" =~ ^[0-9]+$ && \"${!ret}\" -ge 0 && \"${!ret}\" -le 127 ]]; then\n ## This could happen if you don't use `p-err` wrapper,\n ## or used stdout in unexpected ways in your inner\n ## command.\n echo \"Error: last value is not a number, did you finish with an errorlevel ?\" >&2\n read -r -- \"$ret\" <<<\"last-value-not-a-number\"\n fi\n fi\n false\n }\n}\n\n## Simply runs command given as argument and adds errorlevel in the\n## standard output. Is expected to be used in tandem with\n## `read-0-err`.\np-err() {\n local exp=\"$1\"\n \"$@\"\n printf \"%s\" \"$?\"\n}\n\nwyq-r() {\n local exp=\"$1\"\n ./yq e -0 -r=false \"$1\"\n printf \"%s\" \"$?\"\n}\n\ntestBasicUsageRaw() {\n cat >test.yml < expected.out\n\n ## We need to compare binary content here. We have to filter the compared\n ## content through a representation that gets rid of NUL chars but accurately\n ## transcribe the content.\n ## Also as it would be nice to have a pretty output in case the test fails,\n ## we use here 'hd': a widely available shortcut to 'hexdump' that will\n ## pretty-print any binary to it's hexadecimal representation.\n ##\n ## Note that the standard `assertEquals` compare its arguments\n ## value, but they can't hold NUL characters (this comes from the\n ## limitation of the C API of `exec*(..)` functions that requires\n ## `const char *arv[]`). And these are NUL terminated strings. As a\n ## consequence, the NUL characters gets removed in bash arguments.\n assertEquals \"$(hd expected.out)\" \\\n \"$(./yq e -0 '.a, .b' test.yml | hd)\"\n\n rm expected.out\n}\n\ntestBasicUsage() {\n local a b\n cat >test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <\n \n cat\n \n\nEOM\n\n X=$(./yq e --output-format=xml test.yml)\n assertEquals \"$expected\" \"$X\"\n\n X=$(./yq ea --output-format=xml test.yml)\n assertEquals \"$expected\" \"$X\"\n}\n\ntestOutputXmlShort() {\n cat >test.yml <\n \n cat\n \n\nEOM\n\n X=$(./yq e --output-format=x test.yml)\n assertEquals \"$expected\" \"$X\"\n\n X=$(./yq ea --output-format=x test.yml)\n assertEquals \"$expected\" \"$X\"\n}\n\ntestOutputKYaml() {\n cat >test.yml <<'EOL'\n# leading\na: 1 # a line\n# head b\nb: 2\nc:\n # head d\n - d # d line\nEOL\n\n read -r -d '' expected <<'EOM'\n# leading\n{\n a: 1, # a line\n # head b\n b: 2,\n c: [\n # head d\n \"d\", # d line\n ],\n}\nEOM\n\n X=$(./yq e --output-format=kyaml test.yml)\n assertEquals \"$expected\" \"$X\"\n\n X=$(./yq ea --output-format=kyaml test.yml)\n assertEquals \"$expected\" \"$X\"\n}\n\ntestOutputKYamlShort() {\n cat >test.yml <test.yml <\n \n cat\n dog\n \n\nEOM\n\n X=$(./yq e --output-format=x test.yml)\n assertEquals \"$expected\" \"$X\"\n\n X=$(./yq ea --output-format=x test.yml)\n assertEquals \"$expected\" \"$X\"\n}\n\ntestLuaOutputPretty() {\n cat >test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yq <test.yml <test.yml <test.yml <test.yml <test_splitExp.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test2.yml <test.yml <.go` implementing the `Encoder` interface:\n- `Encode()` - Convert a `CandidateNode` to your format and write to the output writer\n- `PrintDocumentSeparator()` - Handle document separators if your format requires them\n- `PrintLeadingContent()` - Handle leading content/comments if supported\n- `CanHandleAliases()` - Return whether your format supports YAML aliases\n\nSee `encoder_json.go` or `encoder_base64.go` for examples.\n\n### Step 2: Create the Decoder File\n\nCreate `pkg/yqlib/decoder_.go` implementing the `Decoder` interface:\n- `Init()` - Initialize the decoder with the input reader and set up any needed state\n- `Decode()` - Decode one document from the input and return a `CandidateNode`, or `io.EOF` when finished\n\nSee `decoder_json.go` or `decoder_base64.go` for examples.\n\n### Step 3: Create Tests (Mandatory)\n\nCreate a test file `pkg/yqlib/_test.go` using the `formatScenario` pattern:\n- Define test scenarios as `formatScenario` structs with fields: `description`, `input`, `expected`, `scenarioType`\n- `scenarioType` can be `\"decode\"` (test decoding to YAML) or `\"roundtrip\"` (encode/decode preservation)\n- Create a helper function `testScenario()` that switches on `scenarioType`\n- Create main test function `TestFormatScenarios()` that iterates over scenarios\n- The main test function should use `documentScenarios` to ensure testcase documentation is generated.\n\nTest coverage must include:\n- Basic data types (scalars, arrays, objects/maps)\n- Nested structures\n- Edge cases (empty inputs, special characters, escape sequences)\n- Format-specific features or syntax\n- Round-trip tests: decode → encode → decode should preserve data\n\nSee `hcl_test.go` for a complete example.\n\n### Step 4: Register the Format in format.go\n\nEdit `pkg/yqlib/format.go`:\n\n1. Add a new format variable:\n - `\"\"` is the formal name (e.g., \"json\", \"yaml\")\n - `[]string{...}` contains short aliases (can be empty)\n - The first function creates an encoder (can be nil for encode-only formats)\n - The second function creates a decoder (can be nil for decode-only formats)\n\n2. Add the format to the `Formats` slice in the same file\n\nSee existing formats in `format.go` for the exact structure.\n\n### Step 5: Handle Encoder Configuration (if needed)\n\nIf your format has preferences/configuration options:\n\n1. Create a preferences struct with your configuration fields\n2. Update the encoder to accept preferences in its factory function\n3. Update `format.go` to pass the configured preferences\n4. Update `operator_encoder_decoder.go` if special indent handling is needed (see existing formats like JSON and YAML for the pattern)\n\nThis pattern is optional and only needed if your format has user-configurable options.\n\n## Build Tags\n\nUse build tags to allow optional compilation of formats:\n- Add `//go:build !yq_no` at the top of your encoder and decoder files\n- Create a no-build version in `pkg/yqlib/no_.go` that returns nil for encoder/decoder factories\n\nThis allows users to compile yq without certain formats using: `go build -tags yq_no`\n\n## Working with CandidateNode\n\nThe `CandidateNode` struct represents a YAML node with:\n- `Kind`: The node type (ScalarNode, SequenceNode, MappingNode)\n- `Tag`: The YAML tag (e.g., \"!!str\", \"!!int\", \"!!map\")\n- `Value`: The scalar value (for ScalarNode only)\n- `Content`: Child nodes (for SequenceNode and MappingNode)\n\nKey methods:\n- `node.guessTagFromCustomType()` - Infer the tag from Go type\n- `node.AsList()` - Convert to a list for processing\n- `node.CreateReplacement()` - Create a new replacement node\n- `NewCandidate()` - Create a new CandidateNode\n\n## Key Points\n\n✅ **DO:**\n- Implement only the `Encoder` and `Decoder` interfaces\n- Register your format in `format.go` only\n- Keep format-specific logic in your encoder/decoder files\n- Use the candidate_node style attribute to store style information for round-trip. Ask if this needs to be updated with new styles.\n- Use build tags for optional compilation\n- Add comprehensive tests\n- Run the specific encoder/decoder test (e.g. _test.go) whenever you make ay changes to the encoder_ or decoder_\n- Handle errors gracefully\n- Add the no build directive, like the xml encoder and decoder, that enables a minimal yq builds. e.g. `//go:build !yq_`. Be sure to also update the build_small-yq.sh and build-tinygo-yq.sh to not include the new format.\n\n❌ **DON'T:**\n- Modify `candidate_node.go` to add format-specific logic\n- Add format-specific fields to `CandidateNode`\n- Create special cases in core navigation or evaluation logic\n- Bypass the encoder/decoder interfaces\n- Use candidate_node tag attribute for anything other than indicate the data type\n\n## Examples\n\nRefer to existing format implementations for patterns:\n\n- **Simple encoder/decoder**: `encoder_json.go`, `decoder_json.go`\n- **Complex with preferences**: `encoder_yaml.go`, `decoder_yaml.go`\n- **Encoder-only**: `encoder_sh.go` (ShFormat has nil decoder)\n- **String-only operations**: `encoder_base64.go`, `decoder_base64.go`\n\n## Testing Your Implementation (Mandatory)\n\nTests must be implemented in `_test.go` following the `formatScenario` pattern:\n\n1. **Create test scenarios** using the `formatScenario` struct with fields:\n - `description`: Brief description of what's being tested\n - `input`: Sample input in your format\n - `expected`: Expected output (typically in YAML for decode tests)\n - `scenarioType`: Either `\"decode\"` or `\"roundtrip\"`\n\n2. **Test coverage must include:**\n - Basic data types (scalars, arrays, objects/maps)\n - Nested structures\n - Edge cases (empty inputs, special characters, escape sequences)\n - Format-specific features or syntax\n - Round-trip tests: decode → encode → decode should preserve data\n\n3. **Test function pattern:**\n - `testScenario()`: Helper function that switches on `scenarioType`\n - `TestFormatScenarios()`: Main test function that iterates over scenarios\n\n4. **Example from existing formats:**\n - See `hcl_test.go` for a complete example\n - See `yaml_test.go` for YAML-specific patterns\n - See `json_test.go` for more complex scenarios\n\n## Common Patterns\n\n### Format with Indentation\nUse preferences to control output formatting:\n```go\ntype Preferences struct {\n Indent int\n}\n\nfunc (prefs *Preferences) Copy() Preferences {\n return *prefs\n}\n```\n\n### Multiple Documents\nDecoders should support reading multiple documents:\n```go\nfunc (dec *Decoder) Decode() (*CandidateNode, error) {\n if dec.finished {\n return nil, io.EOF\n }\n // ... decode next document ...\n if noMoreDocuments {\n dec.finished = true\n }\n return candidate, nil\n}\n```\n\n---\n\n# Adding a New Operator\n\nThis guide explains how to add a new operator to yq. Operators are the core of yq's expression language and process `CandidateNode` objects without requiring modifications to `candidate_node.go` itself.\n\n## Overview\n\nOperators transform data by implementing a handler function that processes a `Context` containing `CandidateNode` objects. Each operator is:\n\n1. Defined as an `operationType` in `operation.go`\n2. Registered in the lexer in `lexer_participle.go`\n3. Implemented in its own `operator_.go` file\n4. Tested in `operator__test.go`\n5. Documented in `pkg/yqlib/doc/operators/headers/.md`\n\n## Architecture\n\n### Key Files\n\n- `pkg/yqlib/operation.go` - Defines `operationType` and operator registry\n- `pkg/yqlib/lexer_participle.go` - Registers operators with their syntax patterns\n- `pkg/yqlib/operator_.go` - Operator implementation\n- `pkg/yqlib/operator__test.go` - Operator tests using `expressionScenario`\n- `pkg/yqlib/doc/operators/headers/.md` - Documentation header\n\n### Core Types\n\n**operationType:**\n```go\ntype operationType struct {\n Type string // Unique operator name (e.g., \"REVERSE\")\n NumArgs uint // Number of arguments (0 for no args)\n Precedence uint // Operator precedence (higher = higher precedence)\n Handler operatorHandler // The function that executes the operator\n CheckForPostTraverse bool // Whether to apply post-traversal logic\n ToString func(*Operation) string // Custom string representation\n}\n```\n\n**operatorHandler signature:**\n```go\ntype operatorHandler func(*dataTreeNavigator, Context, *ExpressionNode) (Context, error)\n```\n\n**expressionScenario for tests:**\n```go\ntype expressionScenario struct {\n description string\n subdescription string\n document string\n expression string\n expected []string\n skipDoc bool\n expectedError string\n}\n```\n\n## Step-by-Step: Adding a New Operator\n\n### Step 1: Create the Operator Implementation File\n\nCreate `pkg/yqlib/operator_.go` implementing the operator handler function:\n- Implement the `operatorHandler` function signature\n- Process nodes from `context.MatchingNodes`\n- Return a new `Context` with results using `context.ChildContext()`\n- Use `candidate.CreateReplacement()` or `candidate.CreateReplacementWithComments()` to create new nodes\n- Handle errors gracefully with meaningful error messages\n\nSee `operator_reverse.go` or `operator_keys.go` for examples.\n\n### Step 2: Register the Operator in operation.go\n\nAdd the operator type definition to `pkg/yqlib/operation.go`:\n\n```go\nvar OpType = &operationType{\n Type: \"\", // All caps, matches pattern in lexer\n NumArgs: 0, // 0 for no args, 1+ for args\n Precedence: 50, // Typical range: 40-55\n Handler: Operator, // Reference to handler function\n}\n```\n\n**Precedence guidelines:**\n- 10-20: Logical operators (OR, AND, UNION)\n- 30: Pipe operator\n- 40: Assignment and comparison operators\n- 42: Arithmetic operators (ADD, SUBTRACT, MULTIPLY, DIVIDE)\n- 50-52: Most other operators\n- 55: High precedence (e.g., GET_VARIABLE)\n\n**Optional fields:**\n- `CheckForPostTraverse: true` - If your operator can have another directly after it without the pipe character. Most of the time this is false.\n- `ToString: customToString` - Custom string representation (rarely needed)\n\n### Step 3: Register the Operator in lexer_participle.go\n\nEdit `pkg/yqlib/lexer_participle.go` to add the operator to the lexer rules:\n- Use `simpleOp()` for simple keyword patterns\n- Use object syntax for regex patterns or complex syntax\n- Support optional characters with `_?` and aliases with `|`\n\nSee existing operators in `lexer_participle.go` for pattern examples.\n\n### Step 4: Create Tests (Mandatory)\n\nCreate `pkg/yqlib/operator__test.go` using the `expressionScenario` pattern:\n- Define test scenarios with `description`, `document`, `expression`, and `expected` fields\n- `expected` is a slice of strings showing output format: `\"D, P[], ()::\\n\"`\n- Set `skipDoc: true` for edge cases you don't want in generated documentation\n- Include `subdescription` for longer test names\n- Set `expectedError` if testing error cases\n- Create main test function that iterates over scenarios\n- The main test function should use `documentScenarios` to ensure testcase documentation is generated.\n\nTest coverage must include:\n- Basic data types and nested structures\n- Edge cases (empty inputs, special characters, type errors)\n- Multiple outputs if applicable\n- Format-specific features\n\nSee `operator_reverse_test.go` for a simple example and `operator_keys_test.go` for complex cases.\n\n### Step 5: Create Documentation Header\n\nCreate `pkg/yqlib/doc/operators/headers/.md`:\n- Use the exact operator name as the title\n- Include a concise 1-2 sentence summary\n- Add additional context or examples if the operator is complex\n\nSee existing headers in `doc/operators/headers/` for examples.\n\n## Working with Context and CandidateNode\n\n### Context Management\n- `context.ChildContext(results)` - Create child context with results\n- `context.GetVariable(\"varName\")` - Get variables stored in context\n- `context.SetVariable(\"varName\", value)` - Set variables in context\n\n### CandidateNode Operations\n- `candidate.CreateReplacement(ScalarNode, \"!!str\", stringValue)` - Create a replacement node\n- `candidate.CreateReplacementWithComments(SequenceNode, \"!!seq\", candidate.Style)` - With style preserved\n- `candidate.Kind` - The node type (ScalarNode, SequenceNode, MappingNode)\n- `candidate.Tag` - The YAML tag (!!str, !!int, etc.)\n- `candidate.Value` - The scalar value (for ScalarNode only)\n- `candidate.Content` - Child nodes (for SequenceNode and MappingNode)\n- `candidate.guessTagFromCustomType()` - Infer the tag from Go type\n- `candidate.AsList()` - Convert to a list representation\n\n## Key Points\n\n✅ **DO:**\n- Implement the operator handler with the correct signature\n- Register in `operation.go` with appropriate precedence\n- Add the lexer pattern in `lexer_participle.go`\n- Write comprehensive tests covering normal and edge cases\n- Create a documentation header in `doc/operators/headers/`\n- Use `Context.ChildContext()` for proper context threading\n- Handle all node types gracefully\n- Return meaningful error messages\n\n❌ **DON'T:**\n- Modify `candidate_node.go` (operators shouldn't need this)\n- Modify core navigation or evaluation logic\n- Bypass the handler function pattern\n- Add format-specific or operator-specific fields to `CandidateNode`\n- Skip tests or documentation\n\n## Examples\n\nRefer to existing operator implementations for patterns:\n\n- **No-argument operator**: `operator_reverse.go` - Processes arrays/sequences\n- **Single-argument operator**: `operator_map.go` - Takes an expression argument\n- **Complex multi-output**: `operator_keys.go` - Produces multiple results\n- **With preferences**: `operator_to_number.go` - Configuration options\n- **Error handling**: `operator_error.go` - Control flow with errors\n- **String operations**: `operator_strings.go` - Multiple related operators\n\n## Testing Patterns\n\nRefer to existing test files for specific patterns:\n- Basic expression tests in `operator_reverse_test.go`\n- Multi-output tests in `operator_keys_test.go`\n- Error handling tests in `operator_error_test.go`\n- Tests with `skipDoc` flag to exclude from generated documentation\n\n## Common Patterns\n\nRefer to existing operator implementations for these patterns:\n- Simple transformation: see `operator_reverse.go`\n- Type checking: see `operator_error.go`\n- Working with arguments: see `operator_map.go`\n- Post-traversal operators: see `operator_with.go`\n" }, { "path": "cmd/completion.go", "content": "package cmd\n\nimport (\n\t\"os\"\n\n\t\"github.com/spf13/cobra\"\n)\n\nvar completionCmd = &cobra.Command{\n\tUse: \"completion [bash|zsh|fish|powershell]\",\n\tAliases: []string{\"shell-completion\"},\n\tShort: \"Generate the autocompletion script for the specified shell\",\n\tLong: `To load completions:\n\nBash:\n\n$ source <(yq completion bash)\n\n# To load completions for each session, execute once:\nLinux:\n $ yq completion bash > /etc/bash_completion.d/yq\nMacOS:\n $ yq completion bash > /usr/local/etc/bash_completion.d/yq\n\nZsh:\n\n# If shell completion is not already enabled in your environment you will need\n# to enable it. You can execute the following once:\n\n$ echo \"autoload -U compinit; compinit\" >> ~/.zshrc\n\n# To load completions for each session, execute once:\n$ yq completion zsh > \"${fpath[1]}/_yq\"\n\n# You will need to start a new shell for this setup to take effect.\n\nFish:\n\n$ yq completion fish | source\n\n# To load completions for each session, execute once:\n$ yq completion fish > ~/.config/fish/completions/yq.fish\n`,\n\tDisableFlagsInUseLine: true,\n\tValidArgs: []string{\"bash\", \"zsh\", \"fish\", \"powershell\"},\n\tArgs: cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs),\n\tRunE: func(cmd *cobra.Command, args []string) error {\n\t\tvar err error = nil\n\t\tswitch args[0] {\n\t\tcase \"bash\":\n\t\t\terr = cmd.Root().GenBashCompletionV2(os.Stdout, true)\n\t\tcase \"zsh\":\n\t\t\terr = cmd.Root().GenZshCompletion(os.Stdout)\n\t\tcase \"fish\":\n\t\t\terr = cmd.Root().GenFishCompletion(os.Stdout, true)\n\t\tcase \"powershell\":\n\t\t\terr = cmd.Root().GenPowerShellCompletion(os.Stdout)\n\t\t}\n\t\treturn err\n\n\t},\n}\n" }, { "path": "cmd/constant.go", "content": "package cmd\n\nvar unwrapScalarFlag = newUnwrapFlag()\n\nvar printNodeInfo = false\n\nvar unwrapScalar = false\n\nvar writeInplace = false\nvar outputToJSON = false\n\nvar outputFormat = \"\"\n\nvar inputFormat = \"\"\n\nvar exitStatus = false\nvar indent = 2\nvar noDocSeparators = false\nvar nullInput = false\nvar nulSepOutput = false\nvar verbose = false\nvar version = false\nvar prettyPrint = false\n\nvar forceColor = false\nvar forceNoColor = false\nvar colorsEnabled = false\n\n// can be either \"\" (off), \"extract\" or \"process\"\nvar frontMatter = \"\"\n\nvar splitFileExp = \"\"\nvar splitFileExpFile = \"\"\n\nvar completedSuccessfully = false\n\nvar forceExpression = \"\"\n\nvar expressionFile = \"\"\n" }, { "path": "cmd/evaluate_all_command.go", "content": "package cmd\n\nimport (\n\t\"errors\"\n\n\t\"github.com/mikefarah/yq/v4/pkg/yqlib\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc createEvaluateAllCommand() *cobra.Command {\n\tvar cmdEvalAll = &cobra.Command{\n\t\tUse: \"eval-all [expression] [yaml_file1]...\",\n\t\tAliases: []string{\"ea\"},\n\t\tShort: \"Loads _all_ yaml documents of _all_ yaml files and runs expression once\",\n\t\tValidArgsFunction: func(_ *cobra.Command, args []string, _ string) ([]string, cobra.ShellCompDirective) {\n\t\t\tif len(args) == 0 {\n\t\t\t\treturn nil, cobra.ShellCompDirectiveNoFileComp\n\t\t\t}\n\t\t\treturn nil, cobra.ShellCompDirectiveDefault\n\t\t},\n\t\tExample: `\n# Merge f2.yml into f1.yml (in place)\nyq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' f1.yml f2.yml\n## the same command and expression using shortened names:\nyq ea -i 'select(fi == 0) * select(fi == 1)' f1.yml f2.yml\n\n\n# Merge all given files\nyq ea '. as $item ireduce ({}; . * $item )' file1.yml file2.yml ...\n\n# Pipe from STDIN\n## use '-' as a filename to pipe from STDIN\ncat file2.yml | yq ea '.a.b' file1.yml - file3.yml\n`,\n\t\tLong: `yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) \nSee https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.\n\n## Evaluate All ##\nThis command loads _all_ yaml documents of _all_ yaml files and runs expression once\nUseful when you need to run an expression across several yaml documents or files (like merge).\nNote that it consumes more memory than eval.\n`,\n\t\tRunE: evaluateAll,\n\t}\n\treturn cmdEvalAll\n}\nfunc evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {\n\t// 0 args, read std in\n\t// 1 arg, null input, process expression\n\t// 1 arg, read file in sequence\n\t// 2+ args, [0] = expression, file the rest\n\n\tvar err error\n\n\texpression, args, err := initCommand(cmd, args)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tout := cmd.OutOrStdout()\n\n\tif writeInplace {\n\t\t// only use colours if its forced\n\t\tcolorsEnabled = forceColor\n\t\twriteInPlaceHandler := yqlib.NewWriteInPlaceHandler(args[0])\n\t\tout, err = writeInPlaceHandler.CreateTempFile()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// need to indirectly call the function so that completedSuccessfully is\n\t\t// passed when we finish execution as opposed to now\n\t\tdefer func() {\n\t\t\tif cmdError == nil {\n\t\t\t\tcmdError = writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully)\n\t\t\t}\n\t\t}()\n\t}\n\n\tformat, err := yqlib.FormatFromString(outputFormat)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdecoder, err := configureDecoder(true)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tprinterWriter, err := configurePrinterWriter(format, out)\n\tif err != nil {\n\t\treturn err\n\t}\n\tencoder, err := configureEncoder()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tprinter := yqlib.NewPrinter(encoder, printerWriter)\n\tif nulSepOutput {\n\t\tprinter.SetNulSepOutput(true)\n\t}\n\n\tif frontMatter != \"\" {\n\t\tfrontMatterHandler := yqlib.NewFrontMatterHandler(args[0])\n\t\terr = frontMatterHandler.Split()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\targs[0] = frontMatterHandler.GetYamlFrontMatterFilename()\n\n\t\tif frontMatter == \"process\" {\n\t\t\treader := frontMatterHandler.GetContentReader()\n\t\t\tprinter.SetAppendix(reader)\n\t\t\tdefer yqlib.SafelyCloseReader(reader)\n\t\t}\n\t\tdefer frontMatterHandler.CleanUp()\n\t}\n\n\tallAtOnceEvaluator := yqlib.NewAllAtOnceEvaluator()\n\n\tswitch len(args) {\n\tcase 0:\n\t\tif nullInput {\n\t\t\terr = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(expression), printer)\n\t\t} else {\n\t\t\tcmd.Println(cmd.UsageString())\n\t\t\treturn nil\n\t\t}\n\tdefault:\n\t\terr = allAtOnceEvaluator.EvaluateFiles(processExpression(expression), args, printer, decoder)\n\t}\n\n\tcompletedSuccessfully = err == nil\n\n\tif err == nil && exitStatus && !printer.PrintedAnything() {\n\t\treturn errors.New(\"no matches found\")\n\t}\n\n\treturn err\n}\n" }, { "path": "cmd/evaluate_all_command_test.go", "content": "package cmd\n\nimport (\n\t\"bytes\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc TestCreateEvaluateAllCommand(t *testing.T) {\n\tcmd := createEvaluateAllCommand()\n\n\tif cmd == nil {\n\t\tt.Fatal(\"createEvaluateAllCommand returned nil\")\n\t}\n\n\t// Test basic command properties\n\tif cmd.Use != \"eval-all [expression] [yaml_file1]...\" {\n\t\tt.Errorf(\"Expected Use to be 'eval-all [expression] [yaml_file1]...', got %q\", cmd.Use)\n\t}\n\n\tif cmd.Short == \"\" {\n\t\tt.Error(\"Expected Short description to be non-empty\")\n\t}\n\n\tif cmd.Long == \"\" {\n\t\tt.Error(\"Expected Long description to be non-empty\")\n\t}\n\n\t// Test aliases\n\texpectedAliases := []string{\"ea\"}\n\tif len(cmd.Aliases) != len(expectedAliases) {\n\t\tt.Errorf(\"Expected %d aliases, got %d\", len(expectedAliases), len(cmd.Aliases))\n\t}\n\n\tfor i, expected := range expectedAliases {\n\t\tif i >= len(cmd.Aliases) || cmd.Aliases[i] != expected {\n\t\t\tt.Errorf(\"Expected alias %d to be %q, got %q\", i, expected, cmd.Aliases[i])\n\t\t}\n\t}\n}\n\nfunc TestEvaluateAll_NoArgs(t *testing.T) {\n\t// Create a temporary command\n\tcmd := createEvaluateAllCommand()\n\n\t// Set up command to capture output\n\tvar output bytes.Buffer\n\tcmd.SetOut(&output)\n\n\t// Test with no arguments and no null input\n\tnullInput = false\n\tdefer func() { nullInput = false }()\n\n\terr := evaluateAll(cmd, []string{})\n\n\t// Should not error, but should print usage\n\tif err != nil {\n\t\tt.Errorf(\"evaluateAll with no args should not error, got: %v\", err)\n\t}\n\n\t// Should have printed usage information\n\tif output.Len() == 0 {\n\t\tt.Error(\"Expected usage information to be printed\")\n\t}\n}\n\nfunc TestEvaluateAll_NullInput(t *testing.T) {\n\t// Create a temporary command\n\tcmd := createEvaluateAllCommand()\n\n\t// Set up command to capture output\n\tvar output bytes.Buffer\n\tcmd.SetOut(&output)\n\n\t// Test with null input\n\tnullInput = true\n\tdefer func() { nullInput = false }()\n\n\terr := evaluateAll(cmd, []string{})\n\n\t// Should not error when using null input\n\tif err != nil {\n\t\tt.Errorf(\"evaluateAll with null input should not error, got: %v\", err)\n\t}\n}\n\nfunc TestEvaluateAll_WithSingleFile(t *testing.T) {\n\t// Create a temporary YAML file\n\ttempDir := t.TempDir()\n\tyamlFile := filepath.Join(tempDir, \"test.yaml\")\n\tyamlContent := []byte(\"name: test\\nage: 25\\n\")\n\terr := os.WriteFile(yamlFile, yamlContent, 0600)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create test YAML file: %v\", err)\n\t}\n\n\t// Create a temporary command\n\tcmd := createEvaluateAllCommand()\n\n\t// Set up command to capture output\n\tvar output bytes.Buffer\n\tcmd.SetOut(&output)\n\n\t// Test with a single file\n\terr = evaluateAll(cmd, []string{yamlFile})\n\n\t// Should not error\n\tif err != nil {\n\t\tt.Errorf(\"evaluateAll with single file should not error, got: %v\", err)\n\t}\n\n\t// Should have some output\n\tif output.Len() == 0 {\n\t\tt.Error(\"Expected output from evaluateAll with single file\")\n\t}\n}\n\nfunc TestEvaluateAll_WithMultipleFiles(t *testing.T) {\n\t// Create temporary YAML files\n\ttempDir := t.TempDir()\n\n\tyamlFile1 := filepath.Join(tempDir, \"test1.yaml\")\n\tyamlContent1 := []byte(\"name: test1\\nage: 25\\n\")\n\terr := os.WriteFile(yamlFile1, yamlContent1, 0600)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create test YAML file 1: %v\", err)\n\t}\n\n\tyamlFile2 := filepath.Join(tempDir, \"test2.yaml\")\n\tyamlContent2 := []byte(\"name: test2\\nage: 30\\n\")\n\terr = os.WriteFile(yamlFile2, yamlContent2, 0600)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create test YAML file 2: %v\", err)\n\t}\n\n\t// Create a temporary command\n\tcmd := createEvaluateAllCommand()\n\n\t// Set up command to capture output\n\tvar output bytes.Buffer\n\tcmd.SetOut(&output)\n\n\t// Test with multiple files\n\terr = evaluateAll(cmd, []string{yamlFile1, yamlFile2})\n\n\t// Should not error\n\tif err != nil {\n\t\tt.Errorf(\"evaluateAll with multiple files should not error, got: %v\", err)\n\t}\n\n\t// Should have output\n\tif output.Len() == 0 {\n\t\tt.Error(\"Expected output from evaluateAll with multiple files\")\n\t}\n}\n\nfunc TestEvaluateAll_WithExpression(t *testing.T) {\n\t// Create a temporary YAML file\n\ttempDir := t.TempDir()\n\tyamlFile := filepath.Join(tempDir, \"test.yaml\")\n\tyamlContent := []byte(\"name: test\\nage: 25\\n\")\n\terr := os.WriteFile(yamlFile, yamlContent, 0600)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create test YAML file: %v\", err)\n\t}\n\n\t// Create a temporary command\n\tcmd := createEvaluateAllCommand()\n\n\t// Set up command to capture output\n\tvar output bytes.Buffer\n\tcmd.SetOut(&output)\n\n\t// Test with expression\n\terr = evaluateAll(cmd, []string{\".name\", yamlFile})\n\n\t// Should not error\n\tif err != nil {\n\t\tt.Errorf(\"evaluateAll with expression should not error, got: %v\", err)\n\t}\n\n\t// Should have output\n\tif output.Len() == 0 {\n\t\tt.Error(\"Expected output from evaluateAll with expression\")\n\t}\n}\n\nfunc TestEvaluateAll_WriteInPlace(t *testing.T) {\n\t// Create a temporary YAML file\n\ttempDir := t.TempDir()\n\tyamlFile := filepath.Join(tempDir, \"test.yaml\")\n\tyamlContent := []byte(\"name: test\\nage: 25\\n\")\n\terr := os.WriteFile(yamlFile, yamlContent, 0600)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create test YAML file: %v\", err)\n\t}\n\n\t// Create a temporary command\n\tcmd := createEvaluateAllCommand()\n\n\t// Set up command to capture output\n\tvar output bytes.Buffer\n\tcmd.SetOut(&output)\n\n\t// Enable write in place\n\toriginalWriteInplace := writeInplace\n\twriteInplace = true\n\tdefer func() { writeInplace = originalWriteInplace }()\n\n\t// Test with write in place\n\terr = evaluateAll(cmd, []string{\".name = \\\"updated\\\"\", yamlFile})\n\n\t// Should not error\n\tif err != nil {\n\t\tt.Errorf(\"evaluateAll with write in place should not error, got: %v\", err)\n\t}\n\n\t// Verify the file was updated\n\tupdatedContent, err := os.ReadFile(yamlFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to read updated file: %v\", err)\n\t}\n\n\t// Should contain the updated content\n\tif !strings.Contains(string(updatedContent), \"updated\") {\n\t\tt.Errorf(\"Expected file to contain 'updated', got: %s\", string(updatedContent))\n\t}\n}\n\nfunc TestEvaluateAll_ExitStatus(t *testing.T) {\n\t// Create a temporary YAML file\n\ttempDir := t.TempDir()\n\tyamlFile := filepath.Join(tempDir, \"test.yaml\")\n\tyamlContent := []byte(\"name: test\\nage: 25\\n\")\n\terr := os.WriteFile(yamlFile, yamlContent, 0600)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create test YAML file: %v\", err)\n\t}\n\n\t// Create a temporary command\n\tcmd := createEvaluateAllCommand()\n\n\t// Set up command to capture output\n\tvar output bytes.Buffer\n\tcmd.SetOut(&output)\n\n\t// Enable exit status\n\toriginalExitStatus := exitStatus\n\texitStatus = true\n\tdefer func() { exitStatus = originalExitStatus }()\n\n\t// Test with expression that should find no matches\n\terr = evaluateAll(cmd, []string{\".nonexistent\", yamlFile})\n\n\t// Should error when no matches found and exit status is enabled\n\tif err == nil {\n\t\tt.Error(\"Expected error when no matches found and exit status is enabled\")\n\t}\n}\n\nfunc TestEvaluateAll_WithMultipleDocuments(t *testing.T) {\n\t// Create a temporary YAML file with multiple documents\n\ttempDir := t.TempDir()\n\tyamlFile := filepath.Join(tempDir, \"test.yaml\")\n\tyamlContent := []byte(\"---\\nname: doc1\\nage: 25\\n---\\nname: doc2\\nage: 30\\n\")\n\terr := os.WriteFile(yamlFile, yamlContent, 0600)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create test YAML file: %v\", err)\n\t}\n\n\t// Create a temporary command\n\tcmd := createEvaluateAllCommand()\n\n\t// Set up command to capture output\n\tvar output bytes.Buffer\n\tcmd.SetOut(&output)\n\n\t// Test with multiple documents\n\terr = evaluateAll(cmd, []string{\".\", yamlFile})\n\n\t// Should not error\n\tif err != nil {\n\t\tt.Errorf(\"evaluateAll with multiple documents should not error, got: %v\", err)\n\t}\n\n\t// Should have output\n\tif output.Len() == 0 {\n\t\tt.Error(\"Expected output from evaluateAll with multiple documents\")\n\t}\n}\n\nfunc TestEvaluateAll_NulSepOutput(t *testing.T) {\n\t// Create a temporary YAML file\n\ttempDir := t.TempDir()\n\tyamlFile := filepath.Join(tempDir, \"test.yaml\")\n\tyamlContent := []byte(\"name: test\\nage: 25\\n\")\n\terr := os.WriteFile(yamlFile, yamlContent, 0600)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create test YAML file: %v\", err)\n\t}\n\n\t// Create a temporary command\n\tcmd := createEvaluateAllCommand()\n\n\t// Set up command to capture output\n\tvar output bytes.Buffer\n\tcmd.SetOut(&output)\n\n\t// Enable nul separator output\n\toriginalNulSepOutput := nulSepOutput\n\tnulSepOutput = true\n\tdefer func() { nulSepOutput = originalNulSepOutput }()\n\n\t// Test with nul separator output\n\terr = evaluateAll(cmd, []string{\".name\", yamlFile})\n\n\t// Should not error\n\tif err != nil {\n\t\tt.Errorf(\"evaluateAll with nul separator output should not error, got: %v\", err)\n\t}\n\n\t// Should have output\n\tif output.Len() == 0 {\n\t\tt.Error(\"Expected output from evaluateAll with nul separator output\")\n\t}\n}\n" }, { "path": "cmd/evaluate_sequence_command.go", "content": "package cmd\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\n\t\"github.com/mikefarah/yq/v4/pkg/yqlib\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc createEvaluateSequenceCommand() *cobra.Command {\n\tvar cmdEvalSequence = &cobra.Command{\n\t\tUse: \"eval [expression] [yaml_file1]...\",\n\t\tAliases: []string{\"e\"},\n\t\tShort: \"(default) Apply the expression to each document in each yaml file in sequence\",\n\t\tValidArgsFunction: func(_ *cobra.Command, args []string, _ string) ([]string, cobra.ShellCompDirective) {\n\t\t\tif len(args) == 0 {\n\t\t\t\treturn nil, cobra.ShellCompDirectiveNoFileComp\n\t\t\t}\n\t\t\treturn nil, cobra.ShellCompDirectiveDefault\n\t\t},\n\t\tExample: `\n# Reads field under the given path for each file\nyq e '.a.b' f1.yml f2.yml \n\n# Prints out the file\nyq e sample.yaml \n\n# Pipe from STDIN\n## use '-' as a filename to pipe from STDIN\ncat file2.yml | yq e '.a.b' file1.yml - file3.yml\n\n# Creates a new yaml document\n## Note that editing an empty file does not work.\nyq e -n '.a.b.c = \"cat\"' \n\n# Update a file in place\nyq e '.a.b = \"cool\"' -i file.yaml \n`,\n\t\tLong: `yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) \nSee https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.\n\n## Evaluate Sequence ##\nThis command iterates over each yaml document from each given file, applies the \nexpression and prints the result in sequence.`,\n\t\tRunE: evaluateSequence,\n\t}\n\treturn cmdEvalSequence\n}\n\nfunc processExpression(expression string) string {\n\n\tif prettyPrint && expression == \"\" {\n\t\treturn yqlib.PrettyPrintExp\n\t} else if prettyPrint {\n\t\treturn fmt.Sprintf(\"%v | %v\", expression, yqlib.PrettyPrintExp)\n\t}\n\treturn expression\n}\n\nfunc evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {\n\t// 0 args, read std in\n\t// 1 arg, null input, process expression\n\t// 1 arg, read file in sequence\n\t// 2+ args, [0] = expression, file the rest\n\n\tout := cmd.OutOrStdout()\n\n\tvar err error\n\n\texpression, args, err := initCommand(cmd, args)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif writeInplace {\n\t\t// only use colours if its forced\n\t\tcolorsEnabled = forceColor\n\t\twriteInPlaceHandler := yqlib.NewWriteInPlaceHandler(args[0])\n\t\tout, err = writeInPlaceHandler.CreateTempFile()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// need to indirectly call the function so that completedSuccessfully is\n\t\t// passed when we finish execution as opposed to now\n\t\tdefer func() {\n\t\t\tif cmdError == nil {\n\t\t\t\tcmdError = writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully)\n\t\t\t}\n\t\t}()\n\t}\n\n\tformat, err := yqlib.FormatFromString(outputFormat)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tprinterWriter, err := configurePrinterWriter(format, out)\n\tif err != nil {\n\t\treturn err\n\t}\n\tencoder, err := configureEncoder()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tprinter := yqlib.NewPrinter(encoder, printerWriter)\n\n\tif printNodeInfo {\n\t\tprinter = yqlib.NewNodeInfoPrinter(printerWriter)\n\t}\n\n\tif nulSepOutput {\n\t\tprinter.SetNulSepOutput(true)\n\t}\n\n\tdecoder, err := configureDecoder(false)\n\tif err != nil {\n\t\treturn err\n\t}\n\tstreamEvaluator := yqlib.NewStreamEvaluator()\n\n\tif frontMatter != \"\" {\n\t\tyqlib.GetLogger().Debug(\"using front matter handler\")\n\t\tfrontMatterHandler := yqlib.NewFrontMatterHandler(args[0])\n\t\terr = frontMatterHandler.Split()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\targs[0] = frontMatterHandler.GetYamlFrontMatterFilename()\n\n\t\tif frontMatter == \"process\" {\n\t\t\treader := frontMatterHandler.GetContentReader()\n\t\t\tprinter.SetAppendix(reader)\n\t\t\tdefer yqlib.SafelyCloseReader(reader)\n\t\t}\n\t\tdefer frontMatterHandler.CleanUp()\n\t}\n\n\tswitch len(args) {\n\tcase 0:\n\t\tif nullInput {\n\t\t\terr = streamEvaluator.EvaluateNew(processExpression(expression), printer)\n\t\t} else {\n\t\t\tcmd.Println(cmd.UsageString())\n\t\t\treturn nil\n\t\t}\n\tdefault:\n\t\terr = streamEvaluator.EvaluateFiles(processExpression(expression), args, printer, decoder)\n\t}\n\tcompletedSuccessfully = err == nil\n\n\tif err == nil && exitStatus && !printer.PrintedAnything() {\n\t\treturn errors.New(\"no matches found\")\n\t}\n\n\treturn err\n}\n" }, { "path": "cmd/evaluate_sequence_command_test.go", "content": "package cmd\n\nimport (\n\t\"bytes\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc TestCreateEvaluateSequenceCommand(t *testing.T) {\n\tcmd := createEvaluateSequenceCommand()\n\n\tif cmd == nil {\n\t\tt.Fatal(\"createEvaluateSequenceCommand returned nil\")\n\t}\n\n\t// Test basic command properties\n\tif cmd.Use != \"eval [expression] [yaml_file1]...\" {\n\t\tt.Errorf(\"Expected Use to be 'eval [expression] [yaml_file1]...', got %q\", cmd.Use)\n\t}\n\n\tif cmd.Short == \"\" {\n\t\tt.Error(\"Expected Short description to be non-empty\")\n\t}\n\n\tif cmd.Long == \"\" {\n\t\tt.Error(\"Expected Long description to be non-empty\")\n\t}\n\n\t// Test aliases\n\texpectedAliases := []string{\"e\"}\n\tif len(cmd.Aliases) != len(expectedAliases) {\n\t\tt.Errorf(\"Expected %d aliases, got %d\", len(expectedAliases), len(cmd.Aliases))\n\t}\n\n\tfor i, expected := range expectedAliases {\n\t\tif i >= len(cmd.Aliases) || cmd.Aliases[i] != expected {\n\t\t\tt.Errorf(\"Expected alias %d to be %q, got %q\", i, expected, cmd.Aliases[i])\n\t\t}\n\t}\n}\n\nfunc TestProcessExpression(t *testing.T) {\n\t// Reset global variables\n\toriginalPrettyPrint := prettyPrint\n\tdefer func() { prettyPrint = originalPrettyPrint }()\n\n\ttests := []struct {\n\t\tname string\n\t\tprettyPrint bool\n\t\texpression string\n\t\texpected string\n\t}{\n\t\t{\n\t\t\tname: \"empty expression without pretty print\",\n\t\t\tprettyPrint: false,\n\t\t\texpression: \"\",\n\t\t\texpected: \"\",\n\t\t},\n\t\t{\n\t\t\tname: \"empty expression with pretty print\",\n\t\t\tprettyPrint: true,\n\t\t\texpression: \"\",\n\t\t\texpected: `(... | (select(tag != \"!!str\"), select(tag == \"!!str\") | select(test(\"(?i)^(y|yes|n|no|on|off)$\") | not)) ) style=\"\"`,\n\t\t},\n\t\t{\n\t\t\tname: \"simple expression without pretty print\",\n\t\t\tprettyPrint: false,\n\t\t\texpression: \".a.b\",\n\t\t\texpected: \".a.b\",\n\t\t},\n\t\t{\n\t\t\tname: \"simple expression with pretty print\",\n\t\t\tprettyPrint: true,\n\t\t\texpression: \".a.b\",\n\t\t\texpected: `.a.b | (... | (select(tag != \"!!str\"), select(tag == \"!!str\") | select(test(\"(?i)^(y|yes|n|no|on|off)$\") | not)) ) style=\"\"`,\n\t\t},\n\t\t{\n\t\t\tname: \"complex expression with pretty print\",\n\t\t\tprettyPrint: true,\n\t\t\texpression: \".items[] | select(.active == true)\",\n\t\t\texpected: `.items[] | select(.active == true) | (... | (select(tag != \"!!str\"), select(tag == \"!!str\") | select(test(\"(?i)^(y|yes|n|no|on|off)$\") | not)) ) style=\"\"`,\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tprettyPrint = tt.prettyPrint\n\t\t\tresult := processExpression(tt.expression)\n\t\t\tif result != tt.expected {\n\t\t\t\tt.Errorf(\"processExpression(%q) = %q, want %q\", tt.expression, result, tt.expected)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestEvaluateSequence_NoArgs(t *testing.T) {\n\t// Create a temporary command\n\tcmd := createEvaluateSequenceCommand()\n\n\t// Set up command to capture output\n\tvar output bytes.Buffer\n\tcmd.SetOut(&output)\n\n\t// Test with no arguments and no null input\n\tnullInput = false\n\tdefer func() { nullInput = false }()\n\n\terr := evaluateSequence(cmd, []string{})\n\n\t// Should not error, but should print usage\n\tif err != nil {\n\t\tt.Errorf(\"evaluateSequence with no args should not error, got: %v\", err)\n\t}\n\n\t// Should have printed usage information\n\tif output.Len() == 0 {\n\t\tt.Error(\"Expected usage information to be printed\")\n\t}\n}\n\nfunc TestEvaluateSequence_NullInput(t *testing.T) {\n\t// Create a temporary command\n\tcmd := createEvaluateSequenceCommand()\n\n\t// Set up command to capture output\n\tvar output bytes.Buffer\n\tcmd.SetOut(&output)\n\n\t// Test with null input\n\tnullInput = true\n\tdefer func() { nullInput = false }()\n\n\terr := evaluateSequence(cmd, []string{})\n\n\t// Should not error when using null input\n\tif err != nil {\n\t\tt.Errorf(\"evaluateSequence with null input should not error, got: %v\", err)\n\t}\n}\n\nfunc TestEvaluateSequence_WithFile(t *testing.T) {\n\t// Create a temporary YAML file\n\ttempDir := t.TempDir()\n\tyamlFile := filepath.Join(tempDir, \"test.yaml\")\n\tyamlContent := []byte(\"name: test\\nage: 25\\n\")\n\terr := os.WriteFile(yamlFile, yamlContent, 0600)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create test YAML file: %v\", err)\n\t}\n\n\t// Create a temporary command\n\tcmd := createEvaluateSequenceCommand()\n\n\t// Set up command to capture output\n\tvar output bytes.Buffer\n\tcmd.SetOut(&output)\n\n\t// Test with a file\n\terr = evaluateSequence(cmd, []string{yamlFile})\n\n\t// Should not error\n\tif err != nil {\n\t\tt.Errorf(\"evaluateSequence with file should not error, got: %v\", err)\n\t}\n\n\t// Should have some output\n\tif output.Len() == 0 {\n\t\tt.Error(\"Expected output from evaluateSequence with file\")\n\t}\n}\n\nfunc TestEvaluateSequence_WithExpressionAndFile(t *testing.T) {\n\t// Create a temporary YAML file\n\ttempDir := t.TempDir()\n\tyamlFile := filepath.Join(tempDir, \"test.yaml\")\n\tyamlContent := []byte(\"name: test\\nage: 25\\n\")\n\terr := os.WriteFile(yamlFile, yamlContent, 0600)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create test YAML file: %v\", err)\n\t}\n\n\t// Create a temporary command\n\tcmd := createEvaluateSequenceCommand()\n\n\t// Set up command to capture output\n\tvar output bytes.Buffer\n\tcmd.SetOut(&output)\n\n\t// Test with expression and file\n\terr = evaluateSequence(cmd, []string{\".name\", yamlFile})\n\n\t// Should not error\n\tif err != nil {\n\t\tt.Errorf(\"evaluateSequence with expression and file should not error, got: %v\", err)\n\t}\n\n\t// Should have output\n\tif output.Len() == 0 {\n\t\tt.Error(\"Expected output from evaluateSequence with expression and file\")\n\t}\n}\n\nfunc TestEvaluateSequence_WriteInPlace(t *testing.T) {\n\t// Create a temporary YAML file\n\ttempDir := t.TempDir()\n\tyamlFile := filepath.Join(tempDir, \"test.yaml\")\n\tyamlContent := []byte(\"name: test\\nage: 25\\n\")\n\terr := os.WriteFile(yamlFile, yamlContent, 0600)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create test YAML file: %v\", err)\n\t}\n\n\t// Create a temporary command\n\tcmd := createEvaluateSequenceCommand()\n\n\t// Set up command to capture output\n\tvar output bytes.Buffer\n\tcmd.SetOut(&output)\n\n\t// Enable write in place\n\toriginalWriteInplace := writeInplace\n\twriteInplace = true\n\tdefer func() { writeInplace = originalWriteInplace }()\n\n\t// Test with write in place\n\terr = evaluateSequence(cmd, []string{\".name = \\\"updated\\\"\", yamlFile})\n\n\t// Should not error\n\tif err != nil {\n\t\tt.Errorf(\"evaluateSequence with write in place should not error, got: %v\", err)\n\t}\n\n\t// Verify the file was updated\n\tupdatedContent, err := os.ReadFile(yamlFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to read updated file: %v\", err)\n\t}\n\n\t// Should contain the updated content\n\tif !strings.Contains(string(updatedContent), \"updated\") {\n\t\tt.Errorf(\"Expected file to contain 'updated', got: %s\", string(updatedContent))\n\t}\n}\n\nfunc TestEvaluateSequence_ExitStatus(t *testing.T) {\n\t// Create a temporary YAML file\n\ttempDir := t.TempDir()\n\tyamlFile := filepath.Join(tempDir, \"test.yaml\")\n\tyamlContent := []byte(\"name: test\\nage: 25\\n\")\n\terr := os.WriteFile(yamlFile, yamlContent, 0600)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create test YAML file: %v\", err)\n\t}\n\n\t// Create a temporary command\n\tcmd := createEvaluateSequenceCommand()\n\n\t// Set up command to capture output\n\tvar output bytes.Buffer\n\tcmd.SetOut(&output)\n\n\t// Enable exit status\n\toriginalExitStatus := exitStatus\n\texitStatus = true\n\tdefer func() { exitStatus = originalExitStatus }()\n\n\t// Test with expression that should find no matches\n\terr = evaluateSequence(cmd, []string{\".nonexistent\", yamlFile})\n\n\t// Should error when no matches found and exit status is enabled\n\tif err == nil {\n\t\tt.Error(\"Expected error when no matches found and exit status is enabled\")\n\t}\n}\n" }, { "path": "cmd/root.go", "content": "package cmd\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\n\t\"github.com/mikefarah/yq/v4/pkg/yqlib\"\n\t\"github.com/spf13/cobra\"\n\tlogging \"gopkg.in/op/go-logging.v1\"\n)\n\ntype runeValue rune\n\nfunc newRuneVar(p *rune) *runeValue {\n\treturn (*runeValue)(p)\n}\n\nfunc (r *runeValue) String() string {\n\treturn string(*r)\n}\n\nfunc (r *runeValue) Set(rawVal string) error {\n\tval := strings.ReplaceAll(rawVal, \"\\\\n\", \"\\n\")\n\tval = strings.ReplaceAll(val, \"\\\\t\", \"\\t\")\n\tval = strings.ReplaceAll(val, \"\\\\r\", \"\\r\")\n\tval = strings.ReplaceAll(val, \"\\\\f\", \"\\f\")\n\tval = strings.ReplaceAll(val, \"\\\\v\", \"\\v\")\n\tif len(val) != 1 {\n\t\treturn fmt.Errorf(\"[%v] is not a valid character. Must be length 1 was %v\", val, len(val))\n\t}\n\t*r = runeValue(rune(val[0]))\n\treturn nil\n}\n\nfunc (r *runeValue) Type() string {\n\treturn \"char\"\n}\n\nfunc New() *cobra.Command {\n\tvar rootCmd = &cobra.Command{\n\t\tUse: \"yq\",\n\t\tShort: \"yq is a lightweight and portable command-line data file processor.\",\n\t\tLong: `yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) \nSee https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,\n\t\tExample: `\n# yq tries to auto-detect the file format based off the extension, and defaults to YAML if it's unknown (or piping through STDIN)\n# Use the '-p/--input-format' flag to specify a format type.\ncat file.xml | yq -p xml\n\n# read the \"stuff\" node from \"myfile.yml\"\nyq '.stuff' < myfile.yml\n\n# update myfile.yml in place\nyq -i '.stuff = \"foo\"' myfile.yml\n\n# print contents of sample.json as idiomatic YAML\nyq -P -oy sample.json\n`,\n\n\t\tRunE: func(cmd *cobra.Command, args []string) error {\n\t\t\tif version {\n\t\t\t\tcmd.Print(GetVersionDisplay())\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\treturn evaluateSequence(cmd, args)\n\n\t\t},\n\t\tPersistentPreRunE: func(cmd *cobra.Command, _ []string) error {\n\t\t\tcmd.SetOut(cmd.OutOrStdout())\n\t\t\tlevel := logging.WARNING\n\t\t\tstringFormat := `[%{level}] %{color}%{time:15:04:05}%{color:reset} %{message}`\n\n\t\t\t// when NO_COLOR environment variable presents and not an empty string the coloured output should be disabled;\n\t\t\t// refer to no-color.org\n\t\t\tforceNoColor = forceNoColor || os.Getenv(\"NO_COLOR\") != \"\"\n\n\t\t\tif verbose && forceNoColor {\n\t\t\t\tlevel = logging.DEBUG\n\t\t\t\tstringFormat = `[%{level:5.5s}] %{time:15:04:05} %{shortfile:-33s} %{shortfunc:-25s} %{message}`\n\t\t\t} else if verbose {\n\t\t\t\tlevel = logging.DEBUG\n\t\t\t\tstringFormat = `[%{level:5.5s}] %{color}%{time:15:04:05}%{color:bold} %{shortfile:-33s} %{shortfunc:-25s}%{color:reset} %{message}`\n\t\t\t} else if forceNoColor {\n\t\t\t\tstringFormat = `[%{level}] %{time:15:04:05} %{message}`\n\t\t\t}\n\n\t\t\tvar format = logging.MustStringFormatter(stringFormat)\n\t\t\tvar backend = logging.AddModuleLevel(\n\t\t\t\tlogging.NewBackendFormatter(logging.NewLogBackend(os.Stderr, \"\", 0), format))\n\n\t\t\tbackend.SetLevel(level, \"\")\n\n\t\t\tlogging.SetBackend(backend)\n\t\t\tyqlib.InitExpressionParser()\n\n\t\t\treturn nil\n\t\t},\n\t}\n\n\trootCmd.PersistentFlags().BoolVarP(&verbose, \"verbose\", \"v\", false, \"verbose mode\")\n\trootCmd.PersistentFlags().BoolVarP(&printNodeInfo, \"debug-node-info\", \"\", false, \"debug node info\")\n\n\trootCmd.PersistentFlags().BoolVarP(&outputToJSON, \"tojson\", \"j\", false, \"(deprecated) output as json. Set indent to 0 to print json in one line.\")\n\terr := rootCmd.PersistentFlags().MarkDeprecated(\"tojson\", \"please use -o=json instead\")\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\trootCmd.PersistentFlags().StringVarP(&outputFormat, \"output-format\", \"o\", \"auto\", fmt.Sprintf(\"[auto|a|%v] output format type.\", yqlib.GetAvailableOutputFormatString()))\n\tvar outputCompletions = []string{\"auto\"}\n\tfor _, formats := range yqlib.GetAvailableOutputFormats() {\n\t\toutputCompletions = append(outputCompletions, formats.FormalName)\n\t}\n\n\tif err = rootCmd.RegisterFlagCompletionFunc(\"output-format\", cobra.FixedCompletions(outputCompletions, cobra.ShellCompDirectiveNoFileComp)); err != nil {\n\t\tpanic(err)\n\t}\n\trootCmd.PersistentFlags().StringVarP(&inputFormat, \"input-format\", \"p\", \"auto\", fmt.Sprintf(\"[auto|a|%v] parse format for input.\", yqlib.GetAvailableInputFormatString()))\n\n\tvar inputCompletions = []string{\"auto\"}\n\tfor _, formats := range yqlib.GetAvailableInputFormats() {\n\t\tinputCompletions = append(inputCompletions, formats.FormalName)\n\t}\n\n\tif err = rootCmd.RegisterFlagCompletionFunc(\"input-format\", cobra.FixedCompletions(inputCompletions, cobra.ShellCompDirectiveNoFileComp)); err != nil {\n\t\tpanic(err)\n\t}\n\n\trootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, \"xml-attribute-prefix\", yqlib.ConfiguredXMLPreferences.AttributePrefix, \"prefix for xml attributes\")\n\tif err = rootCmd.RegisterFlagCompletionFunc(\"xml-attribute-prefix\", cobra.NoFileCompletions); err != nil {\n\t\tpanic(err)\n\t}\n\trootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, \"xml-content-name\", yqlib.ConfiguredXMLPreferences.ContentName, \"name for xml content (if no attribute name is present).\")\n\tif err = rootCmd.RegisterFlagCompletionFunc(\"xml-content-name\", cobra.NoFileCompletions); err != nil {\n\t\tpanic(err)\n\t}\n\trootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.StrictMode, \"xml-strict-mode\", yqlib.ConfiguredXMLPreferences.StrictMode, \"enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.\")\n\trootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.KeepNamespace, \"xml-keep-namespace\", yqlib.ConfiguredXMLPreferences.KeepNamespace, \"enables keeping namespace after parsing attributes\")\n\trootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.UseRawToken, \"xml-raw-token\", yqlib.ConfiguredXMLPreferences.UseRawToken, \"enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.\")\n\trootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ProcInstPrefix, \"xml-proc-inst-prefix\", yqlib.ConfiguredXMLPreferences.ProcInstPrefix, \"prefix for xml processing instructions (e.g. )\")\n\tif err = rootCmd.RegisterFlagCompletionFunc(\"xml-proc-inst-prefix\", cobra.NoFileCompletions); err != nil {\n\t\tpanic(err)\n\t}\n\trootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.DirectiveName, \"xml-directive-name\", yqlib.ConfiguredXMLPreferences.DirectiveName, \"name for xml directives (e.g. )\")\n\tif err = rootCmd.RegisterFlagCompletionFunc(\"xml-directive-name\", cobra.NoFileCompletions); err != nil {\n\t\tpanic(err)\n\t}\n\trootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipProcInst, \"xml-skip-proc-inst\", yqlib.ConfiguredXMLPreferences.SkipProcInst, \"skip over process instructions (e.g. )\")\n\trootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipDirectives, \"xml-skip-directives\", yqlib.ConfiguredXMLPreferences.SkipDirectives, \"skip over directives (e.g. )\")\n\n\trootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredCsvPreferences.AutoParse, \"csv-auto-parse\", yqlib.ConfiguredCsvPreferences.AutoParse, \"parse CSV YAML/JSON values\")\n\trootCmd.PersistentFlags().Var(newRuneVar(&yqlib.ConfiguredCsvPreferences.Separator), \"csv-separator\", \"CSV Separator character\")\n\n\trootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredTsvPreferences.AutoParse, \"tsv-auto-parse\", yqlib.ConfiguredTsvPreferences.AutoParse, \"parse TSV YAML/JSON values\")\n\n\trootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredLuaPreferences.DocPrefix, \"lua-prefix\", yqlib.ConfiguredLuaPreferences.DocPrefix, \"prefix\")\n\tif err = rootCmd.RegisterFlagCompletionFunc(\"lua-prefix\", cobra.NoFileCompletions); err != nil {\n\t\tpanic(err)\n\t}\n\trootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredLuaPreferences.DocSuffix, \"lua-suffix\", yqlib.ConfiguredLuaPreferences.DocSuffix, \"suffix\")\n\tif err = rootCmd.RegisterFlagCompletionFunc(\"lua-suffix\", cobra.NoFileCompletions); err != nil {\n\t\tpanic(err)\n\t}\n\trootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.UnquotedKeys, \"lua-unquoted\", yqlib.ConfiguredLuaPreferences.UnquotedKeys, \"output unquoted string keys (e.g. {foo=\\\"bar\\\"})\")\n\trootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.Globals, \"lua-globals\", yqlib.ConfiguredLuaPreferences.Globals, \"output keys as top-level global variables\")\n\n\trootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredPropertiesPreferences.KeyValueSeparator, \"properties-separator\", yqlib.ConfiguredPropertiesPreferences.KeyValueSeparator, \"separator to use between keys and values\")\n\trootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredPropertiesPreferences.UseArrayBrackets, \"properties-array-brackets\", yqlib.ConfiguredPropertiesPreferences.UseArrayBrackets, \"use [x] in array paths (e.g. for SpringBoot)\")\n\n\trootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredShellVariablesPreferences.KeySeparator, \"shell-key-separator\", yqlib.ConfiguredShellVariablesPreferences.KeySeparator, \"separator for shell variable key paths\")\n\tif err = rootCmd.RegisterFlagCompletionFunc(\"shell-key-separator\", cobra.NoFileCompletions); err != nil {\n\t\tpanic(err)\n\t}\n\n\trootCmd.PersistentFlags().BoolVar(&yqlib.StringInterpolationEnabled, \"string-interpolation\", yqlib.StringInterpolationEnabled, \"Toggles strings interpolation of \\\\(exp)\")\n\n\trootCmd.PersistentFlags().BoolVarP(&nullInput, \"null-input\", \"n\", false, \"Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.\")\n\trootCmd.PersistentFlags().BoolVarP(&noDocSeparators, \"no-doc\", \"N\", false, \"Don't print document separators (---)\")\n\n\trootCmd.PersistentFlags().IntVarP(&indent, \"indent\", \"I\", 2, \"sets indent level for output\")\n\tif err = rootCmd.RegisterFlagCompletionFunc(\"indent\", cobra.NoFileCompletions); err != nil {\n\t\tpanic(err)\n\t}\n\trootCmd.Flags().BoolVarP(&version, \"version\", \"V\", false, \"Print version information and quit\")\n\trootCmd.PersistentFlags().BoolVarP(&writeInplace, \"inplace\", \"i\", false, \"update the file in place of first file given.\")\n\trootCmd.PersistentFlags().VarP(unwrapScalarFlag, \"unwrapScalar\", \"r\", \"unwrap scalar, print the value with no quotes, colours or comments. Defaults to true for yaml\")\n\trootCmd.PersistentFlags().Lookup(\"unwrapScalar\").NoOptDefVal = \"true\"\n\trootCmd.PersistentFlags().BoolVarP(&nulSepOutput, \"nul-output\", \"0\", false, \"Use NUL char to separate values. If unwrap scalar is also set, fail if unwrapped scalar contains NUL char.\")\n\n\trootCmd.PersistentFlags().BoolVarP(&prettyPrint, \"prettyPrint\", \"P\", false, \"pretty print, shorthand for '... style = \\\"\\\"'\")\n\trootCmd.PersistentFlags().BoolVarP(&exitStatus, \"exit-status\", \"e\", false, \"set exit status if there are no matches or null or false is returned\")\n\n\trootCmd.PersistentFlags().BoolVarP(&forceColor, \"colors\", \"C\", false, \"force print with colors\")\n\trootCmd.PersistentFlags().BoolVarP(&forceNoColor, \"no-colors\", \"M\", forceNoColor, \"force print with no colors\")\n\trootCmd.PersistentFlags().StringVarP(&frontMatter, \"front-matter\", \"f\", \"\", \"(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact\")\n\tif err = rootCmd.RegisterFlagCompletionFunc(\"front-matter\", cobra.FixedCompletions([]string{\"extract\", \"process\"}, cobra.ShellCompDirectiveNoFileComp)); err != nil {\n\t\tpanic(err)\n\t}\n\trootCmd.PersistentFlags().StringVarP(&forceExpression, \"expression\", \"\", \"\", \"forcibly set the expression argument. Useful when yq argument detection thinks your expression is a file.\")\n\tif err = rootCmd.RegisterFlagCompletionFunc(\"expression\", cobra.NoFileCompletions); err != nil {\n\t\tpanic(err)\n\t}\n\trootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredYamlPreferences.LeadingContentPreProcessing, \"header-preprocess\", \"\", true, \"Slurp any header comments and separators before processing expression.\")\n\trootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredYamlPreferences.FixMergeAnchorToSpec, \"yaml-fix-merge-anchor-to-spec\", \"\", false, \"Fix merge anchor to match YAML spec. Will default to true in late 2025\")\n\trootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredYamlPreferences.CompactSequenceIndent, \"yaml-compact-seq-indent\", \"c\", false, \"Use compact sequence indentation where '- ' is considered part of the indentation.\")\n\n\trootCmd.PersistentFlags().StringVarP(&splitFileExp, \"split-exp\", \"s\", \"\", \"print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter. The necessary directories will be created.\")\n\tif err = rootCmd.RegisterFlagCompletionFunc(\"split-exp\", cobra.NoFileCompletions); err != nil {\n\t\tpanic(err)\n\t}\n\trootCmd.PersistentFlags().StringVarP(&splitFileExpFile, \"split-exp-file\", \"\", \"\", \"Use a file to specify the split-exp expression.\")\n\tif err = rootCmd.MarkPersistentFlagFilename(\"split-exp-file\"); err != nil {\n\t\tpanic(err)\n\t}\n\n\trootCmd.PersistentFlags().StringVarP(&expressionFile, \"from-file\", \"\", \"\", \"Load expression from specified file.\")\n\tif err = rootCmd.MarkPersistentFlagFilename(\"from-file\"); err != nil {\n\t\tpanic(err)\n\t}\n\n\trootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableEnvOps, \"security-disable-env-ops\", \"\", false, \"Disable env related operations.\")\n\trootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableFileOps, \"security-disable-file-ops\", \"\", false, \"Disable file related operations (e.g. load)\")\n\n\trootCmd.AddCommand(\n\t\tcreateEvaluateSequenceCommand(),\n\t\tcreateEvaluateAllCommand(),\n\t\tcompletionCmd,\n\t)\n\treturn rootCmd\n}\n" }, { "path": "cmd/root_test.go", "content": "package cmd\n\nimport (\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc TestNewRuneVar(t *testing.T) {\n\tvar r rune\n\truneVar := newRuneVar(&r)\n\n\tif runeVar == nil {\n\t\tt.Fatal(\"newRuneVar returned nil\")\n\t}\n}\n\nfunc TestRuneValue_String(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\truneVal rune\n\t\texpected string\n\t}{\n\t\t{\n\t\t\tname: \"simple character\",\n\t\t\truneVal: 'a',\n\t\t\texpected: \"a\",\n\t\t},\n\t\t{\n\t\t\tname: \"special character\",\n\t\t\truneVal: '\\n',\n\t\t\texpected: \"\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"unicode character\",\n\t\t\truneVal: 'ñ',\n\t\t\texpected: \"ñ\",\n\t\t},\n\t\t{\n\t\t\tname: \"zero rune\",\n\t\t\truneVal: 0,\n\t\t\texpected: string(rune(0)),\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\truneVal := runeValue(tt.runeVal)\n\t\t\tresult := runeVal.String()\n\t\t\tif result != tt.expected {\n\t\t\t\tt.Errorf(\"runeValue.String() = %q, want %q\", result, tt.expected)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestRuneValue_Set(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tinput string\n\t\texpected rune\n\t\texpectError bool\n\t}{\n\t\t{\n\t\t\tname: \"simple character\",\n\t\t\tinput: \"a\",\n\t\t\texpected: 'a',\n\t\t\texpectError: false,\n\t\t},\n\t\t{\n\t\t\tname: \"newline escape\",\n\t\t\tinput: \"\\\\n\",\n\t\t\texpected: '\\n',\n\t\t\texpectError: false,\n\t\t},\n\t\t{\n\t\t\tname: \"tab escape\",\n\t\t\tinput: \"\\\\t\",\n\t\t\texpected: '\\t',\n\t\t\texpectError: false,\n\t\t},\n\t\t{\n\t\t\tname: \"carriage return escape\",\n\t\t\tinput: \"\\\\r\",\n\t\t\texpected: '\\r',\n\t\t\texpectError: false,\n\t\t},\n\t\t{\n\t\t\tname: \"form feed escape\",\n\t\t\tinput: \"\\\\f\",\n\t\t\texpected: '\\f',\n\t\t\texpectError: false,\n\t\t},\n\t\t{\n\t\t\tname: \"vertical tab escape\",\n\t\t\tinput: \"\\\\v\",\n\t\t\texpected: '\\v',\n\t\t\texpectError: false,\n\t\t},\n\t\t{\n\t\t\tname: \"empty string\",\n\t\t\tinput: \"\",\n\t\t\texpected: 0,\n\t\t\texpectError: true,\n\t\t},\n\t\t{\n\t\t\tname: \"multiple characters\",\n\t\t\tinput: \"ab\",\n\t\t\texpected: 0,\n\t\t\texpectError: true,\n\t\t},\n\t\t{\n\t\t\tname: \"special character\",\n\t\t\tinput: \"ñ\",\n\t\t\texpected: 'ñ',\n\t\t\texpectError: true, // This will fail because the Set function checks len(val) != 1\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tvar r rune\n\t\t\truneVal := newRuneVar(&r)\n\n\t\t\terr := runeVal.Set(tt.input)\n\n\t\t\tif tt.expectError {\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Errorf(\"Expected error for input %q, but got none\", tt.input)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Errorf(\"Unexpected error for input %q: %v\", tt.input, err)\n\t\t\t\t}\n\t\t\t\tif r != tt.expected {\n\t\t\t\t\tt.Errorf(\"Expected rune %q (%d), got %q (%d)\",\n\t\t\t\t\t\tstring(tt.expected), tt.expected, string(r), r)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestRuneValue_Set_ErrorMessages(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tinput string\n\t\texpectedError string\n\t}{\n\t\t{\n\t\t\tname: \"empty string error\",\n\t\t\tinput: \"\",\n\t\t\texpectedError: \"[] is not a valid character. Must be length 1 was 0\",\n\t\t},\n\t\t{\n\t\t\tname: \"multiple characters error\",\n\t\t\tinput: \"abc\",\n\t\t\texpectedError: \"[abc] is not a valid character. Must be length 1 was 3\",\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tvar r rune\n\t\t\truneVal := newRuneVar(&r)\n\n\t\t\terr := runeVal.Set(tt.input)\n\n\t\t\tif err == nil {\n\t\t\t\tt.Errorf(\"Expected error for input %q, but got none\", tt.input)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tif !strings.Contains(err.Error(), tt.expectedError) {\n\t\t\t\tt.Errorf(\"Expected error message to contain %q, got %q\",\n\t\t\t\t\ttt.expectedError, err.Error())\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestRuneValue_Type(t *testing.T) {\n\tvar r rune\n\truneVal := newRuneVar(&r)\n\n\tresult := runeVal.Type()\n\texpected := \"char\"\n\n\tif result != expected {\n\t\tt.Errorf(\"runeValue.Type() = %q, want %q\", result, expected)\n\t}\n}\n\nfunc TestNew(t *testing.T) {\n\trootCmd := New()\n\n\tif rootCmd == nil {\n\t\tt.Fatal(\"New() returned nil\")\n\t}\n\n\t// Test basic command properties\n\tif rootCmd.Use != \"yq\" {\n\t\tt.Errorf(\"Expected Use to be 'yq', got %q\", rootCmd.Use)\n\t}\n\n\tif rootCmd.Short == \"\" {\n\t\tt.Error(\"Expected Short description to be non-empty\")\n\t}\n\n\tif rootCmd.Long == \"\" {\n\t\tt.Error(\"Expected Long description to be non-empty\")\n\t}\n\n\t// Test that the command has the expected subcommands\n\texpectedCommands := []string{\"eval\", \"eval-all\", \"completion\"}\n\tactualCommands := make([]string, 0, len(rootCmd.Commands()))\n\n\tfor _, cmd := range rootCmd.Commands() {\n\t\tactualCommands = append(actualCommands, cmd.Name())\n\t}\n\n\tfor _, expected := range expectedCommands {\n\t\tfound := false\n\t\tfor _, actual := range actualCommands {\n\t\t\tif actual == expected {\n\t\t\t\tfound = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif !found {\n\t\t\tt.Errorf(\"Expected command %q not found in actual commands: %v\",\n\t\t\t\texpected, actualCommands)\n\t\t}\n\t}\n}\n\nfunc TestNew_FlagCompletions(t *testing.T) {\n\trootCmd := New()\n\n\t// Test that flag completion functions are registered\n\t// This is a basic smoke test - we can't easily test the actual completion logic\n\t// without more complex setup\n\tflags := []string{\n\t\t\"output-format\",\n\t\t\"input-format\",\n\t\t\"xml-attribute-prefix\",\n\t\t\"xml-content-name\",\n\t\t\"xml-proc-inst-prefix\",\n\t\t\"xml-directive-name\",\n\t\t\"lua-prefix\",\n\t\t\"lua-suffix\",\n\t\t\"properties-separator\",\n\t\t\"indent\",\n\t\t\"front-matter\",\n\t\t\"expression\",\n\t\t\"split-exp\",\n\t}\n\n\tfor _, flagName := range flags {\n\t\tflag := rootCmd.PersistentFlags().Lookup(flagName)\n\t\tif flag == nil {\n\t\t\tt.Errorf(\"Expected flag %q to exist\", flagName)\n\t\t}\n\t}\n}\n" }, { "path": "cmd/unwrap_flag.go", "content": "package cmd\n\nimport (\n\t\"strconv\"\n\n\t\"github.com/spf13/pflag\"\n)\n\ntype boolFlag interface {\n\tpflag.Value\n\tIsExplicitlySet() bool\n\tIsSet() bool\n}\n\ntype unwrapScalarFlagStrc struct {\n\texplicitlySet bool\n\tvalue bool\n}\n\nfunc newUnwrapFlag() boolFlag {\n\treturn &unwrapScalarFlagStrc{value: true}\n}\n\nfunc (f *unwrapScalarFlagStrc) IsExplicitlySet() bool {\n\treturn f.explicitlySet\n}\n\nfunc (f *unwrapScalarFlagStrc) IsSet() bool {\n\treturn f.value\n}\n\nfunc (f *unwrapScalarFlagStrc) String() string {\n\treturn strconv.FormatBool(f.value)\n}\n\nfunc (f *unwrapScalarFlagStrc) Set(value string) error {\n\n\tv, err := strconv.ParseBool(value)\n\tf.value = v\n\tf.explicitlySet = true\n\treturn err\n}\n\nfunc (*unwrapScalarFlagStrc) Type() string {\n\treturn \"bool\"\n}\n" }, { "path": "cmd/utils.go", "content": "package cmd\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"strings\"\n\n\t\"github.com/mikefarah/yq/v4/pkg/yqlib\"\n\t\"github.com/spf13/cobra\"\n\t\"gopkg.in/op/go-logging.v1\"\n)\n\nfunc isAutomaticOutputFormat() bool {\n\treturn outputFormat == \"\" || outputFormat == \"auto\" || outputFormat == \"a\"\n}\n\nfunc initCommand(cmd *cobra.Command, args []string) (string, []string, error) {\n\tcmd.SilenceUsage = true\n\n\tsetupColors()\n\n\texpression, args, err := processArgs(args)\n\tif err != nil {\n\t\treturn \"\", nil, err\n\t}\n\n\tif err := loadSplitFileExpression(); err != nil {\n\t\treturn \"\", nil, err\n\t}\n\n\thandleBackwardsCompatibility()\n\n\tif err := validateCommandFlags(args); err != nil {\n\t\treturn \"\", nil, err\n\t}\n\n\tif err := configureFormats(args); err != nil {\n\t\treturn \"\", nil, err\n\t}\n\n\tconfigureUnwrapScalar()\n\n\treturn expression, args, nil\n}\n\nfunc setupColors() {\n\tfileInfo, _ := os.Stdout.Stat()\n\n\tif forceColor || (!forceNoColor && (fileInfo.Mode()&os.ModeCharDevice) != 0) {\n\t\tcolorsEnabled = true\n\t}\n}\n\nfunc loadSplitFileExpression() error {\n\tif splitFileExpFile != \"\" {\n\t\tsplitExpressionBytes, err := os.ReadFile(splitFileExpFile)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tsplitFileExp = string(splitExpressionBytes)\n\t}\n\treturn nil\n}\n\nfunc handleBackwardsCompatibility() {\n\t// backwards compatibility\n\tif outputToJSON {\n\t\toutputFormat = \"json\"\n\t}\n}\n\nfunc validateCommandFlags(args []string) error {\n\tif writeInplace && (len(args) == 0 || args[0] == \"-\") {\n\t\treturn fmt.Errorf(\"write in place flag only applicable when giving an expression and at least one file\")\n\t}\n\n\tif frontMatter != \"\" && len(args) == 0 {\n\t\treturn fmt.Errorf(\"front matter flag only applicable when giving an expression and at least one file\")\n\t}\n\n\tif writeInplace && splitFileExp != \"\" {\n\t\treturn fmt.Errorf(\"write in place cannot be used with split file\")\n\t}\n\n\tif nullInput && len(args) > 0 {\n\t\treturn fmt.Errorf(\"cannot pass files in when using null-input flag\")\n\t}\n\n\treturn nil\n}\n\nfunc configureFormats(args []string) error {\n\tinputFilename := \"\"\n\tif len(args) > 0 {\n\t\tinputFilename = args[0]\n\t}\n\n\tif err := configureInputFormat(inputFilename); err != nil {\n\t\treturn err\n\t}\n\n\tif err := configureOutputFormat(); err != nil {\n\t\treturn err\n\t}\n\n\tyqlib.GetLogger().Debug(\"Using input format %v\", inputFormat)\n\tyqlib.GetLogger().Debug(\"Using output format %v\", outputFormat)\n\n\treturn nil\n}\n\nfunc configureInputFormat(inputFilename string) error {\n\tif inputFormat == \"\" || inputFormat == \"auto\" || inputFormat == \"a\" {\n\t\tinputFormat = yqlib.FormatStringFromFilename(inputFilename)\n\n\t\t_, err := yqlib.FormatFromString(inputFormat)\n\t\tif err != nil {\n\t\t\t// unknown file type, default to yaml\n\t\t\tyqlib.GetLogger().Debug(\"Unknown file format extension '%v', defaulting to yaml\", inputFormat)\n\t\t\tinputFormat = \"yaml\"\n\t\t\tif isAutomaticOutputFormat() {\n\t\t\t\toutputFormat = \"yaml\"\n\t\t\t}\n\t\t} else if isAutomaticOutputFormat() {\n\t\t\toutputFormat = inputFormat\n\t\t}\n\t} else if isAutomaticOutputFormat() {\n\t\t// backwards compatibility -\n\t\t// before this was introduced, `yq -pcsv things.csv`\n\t\t// would produce *yaml* output.\n\t\t//\n\t\toutputFormat = yqlib.FormatStringFromFilename(inputFilename)\n\t\tif inputFilename != \"-\" {\n\t\t\tyqlib.GetLogger().Warning(\"yq default output is now 'auto' (based on the filename extension). Normally yq would output '%v', but for backwards compatibility 'yaml' has been set. Please use -oy to specify yaml, or drop the -p flag.\", outputFormat)\n\t\t}\n\t\toutputFormat = \"yaml\"\n\t}\n\treturn nil\n}\n\nfunc configureOutputFormat() error {\n\toutputFormatType, err := yqlib.FormatFromString(outputFormat)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif outputFormatType == yqlib.YamlFormat ||\n\t\toutputFormatType == yqlib.PropertiesFormat {\n\t\tunwrapScalar = true\n\t}\n\n\treturn nil\n}\n\nfunc configureUnwrapScalar() {\n\tif unwrapScalarFlag.IsExplicitlySet() {\n\t\tunwrapScalar = unwrapScalarFlag.IsSet()\n\t}\n}\n\nfunc configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {\n\tformat, err := yqlib.FormatFromString(inputFormat)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tyqlib.ConfiguredYamlPreferences.EvaluateTogether = evaluateTogether\n\n\tif format.DecoderFactory == nil {\n\t\treturn nil, fmt.Errorf(\"no support for %s input format\", inputFormat)\n\t}\n\tyqlibDecoder := format.DecoderFactory()\n\tif yqlibDecoder == nil {\n\t\treturn nil, fmt.Errorf(\"no support for %s input format\", inputFormat)\n\t}\n\treturn yqlibDecoder, nil\n}\n\nfunc configurePrinterWriter(format *yqlib.Format, out io.Writer) (yqlib.PrinterWriter, error) {\n\n\tvar printerWriter yqlib.PrinterWriter\n\n\tif splitFileExp != \"\" {\n\t\tcolorsEnabled = forceColor\n\t\tsplitExp, err := yqlib.ExpressionParser.ParseExpression(splitFileExp)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"bad split document expression: %w\", err)\n\t\t}\n\t\tprinterWriter = yqlib.NewMultiPrinterWriter(splitExp, format)\n\t} else {\n\t\tprinterWriter = yqlib.NewSinglePrinterWriter(out)\n\t}\n\treturn printerWriter, nil\n}\n\nfunc configureEncoder() (yqlib.Encoder, error) {\n\tyqlibOutputFormat, err := yqlib.FormatFromString(outputFormat)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tyqlib.ConfiguredXMLPreferences.Indent = indent\n\tyqlib.ConfiguredYamlPreferences.Indent = indent\n\tyqlib.ConfiguredKYamlPreferences.Indent = indent\n\tyqlib.ConfiguredJSONPreferences.Indent = indent\n\n\tyqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar\n\tyqlib.ConfiguredKYamlPreferences.UnwrapScalar = unwrapScalar\n\tyqlib.ConfiguredPropertiesPreferences.UnwrapScalar = unwrapScalar\n\tyqlib.ConfiguredJSONPreferences.UnwrapScalar = unwrapScalar\n\tyqlib.ConfiguredShellVariablesPreferences.UnwrapScalar = unwrapScalar\n\n\tyqlib.ConfiguredYamlPreferences.ColorsEnabled = colorsEnabled\n\tyqlib.ConfiguredKYamlPreferences.ColorsEnabled = colorsEnabled\n\tyqlib.ConfiguredJSONPreferences.ColorsEnabled = colorsEnabled\n\tyqlib.ConfiguredHclPreferences.ColorsEnabled = colorsEnabled\n\tyqlib.ConfiguredTomlPreferences.ColorsEnabled = colorsEnabled\n\n\tyqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators\n\tyqlib.ConfiguredKYamlPreferences.PrintDocSeparators = !noDocSeparators\n\n\tencoder := yqlibOutputFormat.EncoderFactory()\n\n\tif encoder == nil {\n\t\treturn nil, fmt.Errorf(\"no support for %s output format\", outputFormat)\n\t}\n\treturn encoder, err\n}\n\n// this is a hack to enable backwards compatibility with githubactions (which pipe /dev/null into everything)\n// and being able to call yq with the filename as a single parameter\n//\n// without this - yq detects there is stdin (thanks githubactions),\n// then tries to parse the filename as an expression\nfunc maybeFile(str string) bool {\n\tyqlib.GetLogger().Debugf(\"checking '%v' is a file\", str)\n\tstat, err := os.Stat(str) // #nosec\n\tresult := err == nil && !stat.IsDir()\n\tif yqlib.GetLogger().IsEnabledFor(logging.DEBUG) {\n\t\tif err != nil {\n\t\t\tyqlib.GetLogger().Debugf(\"error: %v\", err)\n\t\t} else {\n\t\t\tyqlib.GetLogger().Debugf(\"error: %v, dir: %v\", err, stat.IsDir())\n\t\t}\n\t\tyqlib.GetLogger().Debugf(\"result: %v\", result)\n\t}\n\treturn result\n}\n\nfunc processStdInArgs(args []string) []string {\n\tstat, err := os.Stdin.Stat()\n\tif err != nil {\n\t\tyqlib.GetLogger().Debugf(\"error getting stdin: %v\", err)\n\t}\n\tpipingStdin := stat != nil && (stat.Mode()&os.ModeCharDevice) == 0\n\n\t// if we've been given a file, don't automatically\n\t// read from stdin.\n\t// this happens if there is more than one argument\n\t// or only one argument and its a file\n\tif nullInput || !pipingStdin || len(args) > 1 || (len(args) > 0 && maybeFile(args[0])) {\n\t\treturn args\n\t}\n\n\tfor _, arg := range args {\n\t\tif arg == \"-\" {\n\t\t\treturn args\n\t\t}\n\t}\n\tyqlib.GetLogger().Debugf(\"missing '-', adding it to the end\")\n\n\t// we're piping from stdin, but there's no '-' arg\n\t// lets add one to the end\n\treturn append(args, \"-\")\n}\n\nfunc processArgs(originalArgs []string) (string, []string, error) {\n\texpression := forceExpression\n\targs := processStdInArgs(originalArgs)\n\tmaybeFirstArgIsAFile := len(args) > 0 && maybeFile(args[0])\n\n\tif expressionFile == \"\" && maybeFirstArgIsAFile && strings.HasSuffix(args[0], \".yq\") {\n\t\t// lets check if an expression file was given\n\t\tyqlib.GetLogger().Debug(\"Assuming arg %v is an expression file\", args[0])\n\t\texpressionFile = args[0]\n\t\targs = args[1:]\n\t}\n\n\tif expressionFile != \"\" {\n\t\texpressionBytes, err := os.ReadFile(expressionFile)\n\t\tif err != nil {\n\t\t\treturn \"\", nil, err\n\t\t}\n\t\t//replace \\r\\n (windows) with good ol' unix file endings.\n\t\texpression = strings.ReplaceAll(string(expressionBytes), \"\\r\\n\", \"\\n\")\n\t}\n\n\tyqlib.GetLogger().Debugf(\"processed args: %v\", args)\n\tif expression == \"\" && len(args) > 0 && args[0] != \"-\" && !maybeFile(args[0]) {\n\t\tyqlib.GetLogger().Debug(\"assuming expression is '%v'\", args[0])\n\t\texpression = args[0]\n\t\targs = args[1:]\n\t}\n\treturn expression, args, nil\n}\n" }, { "path": "cmd/utils_test.go", "content": "package cmd\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/pkg/yqlib\"\n\t\"github.com/spf13/cobra\"\n)\n\nfunc TestIsAutomaticOutputFormat(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tformat string\n\t\texpected bool\n\t}{\n\t\t{\"empty format\", \"\", true},\n\t\t{\"auto format\", \"auto\", true},\n\t\t{\"short auto format\", \"a\", true},\n\t\t{\"json format\", \"json\", false},\n\t\t{\"yaml format\", \"yaml\", false},\n\t\t{\"xml format\", \"xml\", false},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t// Save original value\n\t\t\toriginalFormat := outputFormat\n\t\t\tdefer func() { outputFormat = originalFormat }()\n\n\t\t\toutputFormat = tt.format\n\t\t\tresult := isAutomaticOutputFormat()\n\t\t\tif result != tt.expected {\n\t\t\t\tt.Errorf(\"isAutomaticOutputFormat() = %v, want %v\", result, tt.expected)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMaybeFile(t *testing.T) {\n\t// Create a temporary file for testing\n\ttempFile, err := os.CreateTemp(\"\", \"test\")\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create temp file: %v\", err)\n\t}\n\tdefer os.Remove(tempFile.Name())\n\ttempFile.Close()\n\n\t// Create a temporary directory for testing\n\ttempDir, err := os.MkdirTemp(\"\", \"test\")\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create temp dir: %v\", err)\n\t}\n\tdefer os.RemoveAll(tempDir)\n\n\ttests := []struct {\n\t\tname string\n\t\tpath string\n\t\texpected bool\n\t}{\n\t\t{\"existing file\", tempFile.Name(), true},\n\t\t{\"existing directory\", tempDir, false},\n\t\t{\"non-existent path\", \"/path/that/does/not/exist\", false},\n\t\t{\"empty string\", \"\", false},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tresult := maybeFile(tt.path)\n\t\t\tif result != tt.expected {\n\t\t\t\tt.Errorf(\"maybeFile(%q) = %v, want %v\", tt.path, result, tt.expected)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestProcessArgs(t *testing.T) {\n\t// Create a temporary file for testing\n\ttempFile, err := os.CreateTemp(\"\", \"test\")\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create temp file: %v\", err)\n\t}\n\tdefer os.Remove(tempFile.Name())\n\ttempFile.Close()\n\n\t// Create a temporary .yq file for testing\n\ttempYqFile, err := os.Create(\"test.yq\")\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create temp yq file: %v\", err)\n\t}\n\tdefer os.Remove(tempYqFile.Name())\n\tif _, err = tempYqFile.WriteString(\".a.b\"); err != nil {\n\t\tt.Fatalf(\"Failed to write to temp yq file: %v\", err)\n\t}\n\ttempYqFile.Close()\n\n\ttests := []struct {\n\t\tname string\n\t\targs []string\n\t\tforceExpression string\n\t\texpressionFile string\n\t\texpectedExpr string\n\t\texpectedArgs []string\n\t\texpectError bool\n\t}{\n\t\t{\n\t\t\tname: \"empty args\",\n\t\t\targs: []string{},\n\t\t\tforceExpression: \"\",\n\t\t\texpressionFile: \"\",\n\t\t\texpectedExpr: \"\",\n\t\t\texpectedArgs: []string{},\n\t\t\texpectError: false,\n\t\t},\n\t\t{\n\t\t\tname: \"force expression\",\n\t\t\targs: []string{\"file1\"},\n\t\t\tforceExpression: \".a.b\",\n\t\t\texpressionFile: \"\",\n\t\t\texpectedExpr: \".a.b\",\n\t\t\texpectedArgs: []string{\"file1\"},\n\t\t\texpectError: false,\n\t\t},\n\t\t{\n\t\t\tname: \"expression as first arg\",\n\t\t\targs: []string{\".a.b\", \"file1\"},\n\t\t\tforceExpression: \"\",\n\t\t\texpressionFile: \"\",\n\t\t\texpectedExpr: \".a.b\",\n\t\t\texpectedArgs: []string{\"file1\"},\n\t\t\texpectError: false,\n\t\t},\n\t\t{\n\t\t\tname: \"file as first arg\",\n\t\t\targs: []string{tempFile.Name()},\n\t\t\tforceExpression: \"\",\n\t\t\texpressionFile: \"\",\n\t\t\texpectedExpr: \"\",\n\t\t\texpectedArgs: []string{tempFile.Name()},\n\t\t\texpectError: false,\n\t\t},\n\t\t{\n\t\t\tname: \"yq file as first arg\",\n\t\t\targs: []string{tempYqFile.Name(), \"things\"},\n\t\t\tforceExpression: \"\",\n\t\t\texpressionFile: \"\",\n\t\t\texpectedExpr: \".a.b\",\n\t\t\texpectedArgs: []string{\"things\"},\n\t\t\texpectError: false,\n\t\t},\n\t\t{\n\t\t\tname: \"dash as first arg\",\n\t\t\targs: []string{\"-\"},\n\t\t\tforceExpression: \"\",\n\t\t\texpressionFile: \"\",\n\t\t\texpectedExpr: \"\",\n\t\t\texpectedArgs: []string{\"-\"},\n\t\t\texpectError: false,\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t// Save original values\n\t\t\toriginalForceExpression := forceExpression\n\t\t\toriginalExpressionFile := expressionFile\n\t\t\tdefer func() {\n\t\t\t\tforceExpression = originalForceExpression\n\t\t\t\texpressionFile = originalExpressionFile\n\t\t\t}()\n\n\t\t\tforceExpression = tt.forceExpression\n\t\t\texpressionFile = tt.expressionFile\n\n\t\t\texpr, args, err := processArgs(tt.args)\n\t\t\tif tt.expectError {\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Errorf(\"processArgs() expected error but got none\")\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"processArgs() unexpected error: %v\", err)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tif expr != tt.expectedExpr {\n\t\t\t\tt.Errorf(\"processArgs() expression = %v, want %v\", expr, tt.expectedExpr)\n\t\t\t}\n\n\t\t\tif !stringsEqual(args, tt.expectedArgs) {\n\t\t\t\tt.Errorf(\"processArgs() args = %v, want %v\", args, tt.expectedArgs)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestConfigureDecoder(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tinputFormat string\n\t\tevaluateTogether bool\n\t\texpectError bool\n\t\texpectType string\n\t}{\n\t\t{\n\t\t\tname: \"yaml format\",\n\t\t\tinputFormat: \"yaml\",\n\t\t\tevaluateTogether: false,\n\t\t\texpectError: false,\n\t\t\texpectType: \"yamlDecoder\",\n\t\t},\n\t\t{\n\t\t\tname: \"json format\",\n\t\t\tinputFormat: \"json\",\n\t\t\tevaluateTogether: true,\n\t\t\texpectError: false,\n\t\t\texpectType: \"jsonDecoder\",\n\t\t},\n\t\t{\n\t\t\tname: \"xml format\",\n\t\t\tinputFormat: \"xml\",\n\t\t\tevaluateTogether: false,\n\t\t\texpectError: false,\n\t\t\texpectType: \"xmlDecoder\",\n\t\t},\n\t\t{\n\t\t\tname: \"invalid format\",\n\t\t\tinputFormat: \"invalid\",\n\t\t\tevaluateTogether: false,\n\t\t\texpectError: true,\n\t\t\texpectType: \"\",\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t// Save original value\n\t\t\toriginalInputFormat := inputFormat\n\t\t\tdefer func() { inputFormat = originalInputFormat }()\n\n\t\t\tinputFormat = tt.inputFormat\n\n\t\t\tdecoder, err := configureDecoder(tt.evaluateTogether)\n\t\t\tif tt.expectError {\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Errorf(\"configureDecoder() expected error but got none\")\n\t\t\t\t}\n\t\t\t\tif decoder != nil {\n\t\t\t\t\tt.Errorf(\"configureDecoder() expected nil decoder but got %v\", decoder)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"configureDecoder() unexpected error: %v\", err)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tif decoder == nil {\n\t\t\t\tt.Errorf(\"configureDecoder() expected decoder but got nil\")\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\ttypeStr := fmt.Sprintf(\"%T\", decoder)\n\t\t\tif !strings.Contains(typeStr, tt.expectType) {\n\t\t\t\tt.Errorf(\"configureDecoder() expected type to contain %q but got %q\", tt.expectType, typeStr)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestConfigurePrinterWriter(t *testing.T) {\n\tyqlib.InitExpressionParser()\n\n\ttests := []struct {\n\t\tname string\n\t\tsplitFileExp string\n\t\tformat *yqlib.Format\n\t\tforceColor bool\n\t\texpectError bool\n\t\texpectMulti bool\n\t\texpectColorsEnabled bool\n\t}{\n\t\t{\n\t\t\tname: \"single printer writer\",\n\t\t\tsplitFileExp: \"\",\n\t\t\tformat: &yqlib.Format{},\n\t\t\tforceColor: false,\n\t\t\texpectError: false,\n\t\t\texpectMulti: false,\n\t\t\texpectColorsEnabled: false,\n\t\t},\n\t\t{\n\t\t\tname: \"multi printer writer with valid expression\",\n\t\t\tsplitFileExp: \".a.b\",\n\t\t\tformat: &yqlib.Format{},\n\t\t\tforceColor: true,\n\t\t\texpectError: false,\n\t\t\texpectMulti: true,\n\t\t\texpectColorsEnabled: true,\n\t\t},\n\t\t{\n\t\t\tname: \"multi printer writer with invalid expression\",\n\t\t\tsplitFileExp: \"[invalid\",\n\t\t\tformat: &yqlib.Format{},\n\t\t\tforceColor: false,\n\t\t\texpectError: true,\n\t\t\texpectMulti: false,\n\t\t\texpectColorsEnabled: false,\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t// Save original values\n\t\t\toriginalSplitFileExp := splitFileExp\n\t\t\toriginalForceColor := forceColor\n\t\t\toriginalColorsEnabled := colorsEnabled\n\t\t\tdefer func() {\n\t\t\t\tsplitFileExp = originalSplitFileExp\n\t\t\t\tforceColor = originalForceColor\n\t\t\t\tcolorsEnabled = originalColorsEnabled\n\t\t\t}()\n\n\t\t\tsplitFileExp = tt.splitFileExp\n\t\t\tforceColor = tt.forceColor\n\t\t\tcolorsEnabled = false // Reset to test the setting\n\n\t\t\twriter, err := configurePrinterWriter(tt.format, os.Stdout)\n\t\t\tif tt.expectError {\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Errorf(\"configurePrinterWriter() expected error but got none\")\n\t\t\t\t}\n\t\t\t\tif writer != nil {\n\t\t\t\t\tt.Errorf(\"configurePrinterWriter() expected nil writer but got %v\", writer)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"configurePrinterWriter() unexpected error: %v\", err)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tif writer == nil {\n\t\t\t\tt.Errorf(\"configurePrinterWriter() expected writer but got nil\")\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\t// Explicitly check colorsEnabled\n\t\t\tif colorsEnabled != tt.expectColorsEnabled {\n\t\t\t\tt.Errorf(\"configurePrinterWriter() colorsEnabled = %v, want %v\", colorsEnabled, tt.expectColorsEnabled)\n\t\t\t}\n\n\t\t\t// Check the type of the returned writer\n\t\t\twriterType := fmt.Sprintf(\"%T\", writer)\n\t\t\tif tt.expectMulti {\n\t\t\t\tif !strings.Contains(writerType, \"multiPrintWriter\") {\n\t\t\t\t\tt.Errorf(\"configurePrinterWriter() expected multiPrintWriter but got %s\", writerType)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif !strings.Contains(writerType, \"singlePrinterWriter\") {\n\t\t\t\t\tt.Errorf(\"configurePrinterWriter() expected singlePrinterWriter but got %s\", writerType)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestConfigureEncoder(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\toutputFormat string\n\t\texpectError bool\n\t\texpectType string\n\t}{\n\t\t{\n\t\t\tname: \"yaml format\",\n\t\t\toutputFormat: \"yaml\",\n\t\t\texpectError: false,\n\t\t\texpectType: \"yamlEncoder\",\n\t\t},\n\t\t{\n\t\t\tname: \"json format\",\n\t\t\toutputFormat: \"json\",\n\t\t\texpectError: false,\n\t\t\texpectType: \"jsonEncoder\",\n\t\t},\n\t\t{\n\t\t\tname: \"xml format\",\n\t\t\toutputFormat: \"xml\",\n\t\t\texpectError: false,\n\t\t\texpectType: \"xmlEncoder\",\n\t\t},\n\t\t{\n\t\t\tname: \"properties format\",\n\t\t\toutputFormat: \"properties\",\n\t\t\texpectError: false,\n\t\t\texpectType: \"propertiesEncoder\",\n\t\t},\n\t\t{\n\t\t\tname: \"invalid format\",\n\t\t\toutputFormat: \"invalid\",\n\t\t\texpectError: true,\n\t\t\texpectType: \"\",\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t// Save original values\n\t\t\toriginalOutputFormat := outputFormat\n\t\t\toriginalIndent := indent\n\t\t\toriginalUnwrapScalar := unwrapScalar\n\t\t\toriginalColorsEnabled := colorsEnabled\n\t\t\toriginalNoDocSeparators := noDocSeparators\n\t\t\tdefer func() {\n\t\t\t\toutputFormat = originalOutputFormat\n\t\t\t\tindent = originalIndent\n\t\t\t\tunwrapScalar = originalUnwrapScalar\n\t\t\t\tcolorsEnabled = originalColorsEnabled\n\t\t\t\tnoDocSeparators = originalNoDocSeparators\n\t\t\t}()\n\n\t\t\toutputFormat = tt.outputFormat\n\t\t\tindent = 2\n\t\t\tunwrapScalar = false\n\t\t\tcolorsEnabled = false\n\t\t\tnoDocSeparators = false\n\n\t\t\tencoder, err := configureEncoder()\n\t\t\tif tt.expectError {\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Errorf(\"configureEncoder() expected error but got none\")\n\t\t\t\t}\n\t\t\t\tif encoder != nil {\n\t\t\t\t\tt.Errorf(\"configureEncoder() expected nil encoder but got %v\", encoder)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"configureEncoder() unexpected error: %v\", err)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tif encoder == nil {\n\t\t\t\tt.Errorf(\"configureEncoder() expected encoder but got nil\")\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\ttypeStr := fmt.Sprintf(\"%T\", encoder)\n\t\t\tif !strings.Contains(typeStr, tt.expectType) {\n\t\t\t\tt.Errorf(\"configureEncoder() expected type to contain %q but got %q\", tt.expectType, typeStr)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestInitCommand(t *testing.T) {\n\t// Create a temporary file for testing\n\ttempFile, err := os.CreateTemp(\"\", \"test\")\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create temp file: %v\", err)\n\t}\n\tdefer os.Remove(tempFile.Name())\n\ttempFile.Close()\n\n\t// Create a temporary split file\n\ttempSplitFile, err := os.CreateTemp(\"\", \"split\")\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create temp split file: %v\", err)\n\t}\n\tdefer os.Remove(tempSplitFile.Name())\n\tif _, err = tempSplitFile.WriteString(\".a.b\"); err != nil {\n\t\tt.Fatalf(\"Failed to write to temp split file: %v\", err)\n\t}\n\ttempSplitFile.Close()\n\n\ttests := []struct {\n\t\tname string\n\t\targs []string\n\t\twriteInplace bool\n\t\tfrontMatter string\n\t\tnullInput bool\n\t\tsplitFileExpFile string\n\t\tsplitFileExp string\n\t\toutputToJSON bool\n\t\texpectError bool\n\t\terrorContains string\n\t\texpectExpr string\n\t\texpectArgs []string\n\t}{\n\t\t{\n\t\t\tname: \"basic command\",\n\t\t\targs: []string{tempFile.Name()},\n\t\t\twriteInplace: false,\n\t\t\tfrontMatter: \"\",\n\t\t\tnullInput: false,\n\t\t\texpectError: false,\n\t\t\texpectExpr: \"\",\n\t\t\texpectArgs: []string{tempFile.Name()},\n\t\t},\n\t\t{\n\t\t\tname: \"write inplace with no args\",\n\t\t\targs: []string{},\n\t\t\twriteInplace: true,\n\t\t\tfrontMatter: \"\",\n\t\t\tnullInput: false,\n\t\t\texpectError: true,\n\t\t\terrorContains: \"write in place flag only applicable when giving an expression and at least one file\",\n\t\t},\n\t\t{\n\t\t\tname: \"split file expression from file\",\n\t\t\targs: []string{tempFile.Name()},\n\t\t\twriteInplace: false,\n\t\t\tfrontMatter: \"\",\n\t\t\tnullInput: false,\n\t\t\tsplitFileExpFile: tempSplitFile.Name(),\n\t\t\texpectError: false,\n\t\t\texpectExpr: \"\",\n\t\t\texpectArgs: []string{tempFile.Name()},\n\t\t},\n\t\t{\n\t\t\tname: \"output to JSON\",\n\t\t\targs: []string{tempFile.Name()},\n\t\t\twriteInplace: false,\n\t\t\tfrontMatter: \"\",\n\t\t\tnullInput: false,\n\t\t\toutputToJSON: true,\n\t\t\texpectError: false,\n\t\t\texpectExpr: \"\",\n\t\t\texpectArgs: []string{tempFile.Name()},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t// Save original values\n\t\t\toriginalWriteInplace := writeInplace\n\t\t\toriginalFrontMatter := frontMatter\n\t\t\toriginalNullInput := nullInput\n\t\t\toriginalSplitFileExpFile := splitFileExpFile\n\t\t\toriginalSplitFileExp := splitFileExp\n\t\t\toriginalOutputToJSON := outputToJSON\n\t\t\toriginalInputFormat := inputFormat\n\t\t\toriginalOutputFormat := outputFormat\n\t\t\toriginalForceColor := forceColor\n\t\t\toriginalForceNoColor := forceNoColor\n\t\t\toriginalColorsEnabled := colorsEnabled\n\t\t\tdefer func() {\n\t\t\t\twriteInplace = originalWriteInplace\n\t\t\t\tfrontMatter = originalFrontMatter\n\t\t\t\tnullInput = originalNullInput\n\t\t\t\tsplitFileExpFile = originalSplitFileExpFile\n\t\t\t\tsplitFileExp = originalSplitFileExp\n\t\t\t\toutputToJSON = originalOutputToJSON\n\t\t\t\tinputFormat = originalInputFormat\n\t\t\t\toutputFormat = originalOutputFormat\n\t\t\t\tforceColor = originalForceColor\n\t\t\t\tforceNoColor = originalForceNoColor\n\t\t\t\tcolorsEnabled = originalColorsEnabled\n\t\t\t}()\n\n\t\t\twriteInplace = tt.writeInplace\n\t\t\tfrontMatter = tt.frontMatter\n\t\t\tnullInput = tt.nullInput\n\t\t\tsplitFileExpFile = tt.splitFileExpFile\n\t\t\tsplitFileExp = tt.splitFileExp\n\t\t\toutputToJSON = tt.outputToJSON\n\t\t\tinputFormat = \"auto\"\n\t\t\toutputFormat = \"auto\"\n\t\t\tforceColor = false\n\t\t\tforceNoColor = false\n\t\t\tcolorsEnabled = false\n\n\t\t\tcmd := &cobra.Command{}\n\t\t\texpr, args, err := initCommand(cmd, tt.args)\n\t\t\tif tt.expectError {\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Errorf(\"initCommand() expected error but got none\")\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tif tt.errorContains != \"\" && !strings.Contains(err.Error(), tt.errorContains) {\n\t\t\t\t\tt.Errorf(\"initCommand() error '%v' does not contain '%v'\", err.Error(), tt.errorContains)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"initCommand() unexpected error: %v\", err)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tif expr != tt.expectExpr {\n\t\t\t\tt.Errorf(\"initCommand() expr = %v, want %v\", expr, tt.expectExpr)\n\t\t\t}\n\t\t\tif !stringsEqual(args, tt.expectArgs) {\n\t\t\t\tt.Errorf(\"initCommand() args = %v, want %v\", args, tt.expectArgs)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestProcessArgsWithExpressionFile(t *testing.T) {\n\t// Create a temporary .yq file with Windows line endings\n\ttempYqFile, err := os.CreateTemp(\"\", \"test.yq\")\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create temp yq file: %v\", err)\n\t}\n\tdefer os.Remove(tempYqFile.Name())\n\tif _, err = tempYqFile.WriteString(\".a.b\\r\\n.c.d\"); err != nil {\n\t\tt.Fatalf(\"Failed to write to temp yq file: %v\", err)\n\t}\n\ttempYqFile.Close()\n\n\t// Save original values\n\toriginalExpressionFile := expressionFile\n\tdefer func() { expressionFile = originalExpressionFile }()\n\n\texpressionFile = tempYqFile.Name()\n\n\texpr, args, err := processArgs([]string{\"file1\"})\n\tif err != nil {\n\t\tt.Errorf(\"processArgs() unexpected error: %v\", err)\n\t\treturn\n\t}\n\n\texpectedExpr := \".a.b\\n.c.d\" // Should convert \\r\\n to \\n\n\tif expr != expectedExpr {\n\t\tt.Errorf(\"processArgs() expression = %v, want %v\", expr, expectedExpr)\n\t}\n\n\texpectedArgs := []string{\"file1\"}\n\tif !stringsEqual(args, expectedArgs) {\n\t\tt.Errorf(\"processArgs() args = %v, want %v\", args, expectedArgs)\n\t}\n}\n\nfunc TestProcessArgsWithNonExistentExpressionFile(t *testing.T) {\n\t// Save original values\n\toriginalExpressionFile := expressionFile\n\tdefer func() { expressionFile = originalExpressionFile }()\n\n\texpressionFile = \"/path/that/does/not/exist\"\n\n\texpr, args, err := processArgs([]string{\"file1\"})\n\tif err == nil {\n\t\tt.Errorf(\"processArgs() expected error but got none\")\n\t}\n\tif expr != \"\" {\n\t\tt.Errorf(\"processArgs() expected empty expression but got %v\", expr)\n\t}\n\tif args != nil {\n\t\tt.Errorf(\"processArgs() expected nil args but got %v\", args)\n\t}\n}\n\nfunc TestInitCommandWithInvalidOutputFormat(t *testing.T) {\n\t// Create a temporary file for testing\n\ttempFile, err := os.CreateTemp(\"\", \"test\")\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create temp file: %v\", err)\n\t}\n\tdefer os.Remove(tempFile.Name())\n\ttempFile.Close()\n\n\t// Save original values\n\toriginalInputFormat := inputFormat\n\toriginalOutputFormat := outputFormat\n\tdefer func() {\n\t\tinputFormat = originalInputFormat\n\t\toutputFormat = originalOutputFormat\n\t}()\n\n\tinputFormat = \"auto\"\n\toutputFormat = \"invalid\"\n\n\tcmd := &cobra.Command{}\n\texpr, args, err := initCommand(cmd, []string{tempFile.Name()})\n\tif err == nil {\n\t\tt.Errorf(\"initCommand() expected error but got none\")\n\t}\n\tif expr != \"\" {\n\t\tt.Errorf(\"initCommand() expected empty expression but got %v\", expr)\n\t}\n\tif args != nil {\n\t\tt.Errorf(\"initCommand() expected nil args but got %v\", args)\n\t}\n}\n\nfunc TestInitCommandWithUnknownInputFormat(t *testing.T) {\n\t// Create a temporary file with unknown extension\n\ttempFile, err := os.CreateTemp(\"\", \"test.unknown\")\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create temp file: %v\", err)\n\t}\n\tdefer os.Remove(tempFile.Name())\n\ttempFile.Close()\n\n\t// Save original values\n\toriginalInputFormat := inputFormat\n\toriginalOutputFormat := outputFormat\n\tdefer func() {\n\t\tinputFormat = originalInputFormat\n\t\toutputFormat = originalOutputFormat\n\t}()\n\n\tinputFormat = \"auto\"\n\toutputFormat = \"auto\"\n\n\tcmd := &cobra.Command{}\n\texpr, args, err := initCommand(cmd, []string{tempFile.Name()})\n\tif err != nil {\n\t\tt.Errorf(\"initCommand() unexpected error: %v\", err)\n\t\treturn\n\t}\n\n\t// expr can be empty when no expression is provided\n\t_ = expr\n\tif args == nil {\n\t\tt.Errorf(\"initCommand() expected non-nil args\")\n\t}\n}\n\nfunc TestConfigurePrinterWriterWithInvalidSplitExpression(t *testing.T) {\n\t// Save original value\n\toriginalSplitFileExp := splitFileExp\n\tdefer func() { splitFileExp = originalSplitFileExp }()\n\n\tsplitFileExp = \"[invalid expression\"\n\n\twriter, err := configurePrinterWriter(&yqlib.Format{}, os.Stdout)\n\tif err == nil {\n\t\tt.Errorf(\"configurePrinterWriter() expected error but got none\")\n\t}\n\tif writer != nil {\n\t\tt.Errorf(\"configurePrinterWriter() expected nil writer but got %v\", writer)\n\t}\n\tif err != nil && !strings.Contains(err.Error(), \"bad split document expression\") {\n\t\tt.Errorf(\"configurePrinterWriter() error '%v' does not contain expected message\", err.Error())\n\t}\n}\n\nfunc TestMaybeFileWithDirectory(t *testing.T) {\n\t// Create a temporary directory\n\ttempDir, err := os.MkdirTemp(\"\", \"test\")\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create temp dir: %v\", err)\n\t}\n\tdefer os.RemoveAll(tempDir)\n\n\tresult := maybeFile(tempDir)\n\tif result {\n\t\tt.Errorf(\"maybeFile(%q) = %v, want false\", tempDir, result)\n\t}\n}\n\nfunc TestProcessStdInArgsWithDash(t *testing.T) {\n\targs := []string{\"-\", \"file1\"}\n\tresult := processStdInArgs(args)\n\tif !stringsEqual(result, args) {\n\t\tt.Errorf(\"processStdInArgs() = %v, want %v\", result, args)\n\t}\n}\n\nfunc TestProcessArgsWithYqFileExtension(t *testing.T) {\n\ttempYqFile, err := os.Create(\"test.yq\")\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create temp yq file: %v\", err)\n\t}\n\tdefer os.Remove(tempYqFile.Name())\n\tif _, err = tempYqFile.WriteString(\".a.b\"); err != nil {\n\t\tt.Fatalf(\"Failed to write to temp yq file: %v\", err)\n\t}\n\ttempYqFile.Close()\n\n\t// Save original values\n\toriginalExpressionFile := expressionFile\n\toriginalForceExpression := forceExpression\n\tdefer func() {\n\t\texpressionFile = originalExpressionFile\n\t\tforceExpression = originalForceExpression\n\t}()\n\n\t// Reset expressionFile to empty to test the auto-detection\n\texpressionFile = \"\"\n\tforceExpression = \"\"\n\n\t// Debug: check the conditions manually\n\tt.Logf(\"expressionFile: %q\", expressionFile)\n\tt.Logf(\"forceExpression: %q\", forceExpression)\n\tt.Logf(\"tempYqFile.Name(): %q\", tempYqFile.Name())\n\tt.Logf(\"strings.HasSuffix(tempYqFile.Name(), '.yq'): %v\", strings.HasSuffix(tempYqFile.Name(), \".yq\"))\n\tt.Logf(\"maybeFile(tempYqFile.Name()): %v\", maybeFile(tempYqFile.Name()))\n\n\t// Test with only the yq file as argument (should be treated as expression file)\n\texpr, args, err := processArgs([]string{tempYqFile.Name()})\n\tif err != nil {\n\t\tt.Errorf(\"processArgs() unexpected error: %v\", err)\n\t\treturn\n\t}\n\n\tif expr != \".a.b\" {\n\t\tt.Errorf(\"processArgs() expression = %v, want .a.b\", expr)\n\t}\n\n\texpectedArgs := []string{}\n\tif !stringsEqual(args, expectedArgs) {\n\t\tt.Errorf(\"processArgs() args = %v, want %v\", args, expectedArgs)\n\t}\n}\n\nfunc TestConfigureEncoderWithYamlFormat(t *testing.T) {\n\t// Save original values\n\toriginalOutputFormat := outputFormat\n\toriginalIndent := indent\n\toriginalUnwrapScalar := unwrapScalar\n\toriginalColorsEnabled := colorsEnabled\n\toriginalNoDocSeparators := noDocSeparators\n\tdefer func() {\n\t\toutputFormat = originalOutputFormat\n\t\tindent = originalIndent\n\t\tunwrapScalar = originalUnwrapScalar\n\t\tcolorsEnabled = originalColorsEnabled\n\t\tnoDocSeparators = originalNoDocSeparators\n\t}()\n\n\toutputFormat = \"yaml\"\n\tindent = 4\n\tunwrapScalar = true\n\tcolorsEnabled = true\n\tnoDocSeparators = true\n\n\tencoder, err := configureEncoder()\n\tif err != nil {\n\t\tt.Errorf(\"configureEncoder() unexpected error: %v\", err)\n\t\treturn\n\t}\n\n\tif encoder == nil {\n\t\tt.Errorf(\"configureEncoder() expected encoder but got nil\")\n\t}\n}\n\nfunc TestConfigureEncoderWithPropertiesFormat(t *testing.T) {\n\t// Save original values\n\toriginalOutputFormat := outputFormat\n\toriginalIndent := indent\n\toriginalUnwrapScalar := unwrapScalar\n\toriginalColorsEnabled := colorsEnabled\n\toriginalNoDocSeparators := noDocSeparators\n\tdefer func() {\n\t\toutputFormat = originalOutputFormat\n\t\tindent = originalIndent\n\t\tunwrapScalar = originalUnwrapScalar\n\t\tcolorsEnabled = originalColorsEnabled\n\t\tnoDocSeparators = originalNoDocSeparators\n\t}()\n\n\toutputFormat = \"properties\"\n\tindent = 2\n\tunwrapScalar = false\n\tcolorsEnabled = false\n\tnoDocSeparators = false\n\n\tencoder, err := configureEncoder()\n\tif err != nil {\n\t\tt.Errorf(\"configureEncoder() unexpected error: %v\", err)\n\t\treturn\n\t}\n\n\tif encoder == nil {\n\t\tt.Errorf(\"configureEncoder() expected encoder but got nil\")\n\t}\n}\n\n// Mock boolFlag for testing\ntype mockBoolFlag struct {\n\texplicitlySet bool\n\tvalue bool\n}\n\nfunc (f *mockBoolFlag) IsExplicitlySet() bool {\n\treturn f.explicitlySet\n}\n\nfunc (f *mockBoolFlag) IsSet() bool {\n\treturn f.value\n}\n\nfunc (f *mockBoolFlag) String() string {\n\treturn \"mock\"\n}\n\nfunc (f *mockBoolFlag) Set(_ string) error {\n\treturn nil\n}\n\nfunc (f *mockBoolFlag) Type() string {\n\treturn \"bool\"\n}\n\n// Helper function to compare string slices\nfunc stringsEqual(a, b []string) bool {\n\tif len(a) != len(b) {\n\t\treturn false\n\t}\n\tfor i := range a {\n\t\tif a[i] != b[i] {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc TestSetupColors(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tforceColor bool\n\t\tforceNoColor bool\n\t\texpectColors bool\n\t}{\n\t\t{\n\t\t\tname: \"force colour enabled\",\n\t\t\tforceColor: true,\n\t\t\tforceNoColor: false,\n\t\t\texpectColors: true,\n\t\t},\n\t\t{\n\t\t\tname: \"force no colour enabled\",\n\t\t\tforceColor: false,\n\t\t\tforceNoColor: true,\n\t\t\texpectColors: false,\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t// Save original values\n\t\t\toriginalForceColor := forceColor\n\t\t\toriginalForceNoColor := forceNoColor\n\t\t\toriginalColorsEnabled := colorsEnabled\n\t\t\tdefer func() {\n\t\t\t\tforceColor = originalForceColor\n\t\t\t\tforceNoColor = originalForceNoColor\n\t\t\t\tcolorsEnabled = originalColorsEnabled\n\t\t\t}()\n\n\t\t\tforceColor = tt.forceColor\n\t\t\tforceNoColor = tt.forceNoColor\n\t\t\tcolorsEnabled = false // Reset to test the setting\n\n\t\t\tsetupColors()\n\n\t\t\tif colorsEnabled != tt.expectColors {\n\t\t\t\tt.Errorf(\"setupColors() colorsEnabled = %v, want %v\", colorsEnabled, tt.expectColors)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestLoadSplitFileExpression(t *testing.T) {\n\t// Create a temporary file with expression content\n\ttempFile, err := os.CreateTemp(\"\", \"split\")\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create temp file: %v\", err)\n\t}\n\tdefer os.Remove(tempFile.Name())\n\tif _, err = tempFile.WriteString(\".a.b\"); err != nil {\n\t\tt.Fatalf(\"Failed to write to temp file: %v\", err)\n\t}\n\ttempFile.Close()\n\n\ttests := []struct {\n\t\tname string\n\t\tsplitFileExpFile string\n\t\texpectError bool\n\t\texpectContent string\n\t}{\n\t\t{\n\t\t\tname: \"load from file\",\n\t\t\tsplitFileExpFile: tempFile.Name(),\n\t\t\texpectError: false,\n\t\t\texpectContent: \".a.b\",\n\t\t},\n\t\t{\n\t\t\tname: \"no file specified\",\n\t\t\tsplitFileExpFile: \"\",\n\t\t\texpectError: false,\n\t\t\texpectContent: \"\",\n\t\t},\n\t\t{\n\t\t\tname: \"non-existent file\",\n\t\t\tsplitFileExpFile: \"/path/that/does/not/exist\",\n\t\t\texpectError: true,\n\t\t\texpectContent: \"\",\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t// Save original value\n\t\t\toriginalSplitFileExpFile := splitFileExpFile\n\t\t\toriginalSplitFileExp := splitFileExp\n\t\t\tdefer func() {\n\t\t\t\tsplitFileExpFile = originalSplitFileExpFile\n\t\t\t\tsplitFileExp = originalSplitFileExp\n\t\t\t}()\n\n\t\t\tsplitFileExpFile = tt.splitFileExpFile\n\t\t\tsplitFileExp = \"\"\n\n\t\t\terr := loadSplitFileExpression()\n\t\t\tif tt.expectError {\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Errorf(\"loadSplitFileExpression() expected error but got none\")\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"loadSplitFileExpression() unexpected error: %v\", err)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tif splitFileExp != tt.expectContent {\n\t\t\t\tt.Errorf(\"loadSplitFileExpression() splitFileExp = %v, want %v\", splitFileExp, tt.expectContent)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestHandleBackwardsCompatibility(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\toutputToJSON bool\n\t\tinitialFormat string\n\t\texpectFormat string\n\t}{\n\t\t{\n\t\t\tname: \"outputToJSON true\",\n\t\t\toutputToJSON: true,\n\t\t\tinitialFormat: \"yaml\",\n\t\t\texpectFormat: \"json\",\n\t\t},\n\t\t{\n\t\t\tname: \"outputToJSON false\",\n\t\t\toutputToJSON: false,\n\t\t\tinitialFormat: \"yaml\",\n\t\t\texpectFormat: \"yaml\",\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t// Save original value\n\t\t\toriginalOutputToJSON := outputToJSON\n\t\t\toriginalOutputFormat := outputFormat\n\t\t\tdefer func() {\n\t\t\t\toutputToJSON = originalOutputToJSON\n\t\t\t\toutputFormat = originalOutputFormat\n\t\t\t}()\n\n\t\t\toutputToJSON = tt.outputToJSON\n\t\t\toutputFormat = tt.initialFormat\n\n\t\t\thandleBackwardsCompatibility()\n\n\t\t\tif outputFormat != tt.expectFormat {\n\t\t\t\tt.Errorf(\"handleBackwardsCompatibility() outputFormat = %v, want %v\", outputFormat, tt.expectFormat)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestValidateCommandFlags(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs []string\n\t\twriteInplace bool\n\t\tfrontMatter string\n\t\tsplitFileExp string\n\t\tnullInput bool\n\t\texpectError bool\n\t\terrorContains string\n\t}{\n\t\t{\n\t\t\tname: \"valid flags\",\n\t\t\targs: []string{\"file.yaml\"},\n\t\t\twriteInplace: false,\n\t\t\tfrontMatter: \"\",\n\t\t\tsplitFileExp: \"\",\n\t\t\tnullInput: false,\n\t\t\texpectError: false,\n\t\t},\n\t\t{\n\t\t\tname: \"write inplace with no args\",\n\t\t\targs: []string{},\n\t\t\twriteInplace: true,\n\t\t\tfrontMatter: \"\",\n\t\t\tsplitFileExp: \"\",\n\t\t\tnullInput: false,\n\t\t\texpectError: true,\n\t\t\terrorContains: \"write in place flag only applicable when giving an expression and at least one file\",\n\t\t},\n\t\t{\n\t\t\tname: \"write inplace with dash\",\n\t\t\targs: []string{\"-\"},\n\t\t\twriteInplace: true,\n\t\t\tfrontMatter: \"\",\n\t\t\tsplitFileExp: \"\",\n\t\t\tnullInput: false,\n\t\t\texpectError: true,\n\t\t\terrorContains: \"write in place flag only applicable when giving an expression and at least one file\",\n\t\t},\n\t\t{\n\t\t\tname: \"front matter with no args\",\n\t\t\targs: []string{},\n\t\t\twriteInplace: false,\n\t\t\tfrontMatter: \"extract\",\n\t\t\tsplitFileExp: \"\",\n\t\t\tnullInput: false,\n\t\t\texpectError: true,\n\t\t\terrorContains: \"front matter flag only applicable when giving an expression and at least one file\",\n\t\t},\n\t\t{\n\t\t\tname: \"write inplace with split file\",\n\t\t\targs: []string{\"file.yaml\"},\n\t\t\twriteInplace: true,\n\t\t\tfrontMatter: \"\",\n\t\t\tsplitFileExp: \".a.b\",\n\t\t\tnullInput: false,\n\t\t\texpectError: true,\n\t\t\terrorContains: \"write in place cannot be used with split file\",\n\t\t},\n\t\t{\n\t\t\tname: \"null input with args\",\n\t\t\targs: []string{\"file.yaml\"},\n\t\t\twriteInplace: false,\n\t\t\tfrontMatter: \"\",\n\t\t\tsplitFileExp: \"\",\n\t\t\tnullInput: true,\n\t\t\texpectError: true,\n\t\t\terrorContains: \"cannot pass files in when using null-input flag\",\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t// Save original values\n\t\t\toriginalWriteInplace := writeInplace\n\t\t\toriginalFrontMatter := frontMatter\n\t\t\toriginalSplitFileExp := splitFileExp\n\t\t\toriginalNullInput := nullInput\n\t\t\tdefer func() {\n\t\t\t\twriteInplace = originalWriteInplace\n\t\t\t\tfrontMatter = originalFrontMatter\n\t\t\t\tsplitFileExp = originalSplitFileExp\n\t\t\t\tnullInput = originalNullInput\n\t\t\t}()\n\n\t\t\twriteInplace = tt.writeInplace\n\t\t\tfrontMatter = tt.frontMatter\n\t\t\tsplitFileExp = tt.splitFileExp\n\t\t\tnullInput = tt.nullInput\n\n\t\t\terr := validateCommandFlags(tt.args)\n\t\t\tif tt.expectError {\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Errorf(\"validateCommandFlags() expected error but got none\")\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tif tt.errorContains != \"\" && !strings.Contains(err.Error(), tt.errorContains) {\n\t\t\t\t\tt.Errorf(\"validateCommandFlags() error '%v' does not contain '%v'\", err.Error(), tt.errorContains)\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"validateCommandFlags() unexpected error: %v\", err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestConfigureFormats(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\targs []string\n\t\tinputFormat string\n\t\toutputFormat string\n\t\texpectError bool\n\t}{\n\t\t{\n\t\t\tname: \"valid formats\",\n\t\t\targs: []string{\"file.yaml\"},\n\t\t\tinputFormat: \"auto\",\n\t\t\toutputFormat: \"auto\",\n\t\t\texpectError: false,\n\t\t},\n\t\t{\n\t\t\tname: \"invalid output format\",\n\t\t\targs: []string{\"file.yaml\"},\n\t\t\tinputFormat: \"auto\",\n\t\t\toutputFormat: \"invalid\",\n\t\t\texpectError: true,\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t// Save original values\n\t\t\toriginalInputFormat := inputFormat\n\t\t\toriginalOutputFormat := outputFormat\n\t\t\tdefer func() {\n\t\t\t\tinputFormat = originalInputFormat\n\t\t\t\toutputFormat = originalOutputFormat\n\t\t\t}()\n\n\t\t\tinputFormat = tt.inputFormat\n\t\t\toutputFormat = tt.outputFormat\n\n\t\t\terr := configureFormats(tt.args)\n\t\t\tif tt.expectError {\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Errorf(\"configureFormats() expected error but got none\")\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"configureFormats() unexpected error: %v\", err)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestConfigureInputFormat(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tinputFilename string\n\t\tinputFormat string\n\t\toutputFormat string\n\t\texpectInput string\n\t\texpectOutput string\n\t}{\n\t\t{\n\t\t\tname: \"auto format with yaml file\",\n\t\t\tinputFilename: \"file.yaml\",\n\t\t\tinputFormat: \"auto\",\n\t\t\toutputFormat: \"auto\",\n\t\t\texpectInput: \"yaml\",\n\t\t\texpectOutput: \"yaml\",\n\t\t},\n\t\t{\n\t\t\tname: \"auto format with json file\",\n\t\t\tinputFilename: \"file.json\",\n\t\t\tinputFormat: \"auto\",\n\t\t\toutputFormat: \"auto\",\n\t\t\texpectInput: \"json\",\n\t\t\texpectOutput: \"json\",\n\t\t},\n\t\t{\n\t\t\tname: \"auto format with unknown file\",\n\t\t\tinputFilename: \"file.unknown\",\n\t\t\tinputFormat: \"auto\",\n\t\t\toutputFormat: \"auto\",\n\t\t\texpectInput: \"yaml\",\n\t\t\texpectOutput: \"yaml\",\n\t\t},\n\t\t{\n\t\t\tname: \"explicit format\",\n\t\t\tinputFilename: \"file.yaml\",\n\t\t\tinputFormat: \"json\",\n\t\t\toutputFormat: \"auto\",\n\t\t\texpectInput: \"json\",\n\t\t\texpectOutput: \"yaml\", // backwards compatibility\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t// Save original values\n\t\t\toriginalInputFormat := inputFormat\n\t\t\toriginalOutputFormat := outputFormat\n\t\t\tdefer func() {\n\t\t\t\tinputFormat = originalInputFormat\n\t\t\t\toutputFormat = originalOutputFormat\n\t\t\t}()\n\n\t\t\tinputFormat = tt.inputFormat\n\t\t\toutputFormat = tt.outputFormat\n\n\t\t\terr := configureInputFormat(tt.inputFilename)\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"configureInputFormat() unexpected error: %v\", err)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tif inputFormat != tt.expectInput {\n\t\t\t\tt.Errorf(\"configureInputFormat() inputFormat = %v, want %v\", inputFormat, tt.expectInput)\n\t\t\t}\n\t\t\tif outputFormat != tt.expectOutput {\n\t\t\t\tt.Errorf(\"configureInputFormat() outputFormat = %v, want %v\", outputFormat, tt.expectOutput)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestConfigureOutputFormat(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\toutputFormat string\n\t\texpectError bool\n\t\texpectUnwrap bool\n\t}{\n\t\t{\n\t\t\tname: \"yaml format\",\n\t\t\toutputFormat: \"yaml\",\n\t\t\texpectError: false,\n\t\t\texpectUnwrap: true,\n\t\t},\n\t\t{\n\t\t\tname: \"properties format\",\n\t\t\toutputFormat: \"properties\",\n\t\t\texpectError: false,\n\t\t\texpectUnwrap: true,\n\t\t},\n\t\t{\n\t\t\tname: \"json format\",\n\t\t\toutputFormat: \"json\",\n\t\t\texpectError: false,\n\t\t\texpectUnwrap: false,\n\t\t},\n\t\t{\n\t\t\tname: \"invalid format\",\n\t\t\toutputFormat: \"invalid\",\n\t\t\texpectError: true,\n\t\t\texpectUnwrap: false,\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t// Save original values\n\t\t\toriginalOutputFormat := outputFormat\n\t\t\toriginalUnwrapScalar := unwrapScalar\n\t\t\tdefer func() {\n\t\t\t\toutputFormat = originalOutputFormat\n\t\t\t\tunwrapScalar = originalUnwrapScalar\n\t\t\t}()\n\n\t\t\toutputFormat = tt.outputFormat\n\t\t\tunwrapScalar = false // Reset to test the setting\n\n\t\t\terr := configureOutputFormat()\n\t\t\tif tt.expectError {\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Errorf(\"configureOutputFormat() expected error but got none\")\n\t\t\t\t}\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"configureOutputFormat() unexpected error: %v\", err)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tif unwrapScalar != tt.expectUnwrap {\n\t\t\t\tt.Errorf(\"configureOutputFormat() unwrapScalar = %v, want %v\", unwrapScalar, tt.expectUnwrap)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestConfigureUnwrapScalar(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\texplicitlySet bool\n\t\tflagValue bool\n\t\tinitialUnwrap bool\n\t\texpectUnwrap bool\n\t}{\n\t\t{\n\t\t\tname: \"flag not explicitly set\",\n\t\t\texplicitlySet: false,\n\t\t\tflagValue: true,\n\t\t\tinitialUnwrap: true,\n\t\t\texpectUnwrap: true, // Should remain unchanged\n\t\t},\n\t\t{\n\t\t\tname: \"flag explicitly set to true\",\n\t\t\texplicitlySet: true,\n\t\t\tflagValue: true,\n\t\t\tinitialUnwrap: false,\n\t\t\texpectUnwrap: true,\n\t\t},\n\t\t{\n\t\t\tname: \"flag explicitly set to false\",\n\t\t\texplicitlySet: true,\n\t\t\tflagValue: false,\n\t\t\tinitialUnwrap: true,\n\t\t\texpectUnwrap: false,\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t// Save original value\n\t\t\toriginalUnwrapScalar := unwrapScalar\n\t\t\toriginalUnwrapScalarFlag := unwrapScalarFlag\n\t\t\tdefer func() {\n\t\t\t\tunwrapScalar = originalUnwrapScalar\n\t\t\t\tunwrapScalarFlag = originalUnwrapScalarFlag\n\t\t\t}()\n\n\t\t\tunwrapScalar = tt.initialUnwrap\n\t\t\tunwrapScalarFlag = &mockBoolFlag{\n\t\t\t\texplicitlySet: tt.explicitlySet,\n\t\t\t\tvalue: tt.flagValue,\n\t\t\t}\n\n\t\t\tconfigureUnwrapScalar()\n\n\t\t\tif unwrapScalar != tt.expectUnwrap {\n\t\t\t\tt.Errorf(\"configureUnwrapScalar() unwrapScalar = %v, want %v\", unwrapScalar, tt.expectUnwrap)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "cmd/version.go", "content": "package cmd\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n)\n\n// The git commit that was compiled. This will be filled in by the compiler.\nvar (\n\tGitCommit string\n\tGitDescribe string\n\n\t// Version is main version number that is being run at the moment.\n\tVersion = \"v4.52.4\"\n\n\t// VersionPrerelease is a pre-release marker for the version. If this is \"\" (empty string)\n\t// then it means that it is a final release. Otherwise, this is a pre-release\n\t// such as \"dev\" (in development), \"beta\", \"rc1\", etc.\n\tVersionPrerelease = \"\"\n)\n\n// ProductName is the name of the product\nconst ProductName = \"yq\"\n\n// GetVersionDisplay composes the parts of the version in a way that's suitable\n// for displaying to humans.\nfunc GetVersionDisplay() string {\n\treturn fmt.Sprintf(\"yq (https://github.com/mikefarah/yq/) version %s\\n\", getHumanVersion())\n}\n\nfunc getHumanVersion() string {\n\tversion := Version\n\tif GitDescribe != \"\" {\n\t\tversion = GitDescribe\n\t}\n\n\trelease := VersionPrerelease\n\tif release != \"\" {\n\t\tif !strings.Contains(version, release) {\n\t\t\tversion += fmt.Sprintf(\"-%s\", release)\n\t\t}\n\t\tif GitCommit != \"\" {\n\t\t\tversion += fmt.Sprintf(\" (%s)\", GitCommit)\n\t\t}\n\t}\n\n\t// Strip off any single quotes added by the git information.\n\treturn strings.ReplaceAll(version, \"'\", \"\")\n}\n" }, { "path": "cmd/version_test.go", "content": "package cmd\n\nimport (\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc TestGetVersionDisplay(t *testing.T) {\n\tvar expectedVersion = ProductName + \" (https://github.com/mikefarah/yq/) version \" + Version\n\tif VersionPrerelease != \"\" {\n\t\texpectedVersion = expectedVersion + \"-\" + VersionPrerelease\n\t}\n\texpectedVersion = expectedVersion + \"\\n\"\n\ttests := []struct {\n\t\tname string\n\t\twant string\n\t}{\n\t\t{\n\t\t\tname: \"Display Version\",\n\t\t\twant: expectedVersion,\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tif got := GetVersionDisplay(); got != tt.want {\n\t\t\tt.Errorf(\"%q. GetVersionDisplay() = %v, want %v\", tt.name, got, tt.want)\n\t\t}\n\t}\n}\n\nfunc Test_getHumanVersion(t *testing.T) {\n\t// Save original values\n\torigGitDescribe := GitDescribe\n\torigGitCommit := GitCommit\n\torigVersionPrerelease := VersionPrerelease\n\n\t// Restore after test\n\tdefer func() {\n\t\tGitDescribe = origGitDescribe\n\t\tGitCommit = origGitCommit\n\t\tVersionPrerelease = origVersionPrerelease\n\t}()\n\n\tGitDescribe = \"e42813d\"\n\tGitCommit = \"e42813d+CHANGES\"\n\tvar wanted string\n\tif VersionPrerelease == \"\" {\n\t\twanted = GitDescribe\n\t} else {\n\t\twanted = \"e42813d-\" + VersionPrerelease + \" (e42813d+CHANGES)\"\n\t}\n\n\ttests := []struct {\n\t\tname string\n\t\twant string\n\t}{\n\t\t{\n\t\t\tname: \"Git Variables defined\",\n\t\t\twant: wanted,\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tif got := getHumanVersion(); got != tt.want {\n\t\t\tt.Errorf(\"%q. getHumanVersion() = %v, want %v\", tt.name, got, tt.want)\n\t\t}\n\t}\n}\n\nfunc Test_getHumanVersion_NoGitDescribe(t *testing.T) {\n\t// Save original values\n\torigGitDescribe := GitDescribe\n\torigGitCommit := GitCommit\n\torigVersionPrerelease := VersionPrerelease\n\n\t// Restore after test\n\tdefer func() {\n\t\tGitDescribe = origGitDescribe\n\t\tGitCommit = origGitCommit\n\t\tVersionPrerelease = origVersionPrerelease\n\t}()\n\n\tGitDescribe = \"\"\n\tGitCommit = \"\"\n\tVersionPrerelease = \"\"\n\n\tgot := getHumanVersion()\n\tif got != Version {\n\t\tt.Errorf(\"getHumanVersion() = %v, want %v\", got, Version)\n\t}\n}\n\nfunc Test_getHumanVersion_WithPrerelease(t *testing.T) {\n\t// Save original values\n\torigGitDescribe := GitDescribe\n\torigGitCommit := GitCommit\n\torigVersionPrerelease := VersionPrerelease\n\n\t// Restore after test\n\tdefer func() {\n\t\tGitDescribe = origGitDescribe\n\t\tGitCommit = origGitCommit\n\t\tVersionPrerelease = origVersionPrerelease\n\t}()\n\n\tGitDescribe = \"\"\n\tGitCommit = \"abc123\"\n\tVersionPrerelease = \"beta\"\n\n\tgot := getHumanVersion()\n\texpected := Version + \"-beta (abc123)\"\n\tif got != expected {\n\t\tt.Errorf(\"getHumanVersion() = %v, want %v\", got, expected)\n\t}\n}\n\nfunc Test_getHumanVersion_PrereleaseInVersion(t *testing.T) {\n\t// Save original values\n\torigGitDescribe := GitDescribe\n\torigGitCommit := GitCommit\n\torigVersionPrerelease := VersionPrerelease\n\n\t// Restore after test\n\tdefer func() {\n\t\tGitDescribe = origGitDescribe\n\t\tGitCommit = origGitCommit\n\t\tVersionPrerelease = origVersionPrerelease\n\t}()\n\n\tGitDescribe = \"v1.2.3-rc1\"\n\tGitCommit = \"xyz789\"\n\tVersionPrerelease = \"rc1\"\n\n\tgot := getHumanVersion()\n\t// Should not duplicate \"rc1\" since it's already in GitDescribe\n\texpected := \"v1.2.3-rc1 (xyz789)\"\n\tif got != expected {\n\t\tt.Errorf(\"getHumanVersion() = %v, want %v\", got, expected)\n\t}\n}\n\nfunc Test_getHumanVersion_StripSingleQuotes(t *testing.T) {\n\t// Save original values\n\torigGitDescribe := GitDescribe\n\torigGitCommit := GitCommit\n\torigVersionPrerelease := VersionPrerelease\n\n\t// Restore after test\n\tdefer func() {\n\t\tGitDescribe = origGitDescribe\n\t\tGitCommit = origGitCommit\n\t\tVersionPrerelease = origVersionPrerelease\n\t}()\n\n\tGitDescribe = \"'v1.2.3'\"\n\tGitCommit = \"'commit123'\"\n\tVersionPrerelease = \"\"\n\n\tgot := getHumanVersion()\n\t// Should strip single quotes\n\tif strings.Contains(got, \"'\") {\n\t\tt.Errorf(\"getHumanVersion() = %v, should not contain single quotes\", got)\n\t}\n\texpected := \"v1.2.3\"\n\tif got != expected {\n\t\tt.Errorf(\"getHumanVersion() = %v, want %v\", got, expected)\n\t}\n}\n\nfunc TestProductName(t *testing.T) {\n\tif ProductName != \"yq\" {\n\t\tt.Errorf(\"ProductName = %v, want yq\", ProductName)\n\t}\n}\n\nfunc TestVersionIsSet(t *testing.T) {\n\tif Version == \"\" {\n\t\tt.Error(\"Version should not be empty\")\n\t}\n\tif !strings.HasPrefix(Version, \"v\") {\n\t\tt.Errorf(\"Version %v should start with 'v'\", Version)\n\t}\n}\n" }, { "path": "cspell.config.yaml", "content": "---\n$schema: https://raw.githubusercontent.com/streetsidesoftware/cspell/main/cspell.schema.json\nversion: '0.2'\nlanguage: en-GB\ndictionaryDefinitions:\n - name: project-words\n path: './project-words.txt'\n addWords: true\ndictionaries:\n - project-words\nignorePaths:\n - 'vendor'\n - 'bin'\n - '/project-words.txt'\n" }, { "path": "examples/array.yaml", "content": "- [cat, dog, frog, cow]\n- [apple, banana, grape, mango]" }, { "path": "examples/bad.yaml", "content": "b:\n d: be gone\n c: 2\n e:\n - name: Billy Bob # comment over here\n\n---\n[123123" }, { "path": "examples/base64.txt", "content": "bXkgc2VjcmV0IGNoaWxsaSByZWNpcGUgaXMuLi4u" }, { "path": "examples/data.lua", "content": "return {\n\t[\"country\"] = \"Australia\"; -- this place\n\t[\"cities\"] = {\n\t\t\"Sydney\",\n\t\t\"Melbourne\",\n\t\t\"Brisbane\",\n\t\t\"Perth\",\n\t};\n};" }, { "path": "examples/data1-no-comments.yaml", "content": "a: simple\nb: [1, 2]\nc:\n test: 1\n" }, { "path": "examples/data1.yaml", "content": "a: apple" }, { "path": "examples/data2.yaml", "content": "# --------------------------------------------------\n# It's a test with comment\n# --------------------------------------------------\ngroups:\n - name: d" }, { "path": "examples/data3.yaml", "content": "a: \"simple\" # just the best\nb: [1, 3]\nc:\n test: 1" }, { "path": "examples/empty-no-comment.yaml", "content": "" }, { "path": "examples/empty.yaml", "content": "# comment\n" }, { "path": "examples/environment.yq", "content": "#! yq\n.[] |(\n ( select(kind == \"scalar\") | key + \"='\" + . + \"'\"),\n ( select(kind == \"seq\") | key + \"=(\" + (map(\"'\" + . + \"'\") | join(\",\")) + \")\")\n)" }, { "path": "examples/example.properties", "content": "# comments on values appear\nperson.name = Mike\n\n# comments on array values appear\nperson.pets.0 = cat\nperson.food.0 = pizza" }, { "path": "examples/front-matter.yaml", "content": "---\na: apple\nb: banana\n---\nhello there\napples: great" }, { "path": "examples/instruction_sample.yaml", "content": "- command: update \n path: b.c\n value:\n #great \n things: frog # wow!\n- command: delete\n path: b.d" }, { "path": "examples/kyaml.yml", "content": "# leading\na: 1 # a line\n# head b\nb: 2\nc:\n # head d\n - d # d line\n" }, { "path": "examples/leading-separator.yaml", "content": "---\na: test" }, { "path": "examples/merge-anchor.yaml", "content": "foo: &foo\n a: foo_a\n thing: foo_thing\n c: foo_c\n\nbar: &bar\n b: bar_b\n thing: bar_thing\n c: bar_c\n\nfoobarList:\n b: foobarList_b\n <<: [*foo,*bar]\n c: foobarList_c\n\nfoobar:\n c: foobar_c\n <<: *foo\n thing: foobar_thing" }, { "path": "examples/mike.xml", "content": "boing" }, { "path": "examples/mike2.xml", "content": " \n \n \n \n \n \n cool\n \n ba2234r\n bar2234233\n " }, { "path": "examples/multiline-text.yaml", "content": "test: |\n abcdefg \n hijklmno\n\n\n" }, { "path": "examples/multiple_docs.yaml", "content": "commonKey: first document\na: Easy! as one two three\nb:\n c: 2\n d: [3, 4]\n e:\n - name: fred\n value: 3\n - name: sam\n value: 4\n---\ncommonKey: second document\nanother:\n document: here\n---\ncommonKey: third document\nwow: \n - here is another\n" }, { "path": "examples/multiple_docs_small.yaml", "content": "a: Easy! as one two three\n---\nanother:\n document: here\n---\n- 1\n- 2" }, { "path": "examples/numbered_keys.yml", "content": "5:\n 6: camel!" }, { "path": "examples/order.yaml", "content": "version: 3\napplication: MyApp" }, { "path": "examples/order.yml", "content": "version: '2'\nservices:\n test:\n image: ubuntu:14.04\n stdin_open: true\n tty: true" }, { "path": "examples/sample.hcl", "content": "# Arithmetic with literals and application-provided variables\nsum = 1 + addend\n\n# String interpolation and templates\nmessage = \"Hello, ${name}!\"\n\n# Application-provided functions\nshouty_message = upper(message)" }, { "path": "examples/sample.ini", "content": "\n; This is a INI document\n\n[owner]\nname = \"Tom Preston-Werner\"\ndob = 1979-05-27T07:32:00-08:00\n\n[database]\ndb_host = \"localhost\"\ndb_port = 5432\ndb_user = \"postgres\"\ndb_password = \"\"\ndb_name = \"postgres\"\n\n" }, { "path": "examples/sample.json", "content": "{\"a\":\"Easy! as one two three\",\"b\":{\"c\":2,\"d\":[3,4],\"e\":[{\"name\":\"fred\",\"value\":3},{\"name\":\"sam\",\"value\":4}]},\"ab\":\"must appear last\"}\n" }, { "path": "examples/sample.tf", "content": "# main.tf\n\n# Define required providers and minimum Terraform version\nterraform {\n required_providers {\n aws = {\n source = \"hashicorp/aws\"\n version = \"~> 5.0\"\n }\n }\n required_version = \">= 1.2\"\n}\n\n# Configure the AWS provider\nprovider \"aws\" {\n region = var.aws_region\n}\n\n# Define an S3 bucket resource\nresource \"aws_s3_bucket\" \"example_bucket\" {\n bucket = var.bucket_name\n\n tags = {\n Environment = \"Development\"\n Project = \"TerraformExample\"\n }\n}\n" }, { "path": "examples/sample.toml", "content": "\n\n# This is a TOML document\n\ntitle = \"TOML Example\"\n\n[owner]\nname = \"Tom Preston-Werner\"\ndob = 1979-05-27T07:32:00-08:00\n\n[database]\nenabled = true\nports = [ 8000, 8001, 8002 ]\ndata = [ [\"delta\", \"phi\"], [3.14] ]\ntemp_targets = { cpu = 79.5, case = 72.0 }\n\n[servers]\n\n[servers.alpha]\nip = \"10.0.0.1\"\nrole = \"frontend\"\n\n[servers.beta]\nip = \"10.0.0.2\"\nrole = \"backend\"\n\n" }, { "path": "examples/sample.yaml", "content": " # things\na: apple" }, { "path": "examples/sample2.hcl", "content": "# Arithmetic with literals and application-provided variables\nsum = 1 + addend\n\n# String interpolation and templates\nmessage = \"Hello, ${name}!\"\n\n# Application-provided functions\nshouty_message = upper(message)" }, { "path": "examples/sample_array.yaml", "content": "[1,2,3]" }, { "path": "examples/sample_array_2.yaml", "content": "- 4\n- 5" }, { "path": "examples/sample_no_sections.ini", "content": "\n; This is a INI document\nname = \"Tom Preston-Werner\"\ndob = 1979-05-27T07:32:00-08:00\nenabled = true\nip = \"10.0.0.1\"\nrole = \"frontend\"\ntreads = 4\n" }, { "path": "examples/sample_objects.csv", "content": "name,numberOfCats,likesApples,height\n,1,true,168.8\nSamantha's Rabbit,2,false,-188.8" }, { "path": "examples/sample_text.yaml", "content": "hi\n" }, { "path": "examples/simple-anchor-exploded.yaml", "content": "foo:\n a: 1\n\nfoobar: \n a: 1" }, { "path": "examples/simple-anchor.yaml", "content": "foo: &foo\n a: 1\n\nfoobar: \n <<: *foo" }, { "path": "examples/small.properties", "content": "this.is = a properties file" }, { "path": "examples/small.xml", "content": "is some xml" }, { "path": "examples/small.yaml", "content": "---\n# comment\n# about things\na: cat" }, { "path": "examples/thing.yml", "content": "a: apple is included\nb: cool." }, { "path": "github-action/Dockerfile", "content": "FROM mikefarah/yq:4\n\nCOPY entrypoint.sh /entrypoint.sh\n\n# github action recommendation is to run as root.\n# https://docs.github.com/en/actions/creating-actions/dockerfile-support-for-github-actions#user\nUSER root\n\nENTRYPOINT [\"/entrypoint.sh\"]\n" }, { "path": "github-action/entrypoint.sh", "content": "#!/bin/sh -l\nset -e\necho \"::debug::\\$cmd: $1\"\nRESULT=$(eval \"$1\")\necho \"::debug::\\$RESULT: $RESULT\"\n# updating from \n# https://github.com/orgs/community/discussions/26288#discussioncomment-3876281\n# https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-output-parameter\ndelimiter=$(cat /proc/sys/kernel/random/uuid)\necho \"result<<${delimiter}\" >> \"${GITHUB_OUTPUT}\"\necho \"${RESULT}\" >> \"${GITHUB_OUTPUT}\"\necho \"${delimiter}\" >> \"${GITHUB_OUTPUT}\"\n" }, { "path": "go.mod", "content": "module github.com/mikefarah/yq/v4\n\nrequire (\n\tgithub.com/a8m/envsubst v1.4.3\n\tgithub.com/alecthomas/participle/v2 v2.1.4\n\tgithub.com/alecthomas/repr v0.5.2\n\tgithub.com/dimchansky/utfbom v1.1.1\n\tgithub.com/elliotchance/orderedmap v1.8.0\n\tgithub.com/fatih/color v1.18.0\n\tgithub.com/go-ini/ini v1.67.0\n\tgithub.com/goccy/go-json v0.10.5\n\tgithub.com/goccy/go-yaml v1.19.2\n\tgithub.com/hashicorp/hcl/v2 v2.24.0\n\tgithub.com/jinzhu/copier v0.4.0\n\tgithub.com/magiconair/properties v1.8.10\n\tgithub.com/pelletier/go-toml/v2 v2.2.4\n\tgithub.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e\n\tgithub.com/spf13/cobra v1.10.2\n\tgithub.com/spf13/pflag v1.0.10\n\tgithub.com/yuin/gopher-lua v1.1.1\n\tgithub.com/zclconf/go-cty v1.17.0\n\tgo.yaml.in/yaml/v4 v4.0.0-rc.3\n\tgolang.org/x/mod v0.33.0\n\tgolang.org/x/net v0.50.0\n\tgolang.org/x/text v0.34.0\n\tgopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473\n)\n\nrequire (\n\tgithub.com/agext/levenshtein v1.2.1 // indirect\n\tgithub.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect\n\tgithub.com/google/go-cmp v0.6.0 // indirect\n\tgithub.com/inconshreveable/mousetrap v1.1.0 // indirect\n\tgithub.com/mattn/go-colorable v0.1.14 // indirect\n\tgithub.com/mattn/go-isatty v0.0.20 // indirect\n\tgithub.com/mitchellh/go-wordwrap v1.0.1 // indirect\n\tgolang.org/x/sync v0.19.0 // indirect\n\tgolang.org/x/sys v0.41.0 // indirect\n\tgolang.org/x/tools v0.41.0 // indirect\n)\n\ngo 1.24.0\n\ntoolchain go1.24.1\n" }, { "path": "go.sum", "content": "github.com/a8m/envsubst v1.4.3 h1:kDF7paGK8QACWYaQo6KtyYBozY2jhQrTuNNuUxQkhJY=\ngithub.com/a8m/envsubst v1.4.3/go.mod h1:4jjHWQlZoaXPoLQUb7H2qT4iLkZDdmEQiOUogdUmqVU=\ngithub.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8=\ngithub.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=\ngithub.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=\ngithub.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=\ngithub.com/alecthomas/participle/v2 v2.1.4 h1:W/H79S8Sat/krZ3el6sQMvMaahJ+XcM9WSI2naI7w2U=\ngithub.com/alecthomas/participle/v2 v2.1.4/go.mod h1:8tqVbpTX20Ru4NfYQgZf4mP18eXPTBViyMWiArNEgGI=\ngithub.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs=\ngithub.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=\ngithub.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=\ngithub.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=\ngithub.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=\ngithub.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=\ngithub.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=\ngithub.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=\ngithub.com/elliotchance/orderedmap v1.8.0 h1:TrOREecvh3JbS+NCgwposXG5ZTFHtEsQiCGOhPElnMw=\ngithub.com/elliotchance/orderedmap v1.8.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=\ngithub.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=\ngithub.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=\ngithub.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=\ngithub.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=\ngithub.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=\ngithub.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=\ngithub.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=\ngithub.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=\ngithub.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM=\ngithub.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=\ngithub.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=\ngithub.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=\ngithub.com/hashicorp/hcl/v2 v2.24.0 h1:2QJdZ454DSsYGoaE6QheQZjtKZSUs9Nh2izTWiwQxvE=\ngithub.com/hashicorp/hcl/v2 v2.24.0/go.mod h1:oGoO1FIQYfn/AgyOhlg9qLC6/nOJPX3qGbkZpYAcqfM=\ngithub.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=\ngithub.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=\ngithub.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=\ngithub.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=\ngithub.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=\ngithub.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=\ngithub.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=\ngithub.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=\ngithub.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=\ngithub.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=\ngithub.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=\ngithub.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=\ngithub.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=\ngithub.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=\ngithub.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=\ngithub.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=\ngithub.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=\ngithub.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=\ngithub.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=\ngithub.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=\ngithub.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=\ngithub.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=\ngithub.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=\ngithub.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=\ngithub.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=\ngithub.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=\ngithub.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=\ngithub.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=\ngithub.com/zclconf/go-cty v1.17.0 h1:seZvECve6XX4tmnvRzWtJNHdscMtYEx5R7bnnVyd/d0=\ngithub.com/zclconf/go-cty v1.17.0/go.mod h1:wqFzcImaLTI6A5HfsRwB0nj5n0MRZFwmey8YoFPPs3U=\ngithub.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo=\ngithub.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM=\ngo.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=\ngo.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go=\ngo.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=\ngolang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=\ngolang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=\ngolang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=\ngolang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=\ngolang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=\ngolang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=\ngolang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=\ngolang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=\ngolang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=\ngolang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=\ngolang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc=\ngolang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW8s2qTSe3wGBtvo0MbVQG/c5k8RE=\ngopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx0Ydtgjl4cqmbRCsY4/+z4cYDeqwZTk6zog=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\n" }, { "path": "go_install_test.go", "content": "package main\n\nimport (\n\t\"io\"\n\t\"testing\"\n\n\t\"golang.org/x/mod/module\"\n\t\"golang.org/x/mod/zip\"\n)\n\n// TestGoInstallCompatibility ensures the module can be zipped for go install.\n// This is an integration test that uses the same zip.CreateFromDir function\n// that go install uses internally. If this test fails, go install will fail.\n// See: https://github.com/mikefarah/yq/issues/2587\nfunc TestGoInstallCompatibility(t *testing.T) {\n\tmod := module.Version{\n\t\tPath: \"github.com/mikefarah/yq/v4\",\n\t\tVersion: \"v4.0.0\", // the actual version doesn't matter for validation\n\t}\n\n\tif err := zip.CreateFromDir(io.Discard, mod, \".\"); err != nil {\n\t\tt.Fatalf(\"Module cannot be zipped for go install: %v\", err)\n\t}\n}\n" }, { "path": "how-it-works.md", "content": "# Expression Syntax: A Visual Guide\nIn `yq`, expressions are made up of operators and pipes. A context of nodes is passed through the expression, and each operation takes the context as input and returns a new context as output. That output is piped in as input for the next operation in the expression.\n\nLet's break down the process step by step using a diagram. We'll start with a single YAML document, apply an expression, and observe how the context changes at each step.\n\nGiven a document like:\n\n```yaml\nroot:\n items:\n - name: apple\n type: fruit\n - name: carrot\n type: vegetable\n - name: banana\n type: fruit\n```\n\nYou can use dot notation to access nested structures. For example, to access the `name` of the first item, you would use the expression `.root.items[0].name`, which would return `apple`.\n\nBut lets see how we could find all the fruit under `items`\n\n## Step 1: Initial Context\nThe context starts at the root of the YAML document. In this case, the entire document is the initial context. \n\n```\nroot\n└── items\n ├── name: apple\n │ type: fruit\n ├── name: carrot\n │ type: vegetable\n └── name: banana\n type: fruit\n```\n\n## Step 2: Splatting the Array\nUsing the expression `.root.items[]`, we \"splat\" the items array. This means each element of the array becomes its own node in the context:\n\n```\nNode 1: { name: apple, type: fruit }\nNode 2: { name: carrot, type: vegetable }\nNode 3: { name: banana, type: fruit }\n```\n\n## Step 3: Filtering the Nodes\nNext, we apply a filter to select only the nodes where type is fruit. The expression `.root.items[] | select(.type == \"fruit\")` filters the nodes:\n\n```\nFiltered Node 1: { name: apple, type: fruit }\nFiltered Node 2: { name: banana, type: fruit }\n```\n\n## Step 4: Extracting a Field\nFinally, we extract the name field from the filtered nodes using `.root.items[] | select(.type == \"fruit\") | .name` This results in:\n\n```\napple\nbanana\n```\n\n## Simple assignment example\n\nGiven a document like:\n\n```yaml\na: cat\nb: dog\n```\n\nwith an expression:\n\n```\n.a = .b\n```\n\nLike math expressions - operator precedence is important. \n\nThe `=` operator takes two arguments, a `lhs` expression, which in this case is `.a` and `rhs` expression which is `.b`. \n\nIt pipes the current, lets call it 'root' context through the `lhs` expression of `.a` to return the node \n\n```yaml\ncat\n```\n\nSide note: this node holds not only its value 'cat', but comments and metadata too, including path and parent information.\n\nThe `=` operator then pipes the 'root' context through the `rhs` expression of `.b` to return the node\n\n```yaml\ndog\n```\n\nBoth sides have now been evaluated, so now the operator copies across the value from the RHS (`.b`) to the LHS (`.a`), and it returns the now updated context:\n\n```yaml\na: dog\nb: dog\n```\n\n## Complex assignment, operator precedence rules\n\nJust like math expressions - `yq` expressions have an order of precedence. The pipe `|` operator has a low order of precedence, so operators with higher precedence will get evaluated first. \n\nMost of the time, this is intuitively what you'd want, for instance `.a = \"cat\" | .b = \"dog\"` is effectively: `(.a = \"cat\") | (.b = \"dog\")`.\n\nHowever, this is not always the case, particularly if you have a complex LHS or RHS expression, for instance if you want to select particular nodes to update. \n\nLets say you had:\n\n```yaml\n- name: bob\n fruit: apple\n- name: sally\n fruit: orange\n\n```\n\nLets say you wanted to update the `sally` entry to have fruit: 'mango'. The _incorrect_ way to do that is:\n`.[] | select(.name == \"sally\") | .fruit = \"mango\"`.\n\nBecause `|` has a low operator precedence, this will be evaluated (_incorrectly_) as : `(.[]) | (select(.name == \"sally\")) | (.fruit = \"mango\")`. What you'll see is only the updated segment returned:\n\n```yaml\nname: sally\nfruit: mango\n```\n\n**Important**: To properly update this YAML, you must wrap the entire LHS in parentheses. Think of it like using brackets in math to ensure the correct order of operations.\n`(.[] | select(.name == \"sally\") | .fruit) = \"mango\"`\n\n\nNow that entire LHS expression is passed to the 'assign' (`=`) operator, and the yaml is correctly updated and returned:\n\n\n```yaml\n- name: bob\n fruit: apple\n- name: sally\n fruit: mango\n\n```\n\n## Relative update (e.g. `|=`)\nThere is another form of the `=` operator which we call the relative form. It's very similar to `=` but with one key difference when evaluating the RHS expression.\n\nIn the plain form, we pass in the 'root' level context to the RHS expression. In relative form, we pass in _each result of the LHS_ to the RHS expression. Let's go through an example.\n\nGiven a document like:\n\n```yaml\na: 1\nb: thing\n```\n\nwith an expression:\n\n```\n.a |= . + 1\n```\n\nSimilar to the `=` operator, `|=` takes two operands, the LHS and RHS.\n\nIt pipes the current context (the whole document) through the LHS expression of `.a` to get the node value:\n\n```\n1\n```\n\nNow it pipes _that LHS context_ into the RHS expression `. + 1` (whereas in the `=` plain form it piped the original document context into the RHS) to yield:\n\n\n```\n2\n```\n\nThe assignment operator then copies across the value from the RHS to the value on the LHS, and it returns the now updated 'root' context:\n\n```yaml\na: 2\nb: thing\n```" }, { "path": "mkdocs.yml", "content": "docs_dir: mkdocs\nsite_dir: docs\nsite_name: Yq\ntheme: 'material'\nrepo_name: 'mikefarah/yq'\nrepo_url: 'https://github.com/mikefarah/yq'\n\n" }, { "path": "pkg/yqlib/all_at_once_evaluator.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n)\n\n// A yaml expression evaluator that runs the expression once against all files/nodes in memory.\ntype Evaluator interface {\n\tEvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error\n\n\t// EvaluateNodes takes an expression and one or more yaml nodes, returning a list of matching candidate nodes\n\tEvaluateNodes(expression string, nodes ...*CandidateNode) (*list.List, error)\n\n\t// EvaluateCandidateNodes takes an expression and list of candidate nodes, returning a list of matching candidate nodes\n\tEvaluateCandidateNodes(expression string, inputCandidateNodes *list.List) (*list.List, error)\n}\n\ntype allAtOnceEvaluator struct {\n\ttreeNavigator DataTreeNavigator\n}\n\nfunc NewAllAtOnceEvaluator() Evaluator {\n\tInitExpressionParser()\n\treturn &allAtOnceEvaluator{treeNavigator: NewDataTreeNavigator()}\n}\n\nfunc (e *allAtOnceEvaluator) EvaluateNodes(expression string, nodes ...*CandidateNode) (*list.List, error) {\n\tinputCandidates := list.New()\n\tfor _, node := range nodes {\n\t\tinputCandidates.PushBack(node)\n\t}\n\treturn e.EvaluateCandidateNodes(expression, inputCandidates)\n}\n\nfunc (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCandidates *list.List) (*list.List, error) {\n\tnode, err := ExpressionParser.ParseExpression(expression)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tcontext, err := e.treeNavigator.GetMatchingNodes(Context{MatchingNodes: inputCandidates}, node)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn context.MatchingNodes, nil\n}\n\nfunc (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error {\n\tfileIndex := 0\n\n\tvar allDocuments = list.New()\n\tfor _, filename := range filenames {\n\t\treader, err := readStream(filename)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tfileDocuments, err := readDocuments(reader, filename, fileIndex, decoder)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tallDocuments.PushBackList(fileDocuments)\n\t\tfileIndex = fileIndex + 1\n\t}\n\n\tif allDocuments.Len() == 0 {\n\t\tcandidateNode := createScalarNode(nil, \"\")\n\t\tallDocuments.PushBack(candidateNode)\n\t}\n\n\tmatches, err := e.EvaluateCandidateNodes(expression, allDocuments)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn printer.PrintResults(matches)\n}\n" }, { "path": "pkg/yqlib/all_at_once_evaluator_test.go", "content": "package yqlib\n\nimport (\n\t\"bufio\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nvar evaluateNodesScenario = []expressionScenario{\n\t{\n\t\tdocument: `a: hello`,\n\t\texpression: `.a`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!str)::hello\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: `a: hello`,\n\t\texpression: `.`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: hello\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: `- a: \"yes\"`,\n\t\texpression: `.[] | has(\"a\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!bool)::true\\n\",\n\t\t},\n\t},\n}\n\nfunc TestAllAtOnceEvaluateNodes(t *testing.T) {\n\tvar evaluator = NewAllAtOnceEvaluator()\n\t// logging.SetLevel(logging.DEBUG, \"\")\n\tfor _, tt := range evaluateNodesScenario {\n\t\tdecoder := NewYamlDecoder(ConfiguredYamlPreferences)\n\t\treader := bufio.NewReader(strings.NewReader(tt.document))\n\t\terr := decoder.Init(reader)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t\treturn\n\t\t}\n\t\tcandidateNode, errorReading := decoder.Decode()\n\n\t\tif errorReading != nil {\n\t\t\tt.Error(errorReading)\n\t\t\treturn\n\t\t}\n\n\t\tlist, _ := evaluator.EvaluateNodes(tt.expression, candidateNode)\n\t\ttest.AssertResultComplex(t, tt.expected, resultsToString(t, list))\n\t}\n}\n" }, { "path": "pkg/yqlib/base64_test.go", "content": "//go:build !yq_nobase64\n\npackage yqlib\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nconst base64EncodedSimple = \"YSBzcGVjaWFsIHN0cmluZw==\"\nconst base64DecodedSimpleExtraSpaces = \"\\n \" + base64EncodedSimple + \" \\n\"\nconst base64DecodedSimple = \"a special string\"\n\nconst base64EncodedUTF8 = \"V29ya3Mgd2l0aCBVVEYtMTYg8J+Yig==\"\nconst base64DecodedUTF8 = \"Works with UTF-16 😊\"\n\nconst base64EncodedYaml = \"YTogYXBwbGUK\"\nconst base64DecodedYaml = \"a: apple\\n\"\n\nconst base64EncodedEmpty = \"\"\nconst base64DecodedEmpty = \"\"\n\nconst base64MissingPadding = \"Y2F0cw\"\nconst base64DecodedMissingPadding = \"cats\"\n\nconst base64EncodedCats = \"Y2F0cw==\"\nconst base64DecodedCats = \"cats\"\n\nvar base64Scenarios = []formatScenario{\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"empty decode\",\n\t\tinput: base64EncodedEmpty,\n\t\texpected: base64DecodedEmpty + \"\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"simple decode\",\n\t\tinput: base64EncodedSimple,\n\t\texpected: base64DecodedSimple + \"\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Decode base64: simple\",\n\t\tsubdescription: \"Decoded data is assumed to be a string.\",\n\t\tinput: base64EncodedSimple,\n\t\texpected: base64DecodedSimple + \"\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Decode base64: UTF-8\",\n\t\tsubdescription: \"Base64 decoding supports UTF-8 encoded strings.\",\n\t\tinput: base64EncodedUTF8,\n\t\texpected: base64DecodedUTF8 + \"\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"decode missing padding\",\n\t\tinput: base64MissingPadding,\n\t\texpected: base64DecodedMissingPadding + \"\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\n\t\tdescription: \"Decode with extra spaces\",\n\t\tsubdescription: \"Extra leading/trailing whitespace is stripped\",\n\t\tinput: base64DecodedSimpleExtraSpaces,\n\t\texpected: base64DecodedSimple + \"\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"decode with padding\",\n\t\tinput: base64EncodedCats,\n\t\texpected: base64DecodedCats + \"\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"decode yaml document\",\n\t\tinput: base64EncodedYaml,\n\t\texpected: base64DecodedYaml + \"\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Encode base64: string\",\n\t\tinput: \"\\\"\" + base64DecodedSimple + \"\\\"\",\n\t\texpected: base64EncodedSimple,\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tdescription: \"Encode base64: string from document\",\n\t\tsubdescription: \"Extract a string field and encode it to base64.\",\n\t\tinput: \"coolData: \\\"\" + base64DecodedSimple + \"\\\"\",\n\t\texpression: \".coolData\",\n\t\texpected: base64EncodedSimple,\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"encode empty string\",\n\t\tinput: \"\\\"\\\"\",\n\t\texpected: \"\",\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"encode UTF-8 string\",\n\t\tinput: \"\\\"\" + base64DecodedUTF8 + \"\\\"\",\n\t\texpected: base64EncodedUTF8,\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"encode cats\",\n\t\tinput: \"\\\"\" + base64DecodedCats + \"\\\"\",\n\t\texpected: base64EncodedCats,\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip: simple\",\n\t\tskipDoc: true,\n\t\tinput: base64EncodedSimple,\n\t\texpected: base64EncodedSimple,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip: UTF-8\",\n\t\tskipDoc: true,\n\t\tinput: base64EncodedUTF8,\n\t\texpected: base64EncodedUTF8,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip: missing padding\",\n\t\tskipDoc: true,\n\t\tinput: base64MissingPadding,\n\t\texpected: base64EncodedCats,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip: empty\",\n\t\tskipDoc: true,\n\t\tinput: base64EncodedEmpty,\n\t\texpected: base64EncodedEmpty,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Encode error: non-string\",\n\t\tskipDoc: true,\n\t\tinput: \"123\",\n\t\texpectedError: \"cannot encode !!int as base64, can only operate on strings\",\n\t\tscenarioType: \"encode-error\",\n\t},\n\t{\n\t\tdescription: \"Encode error: array\",\n\t\tskipDoc: true,\n\t\tinput: \"[1, 2, 3]\",\n\t\texpectedError: \"cannot encode !!seq as base64, can only operate on strings\",\n\t\tscenarioType: \"encode-error\",\n\t},\n\t{\n\t\tdescription: \"Encode error: map\",\n\t\tskipDoc: true,\n\t\tinput: \"{b: c}\",\n\t\texpectedError: \"cannot encode !!map as base64, can only operate on strings\",\n\t\tscenarioType: \"encode-error\",\n\t},\n}\n\nfunc testBase64Scenario(t *testing.T, s formatScenario) {\n\tswitch s.scenarioType {\n\tcase \"\", \"decode\":\n\t\tyamlPrefs := ConfiguredYamlPreferences.Copy()\n\t\tyamlPrefs.Indent = 4\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewBase64Decoder(), NewYamlEncoder(yamlPrefs)), s.description)\n\tcase \"encode\":\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewBase64Encoder()), s.description)\n\tcase \"roundtrip\":\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewBase64Decoder(), NewBase64Encoder()), s.description)\n\tcase \"encode-error\":\n\t\tresult, err := processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewBase64Encoder())\n\t\tif err == nil {\n\t\t\tt.Errorf(\"Expected error '%v' but it worked: %v\", s.expectedError, result)\n\t\t} else {\n\t\t\ttest.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description)\n\t\t}\n\n\tdefault:\n\t\tpanic(fmt.Sprintf(\"unhandled scenario type %q\", s.scenarioType))\n\t}\n}\n\nfunc documentBase64Scenario(_ *testing.T, w *bufio.Writer, i interface{}) {\n\ts := i.(formatScenario)\n\n\tif s.skipDoc {\n\t\treturn\n\t}\n\tswitch s.scenarioType {\n\tcase \"\", \"decode\":\n\t\tdocumentBase64DecodeScenario(w, s)\n\tcase \"encode\":\n\t\tdocumentBase64EncodeScenario(w, s)\n\n\tdefault:\n\t\tpanic(fmt.Sprintf(\"unhandled scenario type %q\", s.scenarioType))\n\t}\n}\n\nfunc documentBase64DecodeScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.txt file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\texpression := s.expression\n\tif expression == \"\" {\n\t\texpression = \".\"\n\t}\n\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -p=base64 -oy '%v' sample.txt\\n```\\n\", expression))\n\twriteOrPanic(w, \"will output\\n\")\n\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewBase64Decoder(), NewYamlEncoder(ConfiguredYamlPreferences))))\n}\n\nfunc documentBase64EncodeScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.yml file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\texpression := s.expression\n\tif expression == \"\" {\n\t\texpression = \".\"\n\t}\n\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -o=base64 '%v' sample.yml\\n```\\n\", expression))\n\twriteOrPanic(w, \"will output\\n\")\n\n\twriteOrPanic(w, fmt.Sprintf(\"```\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewBase64Encoder())))\n}\n\nfunc TestBase64Scenarios(t *testing.T) {\n\tfor _, tt := range base64Scenarios {\n\t\ttestBase64Scenario(t, tt)\n\t}\n\tgenericScenarios := make([]interface{}, len(base64Scenarios))\n\tfor i, s := range base64Scenarios {\n\t\tgenericScenarios[i] = s\n\t}\n\tdocumentScenarios(t, \"usage\", \"base64\", genericScenarios, documentBase64Scenario)\n}\n" }, { "path": "pkg/yqlib/candidate_node.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n)\n\ntype Kind uint32\n\nconst (\n\tSequenceNode Kind = 1 << iota\n\tMappingNode\n\tScalarNode\n\tAliasNode\n)\n\ntype Style uint32\n\nconst (\n\tTaggedStyle Style = 1 << iota\n\tDoubleQuotedStyle\n\tSingleQuotedStyle\n\tLiteralStyle\n\tFoldedStyle\n\tFlowStyle\n)\n\nfunc createStringScalarNode(stringValue string) *CandidateNode {\n\tvar node = &CandidateNode{Kind: ScalarNode}\n\tnode.Value = stringValue\n\tnode.Tag = \"!!str\"\n\treturn node\n}\n\nfunc createScalarNode(value interface{}, stringValue string) *CandidateNode {\n\tvar node = &CandidateNode{Kind: ScalarNode}\n\tnode.Value = stringValue\n\n\tswitch value.(type) {\n\tcase float32, float64:\n\t\tnode.Tag = \"!!float\"\n\tcase int, int64, int32:\n\t\tnode.Tag = \"!!int\"\n\tcase bool:\n\t\tnode.Tag = \"!!bool\"\n\tcase string:\n\t\tnode.Tag = \"!!str\"\n\tcase nil:\n\t\tnode.Tag = \"!!null\"\n\t}\n\treturn node\n}\n\ntype NodeInfo struct {\n\tKind string `yaml:\"kind\"`\n\tStyle string `yaml:\"style,omitempty\"`\n\tAnchor string `yaml:\"anchor,omitempty\"`\n\tTag string `yaml:\"tag,omitempty\"`\n\tHeadComment string `yaml:\"headComment,omitempty\"`\n\tLineComment string `yaml:\"lineComment,omitempty\"`\n\tFootComment string `yaml:\"footComment,omitempty\"`\n\tValue string `yaml:\"value,omitempty\"`\n\tLine int `yaml:\"line,omitempty\"`\n\tColumn int `yaml:\"column,omitempty\"`\n\tContent []*NodeInfo `yaml:\"content,omitempty\"`\n}\n\ntype CandidateNode struct {\n\tKind Kind\n\tStyle Style\n\n\tTag string\n\tValue string\n\tAnchor string\n\tAlias *CandidateNode\n\tContent []*CandidateNode\n\n\tHeadComment string\n\tLineComment string\n\tFootComment string\n\n\tParent *CandidateNode // parent node\n\tKey *CandidateNode // node key, if this is a value from a map (or index in an array)\n\n\tLeadingContent string\n\n\tdocument uint // the document index of this node\n\tfilename string\n\n\tLine int\n\tColumn int\n\n\tfileIndex int\n\t// when performing op against all nodes given, this will treat all the nodes as one\n\t// (e.g. top level cross document merge). This property does not propagate to child nodes.\n\tEvaluateTogether bool\n\tIsMapKey bool\n\t// For formats like HCL and TOML: indicates that child entries should be emitted as separate blocks/tables\n\t// rather than consolidated into nested mappings (default behaviour)\n\tEncodeSeparate bool\n}\n\nfunc (n *CandidateNode) CreateChild() *CandidateNode {\n\treturn &CandidateNode{\n\t\tParent: n,\n\t}\n}\n\nfunc (n *CandidateNode) SetDocument(idx uint) {\n\tn.document = idx\n}\n\nfunc (n *CandidateNode) GetDocument() uint {\n\t// defer to parent\n\tif n.Parent != nil {\n\t\treturn n.Parent.GetDocument()\n\t}\n\treturn n.document\n}\n\nfunc (n *CandidateNode) SetFilename(name string) {\n\tn.filename = name\n}\n\nfunc (n *CandidateNode) GetFilename() string {\n\tif n.Parent != nil {\n\t\treturn n.Parent.GetFilename()\n\t}\n\treturn n.filename\n}\n\nfunc (n *CandidateNode) SetFileIndex(idx int) {\n\tn.fileIndex = idx\n}\n\nfunc (n *CandidateNode) GetFileIndex() int {\n\tif n.Parent != nil {\n\t\treturn n.Parent.GetFileIndex()\n\t}\n\treturn n.fileIndex\n}\n\nfunc (n *CandidateNode) GetKey() string {\n\tkeyPrefix := \"\"\n\tif n.IsMapKey {\n\t\tkeyPrefix = fmt.Sprintf(\"key-%v-\", n.Value)\n\t}\n\tkey := \"\"\n\tif n.Key != nil {\n\t\tkey = n.Key.Value\n\t}\n\treturn fmt.Sprintf(\"%v%v - %v\", keyPrefix, n.GetDocument(), key)\n}\n\nfunc (n *CandidateNode) getParsedKey() interface{} {\n\tif n.IsMapKey {\n\t\treturn n.Value\n\t}\n\tif n.Key == nil {\n\t\treturn nil\n\t}\n\tif n.Key.Tag == \"!!str\" {\n\t\treturn n.Key.Value\n\t}\n\tindex, err := parseInt(n.Key.Value)\n\tif err != nil {\n\t\treturn n.Key.Value\n\t}\n\treturn index\n\n}\n\nfunc (n *CandidateNode) FilterMapContentByKey(keyPredicate func(*CandidateNode) bool) []*CandidateNode {\n\tvar result []*CandidateNode\n\tfor index := 0; index < len(n.Content); index = index + 2 {\n\t\tkeyNode := n.Content[index]\n\t\tvalueNode := n.Content[index+1]\n\t\tif keyPredicate(keyNode) {\n\t\t\tresult = append(result, keyNode, valueNode)\n\t\t}\n\t}\n\treturn result\n}\n\nfunc (n *CandidateNode) GetPath() []interface{} {\n\tkey := n.getParsedKey()\n\tif n.Parent != nil && key != nil {\n\t\treturn append(n.Parent.GetPath(), key)\n\t}\n\n\tif key != nil {\n\t\treturn []interface{}{key}\n\t}\n\treturn make([]interface{}, 0)\n}\n\nfunc (n *CandidateNode) GetNicePath() string {\n\tvar sb strings.Builder\n\tpath := n.GetPath()\n\tfor i, element := range path {\n\t\telementStr := fmt.Sprintf(\"%v\", element)\n\t\tswitch element.(type) {\n\t\tcase int:\n\t\t\tsb.WriteString(\"[\" + elementStr + \"]\")\n\t\tdefault:\n\t\t\tif i == 0 {\n\t\t\t\tsb.WriteString(elementStr)\n\t\t\t} else if strings.ContainsRune(elementStr, '.') {\n\t\t\t\tsb.WriteString(\"[\" + elementStr + \"]\")\n\t\t\t} else {\n\t\t\t\tsb.WriteString(\".\" + elementStr)\n\t\t\t}\n\t\t}\n\t}\n\treturn sb.String()\n}\n\nfunc (n *CandidateNode) AsList() *list.List {\n\telMap := list.New()\n\telMap.PushBack(n)\n\treturn elMap\n}\n\nfunc (n *CandidateNode) SetParent(parent *CandidateNode) {\n\tn.Parent = parent\n}\n\ntype ValueVisitor func(*CandidateNode) error\n\nfunc (n *CandidateNode) VisitValues(visitor ValueVisitor) error {\n\tswitch n.Kind {\n\tcase MappingNode:\n\t\tfor i := 1; i < len(n.Content); i = i + 2 {\n\t\t\tif err := visitor(n.Content[i]); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\tcase SequenceNode:\n\t\tfor i := 0; i < len(n.Content); i = i + 1 {\n\t\t\tif err := visitor(n.Content[i]); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (n *CandidateNode) CanVisitValues() bool {\n\treturn n.Kind == MappingNode || n.Kind == SequenceNode\n}\n\nfunc (n *CandidateNode) AddKeyValueChild(rawKey *CandidateNode, rawValue *CandidateNode) (*CandidateNode, *CandidateNode) {\n\tkey := rawKey.Copy()\n\tkey.SetParent(n)\n\tkey.IsMapKey = true\n\n\tvalue := rawValue.Copy()\n\tvalue.SetParent(n)\n\tvalue.IsMapKey = false // force this, incase we are creating a value from a key\n\tvalue.Key = key\n\n\tn.Content = append(n.Content, key, value)\n\treturn key, value\n}\n\nfunc (n *CandidateNode) AddChild(rawChild *CandidateNode) {\n\tvalue := rawChild.Copy()\n\tvalue.SetParent(n)\n\tvalue.IsMapKey = false\n\n\tindex := len(n.Content)\n\tkeyNode := createScalarNode(index, fmt.Sprintf(\"%v\", index))\n\tkeyNode.SetParent(n)\n\tvalue.Key = keyNode\n\n\tn.Content = append(n.Content, value)\n}\n\nfunc (n *CandidateNode) AddChildren(children []*CandidateNode) {\n\tif n.Kind == MappingNode {\n\t\tfor i := 0; i < len(children); i += 2 {\n\t\t\tkey := children[i]\n\t\t\tvalue := children[i+1]\n\t\t\tn.AddKeyValueChild(key, value)\n\t\t}\n\n\t} else {\n\t\tfor _, rawChild := range children {\n\t\t\tn.AddChild(rawChild)\n\t\t}\n\t}\n}\n\nfunc (n *CandidateNode) GetValueRep() (interface{}, error) {\n\tlog.Debugf(\"GetValueRep for %v value: %v\", n.GetNicePath(), n.Value)\n\trealTag := n.guessTagFromCustomType()\n\n\tswitch realTag {\n\tcase \"!!int\":\n\t\t_, val, err := parseInt64(n.Value)\n\t\treturn val, err\n\tcase \"!!float\":\n\t\t// need to test this\n\t\treturn strconv.ParseFloat(n.Value, 64)\n\tcase \"!!bool\":\n\t\treturn isTruthyNode(n), nil\n\tcase \"!!null\":\n\t\treturn nil, nil\n\t}\n\n\treturn n.Value, nil\n}\n\nfunc (n *CandidateNode) guessTagFromCustomType() string {\n\tif strings.HasPrefix(n.Tag, \"!!\") {\n\t\treturn n.Tag\n\t} else if n.Value == \"\" {\n\t\tlog.Debug(\"guessTagFromCustomType: node has no value to guess the type with\")\n\t\treturn n.Tag\n\t}\n\tdataBucket, errorReading := parseSnippet(n.Value)\n\n\tif errorReading != nil {\n\t\tlog.Debug(\"guessTagFromCustomType: could not guess underlying tag type %v\", errorReading)\n\t\treturn n.Tag\n\t}\n\tguessedTag := dataBucket.Tag\n\tlog.Info(\"im guessing the tag %v is a %v\", n.Tag, guessedTag)\n\treturn guessedTag\n}\n\nfunc (n *CandidateNode) CreateReplacement(kind Kind, tag string, value string) *CandidateNode {\n\tnode := &CandidateNode{\n\t\tKind: kind,\n\t\tTag: tag,\n\t\tValue: value,\n\t}\n\treturn n.CopyAsReplacement(node)\n}\n\nfunc (n *CandidateNode) CopyAsReplacement(replacement *CandidateNode) *CandidateNode {\n\tnewCopy := replacement.Copy()\n\tnewCopy.Parent = n.Parent\n\n\tif n.IsMapKey {\n\t\tnewCopy.Key = n\n\t} else {\n\t\tnewCopy.Key = n.Key\n\t}\n\n\treturn newCopy\n}\n\nfunc (n *CandidateNode) CreateReplacementWithComments(kind Kind, tag string, style Style) *CandidateNode {\n\treplacement := n.CreateReplacement(kind, tag, \"\")\n\treplacement.LeadingContent = n.LeadingContent\n\treplacement.HeadComment = n.HeadComment\n\treplacement.LineComment = n.LineComment\n\treplacement.FootComment = n.FootComment\n\treplacement.Style = style\n\treturn replacement\n}\n\nfunc (n *CandidateNode) Copy() *CandidateNode {\n\treturn n.doCopy(true)\n}\n\nfunc (n *CandidateNode) CopyWithoutContent() *CandidateNode {\n\treturn n.doCopy(false)\n}\n\nfunc (n *CandidateNode) doCopy(cloneContent bool) *CandidateNode {\n\tvar content []*CandidateNode\n\n\tvar copyKey *CandidateNode\n\tif n.Key != nil {\n\t\tcopyKey = n.Key.Copy()\n\t}\n\n\tclone := &CandidateNode{\n\t\tKind: n.Kind,\n\t\tStyle: n.Style,\n\n\t\tTag: n.Tag,\n\t\tValue: n.Value,\n\t\tAnchor: n.Anchor,\n\n\t\t// ok not to clone this,\n\t\t// as its a reference to somewhere else.\n\t\tAlias: n.Alias,\n\t\tContent: content,\n\n\t\tHeadComment: n.HeadComment,\n\t\tLineComment: n.LineComment,\n\t\tFootComment: n.FootComment,\n\n\t\tParent: n.Parent,\n\t\tKey: copyKey,\n\n\t\tLeadingContent: n.LeadingContent,\n\n\t\tdocument: n.document,\n\t\tfilename: n.filename,\n\t\tfileIndex: n.fileIndex,\n\n\t\tLine: n.Line,\n\t\tColumn: n.Column,\n\n\t\tEvaluateTogether: n.EvaluateTogether,\n\t\tIsMapKey: n.IsMapKey,\n\n\t\tEncodeSeparate: n.EncodeSeparate,\n\t}\n\n\tif cloneContent {\n\t\tclone.AddChildren(n.Content)\n\t}\n\n\treturn clone\n}\n\n// updates this candidate from the given candidate node\nfunc (n *CandidateNode) UpdateFrom(other *CandidateNode, prefs assignPreferences) {\n\tif n == other {\n\t\tlog.Debugf(\"UpdateFrom, no need to update from myself.\")\n\t\treturn\n\t}\n\t// if this is an empty map or empty array, use the style of other node.\n\tif (n.Kind != ScalarNode && len(n.Content) == 0) ||\n\t\t// if the tag has changed (e.g. from str to bool)\n\t\t(n.guessTagFromCustomType() != other.guessTagFromCustomType()) {\n\t\tn.Style = other.Style\n\t}\n\n\tn.Content = make([]*CandidateNode, 0)\n\tn.Kind = other.Kind\n\tn.AddChildren(other.Content)\n\n\tn.Value = other.Value\n\n\tn.UpdateAttributesFrom(other, prefs)\n\n}\n\nfunc (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignPreferences) {\n\tlog.Debug(\"UpdateAttributesFrom: n: %v other: %v\", NodeToString(n), NodeToString(other))\n\tif n.Kind != other.Kind {\n\t\t// clear out the contents when switching to a different type\n\t\t// e.g. map to array\n\t\tn.Content = make([]*CandidateNode, 0)\n\t\tn.Value = \"\"\n\t}\n\tn.Kind = other.Kind\n\n\t// don't clobber custom tags...\n\tif prefs.ClobberCustomTags || strings.HasPrefix(n.Tag, \"!!\") || n.Tag == \"\" {\n\t\tn.Tag = other.Tag\n\t}\n\n\tn.Alias = other.Alias\n\n\tif !prefs.DontOverWriteAnchor {\n\t\tn.Anchor = other.Anchor\n\t}\n\n\t// Preserve EncodeSeparate flag for format-specific encoding hints\n\tn.EncodeSeparate = other.EncodeSeparate\n\n\t// merge will pickup the style of the new thing\n\t// when autocreating nodes\n\n\tif n.Style == 0 {\n\t\tn.Style = other.Style\n\t}\n\n\tif other.FootComment != \"\" {\n\t\tn.FootComment = other.FootComment\n\t}\n\tif other.HeadComment != \"\" {\n\t\tn.HeadComment = other.HeadComment\n\t}\n\tif other.LineComment != \"\" {\n\t\tn.LineComment = other.LineComment\n\t}\n}\n\nfunc (n *CandidateNode) ConvertToNodeInfo() *NodeInfo {\n\tinfo := &NodeInfo{\n\t\tKind: kindToString(n.Kind),\n\t\tStyle: styleToString(n.Style),\n\t\tAnchor: n.Anchor,\n\t\tTag: n.Tag,\n\t\tHeadComment: n.HeadComment,\n\t\tLineComment: n.LineComment,\n\t\tFootComment: n.FootComment,\n\t\tValue: n.Value,\n\t\tLine: n.Line,\n\t\tColumn: n.Column,\n\t}\n\tif len(n.Content) > 0 {\n\t\tinfo.Content = make([]*NodeInfo, len(n.Content))\n\t\tfor i, child := range n.Content {\n\t\t\tinfo.Content[i] = child.ConvertToNodeInfo()\n\t\t}\n\t}\n\treturn info\n}\n\n// Helper functions to convert Kind and Style to string for NodeInfo\nfunc kindToString(k Kind) string {\n\tswitch k {\n\tcase SequenceNode:\n\t\treturn \"SequenceNode\"\n\tcase MappingNode:\n\t\treturn \"MappingNode\"\n\tcase ScalarNode:\n\t\treturn \"ScalarNode\"\n\tcase AliasNode:\n\t\treturn \"AliasNode\"\n\tdefault:\n\t\treturn \"Unknown\"\n\t}\n}\n\nfunc styleToString(s Style) string {\n\tvar styles []string\n\tif s&TaggedStyle != 0 {\n\t\tstyles = append(styles, \"TaggedStyle\")\n\t}\n\tif s&DoubleQuotedStyle != 0 {\n\t\tstyles = append(styles, \"DoubleQuotedStyle\")\n\t}\n\tif s&SingleQuotedStyle != 0 {\n\t\tstyles = append(styles, \"SingleQuotedStyle\")\n\t}\n\tif s&LiteralStyle != 0 {\n\t\tstyles = append(styles, \"LiteralStyle\")\n\t}\n\tif s&FoldedStyle != 0 {\n\t\tstyles = append(styles, \"FoldedStyle\")\n\t}\n\tif s&FlowStyle != 0 {\n\t\tstyles = append(styles, \"FlowStyle\")\n\t}\n\treturn strings.Join(styles, \",\")\n}\n" }, { "path": "pkg/yqlib/candidate_node_goccy_yaml.go", "content": "package yqlib\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\n\tyaml \"github.com/goccy/go-yaml\"\n\t\"github.com/goccy/go-yaml/ast\"\n\tgoccyToken \"github.com/goccy/go-yaml/token\"\n)\n\nfunc (o *CandidateNode) goccyDecodeIntoChild(childNode ast.Node, cm yaml.CommentMap, anchorMap map[string]*CandidateNode) (*CandidateNode, error) {\n\tnewChild := o.CreateChild()\n\n\terr := newChild.UnmarshalGoccyYAML(childNode, cm, anchorMap)\n\treturn newChild, err\n}\n\nfunc (o *CandidateNode) UnmarshalGoccyYAML(node ast.Node, cm yaml.CommentMap, anchorMap map[string]*CandidateNode) error {\n\tlog.Debugf(\"UnmarshalYAML %v\", node)\n\tlog.Debugf(\"UnmarshalYAML %v\", node.Type().String())\n\tlog.Debugf(\"UnmarshalYAML Node Value: %v\", node.String())\n\tlog.Debugf(\"UnmarshalYAML Node GetComment: %v\", node.GetComment())\n\n\tif node.GetComment() != nil {\n\t\tcommentMapComments := cm[node.GetPath()]\n\t\tfor _, comment := range node.GetComment().Comments {\n\t\t\t// need to use the comment map to find the position :/\n\t\t\tlog.Debugf(\"%v has a comment of [%v]\", node.GetPath(), comment.Token.Value)\n\t\t\tfor _, commentMapComment := range commentMapComments {\n\t\t\t\tcommentMapValue := strings.Join(commentMapComment.Texts, \"\\n\")\n\t\t\t\tif commentMapValue == comment.Token.Value {\n\t\t\t\t\tlog.Debug(\"found a matching entry in comment map\")\n\t\t\t\t\t// we found the comment in the comment map,\n\t\t\t\t\t// now we can process the position\n\t\t\t\t\tswitch commentMapComment.Position {\n\t\t\t\t\tcase yaml.CommentHeadPosition:\n\t\t\t\t\t\to.HeadComment = comment.String()\n\t\t\t\t\t\tlog.Debug(\"its a head comment %v\", comment.String())\n\t\t\t\t\tcase yaml.CommentLinePosition:\n\t\t\t\t\t\to.LineComment = comment.String()\n\t\t\t\t\t\tlog.Debug(\"its a line comment %v\", comment.String())\n\t\t\t\t\tcase yaml.CommentFootPosition:\n\t\t\t\t\t\to.FootComment = comment.String()\n\t\t\t\t\t\tlog.Debug(\"its a foot comment %v\", comment.String())\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t}\n\t}\n\n\to.Value = node.String()\n\to.Line = node.GetToken().Position.Line\n\to.Column = node.GetToken().Position.Column\n\n\tswitch node.Type() {\n\tcase ast.IntegerType:\n\t\to.Kind = ScalarNode\n\t\to.Tag = \"!!int\"\n\tcase ast.FloatType:\n\t\to.Kind = ScalarNode\n\t\to.Tag = \"!!float\"\n\tcase ast.BoolType:\n\t\to.Kind = ScalarNode\n\t\to.Tag = \"!!bool\"\n\tcase ast.NullType:\n\t\tlog.Debugf(\"its a null type with value %v\", node.GetToken().Value)\n\t\to.Kind = ScalarNode\n\t\to.Tag = \"!!null\"\n\t\to.Value = node.GetToken().Value\n\t\tif node.GetToken().Type == goccyToken.ImplicitNullType {\n\t\t\to.Value = \"\"\n\t\t}\n\tcase ast.StringType:\n\t\to.Kind = ScalarNode\n\t\to.Tag = \"!!str\"\n\t\tswitch node.GetToken().Type {\n\t\tcase goccyToken.SingleQuoteType:\n\t\t\to.Style = SingleQuotedStyle\n\t\tcase goccyToken.DoubleQuoteType:\n\t\t\to.Style = DoubleQuotedStyle\n\t\t}\n\t\to.Value = node.(*ast.StringNode).Value\n\t\tlog.Debugf(\"string value %v\", node.(*ast.StringNode).Value)\n\tcase ast.LiteralType:\n\t\to.Kind = ScalarNode\n\t\to.Tag = \"!!str\"\n\t\to.Style = LiteralStyle\n\t\tastLiteral := node.(*ast.LiteralNode)\n\t\tlog.Debugf(\"astLiteral.Start.Type %v\", astLiteral.Start.Type)\n\t\tif astLiteral.Start.Type == goccyToken.FoldedType {\n\t\t\tlog.Debugf(\"folded Type %v\", astLiteral.Start.Type)\n\t\t\to.Style = FoldedStyle\n\t\t}\n\t\tlog.Debug(\"start value: %v \", node.(*ast.LiteralNode).Start.Value)\n\t\tlog.Debug(\"start value: %v \", node.(*ast.LiteralNode).Start.Type)\n\t\t// TODO: here I could put the original value with line breaks\n\t\t// to solve the multiline > problem\n\t\to.Value = astLiteral.Value.Value\n\tcase ast.TagType:\n\t\tif err := o.UnmarshalGoccyYAML(node.(*ast.TagNode).Value, cm, anchorMap); err != nil {\n\t\t\treturn err\n\t\t}\n\t\to.Tag = node.(*ast.TagNode).Start.Value\n\tcase ast.MappingType:\n\t\tlog.Debugf(\"UnmarshalYAML - a mapping node\")\n\t\to.Kind = MappingNode\n\t\to.Tag = \"!!map\"\n\n\t\tmappingNode := node.(*ast.MappingNode)\n\t\tif mappingNode.IsFlowStyle {\n\t\t\to.Style = FlowStyle\n\t\t}\n\t\tfor _, mappingValueNode := range mappingNode.Values {\n\t\t\terr := o.goccyProcessMappingValueNode(mappingValueNode, cm, anchorMap)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\tif mappingNode.FootComment != nil {\n\t\t\tlog.Debugf(\"mapping node has a foot comment of: %v\", mappingNode.FootComment)\n\t\t\to.FootComment = mappingNode.FootComment.String()\n\t\t}\n\tcase ast.MappingValueType:\n\t\tlog.Debugf(\"UnmarshalYAML - a mapping node\")\n\t\to.Kind = MappingNode\n\t\to.Tag = \"!!map\"\n\t\tmappingValueNode := node.(*ast.MappingValueNode)\n\t\terr := o.goccyProcessMappingValueNode(mappingValueNode, cm, anchorMap)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\tcase ast.SequenceType:\n\t\tlog.Debugf(\"UnmarshalYAML - a sequence node\")\n\t\to.Kind = SequenceNode\n\t\to.Tag = \"!!seq\"\n\t\tsequenceNode := node.(*ast.SequenceNode)\n\t\tif sequenceNode.IsFlowStyle {\n\t\t\to.Style = FlowStyle\n\t\t}\n\t\tastSeq := sequenceNode.Values\n\t\to.Content = make([]*CandidateNode, len(astSeq))\n\t\tfor i := 0; i < len(astSeq); i++ {\n\t\t\tkeyNode := o.CreateChild()\n\t\t\tkeyNode.IsMapKey = true\n\t\t\tkeyNode.Tag = \"!!int\"\n\t\t\tkeyNode.Kind = ScalarNode\n\t\t\tkeyNode.Value = fmt.Sprintf(\"%v\", i)\n\n\t\t\tvalueNode, err := o.goccyDecodeIntoChild(astSeq[i], cm, anchorMap)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\tvalueNode.Key = keyNode\n\t\t\to.Content[i] = valueNode\n\t\t}\n\tcase ast.AnchorType:\n\t\tlog.Debugf(\"UnmarshalYAML - an anchor node\")\n\t\tanchorNode := node.(*ast.AnchorNode)\n\t\terr := o.UnmarshalGoccyYAML(anchorNode.Value, cm, anchorMap)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\to.Anchor = anchorNode.Name.String()\n\t\tanchorMap[o.Anchor] = o\n\n\tcase ast.AliasType:\n\t\tlog.Debugf(\"UnmarshalYAML - an alias node\")\n\t\taliasNode := node.(*ast.AliasNode)\n\t\to.Kind = AliasNode\n\t\to.Value = aliasNode.Value.String()\n\t\to.Alias = anchorMap[o.Value]\n\n\tcase ast.MergeKeyType:\n\t\tlog.Debugf(\"UnmarshalYAML - a merge key\")\n\t\to.Kind = ScalarNode\n\t\to.Tag = \"!!merge\" // note - I should be able to get rid of this.\n\t\to.Value = \"<<\"\n\n\tdefault:\n\t\tlog.Debugf(\"UnmarshalYAML - no idea of the type!!\\n%v: %v\", node.Type(), node.String())\n\t}\n\tlog.Debugf(\"KIND: %v\", o.Kind)\n\treturn nil\n}\n\nfunc (o *CandidateNode) goccyProcessMappingValueNode(mappingEntry *ast.MappingValueNode, cm yaml.CommentMap, anchorMap map[string]*CandidateNode) error {\n\tlog.Debug(\"UnmarshalYAML MAP KEY entry %v\", mappingEntry.Key)\n\n\t// AddKeyValueFirst because it clones the nodes, and we want to have the real refs when Unmarshalling\n\t// particularly for the anchorMap\n\tkeyNode, valueNode := o.AddKeyValueChild(&CandidateNode{}, &CandidateNode{})\n\n\tif err := keyNode.UnmarshalGoccyYAML(mappingEntry.Key, cm, anchorMap); err != nil {\n\t\treturn err\n\t}\n\n\tlog.Debug(\"UnmarshalYAML MAP VALUE entry %v\", mappingEntry.Value)\n\tif err := valueNode.UnmarshalGoccyYAML(mappingEntry.Value, cm, anchorMap); err != nil {\n\t\treturn err\n\t}\n\n\tif mappingEntry.FootComment != nil {\n\t\tvalueNode.FootComment = mappingEntry.FootComment.String()\n\t}\n\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/candidate_node_test.go", "content": "package yqlib\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\ntype valueRepScenario struct {\n\tinput string\n\ttag string\n\texpected interface{}\n}\n\nvar valueRepScenarios = []valueRepScenario{\n\t{\n\t\tinput: `\"cat\"`,\n\t\texpected: `\"cat\"`,\n\t},\n\t{\n\t\tinput: `3`,\n\t\texpected: int64(3),\n\t},\n\t{\n\t\tinput: `3.1`,\n\t\texpected: float64(3.1),\n\t},\n\t{\n\t\tinput: `true`,\n\t\texpected: true,\n\t},\n\t{\n\t\tinput: `y`,\n\t\ttag: \"!!bool\",\n\t\texpected: true,\n\t},\n\t{\n\t\ttag: \"!!null\",\n\t\texpected: nil,\n\t},\n}\n\nfunc TestCandidateNodeGetValueRepScenarios(t *testing.T) {\n\tfor _, tt := range valueRepScenarios {\n\t\tnode := CandidateNode{Value: tt.input, Tag: tt.tag}\n\t\tactual, err := node.GetValueRep()\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t\treturn\n\t\t}\n\t\ttest.AssertResult(t, tt.expected, actual)\n\t}\n}\n\nfunc TestCandidateNodeChildWhenParentUpdated(t *testing.T) {\n\tparent := CandidateNode{}\n\tchild := parent.CreateChild()\n\tparent.SetDocument(1)\n\tparent.SetFileIndex(2)\n\tparent.SetFilename(\"meow\")\n\ttest.AssertResultWithContext(t, \"meow\", child.GetFilename(), \"filename\")\n\ttest.AssertResultWithContext(t, 2, child.GetFileIndex(), \"file index\")\n\ttest.AssertResultWithContext(t, uint(1), child.GetDocument(), \"document index\")\n}\n\ntype createScalarNodeScenario struct {\n\tvalue interface{}\n\tstringValue string\n\texpectedTag string\n}\n\nvar createScalarScenarios = []createScalarNodeScenario{\n\t{\n\t\tvalue: \"mike\",\n\t\tstringValue: \"mike\",\n\t\texpectedTag: \"!!str\",\n\t},\n\t{\n\t\tvalue: 3,\n\t\tstringValue: \"3\",\n\t\texpectedTag: \"!!int\",\n\t},\n\t{\n\t\tvalue: 3.1,\n\t\tstringValue: \"3.1\",\n\t\texpectedTag: \"!!float\",\n\t},\n\t{\n\t\tvalue: true,\n\t\tstringValue: \"true\",\n\t\texpectedTag: \"!!bool\",\n\t},\n\t{\n\t\tvalue: nil,\n\t\tstringValue: \"~\",\n\t\texpectedTag: \"!!null\",\n\t},\n}\n\nfunc TestCreateScalarNodeScenarios(t *testing.T) {\n\tfor _, tt := range createScalarScenarios {\n\t\tactual := createScalarNode(tt.value, tt.stringValue)\n\t\ttest.AssertResultWithContext(t, tt.stringValue, actual.Value, fmt.Sprintf(\"Value for: Value: [%v], String: %v\", tt.value, tt.stringValue))\n\t\ttest.AssertResultWithContext(t, tt.expectedTag, actual.Tag, fmt.Sprintf(\"Value for: Value: [%v], String: %v\", tt.value, tt.stringValue))\n\t}\n}\n\nfunc TestGetKeyForMapValue(t *testing.T) {\n\tkey := createStringScalarNode(\"yourKey\")\n\tn := CandidateNode{Key: key, Value: \"meow\", document: 3}\n\ttest.AssertResult(t, \"3 - yourKey\", n.GetKey())\n}\n\nfunc TestGetKeyForMapKey(t *testing.T) {\n\tkey := createStringScalarNode(\"yourKey\")\n\tkey.IsMapKey = true\n\tkey.document = 3\n\ttest.AssertResult(t, \"key-yourKey-3 - \", key.GetKey())\n}\n\nfunc TestGetKeyForValue(t *testing.T) {\n\tn := CandidateNode{Value: \"meow\", document: 3}\n\ttest.AssertResult(t, \"3 - \", n.GetKey())\n}\n\nfunc TestGetParsedKeyForMapKey(t *testing.T) {\n\tkey := createStringScalarNode(\"yourKey\")\n\tkey.IsMapKey = true\n\tkey.document = 3\n\ttest.AssertResult(t, \"yourKey\", key.getParsedKey())\n}\n\nfunc TestGetParsedKeyForLooseValue(t *testing.T) {\n\tn := CandidateNode{Value: \"meow\", document: 3}\n\ttest.AssertResult(t, nil, n.getParsedKey())\n}\n\nfunc TestGetParsedKeyForMapValue(t *testing.T) {\n\tkey := createStringScalarNode(\"yourKey\")\n\tn := CandidateNode{Key: key, Value: \"meow\", document: 3}\n\ttest.AssertResult(t, \"yourKey\", n.getParsedKey())\n}\n\nfunc TestGetParsedKeyForArrayValue(t *testing.T) {\n\tkey := createScalarNode(4, \"4\")\n\tn := CandidateNode{Key: key, Value: \"meow\", document: 3}\n\ttest.AssertResult(t, 4, n.getParsedKey())\n}\n\nfunc TestCandidateNodeAddKeyValueChild(t *testing.T) {\n\tkey := CandidateNode{Value: \"cool\", IsMapKey: true}\n\tnode := CandidateNode{}\n\n\t_, keyIsValueNow := node.AddKeyValueChild(&CandidateNode{Value: \"newKey\"}, &key)\n\n\ttest.AssertResult(t, keyIsValueNow.IsMapKey, false)\n\ttest.AssertResult(t, key.IsMapKey, true)\n\n}\n\nfunc TestConvertToNodeInfo(t *testing.T) {\n\tchild := &CandidateNode{\n\t\tKind: ScalarNode,\n\t\tStyle: DoubleQuotedStyle,\n\t\tTag: \"!!str\",\n\t\tValue: \"childValue\",\n\t\tLine: 2,\n\t\tColumn: 3,\n\t}\n\tparent := &CandidateNode{\n\t\tKind: MappingNode,\n\t\tStyle: TaggedStyle,\n\t\tTag: \"!!map\",\n\t\tValue: \"\",\n\t\tLine: 1,\n\t\tColumn: 1,\n\t\tContent: []*CandidateNode{child},\n\t\tHeadComment: \"head\",\n\t\tLineComment: \"line\",\n\t\tFootComment: \"foot\",\n\t\tAnchor: \"anchor\",\n\t}\n\tinfo := parent.ConvertToNodeInfo()\n\ttest.AssertResult(t, \"MappingNode\", info.Kind)\n\ttest.AssertResult(t, \"TaggedStyle\", info.Style)\n\ttest.AssertResult(t, \"!!map\", info.Tag)\n\ttest.AssertResult(t, \"head\", info.HeadComment)\n\ttest.AssertResult(t, \"line\", info.LineComment)\n\ttest.AssertResult(t, \"foot\", info.FootComment)\n\ttest.AssertResult(t, \"anchor\", info.Anchor)\n\ttest.AssertResult(t, 1, info.Line)\n\ttest.AssertResult(t, 1, info.Column)\n\tif len(info.Content) != 1 {\n\t\tt.Fatalf(\"Expected 1 child, got %d\", len(info.Content))\n\t}\n\tchildInfo := info.Content[0]\n\ttest.AssertResult(t, \"ScalarNode\", childInfo.Kind)\n\ttest.AssertResult(t, \"DoubleQuotedStyle\", childInfo.Style)\n\ttest.AssertResult(t, \"!!str\", childInfo.Tag)\n\ttest.AssertResult(t, \"childValue\", childInfo.Value)\n\ttest.AssertResult(t, 2, childInfo.Line)\n\ttest.AssertResult(t, 3, childInfo.Column)\n}\n\nfunc TestCandidateNodeGetPath(t *testing.T) {\n\t// Test root node with no parent\n\troot := CandidateNode{Value: \"root\"}\n\tpath := root.GetPath()\n\ttest.AssertResult(t, 0, len(path))\n\n\t// Test node with key\n\tkey := createStringScalarNode(\"myKey\")\n\tnode := CandidateNode{Key: key, Value: \"myValue\"}\n\tpath = node.GetPath()\n\ttest.AssertResult(t, 1, len(path))\n\ttest.AssertResult(t, \"myKey\", path[0])\n\n\t// Test nested path\n\tparent := CandidateNode{}\n\tparentKey := createStringScalarNode(\"parent\")\n\tparent.Key = parentKey\n\tnode.Parent = &parent\n\tpath = node.GetPath()\n\ttest.AssertResult(t, 2, len(path))\n\ttest.AssertResult(t, \"parent\", path[0])\n\ttest.AssertResult(t, \"myKey\", path[1])\n}\n\nfunc TestCandidateNodeGetNicePath(t *testing.T) {\n\t// Test simple key\n\tkey := createStringScalarNode(\"simple\")\n\tnode := CandidateNode{Key: key}\n\tnicePath := node.GetNicePath()\n\ttest.AssertResult(t, \"simple\", nicePath)\n\n\t// Test array index\n\tarrayKey := createScalarNode(0, \"0\")\n\tarrayNode := CandidateNode{Key: arrayKey}\n\tnicePath = arrayNode.GetNicePath()\n\ttest.AssertResult(t, \"[0]\", nicePath)\n\n\tdotKey := createStringScalarNode(\"key.with.dots\")\n\tdotNode := CandidateNode{Key: dotKey}\n\tnicePath = dotNode.GetNicePath()\n\ttest.AssertResult(t, \"key.with.dots\", nicePath)\n\n\t// Test nested path\n\tparentKey := createStringScalarNode(\"parent\")\n\tparent := CandidateNode{Key: parentKey}\n\tchildKey := createStringScalarNode(\"child\")\n\tchild := CandidateNode{Key: childKey, Parent: &parent}\n\tnicePath = child.GetNicePath()\n\ttest.AssertResult(t, \"parent.child\", nicePath)\n}\n\nfunc TestCandidateNodeFilterMapContentByKey(t *testing.T) {\n\t// Create a map with multiple key-value pairs\n\tkey1 := createStringScalarNode(\"key1\")\n\tvalue1 := createStringScalarNode(\"value1\")\n\tkey2 := createStringScalarNode(\"key2\")\n\tvalue2 := createStringScalarNode(\"value2\")\n\tkey3 := createStringScalarNode(\"key3\")\n\tvalue3 := createStringScalarNode(\"value3\")\n\n\tmapNode := &CandidateNode{\n\t\tKind: MappingNode,\n\t\tContent: []*CandidateNode{key1, value1, key2, value2, key3, value3},\n\t}\n\n\t// Filter by key predicate that matches key1 and key3\n\tfiltered := mapNode.FilterMapContentByKey(func(key *CandidateNode) bool {\n\t\treturn key.Value == \"key1\" || key.Value == \"key3\"\n\t})\n\n\t// Should return key1, value1, key3, value3\n\ttest.AssertResult(t, 4, len(filtered))\n\ttest.AssertResult(t, \"key1\", filtered[0].Value)\n\ttest.AssertResult(t, \"value1\", filtered[1].Value)\n\ttest.AssertResult(t, \"key3\", filtered[2].Value)\n\ttest.AssertResult(t, \"value3\", filtered[3].Value)\n}\n\nfunc TestCandidateNodeVisitValues(t *testing.T) {\n\t// Test mapping node\n\tkey1 := createStringScalarNode(\"key1\")\n\tvalue1 := createStringScalarNode(\"value1\")\n\tkey2 := createStringScalarNode(\"key2\")\n\tvalue2 := createStringScalarNode(\"value2\")\n\n\tmapNode := &CandidateNode{\n\t\tKind: MappingNode,\n\t\tContent: []*CandidateNode{key1, value1, key2, value2},\n\t}\n\n\tvar visited []string\n\terr := mapNode.VisitValues(func(node *CandidateNode) error {\n\t\tvisited = append(visited, node.Value)\n\t\treturn nil\n\t})\n\n\ttest.AssertResult(t, nil, err)\n\ttest.AssertResult(t, 2, len(visited))\n\ttest.AssertResult(t, \"value1\", visited[0])\n\ttest.AssertResult(t, \"value2\", visited[1])\n\n\t// Test sequence node\n\titem1 := createStringScalarNode(\"item1\")\n\titem2 := createStringScalarNode(\"item2\")\n\n\tseqNode := &CandidateNode{\n\t\tKind: SequenceNode,\n\t\tContent: []*CandidateNode{item1, item2},\n\t}\n\n\tvisited = []string{}\n\terr = seqNode.VisitValues(func(node *CandidateNode) error {\n\t\tvisited = append(visited, node.Value)\n\t\treturn nil\n\t})\n\n\ttest.AssertResult(t, nil, err)\n\ttest.AssertResult(t, 2, len(visited))\n\ttest.AssertResult(t, \"item1\", visited[0])\n\ttest.AssertResult(t, \"item2\", visited[1])\n\n\t// Test scalar node (should not visit anything)\n\tscalarNode := &CandidateNode{\n\t\tKind: ScalarNode,\n\t\tValue: \"scalar\",\n\t}\n\n\tvisited = []string{}\n\terr = scalarNode.VisitValues(func(node *CandidateNode) error {\n\t\tvisited = append(visited, node.Value)\n\t\treturn nil\n\t})\n\n\ttest.AssertResult(t, nil, err)\n\ttest.AssertResult(t, 0, len(visited))\n}\n\nfunc TestCandidateNodeCanVisitValues(t *testing.T) {\n\tmapNode := &CandidateNode{Kind: MappingNode}\n\tseqNode := &CandidateNode{Kind: SequenceNode}\n\tscalarNode := &CandidateNode{Kind: ScalarNode}\n\n\ttest.AssertResult(t, true, mapNode.CanVisitValues())\n\ttest.AssertResult(t, true, seqNode.CanVisitValues())\n\ttest.AssertResult(t, false, scalarNode.CanVisitValues())\n}\n\nfunc TestCandidateNodeAddChild(t *testing.T) {\n\tparent := &CandidateNode{Kind: SequenceNode}\n\tchild := createStringScalarNode(\"child\")\n\n\tparent.AddChild(child)\n\n\ttest.AssertResult(t, 1, len(parent.Content))\n\ttest.AssertResult(t, false, parent.Content[0].IsMapKey)\n\ttest.AssertResult(t, \"0\", parent.Content[0].Key.Value)\n\t// Check that parent is set correctly\n\tif parent.Content[0].Parent != parent {\n\t\tt.Errorf(\"Expected parent to be set correctly\")\n\t}\n}\n\nfunc TestCandidateNodeAddChildren(t *testing.T) {\n\t// Test sequence node\n\tparent := &CandidateNode{Kind: SequenceNode}\n\tchild1 := createStringScalarNode(\"child1\")\n\tchild2 := createStringScalarNode(\"child2\")\n\n\tparent.AddChildren([]*CandidateNode{child1, child2})\n\n\ttest.AssertResult(t, 2, len(parent.Content))\n\ttest.AssertResult(t, \"child1\", parent.Content[0].Value)\n\ttest.AssertResult(t, \"child2\", parent.Content[1].Value)\n\n\t// Test mapping node\n\tmapParent := &CandidateNode{Kind: MappingNode}\n\tkey1 := createStringScalarNode(\"key1\")\n\tvalue1 := createStringScalarNode(\"value1\")\n\tkey2 := createStringScalarNode(\"key2\")\n\tvalue2 := createStringScalarNode(\"value2\")\n\n\tmapParent.AddChildren([]*CandidateNode{key1, value1, key2, value2})\n\n\ttest.AssertResult(t, 4, len(mapParent.Content))\n\ttest.AssertResult(t, true, mapParent.Content[0].IsMapKey) // key1\n\ttest.AssertResult(t, false, mapParent.Content[1].IsMapKey) // value1\n\ttest.AssertResult(t, true, mapParent.Content[2].IsMapKey) // key2\n\ttest.AssertResult(t, false, mapParent.Content[3].IsMapKey) // value2\n}\n" }, { "path": "pkg/yqlib/candidate_node_yaml.go", "content": "package yqlib\n\nimport (\n\t\"fmt\"\n\n\tyaml \"go.yaml.in/yaml/v4\"\n)\n\nfunc MapYamlStyle(original yaml.Style) Style {\n\tswitch original {\n\tcase yaml.TaggedStyle:\n\t\treturn TaggedStyle\n\tcase yaml.DoubleQuotedStyle:\n\t\treturn DoubleQuotedStyle\n\tcase yaml.SingleQuotedStyle:\n\t\treturn SingleQuotedStyle\n\tcase yaml.LiteralStyle:\n\t\treturn LiteralStyle\n\tcase yaml.FoldedStyle:\n\t\treturn FoldedStyle\n\tcase yaml.FlowStyle:\n\t\treturn FlowStyle\n\tcase 0:\n\t\treturn 0\n\t}\n\treturn Style(original)\n}\n\nfunc MapToYamlStyle(original Style) yaml.Style {\n\tswitch original {\n\tcase TaggedStyle:\n\t\treturn yaml.TaggedStyle\n\tcase DoubleQuotedStyle:\n\t\treturn yaml.DoubleQuotedStyle\n\tcase SingleQuotedStyle:\n\t\treturn yaml.SingleQuotedStyle\n\tcase LiteralStyle:\n\t\treturn yaml.LiteralStyle\n\tcase FoldedStyle:\n\t\treturn yaml.FoldedStyle\n\tcase FlowStyle:\n\t\treturn yaml.FlowStyle\n\tcase 0:\n\t\treturn 0\n\t}\n\treturn yaml.Style(original)\n}\n\nfunc (o *CandidateNode) copyFromYamlNode(node *yaml.Node, anchorMap map[string]*CandidateNode) {\n\to.Style = MapYamlStyle(node.Style)\n\n\to.Tag = node.Tag\n\to.Value = node.Value\n\to.Anchor = node.Anchor\n\n\tif o.Anchor != \"\" {\n\t\tanchorMap[o.Anchor] = o\n\t\tlog.Debug(\"set anchor %v to %v\", o.Anchor, NodeToString(o))\n\t}\n\n\t// its a single alias\n\tif node.Alias != nil && node.Alias.Anchor != \"\" {\n\t\to.Alias = anchorMap[node.Alias.Anchor]\n\t\tlog.Debug(\"set alias to %v\", NodeToString(anchorMap[node.Alias.Anchor]))\n\t}\n\to.HeadComment = node.HeadComment\n\to.LineComment = node.LineComment\n\to.FootComment = node.FootComment\n\n\to.Line = node.Line\n\to.Column = node.Column\n}\n\nfunc (o *CandidateNode) copyToYamlNode(node *yaml.Node) {\n\tnode.Style = MapToYamlStyle(o.Style)\n\n\tnode.Tag = o.Tag\n\tnode.Value = o.Value\n\tnode.Anchor = o.Anchor\n\n\tnode.HeadComment = o.HeadComment\n\n\tnode.LineComment = o.LineComment\n\tnode.FootComment = o.FootComment\n\n\tnode.Line = o.Line\n\tnode.Column = o.Column\n}\n\nfunc (o *CandidateNode) decodeIntoChild(childNode *yaml.Node, anchorMap map[string]*CandidateNode) (*CandidateNode, error) {\n\tnewChild := o.CreateChild()\n\n\t// null yaml.Nodes to not end up calling UnmarshalYAML\n\t// so we call it explicitly\n\tif childNode.Tag == \"!!null\" {\n\t\tnewChild.Kind = ScalarNode\n\t\tnewChild.copyFromYamlNode(childNode, anchorMap)\n\t\treturn newChild, nil\n\t}\n\n\terr := newChild.UnmarshalYAML(childNode, anchorMap)\n\treturn newChild, err\n}\n\nfunc (o *CandidateNode) UnmarshalYAML(node *yaml.Node, anchorMap map[string]*CandidateNode) error {\n\tlog.Debugf(\"UnmarshalYAML %v\", node.Tag)\n\tswitch node.Kind {\n\tcase yaml.AliasNode:\n\t\tlog.Debug(\"UnmarshalYAML - alias from yaml: %v\", o.Tag)\n\t\to.Kind = AliasNode\n\t\to.copyFromYamlNode(node, anchorMap)\n\t\treturn nil\n\tcase yaml.ScalarNode:\n\t\tlog.Debugf(\"UnmarshalYAML - a scalar\")\n\t\to.Kind = ScalarNode\n\t\to.copyFromYamlNode(node, anchorMap)\n\t\treturn nil\n\tcase yaml.MappingNode:\n\t\tlog.Debugf(\"UnmarshalYAML - a mapping node\")\n\t\to.Kind = MappingNode\n\t\to.copyFromYamlNode(node, anchorMap)\n\t\to.Content = make([]*CandidateNode, len(node.Content))\n\t\tfor i := 0; i < len(node.Content); i += 2 {\n\n\t\t\tkeyNode, err := o.decodeIntoChild(node.Content[i], anchorMap)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\tkeyNode.IsMapKey = true\n\n\t\t\tvalueNode, err := o.decodeIntoChild(node.Content[i+1], anchorMap)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\tvalueNode.Key = keyNode\n\n\t\t\to.Content[i] = keyNode\n\t\t\to.Content[i+1] = valueNode\n\t\t}\n\t\tlog.Debugf(\"UnmarshalYAML - finished mapping node\")\n\t\treturn nil\n\tcase yaml.SequenceNode:\n\t\tlog.Debugf(\"UnmarshalYAML - a sequence: %v\", len(node.Content))\n\t\to.Kind = SequenceNode\n\n\t\to.copyFromYamlNode(node, anchorMap)\n\t\tlog.Debugf(\"node Style: %v\", node.Style)\n\t\tlog.Debugf(\"o Style: %v\", o.Style)\n\t\to.Content = make([]*CandidateNode, len(node.Content))\n\t\tfor i := 0; i < len(node.Content); i++ {\n\t\t\tkeyNode := o.CreateChild()\n\t\t\tkeyNode.IsMapKey = true\n\t\t\tkeyNode.Tag = \"!!int\"\n\t\t\tkeyNode.Kind = ScalarNode\n\t\t\tkeyNode.Value = fmt.Sprintf(\"%v\", i)\n\n\t\t\tvalueNode, err := o.decodeIntoChild(node.Content[i], anchorMap)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\tvalueNode.Key = keyNode\n\t\t\to.Content[i] = valueNode\n\t\t}\n\t\treturn nil\n\tcase 0:\n\t\t// not sure when this happens\n\t\to.copyFromYamlNode(node, anchorMap)\n\t\tlog.Debugf(\"UnmarshalYAML - err.. %v\", NodeToString(o))\n\t\treturn nil\n\tdefault:\n\t\treturn fmt.Errorf(\"orderedMap: invalid yaml node\")\n\t}\n}\n\nfunc (o *CandidateNode) MarshalYAML() (*yaml.Node, error) {\n\tlog.Debug(\"MarshalYAML to yaml: %v\", o.Tag)\n\tswitch o.Kind {\n\tcase AliasNode:\n\t\tlog.Debug(\"MarshalYAML - alias to yaml: %v\", o.Tag)\n\t\ttarget := &yaml.Node{Kind: yaml.AliasNode}\n\t\to.copyToYamlNode(target)\n\t\treturn target, nil\n\tcase ScalarNode:\n\t\tlog.Debug(\"MarshalYAML - scalar: %v\", o.Value)\n\t\ttarget := &yaml.Node{Kind: yaml.ScalarNode}\n\t\to.copyToYamlNode(target)\n\t\treturn target, nil\n\tcase MappingNode, SequenceNode:\n\t\ttargetKind := yaml.MappingNode\n\t\tif o.Kind == SequenceNode {\n\t\t\ttargetKind = yaml.SequenceNode\n\t\t}\n\t\ttarget := &yaml.Node{Kind: targetKind}\n\t\to.copyToYamlNode(target)\n\t\tlog.Debugf(\"original style: %v\", o.Style)\n\t\tlog.Debugf(\"original: %v, tag: %v, style: %v, kind: %v\", NodeToString(o), target.Tag, target.Style, target.Kind == yaml.SequenceNode)\n\t\ttarget.Content = make([]*yaml.Node, len(o.Content))\n\t\tfor i := 0; i < len(o.Content); i++ {\n\n\t\t\tchild, err := o.Content[i].MarshalYAML()\n\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\ttarget.Content[i] = child\n\t\t}\n\t\treturn target, nil\n\t}\n\ttarget := &yaml.Node{}\n\to.copyToYamlNode(target)\n\treturn target, nil\n}\n" }, { "path": "pkg/yqlib/candidiate_node_json.go", "content": "//go:build !yq_nojson\n\npackage yqlib\n\nimport (\n\t\"bytes\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\n\t\"github.com/goccy/go-json\"\n)\n\nfunc (o *CandidateNode) setScalarFromJson(value interface{}) error {\n\to.Kind = ScalarNode\n\tswitch rawData := value.(type) {\n\tcase nil:\n\t\to.Tag = \"!!null\"\n\t\to.Value = \"null\"\n\tcase float64, float32:\n\t\to.Value = fmt.Sprintf(\"%v\", value)\n\t\to.Tag = \"!!float\"\n\t\t// json decoder returns ints as float.\n\t\tif value == float64(int64(rawData.(float64))) {\n\t\t\t// aha it's an int disguised as a float\n\t\t\to.Tag = \"!!int\"\n\t\t\to.Value = fmt.Sprintf(\"%v\", int64(value.(float64)))\n\t\t}\n\tcase int, int64, int32:\n\t\to.Value = fmt.Sprintf(\"%v\", value)\n\t\to.Tag = \"!!int\"\n\tcase bool:\n\t\to.Value = fmt.Sprintf(\"%v\", value)\n\t\to.Tag = \"!!bool\"\n\tcase string:\n\t\to.Value = rawData\n\t\to.Tag = \"!!str\"\n\tdefault:\n\t\treturn fmt.Errorf(\"unrecognised type :( %v\", rawData)\n\t}\n\treturn nil\n}\n\nfunc (o *CandidateNode) UnmarshalJSON(data []byte) error {\n\tlog.Debug(\"UnmarshalJSON\")\n\tswitch data[0] {\n\tcase '{':\n\t\tlog.Debug(\"UnmarshalJSON - its a map!\")\n\t\t// its a map\n\t\to.Kind = MappingNode\n\t\to.Tag = \"!!map\"\n\n\t\tdec := json.NewDecoder(bytes.NewReader(data))\n\t\t_, err := dec.Token() // open object\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\t// cycle through k/v\n\t\tvar tok json.Token\n\t\tfor tok, err = dec.Token(); err == nil; tok, err = dec.Token() {\n\t\t\t// we can expect two types: string or Delim. Delim automatically means\n\t\t\t// that it is the closing bracket of the object, whereas string means\n\t\t\t// that there is another key.\n\t\t\tif _, ok := tok.(json.Delim); ok {\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t\tchildKey := o.CreateChild()\n\t\t\tchildKey.IsMapKey = true\n\t\t\tchildKey.Value = tok.(string)\n\t\t\tchildKey.Kind = ScalarNode\n\t\t\tchildKey.Tag = \"!!str\"\n\n\t\t\tchildValue := o.CreateChild()\n\t\t\tchildValue.Key = childKey\n\n\t\t\tif err := dec.Decode(childValue); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\to.Content = append(o.Content, childKey, childValue)\n\t\t}\n\t\t// unexpected error\n\t\tif err != nil && !errors.Is(err, io.EOF) {\n\t\t\treturn err\n\t\t}\n\t\treturn nil\n\tcase '[':\n\t\to.Kind = SequenceNode\n\t\to.Tag = \"!!seq\"\n\t\tlog.Debug(\"UnmarshalJSON - its an array!\")\n\t\tvar children []*CandidateNode\n\t\tif err := json.Unmarshal(data, &children); err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// now we put the children into the content, and set a key value for them\n\t\tfor i, child := range children {\n\n\t\t\tif child == nil {\n\t\t\t\t// need to represent it as a null scalar\n\t\t\t\tchild = createScalarNode(nil, \"null\")\n\t\t\t}\n\t\t\tchildKey := o.CreateChild()\n\t\t\tchildKey.Kind = ScalarNode\n\t\t\tchildKey.Tag = \"!!int\"\n\t\t\tchildKey.Value = fmt.Sprintf(\"%v\", i)\n\t\t\tchildKey.IsMapKey = true\n\n\t\t\tchild.Parent = o\n\t\t\tchild.Key = childKey\n\t\t\to.Content = append(o.Content, child)\n\t\t}\n\t\treturn nil\n\t}\n\tlog.Debug(\"UnmarshalJSON - its a scalar!\")\n\t// otherwise, must be a scalar\n\tvar scalar interface{}\n\terr := json.Unmarshal(data, &scalar)\n\n\tif err != nil {\n\t\treturn err\n\t}\n\tlog.Debug(\"UnmarshalJSON - scalar is %v\", scalar)\n\n\treturn o.setScalarFromJson(scalar)\n\n}\n\nfunc (o *CandidateNode) MarshalJSON() ([]byte, error) {\n\tlog.Debugf(\"MarshalJSON %v\", NodeToString(o))\n\tbuf := new(bytes.Buffer)\n\tenc := json.NewEncoder(buf)\n\tenc.SetIndent(\"\", \" \")\n\tenc.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >\n\n\tswitch o.Kind {\n\tcase AliasNode:\n\t\tlog.Debugf(\"MarshalJSON AliasNode\")\n\t\terr := enc.Encode(o.Alias)\n\t\treturn buf.Bytes(), err\n\tcase ScalarNode:\n\t\tlog.Debugf(\"MarshalJSON ScalarNode\")\n\t\tvalue, err := o.GetValueRep()\n\t\tif err != nil {\n\t\t\treturn buf.Bytes(), err\n\t\t}\n\t\terr = enc.Encode(value)\n\t\treturn buf.Bytes(), err\n\tcase MappingNode:\n\t\tlog.Debugf(\"MarshalJSON MappingNode\")\n\t\tbuf.WriteByte('{')\n\t\tfor i := 0; i < len(o.Content); i += 2 {\n\t\t\tif err := enc.Encode(o.Content[i].Value); err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tbuf.WriteByte(':')\n\t\t\tif err := enc.Encode(o.Content[i+1]); err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tif i != len(o.Content)-2 {\n\t\t\t\tbuf.WriteByte(',')\n\t\t\t}\n\t\t}\n\t\tbuf.WriteByte('}')\n\t\treturn buf.Bytes(), nil\n\tcase SequenceNode:\n\t\tlog.Debugf(\"MarshalJSON SequenceNode, %v, len: %v\", o.Content, len(o.Content))\n\t\tvar err error\n\t\tif len(o.Content) == 0 {\n\t\t\tbuf.WriteString(\"[]\")\n\t\t} else {\n\t\t\terr = enc.Encode(o.Content)\n\t\t}\n\t\treturn buf.Bytes(), err\n\tdefault:\n\t\terr := enc.Encode(nil)\n\t\treturn buf.Bytes(), err\n\t}\n}\n" }, { "path": "pkg/yqlib/chown_linux.go", "content": "//go:build linux\n\npackage yqlib\n\nimport (\n\t\"io/fs\"\n\t\"os\"\n\t\"syscall\"\n)\n\nfunc changeOwner(info fs.FileInfo, file *os.File) error {\n\tif stat, ok := info.Sys().(*syscall.Stat_t); ok {\n\t\tuid := int(stat.Uid)\n\t\tgid := int(stat.Gid)\n\n\t\terr := os.Chown(file.Name(), uid, gid)\n\t\tif err != nil {\n\t\t\t// this happens with snap confinement\n\t\t\t// not really a big issue as users can chown\n\t\t\t// the file themselves if required.\n\t\t\tlog.Info(\"Skipping chown: %v\", err)\n\t\t}\n\t}\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/chown_linux_test.go", "content": "//go:build linux\n\npackage yqlib\n\nimport (\n\t\"os\"\n\t\"path/filepath\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestChangeOwner(t *testing.T) {\n\t// Create a temporary file for testing\n\ttempDir := t.TempDir()\n\ttestFile := filepath.Join(tempDir, \"testfile.txt\")\n\n\t// Create a test file\n\terr := os.WriteFile(testFile, []byte(\"test content\"), 0600)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create test file: %v\", err)\n\t}\n\n\t// Get file info\n\tinfo, err := os.Stat(testFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to stat test file: %v\", err)\n\t}\n\n\t// Create another temporary file to change ownership of\n\ttempFile, err := os.CreateTemp(tempDir, \"chown_test_*.txt\")\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create temp file: %v\", err)\n\t}\n\tdefer os.Remove(tempFile.Name())\n\ttempFile.Close()\n\n\t// Test changeOwner function\n\terr = changeOwner(info, tempFile)\n\tif err != nil {\n\t\tt.Errorf(\"changeOwner failed: %v\", err)\n\t}\n\n\t// Verify that the function doesn't panic with valid input\n\ttempFile2, err := os.CreateTemp(tempDir, \"chown_test2_*.txt\")\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create second temp file: %v\", err)\n\t}\n\tdefer os.Remove(tempFile2.Name())\n\ttempFile2.Close()\n\n\t// Test with the second file\n\terr = changeOwner(info, tempFile2)\n\tif err != nil {\n\t\tt.Errorf(\"changeOwner failed on second file: %v\", err)\n\t}\n}\n\nfunc TestChangeOwnerWithInvalidFileInfo(t *testing.T) {\n\t// Create a mock file info that doesn't have syscall.Stat_t\n\tmockInfo := &mockFileInfo{\n\t\tname: \"mock\",\n\t\tsize: 0,\n\t\tmode: 0600,\n\t}\n\n\t// Create a temporary file\n\ttempFile, err := os.CreateTemp(t.TempDir(), \"chown_test_*.txt\")\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create temp file: %v\", err)\n\t}\n\tdefer os.Remove(tempFile.Name())\n\ttempFile.Close()\n\n\t// Test changeOwner with mock file info (should not panic)\n\terr = changeOwner(mockInfo, tempFile)\n\tif err != nil {\n\t\tt.Errorf(\"changeOwner failed with mock file info: %v\", err)\n\t}\n}\n\nfunc TestChangeOwnerWithNonExistentFile(t *testing.T) {\n\t// Create a temporary file\n\ttempFile, err := os.CreateTemp(t.TempDir(), \"chown_test_*.txt\")\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create temp file: %v\", err)\n\t}\n\tdefer os.Remove(tempFile.Name())\n\ttempFile.Close()\n\n\t// Get file info\n\tinfo, err := os.Stat(tempFile.Name())\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to stat temp file: %v\", err)\n\t}\n\n\t// Remove the file\n\tos.Remove(tempFile.Name())\n\n\terr = changeOwner(info, tempFile)\n\t// The function should not panic even if the file doesn't exist\n\tif err != nil {\n\t\tt.Logf(\"Expected error when changing owner of non-existent file: %v\", err)\n\t}\n}\n\n// mockFileInfo implements fs.FileInfo but doesn't have syscall.Stat_t\ntype mockFileInfo struct {\n\tname string\n\tsize int64\n\tmode os.FileMode\n}\n\nfunc (m *mockFileInfo) Name() string { return m.name }\nfunc (m *mockFileInfo) Size() int64 { return m.size }\nfunc (m *mockFileInfo) Mode() os.FileMode { return m.mode }\nfunc (m *mockFileInfo) ModTime() time.Time { return time.Time{} }\nfunc (m *mockFileInfo) IsDir() bool { return false }\nfunc (m *mockFileInfo) Sys() interface{} { return nil } // This will cause the type assertion to fail\n\nfunc TestChangeOwnerWithSyscallStatT(t *testing.T) {\n\t// Create a temporary file\n\ttempFile, err := os.CreateTemp(t.TempDir(), \"chown_test_*.txt\")\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create temp file: %v\", err)\n\t}\n\tdefer os.Remove(tempFile.Name())\n\ttempFile.Close()\n\n\t// Get file info\n\tinfo, err := os.Stat(tempFile.Name())\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to stat temp file: %v\", err)\n\t}\n\n\terr = changeOwner(info, tempFile)\n\tif err != nil {\n\t\tt.Logf(\"changeOwner returned error (this might be expected in some environments): %v\", err)\n\t}\n}\n" }, { "path": "pkg/yqlib/chown_not_linux_os.go", "content": "//go:build !linux\n\npackage yqlib\n\nimport (\n\t\"io/fs\"\n\t\"os\"\n)\n\nfunc changeOwner(_ fs.FileInfo, _ *os.File) error {\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/color_print.go", "content": "package yqlib\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\n\t\"github.com/fatih/color\"\n\t\"github.com/goccy/go-yaml/lexer\"\n\t\"github.com/goccy/go-yaml/printer\"\n)\n\n// Thanks @risentveber!\n\nconst escape = \"\\x1b\"\n\nfunc format(attr color.Attribute) string {\n\treturn fmt.Sprintf(\"%s[%dm\", escape, attr)\n}\n\nfunc colorizeAndPrint(yamlBytes []byte, writer io.Writer) error {\n\ttokens := lexer.Tokenize(string(yamlBytes))\n\tvar p printer.Printer\n\tp.Bool = func() *printer.Property {\n\t\treturn &printer.Property{\n\t\t\tPrefix: format(color.FgHiMagenta),\n\t\t\tSuffix: format(color.Reset),\n\t\t}\n\t}\n\tp.Number = func() *printer.Property {\n\t\treturn &printer.Property{\n\t\t\tPrefix: format(color.FgHiMagenta),\n\t\t\tSuffix: format(color.Reset),\n\t\t}\n\t}\n\tp.MapKey = func() *printer.Property {\n\t\treturn &printer.Property{\n\t\t\tPrefix: format(color.FgCyan),\n\t\t\tSuffix: format(color.Reset),\n\t\t}\n\t}\n\tp.Anchor = func() *printer.Property {\n\t\treturn &printer.Property{\n\t\t\tPrefix: format(color.FgHiYellow),\n\t\t\tSuffix: format(color.Reset),\n\t\t}\n\t}\n\tp.Alias = func() *printer.Property {\n\t\treturn &printer.Property{\n\t\t\tPrefix: format(color.FgHiYellow),\n\t\t\tSuffix: format(color.Reset),\n\t\t}\n\t}\n\tp.String = func() *printer.Property {\n\t\treturn &printer.Property{\n\t\t\tPrefix: format(color.FgGreen),\n\t\t\tSuffix: format(color.Reset),\n\t\t}\n\t}\n\tp.Comment = func() *printer.Property {\n\t\treturn &printer.Property{\n\t\t\tPrefix: format(color.FgHiBlack),\n\t\t\tSuffix: format(color.Reset),\n\t\t}\n\t}\n\t_, err := writer.Write([]byte(p.PrintTokens(tokens) + \"\\n\"))\n\treturn err\n}\n" }, { "path": "pkg/yqlib/color_print_test.go", "content": "package yqlib\n\nimport (\n\t\"bytes\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/fatih/color\"\n)\n\nfunc TestFormat(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tattr color.Attribute\n\t\texpected string\n\t}{\n\t\t{\n\t\t\tname: \"reset color\",\n\t\t\tattr: color.Reset,\n\t\t\texpected: \"\\x1b[0m\",\n\t\t},\n\t\t{\n\t\t\tname: \"red color\",\n\t\t\tattr: color.FgRed,\n\t\t\texpected: \"\\x1b[31m\",\n\t\t},\n\t\t{\n\t\t\tname: \"green color\",\n\t\t\tattr: color.FgGreen,\n\t\t\texpected: \"\\x1b[32m\",\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tresult := format(tt.attr)\n\t\t\tif result != tt.expected {\n\t\t\t\tt.Errorf(\"format(%d) = %q, want %q\", tt.attr, result, tt.expected)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestColorizeAndPrint(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tyamlBytes []byte\n\t\texpectErr bool\n\t}{\n\t\t{\n\t\t\tname: \"simple yaml\",\n\t\t\tyamlBytes: []byte(\"name: test\\nage: 25\\n\"),\n\t\t\texpectErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"yaml with strings\",\n\t\t\tyamlBytes: []byte(\"name: \\\"hello world\\\"\\nactive: true\\ncount: 42\\n\"),\n\t\t\texpectErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"yaml with anchors and aliases\",\n\t\t\tyamlBytes: []byte(\"default: &default\\n name: test\\nuser: *default\\n\"),\n\t\t\texpectErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"yaml with comments\",\n\t\t\tyamlBytes: []byte(\"# This is a comment\\nname: test\\n\"),\n\t\t\texpectErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"empty yaml\",\n\t\t\tyamlBytes: []byte(\"\"),\n\t\t\texpectErr: false,\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tvar buf bytes.Buffer\n\t\t\terr := colorizeAndPrint(tt.yamlBytes, &buf)\n\n\t\t\tif tt.expectErr && err == nil {\n\t\t\t\tt.Error(\"Expected error but got none\")\n\t\t\t}\n\t\t\tif !tt.expectErr && err != nil {\n\t\t\t\tt.Errorf(\"Unexpected error: %v\", err)\n\t\t\t}\n\n\t\t\t// Check that output contains escape sequences (color codes)\n\t\t\tif !tt.expectErr && len(tt.yamlBytes) > 0 {\n\t\t\t\toutput := buf.String()\n\t\t\t\tif !strings.Contains(output, \"\\x1b[\") {\n\t\t\t\t\tt.Error(\"Expected output to contain color escape sequences\")\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestColorizeAndPrintWithDifferentYamlTypes(t *testing.T) {\n\ttestCases := []struct {\n\t\tname string\n\t\tyaml string\n\t\texpectErr bool\n\t}{\n\t\t{\n\t\t\tname: \"boolean values\",\n\t\t\tyaml: \"active: true\\ninactive: false\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"numeric values\",\n\t\t\tyaml: \"integer: 42\\nfloat: 3.14\\nnegative: -10\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"map keys\",\n\t\t\tyaml: \"user:\\n name: john\\n age: 30\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"string values\",\n\t\t\tyaml: \"message: \\\"hello world\\\"\\ndescription: 'single quotes'\\n\",\n\t\t},\n\t\t{\n\t\t\tname: \"mixed types\",\n\t\t\tyaml: \"config:\\n debug: true\\n port: 8080\\n host: \\\"localhost\\\"\\n\",\n\t\t},\n\t}\n\n\tfor _, tc := range testCases {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tvar buf bytes.Buffer\n\t\t\terr := colorizeAndPrint([]byte(tc.yaml), &buf)\n\n\t\t\tif tc.expectErr && err == nil {\n\t\t\t\tt.Error(\"Expected error but got none\")\n\t\t\t}\n\t\t\tif !tc.expectErr && err != nil {\n\t\t\t\tt.Errorf(\"Unexpected error: %v\", err)\n\t\t\t}\n\n\t\t\t// Verify output contains color codes\n\t\t\tif !tc.expectErr {\n\t\t\t\toutput := buf.String()\n\t\t\t\tif !strings.Contains(output, \"\\x1b[\") {\n\t\t\t\t\tt.Error(\"Expected output to contain color escape sequences\")\n\t\t\t\t}\n\t\t\t\t// Should end with newline\n\t\t\t\tif !strings.HasSuffix(output, \"\\n\") {\n\t\t\t\t\tt.Error(\"Expected output to end with newline\")\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "pkg/yqlib/context.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n\t\"time\"\n\n\tlogging \"gopkg.in/op/go-logging.v1\"\n)\n\ntype Context struct {\n\tMatchingNodes *list.List\n\tVariables map[string]*list.List\n\tDontAutoCreate bool\n\tdatetimeLayout string\n}\n\nfunc (n *Context) SingleReadonlyChildContext(candidate *CandidateNode) Context {\n\tlist := list.New()\n\tlist.PushBack(candidate)\n\tnewContext := n.ChildContext(list)\n\tnewContext.DontAutoCreate = true\n\treturn newContext\n}\n\nfunc (n *Context) SingleChildContext(candidate *CandidateNode) Context {\n\tlist := list.New()\n\tlist.PushBack(candidate)\n\treturn n.ChildContext(list)\n}\n\nfunc (n *Context) SetDateTimeLayout(newDateTimeLayout string) {\n\tn.datetimeLayout = newDateTimeLayout\n}\n\nfunc (n *Context) GetDateTimeLayout() string {\n\tif n.datetimeLayout != \"\" {\n\t\treturn n.datetimeLayout\n\t}\n\treturn time.RFC3339\n}\n\nfunc (n *Context) GetVariable(name string) *list.List {\n\tif n.Variables == nil {\n\t\treturn nil\n\t}\n\treturn n.Variables[name]\n}\n\nfunc (n *Context) SetVariable(name string, value *list.List) {\n\tif n.Variables == nil {\n\t\tn.Variables = make(map[string]*list.List)\n\t}\n\tn.Variables[name] = value\n}\n\nfunc (n *Context) ChildContext(results *list.List) Context {\n\tclone := Context{DontAutoCreate: n.DontAutoCreate, datetimeLayout: n.datetimeLayout}\n\tclone.Variables = make(map[string]*list.List)\n\tfor variableKey, originalValueList := range n.Variables {\n\n\t\tvariableCopyList := list.New()\n\t\tfor el := originalValueList.Front(); el != nil; el = el.Next() {\n\t\t\t// note that we dont make a copy of the candidate node\n\t\t\t// this is so the 'ref' operator can work correctly.\n\t\t\tclonedNode := el.Value.(*CandidateNode)\n\t\t\tvariableCopyList.PushBack(clonedNode)\n\t\t}\n\n\t\tclone.Variables[variableKey] = variableCopyList\n\t}\n\n\tclone.MatchingNodes = results\n\treturn clone\n}\n\nfunc (n *Context) ToString() string {\n\tif !log.IsEnabledFor(logging.DEBUG) {\n\t\treturn \"\"\n\t}\n\tresult := fmt.Sprintf(\"Context\\nDontAutoCreate: %v\\n\", n.DontAutoCreate)\n\treturn result + NodesToString(n.MatchingNodes)\n}\n\nfunc (n *Context) DeepClone() Context {\n\n\tclonedContent := list.New()\n\tfor el := n.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tclonedNode := el.Value.(*CandidateNode).Copy()\n\t\tclonedContent.PushBack(clonedNode)\n\t}\n\n\treturn n.ChildContext(clonedContent)\n}\n\nfunc (n *Context) Clone() Context {\n\treturn n.ChildContext(n.MatchingNodes)\n}\n\nfunc (n *Context) ReadOnlyClone() Context {\n\tclone := n.Clone()\n\tclone.DontAutoCreate = true\n\treturn clone\n}\n\nfunc (n *Context) WritableClone() Context {\n\tclone := n.Clone()\n\tclone.DontAutoCreate = false\n\treturn clone\n}\n" }, { "path": "pkg/yqlib/context_test.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n\tlogging \"gopkg.in/op/go-logging.v1\"\n)\n\nfunc TestChildContext(t *testing.T) {\n\n\texpectedOriginal := make(map[string]*list.List)\n\texpectedOriginal[\"dog\"] = list.New()\n\texpectedOriginal[\"dog\"].PushBack(&CandidateNode{Value: \"woof\"})\n\n\toriginalVariables := make(map[string]*list.List)\n\toriginalVariables[\"dog\"] = list.New()\n\toriginalVariables[\"dog\"].PushBack(&CandidateNode{Value: \"woof\"})\n\n\toriginal := Context{\n\t\tDontAutoCreate: true,\n\t\tdatetimeLayout: \"cat\",\n\t\tVariables: originalVariables,\n\t}\n\n\tnewResults := list.New()\n\tnewResults.PushBack(&CandidateNode{Value: \"bar\"})\n\n\tclone := original.ChildContext(newResults)\n\ttest.AssertResultComplex(t, originalVariables, clone.Variables)\n\n\tclone.Variables[\"dog\"].PushBack(\"bark\")\n\t// ensure this is a separate copy\n\ttest.AssertResultComplex(t, 1, originalVariables[\"dog\"].Len())\n\n}\n\nfunc TestChildContextNoVariables(t *testing.T) {\n\n\toriginal := Context{\n\t\tDontAutoCreate: true,\n\t\tdatetimeLayout: \"cat\",\n\t}\n\n\tnewResults := list.New()\n\tnewResults.PushBack(&CandidateNode{Value: \"bar\"})\n\n\tclone := original.ChildContext(newResults)\n\ttest.AssertResultComplex(t, make(map[string]*list.List), clone.Variables)\n\n}\n\nfunc TestSingleReadonlyChildContext(t *testing.T) {\n\toriginal := Context{\n\t\tDontAutoCreate: false,\n\t\tdatetimeLayout: \"2006-01-02\",\n\t}\n\n\tcandidate := &CandidateNode{Value: \"test\"}\n\tclone := original.SingleReadonlyChildContext(candidate)\n\n\t// Should have DontAutoCreate set to true\n\ttest.AssertResultComplex(t, true, clone.DontAutoCreate)\n\n\t// Should have the candidate node in MatchingNodes\n\ttest.AssertResultComplex(t, 1, clone.MatchingNodes.Len())\n\ttest.AssertResultComplex(t, candidate, clone.MatchingNodes.Front().Value)\n}\n\nfunc TestSingleChildContext(t *testing.T) {\n\toriginal := Context{\n\t\tDontAutoCreate: true,\n\t\tdatetimeLayout: \"2006-01-02\",\n\t}\n\n\tcandidate := &CandidateNode{Value: \"test\"}\n\tclone := original.SingleChildContext(candidate)\n\n\t// Should preserve DontAutoCreate\n\ttest.AssertResultComplex(t, true, clone.DontAutoCreate)\n\n\t// Should have the candidate node in MatchingNodes\n\ttest.AssertResultComplex(t, 1, clone.MatchingNodes.Len())\n\ttest.AssertResultComplex(t, candidate, clone.MatchingNodes.Front().Value)\n}\n\nfunc TestSetDateTimeLayout(t *testing.T) {\n\tcontext := Context{}\n\n\t// Test setting datetime layout\n\tcontext.SetDateTimeLayout(\"2006-01-02T15:04:05Z07:00\")\n\ttest.AssertResultComplex(t, \"2006-01-02T15:04:05Z07:00\", context.datetimeLayout)\n}\n\nfunc TestGetDateTimeLayout(t *testing.T) {\n\t// Test with custom layout\n\tcontext := Context{datetimeLayout: \"2006-01-02\"}\n\tresult := context.GetDateTimeLayout()\n\ttest.AssertResultComplex(t, \"2006-01-02\", result)\n\n\t// Test with empty layout (should return default)\n\tcontext = Context{}\n\tresult = context.GetDateTimeLayout()\n\ttest.AssertResultComplex(t, \"2006-01-02T15:04:05Z07:00\", result)\n}\n\nfunc TestGetVariable(t *testing.T) {\n\t// Test with nil Variables\n\tcontext := Context{}\n\tresult := context.GetVariable(\"nonexistent\")\n\ttest.AssertResultComplex(t, (*list.List)(nil), result)\n\n\t// Test with existing variable\n\tvariables := make(map[string]*list.List)\n\tvariables[\"test\"] = list.New()\n\tvariables[\"test\"].PushBack(&CandidateNode{Value: \"value\"})\n\n\tcontext = Context{Variables: variables}\n\tresult = context.GetVariable(\"test\")\n\ttest.AssertResultComplex(t, variables[\"test\"], result)\n\n\t// Test with non-existent variable\n\tresult = context.GetVariable(\"nonexistent\")\n\ttest.AssertResultComplex(t, (*list.List)(nil), result)\n}\n\nfunc TestSetVariable(t *testing.T) {\n\t// Test setting variable when Variables is nil\n\tcontext := Context{}\n\tvalue := list.New()\n\tvalue.PushBack(&CandidateNode{Value: \"test\"})\n\n\tcontext.SetVariable(\"key\", value)\n\ttest.AssertResultComplex(t, value, context.Variables[\"key\"])\n\n\t// Test setting variable when Variables already exists\n\tcontext.SetVariable(\"key2\", value)\n\ttest.AssertResultComplex(t, value, context.Variables[\"key2\"])\n}\n\nfunc TestToString(t *testing.T) {\n\tcontext := Context{\n\t\tDontAutoCreate: true,\n\t\tMatchingNodes: list.New(),\n\t}\n\n\t// Add a node to test the full string representation\n\tnode := &CandidateNode{Value: \"test\"}\n\tcontext.MatchingNodes.PushBack(node)\n\n\t// Test with debug logging disabled (default)\n\tresult := context.ToString()\n\ttest.AssertResultComplex(t, \"\", result)\n\n\t// Test with debug logging enabled\n\tlogging.SetLevel(logging.DEBUG, \"\")\n\tdefer logging.SetLevel(logging.INFO, \"\") // Reset to default\n\n\tresult2 := context.ToString()\n\ttest.AssertResultComplex(t, true, len(result2) > 0)\n\ttest.AssertResultComplex(t, true, strings.Contains(result2, \"Context\"))\n\ttest.AssertResultComplex(t, true, strings.Contains(result2, \"DontAutoCreate: true\"))\n}\n\nfunc TestDeepClone(t *testing.T) {\n\t// Create original context with variables and matching nodes\n\toriginalVariables := make(map[string]*list.List)\n\toriginalVariables[\"test\"] = list.New()\n\toriginalVariables[\"test\"].PushBack(&CandidateNode{Value: \"original\"})\n\n\toriginal := Context{\n\t\tDontAutoCreate: true,\n\t\tdatetimeLayout: \"2006-01-02\",\n\t\tVariables: originalVariables,\n\t\tMatchingNodes: list.New(),\n\t}\n\n\t// Add a node to MatchingNodes\n\tnode := &CandidateNode{Value: \"test\"}\n\toriginal.MatchingNodes.PushBack(node)\n\n\tclone := original.DeepClone()\n\n\t// Should preserve DontAutoCreate and datetimeLayout\n\ttest.AssertResultComplex(t, original.DontAutoCreate, clone.DontAutoCreate)\n\ttest.AssertResultComplex(t, original.datetimeLayout, clone.datetimeLayout)\n\n\t// Should have copied variables\n\ttest.AssertResultComplex(t, 1, len(clone.Variables))\n\ttest.AssertResultComplex(t, \"original\", clone.Variables[\"test\"].Front().Value.(*CandidateNode).Value)\n\n\t// Should have deep copied MatchingNodes\n\ttest.AssertResultComplex(t, 1, clone.MatchingNodes.Len())\n\n\t// Verify it's a deep copy by modifying the original\n\toriginal.MatchingNodes.Front().Value.(*CandidateNode).Value = \"modified\"\n\ttest.AssertResultComplex(t, \"test\", clone.MatchingNodes.Front().Value.(*CandidateNode).Value)\n}\n\nfunc TestClone(t *testing.T) {\n\t// Create original context\n\toriginal := Context{\n\t\tDontAutoCreate: true,\n\t\tdatetimeLayout: \"2006-01-02\",\n\t\tMatchingNodes: list.New(),\n\t}\n\n\tnode := &CandidateNode{Value: \"test\"}\n\toriginal.MatchingNodes.PushBack(node)\n\n\tclone := original.Clone()\n\n\t// Should preserve DontAutoCreate and datetimeLayout\n\ttest.AssertResultComplex(t, original.DontAutoCreate, clone.DontAutoCreate)\n\ttest.AssertResultComplex(t, original.datetimeLayout, clone.datetimeLayout)\n\n\t// Should have the same MatchingNodes reference\n\ttest.AssertResultComplex(t, original.MatchingNodes, clone.MatchingNodes)\n}\n\nfunc TestReadOnlyClone(t *testing.T) {\n\toriginal := Context{\n\t\tDontAutoCreate: false,\n\t\tdatetimeLayout: \"2006-01-02\",\n\t\tMatchingNodes: list.New(),\n\t}\n\n\tnode := &CandidateNode{Value: \"test\"}\n\toriginal.MatchingNodes.PushBack(node)\n\n\tclone := original.ReadOnlyClone()\n\n\t// Should set DontAutoCreate to true\n\ttest.AssertResultComplex(t, true, clone.DontAutoCreate)\n\n\t// Should preserve other fields\n\ttest.AssertResultComplex(t, original.datetimeLayout, clone.datetimeLayout)\n\ttest.AssertResultComplex(t, original.MatchingNodes, clone.MatchingNodes)\n}\n\nfunc TestWritableClone(t *testing.T) {\n\toriginal := Context{\n\t\tDontAutoCreate: true,\n\t\tdatetimeLayout: \"2006-01-02\",\n\t\tMatchingNodes: list.New(),\n\t}\n\n\tnode := &CandidateNode{Value: \"test\"}\n\toriginal.MatchingNodes.PushBack(node)\n\n\tclone := original.WritableClone()\n\n\t// Should set DontAutoCreate to false\n\ttest.AssertResultComplex(t, false, clone.DontAutoCreate)\n\n\t// Should preserve other fields\n\ttest.AssertResultComplex(t, original.datetimeLayout, clone.datetimeLayout)\n\ttest.AssertResultComplex(t, original.MatchingNodes, clone.MatchingNodes)\n}\n" }, { "path": "pkg/yqlib/csv.go", "content": "package yqlib\n\ntype CsvPreferences struct {\n\tSeparator rune\n\tAutoParse bool\n}\n\nfunc NewDefaultCsvPreferences() CsvPreferences {\n\treturn CsvPreferences{\n\t\tSeparator: ',',\n\t\tAutoParse: true,\n\t}\n}\n\nfunc NewDefaultTsvPreferences() CsvPreferences {\n\treturn CsvPreferences{\n\t\tSeparator: '\\t',\n\t\tAutoParse: true,\n\t}\n}\n\nvar ConfiguredCsvPreferences = NewDefaultCsvPreferences()\nvar ConfiguredTsvPreferences = NewDefaultTsvPreferences()\n" }, { "path": "pkg/yqlib/csv_test.go", "content": "package yqlib\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nconst csvSimple = `name,numberOfCats,likesApples,height\nGary,1,true,168.8\nSamantha's Rabbit,2,false,-188.8\n`\n\nconst csvSimpleWithObject = `name,numberOfCats,likesApples,height,facts\nGary,1,true,168.8,cool: true\nSamantha's Rabbit,2,false,-188.8,tall: indeed\n`\nconst csvMissing = `name,numberOfCats,likesApples,height\n,null,,168.8\n`\nconst expectedUpdatedSimpleCsv = `name,numberOfCats,likesApples,height\nGary,3,true,168.8\nSamantha's Rabbit,2,false,-188.8\n`\n\nconst csvSimpleShort = `Name,Number of Cats\nGary,1\nSamantha's Rabbit,2\n`\n\nconst tsvSimple = `name\tnumberOfCats\tlikesApples\theight\nGary\t1\ttrue\t168.8\nSamantha's Rabbit\t2\tfalse\t-188.8\n`\n\nconst expectedYamlFromCSV = `- name: Gary\n numberOfCats: 1\n likesApples: true\n height: 168.8\n- name: Samantha's Rabbit\n numberOfCats: 2\n likesApples: false\n height: -188.8\n`\nconst expectedYamlFromCSVWithObject = `- name: Gary\n numberOfCats: 1\n likesApples: true\n height: 168.8\n facts:\n cool: true\n- name: Samantha's Rabbit\n numberOfCats: 2\n likesApples: false\n height: -188.8\n facts:\n tall: indeed\n`\n\nconst expectedYamlFromCSVNoParsing = `- name: Gary\n numberOfCats: 1\n likesApples: true\n height: 168.8\n facts: 'cool: true'\n- name: Samantha's Rabbit\n numberOfCats: 2\n likesApples: false\n height: -188.8\n facts: 'tall: indeed'\n`\n\nconst expectedYamlFromCSVMissingData = `- name: Gary\n numberOfCats: 1\n height: 168.8\n- name: Samantha's Rabbit\n height: -188.8\n likesApples: false\n`\n\nconst csvSimpleMissingData = `name,numberOfCats,height\nGary,1,168.8\nSamantha's Rabbit,,-188.8\n`\n\nconst csvTestSimpleYaml = `- [i, like, csv]\n- [because, excel, is, cool]`\n\nconst expectedSimpleCsv = `i,like,csv\nbecause,excel,is,cool\n`\n\nconst tsvTestExpectedSimpleCsv = `i\tlike\tcsv\nbecause\texcel\tis\tcool\n`\n\nvar csvScenarios = []formatScenario{\n\t{\n\t\tdescription: \"Encode CSV simple\",\n\t\tinput: csvTestSimpleYaml,\n\t\texpected: expectedSimpleCsv,\n\t\tscenarioType: \"encode-csv\",\n\t},\n\t{\n\t\tdescription: \"Encode TSV simple\",\n\t\tinput: csvTestSimpleYaml,\n\t\texpected: tsvTestExpectedSimpleCsv,\n\t\tscenarioType: \"encode-tsv\",\n\t},\n\t{\n\t\tdescription: \"Encode Empty\",\n\t\tskipDoc: true,\n\t\tinput: `[]`,\n\t\texpected: \"\",\n\t\tscenarioType: \"encode-csv\",\n\t},\n\t{\n\t\tdescription: \"Comma in value\",\n\t\tskipDoc: true,\n\t\tinput: `[\"comma, in, value\", things]`,\n\t\texpected: \"\\\"comma, in, value\\\",things\\n\",\n\t\tscenarioType: \"encode-csv\",\n\t},\n\t{\n\t\tdescription: \"Encode array of objects to csv\",\n\t\tinput: expectedYamlFromCSV,\n\t\texpected: csvSimple,\n\t\tscenarioType: \"encode-csv\",\n\t},\n\t{\n\t\tdescription: \"Encode array of objects to custom csv format\",\n\t\tsubdescription: \"Add the header row manually, then the we convert each object into an array of values - resulting in an array of arrays. Pick the columns and call the header whatever you like.\",\n\t\tinput: expectedYamlFromCSV,\n\t\texpected: csvSimpleShort,\n\t\texpression: `[[\"Name\", \"Number of Cats\"]] + [.[] | [.name, .numberOfCats ]]`,\n\t\tscenarioType: \"encode-csv\",\n\t},\n\t{\n\t\tdescription: \"Encode array of objects to csv - missing fields behaviour\",\n\t\tsubdescription: \"First entry is used to determine the headers, and it is missing 'likesApples', so it is not included in the csv. Second entry does not have 'numberOfCats' so that is blank\",\n\t\tinput: expectedYamlFromCSVMissingData,\n\t\texpected: csvSimpleMissingData,\n\t\tscenarioType: \"encode-csv\",\n\t},\n\t{\n\t\tdescription: \"decode csv missing\",\n\t\tskipDoc: true,\n\t\tinput: csvMissing,\n\t\texpected: csvMissing,\n\t\tscenarioType: \"roundtrip-csv\",\n\t},\n\t{\n\t\tdescription: \"decode csv key\",\n\t\tskipDoc: true,\n\t\tinput: csvSimple,\n\t\texpression: \".[0].name | key\",\n\t\texpected: \"name\\n\",\n\t\tscenarioType: \"decode-csv\",\n\t},\n\t{\n\t\tdescription: \"decode csv parent\",\n\t\tskipDoc: true,\n\t\tinput: csvSimple,\n\t\texpression: \".[0].name | parent | .height\",\n\t\texpected: \"168.8\\n\",\n\t\tscenarioType: \"decode-csv\",\n\t},\n\t{\n\t\tdescription: \"Parse CSV into an array of objects\",\n\t\tsubdescription: \"First row is assumed to be the header row. By default, entries with YAML/JSON formatting will be parsed!\",\n\t\tinput: csvSimpleWithObject,\n\t\texpected: expectedYamlFromCSVWithObject,\n\t\tscenarioType: \"decode-csv\",\n\t},\n\t{\n\t\tdescription: \"Decode CSV line breaks\",\n\t\tskipDoc: true,\n\t\tinput: \"heading1\\n\\\"some data\\nwith a line break\\\"\\n\",\n\t\texpected: \"- heading1: |-\\n some data\\n with a line break\\n\",\n\t\tscenarioType: \"decode-csv\",\n\t},\n\t{\n\t\tdescription: \"Parse CSV into an array of objects, no auto-parsing\",\n\t\tsubdescription: \"First row is assumed to be the header row. Entries with YAML/JSON will be left as strings.\",\n\t\tinput: csvSimpleWithObject,\n\t\texpected: expectedYamlFromCSVNoParsing,\n\t\tscenarioType: \"decode-csv-no-auto\",\n\t},\n\t{\n\t\tdescription: \"values starting with #, no auto parse\",\n\t\tskipDoc: true,\n\t\tinput: \"value\\n#ffff\",\n\t\texpected: \"- value: '#ffff'\\n\",\n\t\tscenarioType: \"decode-csv-no-auto\",\n\t},\n\t{\n\t\tdescription: \"values starting with #\",\n\t\tskipDoc: true,\n\t\tinput: \"value\\n#ffff\",\n\t\texpected: \"- value: #ffff\\n\",\n\t\tscenarioType: \"decode-csv\",\n\t},\n\t{\n\t\tdescription: \"Scalar roundtrip\",\n\t\tskipDoc: true,\n\t\tinput: \"mike\\ncat\",\n\t\texpression: \".[0].mike\",\n\t\texpected: \"cat\\n\",\n\t\tscenarioType: \"roundtrip-csv\",\n\t},\n\t{\n\t\tdescription: \"Parse TSV into an array of objects\",\n\t\tsubdescription: \"First row is assumed to be the header row.\",\n\t\tinput: tsvSimple,\n\t\texpected: expectedYamlFromCSV,\n\t\tscenarioType: \"decode-tsv-object\",\n\t},\n\t{\n\t\tdescription: \"Round trip\",\n\t\tinput: csvSimple,\n\t\texpected: expectedUpdatedSimpleCsv,\n\t\texpression: `(.[] | select(.name == \"Gary\") | .numberOfCats) = 3`,\n\t\tscenarioType: \"roundtrip-csv\",\n\t},\n}\n\nfunc testCSVScenario(t *testing.T, s formatScenario) {\n\tswitch s.scenarioType {\n\tcase \"encode-csv\":\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(ConfiguredCsvPreferences)), s.description)\n\tcase \"encode-tsv\":\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(ConfiguredTsvPreferences)), s.description)\n\tcase \"decode-csv\":\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredCsvPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)\n\tcase \"decode-csv-no-auto\":\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: ',', AutoParse: false}), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)\n\tcase \"decode-tsv-object\":\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredTsvPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)\n\tcase \"roundtrip-csv\":\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredCsvPreferences), NewCsvEncoder(ConfiguredCsvPreferences)), s.description)\n\tdefault:\n\t\tpanic(fmt.Sprintf(\"unhandled scenario type %q\", s.scenarioType))\n\t}\n}\n\nfunc documentCSVDecodeObjectScenario(w *bufio.Writer, s formatScenario, formatType string) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, fmt.Sprintf(\"Given a sample.%v file of:\\n\", formatType))\n\twriteOrPanic(w, fmt.Sprintf(\"```%v\\n%v\\n```\\n\", formatType, s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -p=%v sample.%v\\n```\\n\", formatType, formatType))\n\twriteOrPanic(w, \"will output\\n\")\n\n\tseparator := ','\n\tif formatType == \"tsv\" {\n\t\tseparator = '\\t'\n\t}\n\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v```\\n\\n\",\n\t\tmustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: true}), NewYamlEncoder(ConfiguredYamlPreferences))),\n\t)\n}\n\nfunc documentCSVDecodeObjectNoAutoScenario(w *bufio.Writer, s formatScenario, formatType string) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, fmt.Sprintf(\"Given a sample.%v file of:\\n\", formatType))\n\twriteOrPanic(w, fmt.Sprintf(\"```%v\\n%v\\n```\\n\", formatType, s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -p=%v --csv-auto-parse=f sample.%v\\n```\\n\", formatType, formatType))\n\twriteOrPanic(w, \"will output\\n\")\n\n\tseparator := ','\n\tif formatType == \"tsv\" {\n\t\tseparator = '\\t'\n\t}\n\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v```\\n\\n\",\n\t\tmustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: false}), NewYamlEncoder(ConfiguredYamlPreferences))),\n\t)\n}\n\nfunc documentCSVEncodeScenario(w *bufio.Writer, s formatScenario, formatType string) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.yml file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\n\texpression := s.expression\n\n\tif expression != \"\" {\n\t\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -o=%v '%v' sample.yml\\n```\\n\", formatType, expression))\n\t} else {\n\t\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -o=%v sample.yml\\n```\\n\", formatType))\n\t}\n\twriteOrPanic(w, \"will output\\n\")\n\n\tseparator := ','\n\tif formatType == \"tsv\" {\n\t\tseparator = '\\t'\n\t}\n\tcsvPrefs := NewDefaultCsvPreferences()\n\tcsvPrefs.Separator = separator\n\twriteOrPanic(w, fmt.Sprintf(\"```%v\\n%v```\\n\\n\", formatType,\n\t\tmustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(csvPrefs))),\n\t)\n}\n\nfunc documentCSVRoundTripScenario(w *bufio.Writer, s formatScenario, formatType string) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, fmt.Sprintf(\"Given a sample.%v file of:\\n\", formatType))\n\twriteOrPanic(w, fmt.Sprintf(\"```%v\\n%v\\n```\\n\", formatType, s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\n\texpression := s.expression\n\n\tif expression != \"\" {\n\t\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -p=%v -o=%v '%v' sample.%v\\n```\\n\", formatType, formatType, expression, formatType))\n\t} else {\n\t\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -p=%v -o=%v sample.%v\\n```\\n\", formatType, formatType, formatType))\n\t}\n\twriteOrPanic(w, \"will output\\n\")\n\n\tseparator := ','\n\tif formatType == \"tsv\" {\n\t\tseparator = '\\t'\n\t}\n\n\tcsvPrefs := NewDefaultCsvPreferences()\n\tcsvPrefs.Separator = separator\n\n\twriteOrPanic(w, fmt.Sprintf(\"```%v\\n%v```\\n\\n\", formatType,\n\t\tmustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: true}), NewCsvEncoder(csvPrefs))),\n\t)\n}\n\nfunc documentCSVScenario(_ *testing.T, w *bufio.Writer, i interface{}) {\n\ts := i.(formatScenario)\n\tif s.skipDoc {\n\t\treturn\n\t}\n\tswitch s.scenarioType {\n\tcase \"encode-csv\":\n\t\tdocumentCSVEncodeScenario(w, s, \"csv\")\n\tcase \"encode-tsv\":\n\t\tdocumentCSVEncodeScenario(w, s, \"tsv\")\n\tcase \"decode-csv\":\n\t\tdocumentCSVDecodeObjectScenario(w, s, \"csv\")\n\tcase \"decode-csv-no-auto\":\n\t\tdocumentCSVDecodeObjectNoAutoScenario(w, s, \"csv\")\n\tcase \"decode-tsv-object\":\n\t\tdocumentCSVDecodeObjectScenario(w, s, \"tsv\")\n\tcase \"roundtrip-csv\":\n\t\tdocumentCSVRoundTripScenario(w, s, \"csv\")\n\n\tdefault:\n\t\tpanic(fmt.Sprintf(\"unhandled scenario type %q\", s.scenarioType))\n\t}\n}\n\nfunc TestCSVScenarios(t *testing.T) {\n\tfor _, tt := range csvScenarios {\n\t\ttestCSVScenario(t, tt)\n\t}\n\tgenericScenarios := make([]interface{}, len(csvScenarios))\n\tfor i, s := range csvScenarios {\n\t\tgenericScenarios[i] = s\n\t}\n\tdocumentScenarios(t, \"usage\", \"csv-tsv\", genericScenarios, documentCSVScenario)\n}\n" }, { "path": "pkg/yqlib/data_tree_navigator.go", "content": "package yqlib\n\nimport (\n\t\"fmt\"\n\n\tlogging \"gopkg.in/op/go-logging.v1\"\n)\n\ntype DataTreeNavigator interface {\n\t// given the context and an expressionNode,\n\t// this will process the against the given expressionNode and return\n\t// a new context of matching candidates\n\tGetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error)\n\n\tDeeplyAssign(context Context, path []interface{}, rhsNode *CandidateNode) error\n}\n\ntype dataTreeNavigator struct {\n}\n\nfunc NewDataTreeNavigator() DataTreeNavigator {\n\treturn &dataTreeNavigator{}\n}\n\nfunc (d *dataTreeNavigator) DeeplyAssign(context Context, path []interface{}, rhsCandidateNode *CandidateNode) error {\n\n\tassignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}\n\n\tif rhsCandidateNode.Kind == MappingNode {\n\t\tlog.Debug(\"DeeplyAssign: deeply merging object\")\n\t\t// if the rhs is a map, we need to deeply merge it in.\n\t\t// otherwise we'll clobber any existing fields\n\t\tassignmentOp = &Operation{OperationType: multiplyAssignOpType, Preferences: multiplyPreferences{\n\t\t\tAppendArrays: true,\n\t\t\tTraversePrefs: traversePreferences{DontFollowAlias: true},\n\t\t\tAssignPrefs: assignPreferences{},\n\t\t}}\n\t}\n\n\trhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}\n\n\tassignmentOpNode := &ExpressionNode{\n\t\tOperation: assignmentOp,\n\t\tLHS: createTraversalTree(path, traversePreferences{}, false),\n\t\tRHS: &ExpressionNode{Operation: rhsOp},\n\t}\n\n\t_, err := d.GetMatchingNodes(context, assignmentOpNode)\n\treturn err\n}\n\nfunc (d *dataTreeNavigator) GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error) {\n\tif expressionNode == nil {\n\t\tlog.Debugf(\"getMatchingNodes - nothing to do\")\n\t\treturn context, nil\n\t}\n\tlog.Debugf(\"Processing Op: %v\", expressionNode.Operation.toString())\n\tif log.IsEnabledFor(logging.DEBUG) {\n\t\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\t\tlog.Debug(NodeToString(el.Value.(*CandidateNode)))\n\t\t}\n\t}\n\thandler := expressionNode.Operation.OperationType.Handler\n\tif handler != nil {\n\t\treturn handler(d, context, expressionNode)\n\t}\n\treturn Context{}, fmt.Errorf(\"unknown operator %v\", expressionNode.Operation.OperationType.Type)\n\n}\n" }, { "path": "pkg/yqlib/data_tree_navigator_test.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nfunc TestGetMatchingNodes_NilExpressionNode(t *testing.T) {\n\tnavigator := NewDataTreeNavigator()\n\tcontext := Context{\n\t\tMatchingNodes: list.New(),\n\t}\n\n\tresult, err := navigator.GetMatchingNodes(context, nil)\n\n\ttest.AssertResult(t, nil, err)\n\ttest.AssertResultComplex(t, context, result)\n}\n\nfunc TestGetMatchingNodes_UnknownOperator(t *testing.T) {\n\tnavigator := NewDataTreeNavigator()\n\tcontext := Context{\n\t\tMatchingNodes: list.New(),\n\t}\n\n\t// Create an expression node with an unknown operation type\n\tunknownOpType := &operationType{Type: \"UNKNOWN\", Handler: nil}\n\texpressionNode := &ExpressionNode{\n\t\tOperation: &Operation{OperationType: unknownOpType},\n\t}\n\n\tresult, err := navigator.GetMatchingNodes(context, expressionNode)\n\n\ttest.AssertResult(t, \"unknown operator UNKNOWN\", err.Error())\n\ttest.AssertResultComplex(t, Context{}, result)\n}\n\nfunc TestGetMatchingNodes_ValidOperator(t *testing.T) {\n\tnavigator := NewDataTreeNavigator()\n\n\t// Create a simple context with a scalar node\n\tscalarNode := &CandidateNode{\n\t\tKind: ScalarNode,\n\t\tTag: \"!!str\",\n\t\tValue: \"test\",\n\t}\n\tcontext := Context{\n\t\tMatchingNodes: list.New(),\n\t}\n\tcontext.MatchingNodes.PushBack(scalarNode)\n\n\t// Create an expression node with a valid operation (self reference)\n\texpressionNode := &ExpressionNode{\n\t\tOperation: &Operation{OperationType: selfReferenceOpType},\n\t}\n\n\tresult, err := navigator.GetMatchingNodes(context, expressionNode)\n\n\ttest.AssertResult(t, nil, err)\n\ttest.AssertResult(t, 1, result.MatchingNodes.Len())\n\n\t// Verify the result contains the same node\n\tresultNode := result.MatchingNodes.Front().Value.(*CandidateNode)\n\ttest.AssertResult(t, scalarNode, resultNode)\n}\n\nfunc TestDeeplyAssign_ScalarNode(t *testing.T) {\n\tnavigator := NewDataTreeNavigator()\n\n\t// Create a context with a root mapping node\n\trootNode := &CandidateNode{\n\t\tKind: MappingNode,\n\t\tTag: \"!!map\",\n\t\tContent: []*CandidateNode{\n\t\t\t{Kind: ScalarNode, Tag: \"!!str\", Value: \"existing\", IsMapKey: true},\n\t\t\t{Kind: ScalarNode, Tag: \"!!str\", Value: \"old_value\"},\n\t\t},\n\t}\n\tcontext := Context{\n\t\tMatchingNodes: list.New(),\n\t}\n\tcontext.MatchingNodes.PushBack(rootNode)\n\n\t// Create a scalar node to assign\n\tscalarNode := &CandidateNode{\n\t\tKind: ScalarNode,\n\t\tTag: \"!!str\",\n\t\tValue: \"new_value\",\n\t}\n\n\t// Assign to path [\"new_key\"]\n\tpath := []interface{}{\"new_key\"}\n\terr := navigator.DeeplyAssign(context, path, scalarNode)\n\n\ttest.AssertResult(t, nil, err)\n\n\t// Verify the assignment was made\n\t// The root node should now have the new key-value pair\n\ttest.AssertResult(t, 4, len(rootNode.Content)) // 2 original + 2 new\n\n\t// Find the new key-value pair\n\tfound := false\n\tfor i := 0; i < len(rootNode.Content)-1; i += 2 {\n\t\tkey := rootNode.Content[i]\n\t\tvalue := rootNode.Content[i+1]\n\t\tif key.Value == \"new_key\" && value.Value == \"new_value\" {\n\t\t\tfound = true\n\t\t\tbreak\n\t\t}\n\t}\n\ttest.AssertResult(t, true, found)\n}\n\nfunc TestDeeplyAssign_MappingNode(t *testing.T) {\n\tnavigator := NewDataTreeNavigator()\n\n\t// Create a context with a root mapping node\n\trootNode := &CandidateNode{\n\t\tKind: MappingNode,\n\t\tTag: \"!!map\",\n\t\tContent: []*CandidateNode{\n\t\t\t{Kind: ScalarNode, Tag: \"!!str\", Value: \"existing\", IsMapKey: true},\n\t\t\t{Kind: ScalarNode, Tag: \"!!str\", Value: \"old_value\"},\n\t\t},\n\t}\n\tcontext := Context{\n\t\tMatchingNodes: list.New(),\n\t}\n\tcontext.MatchingNodes.PushBack(rootNode)\n\n\t// Create a mapping node to assign (this should trigger deep merge)\n\tmappingNode := &CandidateNode{\n\t\tKind: MappingNode,\n\t\tTag: \"!!map\",\n\t\tContent: []*CandidateNode{\n\t\t\t{Kind: ScalarNode, Tag: \"!!str\", Value: \"nested_key\", IsMapKey: true},\n\t\t\t{Kind: ScalarNode, Tag: \"!!str\", Value: \"nested_value\"},\n\t\t},\n\t}\n\n\t// Assign to path [\"new_map\"]\n\tpath := []interface{}{\"new_map\"}\n\terr := navigator.DeeplyAssign(context, path, mappingNode)\n\n\ttest.AssertResult(t, nil, err)\n\n\t// Verify the assignment was made\n\t// The root node should now have the new mapping\n\ttest.AssertResult(t, 4, len(rootNode.Content)) // 2 original + 2 new\n\n\t// Find the new mapping\n\tfound := false\n\tfor i := 0; i < len(rootNode.Content); i += 2 {\n\t\tif i+1 < len(rootNode.Content) {\n\t\t\tkey := rootNode.Content[i]\n\t\t\tvalue := rootNode.Content[i+1]\n\t\t\tif key.Value == \"new_map\" && value.Kind == MappingNode {\n\t\t\t\tfound = true\n\t\t\t\t// Verify the nested content\n\t\t\t\ttest.AssertResult(t, 2, len(value.Content))\n\t\t\t\ttest.AssertResult(t, \"nested_key\", value.Content[0].Value)\n\t\t\t\ttest.AssertResult(t, \"nested_value\", value.Content[1].Value)\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\ttest.AssertResult(t, true, found)\n}\n\nfunc TestDeeplyAssign_DeepPath(t *testing.T) {\n\tnavigator := NewDataTreeNavigator()\n\n\t// Create a context with a root mapping node\n\trootNode := &CandidateNode{\n\t\tKind: MappingNode,\n\t\tTag: \"!!map\",\n\t\tContent: []*CandidateNode{\n\t\t\t{Kind: ScalarNode, Tag: \"!!str\", Value: \"level1\", IsMapKey: true},\n\t\t\t{Kind: MappingNode, Tag: \"!!map\", Content: []*CandidateNode{}},\n\t\t},\n\t}\n\tcontext := Context{\n\t\tMatchingNodes: list.New(),\n\t}\n\tcontext.MatchingNodes.PushBack(rootNode)\n\n\t// Create a scalar node to assign\n\tscalarNode := &CandidateNode{\n\t\tKind: ScalarNode,\n\t\tTag: \"!!str\",\n\t\tValue: \"deep_value\",\n\t}\n\n\t// Assign to deep path [\"level1\", \"level2\", \"level3\"]\n\tpath := []interface{}{\"level1\", \"level2\", \"level3\"}\n\terr := navigator.DeeplyAssign(context, path, scalarNode)\n\n\ttest.AssertResult(t, nil, err)\n\n\t// Verify the deep assignment was made\n\tlevel1Node := rootNode.Content[1] // The mapping node\n\ttest.AssertResult(t, 2, len(level1Node.Content)) // Should have level2 key-value\n\n\tlevel2Key := level1Node.Content[0]\n\tlevel2Value := level1Node.Content[1]\n\ttest.AssertResult(t, \"level2\", level2Key.Value)\n\ttest.AssertResult(t, MappingNode, level2Value.Kind)\n\n\tlevel3Key := level2Value.Content[0]\n\tlevel3Value := level2Value.Content[1]\n\ttest.AssertResult(t, \"level3\", level3Key.Value)\n\ttest.AssertResult(t, \"deep_value\", level3Value.Value)\n}\n\nfunc TestDeeplyAssign_ArrayPath(t *testing.T) {\n\tnavigator := NewDataTreeNavigator()\n\n\t// Create a context with a root mapping node containing an array\n\trootNode := &CandidateNode{\n\t\tKind: MappingNode,\n\t\tTag: \"!!map\",\n\t\tContent: []*CandidateNode{\n\t\t\t{Kind: ScalarNode, Tag: \"!!str\", Value: \"array\", IsMapKey: true},\n\t\t\t{Kind: SequenceNode, Tag: \"!!seq\", Content: []*CandidateNode{}},\n\t\t},\n\t}\n\tcontext := Context{\n\t\tMatchingNodes: list.New(),\n\t}\n\tcontext.MatchingNodes.PushBack(rootNode)\n\n\t// Create a scalar node to assign\n\tscalarNode := &CandidateNode{\n\t\tKind: ScalarNode,\n\t\tTag: \"!!str\",\n\t\tValue: \"array_value\",\n\t}\n\n\t// Assign to array path [\"array\", 0]\n\tpath := []interface{}{\"array\", 0}\n\terr := navigator.DeeplyAssign(context, path, scalarNode)\n\n\ttest.AssertResult(t, nil, err)\n\n\t// Verify the array assignment was made\n\tarrayNode := rootNode.Content[1] // The sequence node\n\ttest.AssertResult(t, 1, len(arrayNode.Content)) // Should have one element\n\n\tarrayElement := arrayNode.Content[0]\n\ttest.AssertResult(t, \"array_value\", arrayElement.Value)\n}\n\nfunc TestDeeplyAssign_OverwriteExisting(t *testing.T) {\n\tnavigator := NewDataTreeNavigator()\n\n\t// Create a context with a root mapping node\n\trootNode := &CandidateNode{\n\t\tKind: MappingNode,\n\t\tTag: \"!!map\",\n\t\tContent: []*CandidateNode{\n\t\t\t{Kind: ScalarNode, Tag: \"!!str\", Value: \"key\", IsMapKey: true},\n\t\t\t{Kind: ScalarNode, Tag: \"!!str\", Value: \"old_value\"},\n\t\t},\n\t}\n\tcontext := Context{\n\t\tMatchingNodes: list.New(),\n\t}\n\tcontext.MatchingNodes.PushBack(rootNode)\n\n\t// Create a scalar node to assign\n\tscalarNode := &CandidateNode{\n\t\tKind: ScalarNode,\n\t\tTag: \"!!str\",\n\t\tValue: \"new_value\",\n\t}\n\n\t// Assign to existing path [\"key\"]\n\tpath := []interface{}{\"key\"}\n\terr := navigator.DeeplyAssign(context, path, scalarNode)\n\n\ttest.AssertResult(t, nil, err)\n\n\t// Verify the value was overwritten\n\ttest.AssertResult(t, 2, len(rootNode.Content)) // Should still have 2 elements\n\n\tkey := rootNode.Content[0]\n\tvalue := rootNode.Content[1]\n\ttest.AssertResult(t, \"key\", key.Value)\n\ttest.AssertResult(t, \"new_value\", value.Value) // Should be overwritten\n}\n\nfunc TestDeeplyAssign_ErrorHandling(t *testing.T) {\n\tnavigator := NewDataTreeNavigator()\n\n\t// Create a context with a scalar node (not a mapping)\n\tscalarNode := &CandidateNode{\n\t\tKind: ScalarNode,\n\t\tTag: \"!!str\",\n\t\tValue: \"not_a_map\",\n\t}\n\tcontext := Context{\n\t\tMatchingNodes: list.New(),\n\t}\n\tcontext.MatchingNodes.PushBack(scalarNode)\n\n\t// Create a scalar node to assign\n\tassignNode := &CandidateNode{\n\t\tKind: ScalarNode,\n\t\tTag: \"!!str\",\n\t\tValue: \"value\",\n\t}\n\n\tpath := []interface{}{\"key\"}\n\terr := navigator.DeeplyAssign(context, path, assignNode)\n\n\t// Print the actual error for debugging\n\tif err != nil {\n\t\tt.Logf(\"Actual error: %v\", err)\n\t}\n\n\ttest.AssertResult(t, nil, err)\n}\n\nfunc TestGetMatchingNodes_WithVariables(t *testing.T) {\n\tnavigator := NewDataTreeNavigator()\n\n\t// Create a context with variables\n\tvariables := make(map[string]*list.List)\n\tvarList := list.New()\n\tvarList.PushBack(&CandidateNode{Kind: ScalarNode, Tag: \"!!str\", Value: \"var_value\"})\n\tvariables[\"test_var\"] = varList\n\n\tcontext := Context{\n\t\tMatchingNodes: list.New(),\n\t\tVariables: variables,\n\t}\n\n\t// Create an expression node that gets a variable\n\texpressionNode := &ExpressionNode{\n\t\tOperation: &Operation{OperationType: getVariableOpType, StringValue: \"test_var\"},\n\t}\n\n\tresult, err := navigator.GetMatchingNodes(context, expressionNode)\n\n\ttest.AssertResult(t, nil, err)\n\ttest.AssertResult(t, 1, result.MatchingNodes.Len())\n\n\t// Verify the variable was retrieved\n\tresultNode := result.MatchingNodes.Front().Value.(*CandidateNode)\n\ttest.AssertResult(t, \"var_value\", resultNode.Value)\n}\n\nfunc TestGetMatchingNodes_EmptyContext(t *testing.T) {\n\tnavigator := NewDataTreeNavigator()\n\n\t// Create an empty context\n\tcontext := Context{\n\t\tMatchingNodes: list.New(),\n\t}\n\n\t// Create an expression node with self reference\n\texpressionNode := &ExpressionNode{\n\t\tOperation: &Operation{OperationType: selfReferenceOpType},\n\t}\n\n\tresult, err := navigator.GetMatchingNodes(context, expressionNode)\n\n\ttest.AssertResult(t, nil, err)\n\ttest.AssertResult(t, 0, result.MatchingNodes.Len())\n}\n\nfunc TestDeeplyAssign_ComplexMappingMerge(t *testing.T) {\n\tnavigator := NewDataTreeNavigator()\n\n\t// Create a context with a root mapping node containing nested data\n\trootNode := &CandidateNode{\n\t\tKind: MappingNode,\n\t\tTag: \"!!map\",\n\t\tContent: []*CandidateNode{\n\t\t\t{Kind: ScalarNode, Tag: \"!!str\", Value: \"config\", IsMapKey: true},\n\t\t\t{Kind: MappingNode, Tag: \"!!map\", Content: []*CandidateNode{\n\t\t\t\t{Kind: ScalarNode, Tag: \"!!str\", Value: \"existing_key\", IsMapKey: true},\n\t\t\t\t{Kind: ScalarNode, Tag: \"!!str\", Value: \"existing_value\"},\n\t\t\t}},\n\t\t},\n\t}\n\tcontext := Context{\n\t\tMatchingNodes: list.New(),\n\t}\n\tcontext.MatchingNodes.PushBack(rootNode)\n\n\t// Create a mapping node to merge\n\tmappingNode := &CandidateNode{\n\t\tKind: MappingNode,\n\t\tTag: \"!!map\",\n\t\tContent: []*CandidateNode{\n\t\t\t{Kind: ScalarNode, Tag: \"!!str\", Value: \"new_key\", IsMapKey: true},\n\t\t\t{Kind: ScalarNode, Tag: \"!!str\", Value: \"new_value\"},\n\t\t\t{Kind: ScalarNode, Tag: \"!!str\", Value: \"existing_key\", IsMapKey: true},\n\t\t\t{Kind: ScalarNode, Tag: \"!!str\", Value: \"updated_value\"},\n\t\t},\n\t}\n\n\t// Assign to path [\"config\"] (should merge with existing mapping)\n\tpath := []interface{}{\"config\"}\n\terr := navigator.DeeplyAssign(context, path, mappingNode)\n\n\ttest.AssertResult(t, nil, err)\n\n\t// Verify the merge was successful\n\tconfigNode := rootNode.Content[1] // The config mapping node\n\ttest.AssertResult(t, 4, len(configNode.Content)) // Should have 2 key-value pairs\n\n\t// Check that both existing and new keys are present\n\tfoundExisting := false\n\tfoundNew := false\n\tfor i := 0; i < len(configNode.Content); i += 2 {\n\t\tif i+1 < len(configNode.Content) {\n\t\t\tkey := configNode.Content[i]\n\t\t\tvalue := configNode.Content[i+1]\n\t\t\tswitch key.Value {\n\t\t\tcase \"existing_key\":\n\t\t\t\tfoundExisting = true\n\t\t\t\ttest.AssertResult(t, \"updated_value\", value.Value) // Should be updated\n\t\t\tcase \"new_key\":\n\t\t\t\tfoundNew = true\n\t\t\t\ttest.AssertResult(t, \"new_value\", value.Value)\n\t\t\t}\n\t\t}\n\t}\n\ttest.AssertResult(t, true, foundExisting)\n\ttest.AssertResult(t, true, foundNew)\n}\n" }, { "path": "pkg/yqlib/decoder.go", "content": "package yqlib\n\nimport (\n\t\"io\"\n)\n\ntype Decoder interface {\n\tInit(reader io.Reader) error\n\tDecode() (*CandidateNode, error)\n}\n" }, { "path": "pkg/yqlib/decoder_base64.go", "content": "//go:build !yq_nobase64\n\npackage yqlib\n\nimport (\n\t\"bytes\"\n\t\"encoding/base64\"\n\t\"io\"\n\t\"strings\"\n)\n\ntype base64Decoder struct {\n\treader io.Reader\n\tfinished bool\n\treadAnything bool\n\tencoding base64.Encoding\n}\n\nfunc NewBase64Decoder() Decoder {\n\treturn &base64Decoder{finished: false, encoding: *base64.StdEncoding}\n}\n\nfunc (dec *base64Decoder) Init(reader io.Reader) error {\n\t// Read all data from the reader and strip leading/trailing whitespace\n\t// This is necessary because base64 decoding needs to see the complete input\n\t// to handle padding correctly, and we need to strip whitespace before decoding.\n\tbuf := new(bytes.Buffer)\n\tif _, err := buf.ReadFrom(reader); err != nil {\n\t\treturn err\n\t}\n\n\t// Strip leading and trailing whitespace\n\tstripped := strings.TrimSpace(buf.String())\n\n\t// Add padding if needed (base64 strings should be a multiple of 4 characters)\n\tpadLen := len(stripped) % 4\n\tif padLen > 0 {\n\t\tstripped += strings.Repeat(\"=\", 4-padLen)\n\t}\n\n\t// Create a new reader from the stripped and padded data\n\tdec.reader = strings.NewReader(stripped)\n\tdec.readAnything = false\n\tdec.finished = false\n\treturn nil\n}\n\nfunc (dec *base64Decoder) Decode() (*CandidateNode, error) {\n\tif dec.finished {\n\t\treturn nil, io.EOF\n\t}\n\tbase64Reader := base64.NewDecoder(&dec.encoding, dec.reader)\n\tbuf := new(bytes.Buffer)\n\n\tif _, err := buf.ReadFrom(base64Reader); err != nil {\n\t\treturn nil, err\n\t}\n\tif buf.Len() == 0 {\n\t\tdec.finished = true\n\n\t\t// if we've read _only_ an empty string, lets return that\n\t\t// otherwise if we've already read some bytes, and now we get\n\t\t// an empty string, then we are done.\n\t\tif dec.readAnything {\n\t\t\treturn nil, io.EOF\n\t\t}\n\t}\n\tdec.readAnything = true\n\treturn createStringScalarNode(buf.String()), nil\n}\n" }, { "path": "pkg/yqlib/decoder_csv_object.go", "content": "//go:build !yq_nocsv\n\npackage yqlib\n\nimport (\n\t\"encoding/csv\"\n\t\"errors\"\n\t\"io\"\n\n\t\"github.com/dimchansky/utfbom\"\n)\n\ntype csvObjectDecoder struct {\n\tprefs CsvPreferences\n\treader csv.Reader\n\tfinished bool\n}\n\nfunc NewCSVObjectDecoder(prefs CsvPreferences) Decoder {\n\treturn &csvObjectDecoder{prefs: prefs}\n}\n\nfunc (dec *csvObjectDecoder) Init(reader io.Reader) error {\n\tcleanReader, enc := utfbom.Skip(reader)\n\tlog.Debugf(\"Detected encoding: %s\\n\", enc)\n\tdec.reader = *csv.NewReader(cleanReader)\n\tdec.reader.Comma = dec.prefs.Separator\n\tdec.finished = false\n\treturn nil\n}\n\nfunc (dec *csvObjectDecoder) convertToNode(content string) *CandidateNode {\n\tnode, err := parseSnippet(content)\n\t// if we're not auto-parsing, then we wont put in parsed objects or arrays\n\t// but we still parse scalars\n\tif err != nil || (!dec.prefs.AutoParse && (node.Kind != ScalarNode || node.Value != content)) {\n\t\treturn createScalarNode(content, content)\n\t}\n\treturn node\n}\n\nfunc (dec *csvObjectDecoder) createObject(headerRow []string, contentRow []string) *CandidateNode {\n\tobjectNode := &CandidateNode{Kind: MappingNode, Tag: \"!!map\"}\n\n\tfor i, header := range headerRow {\n\t\tobjectNode.AddKeyValueChild(createScalarNode(header, header), dec.convertToNode(contentRow[i]))\n\t}\n\treturn objectNode\n}\n\nfunc (dec *csvObjectDecoder) Decode() (*CandidateNode, error) {\n\tif dec.finished {\n\t\treturn nil, io.EOF\n\t}\n\theaderRow, err := dec.reader.Read()\n\tlog.Debugf(\": headerRow%v\", headerRow)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\trootArray := &CandidateNode{Kind: SequenceNode, Tag: \"!!seq\"}\n\n\tcontentRow, err := dec.reader.Read()\n\n\tfor err == nil && len(contentRow) > 0 {\n\t\tlog.Debugf(\"Adding contentRow: %v\", contentRow)\n\t\trootArray.AddChild(dec.createObject(headerRow, contentRow))\n\t\tcontentRow, err = dec.reader.Read()\n\t\tlog.Debugf(\"Read next contentRow: %v, %v\", contentRow, err)\n\t}\n\tif !errors.Is(err, io.EOF) {\n\t\treturn nil, err\n\t}\n\n\treturn rootArray, nil\n}\n" }, { "path": "pkg/yqlib/decoder_goccy_yaml.go", "content": "//go:build !yq_noyaml\n\n//\n// NOTE this is still a WIP - not yet ready.\n//\n\npackage yqlib\n\nimport (\n\t\"io\"\n\n\tyaml \"github.com/goccy/go-yaml\"\n\t\"github.com/goccy/go-yaml/ast\"\n)\n\ntype goccyYamlDecoder struct {\n\tdecoder yaml.Decoder\n\tcm yaml.CommentMap\n\t// anchor map persists over multiple documents for convenience.\n\tanchorMap map[string]*CandidateNode\n}\n\nfunc NewGoccyYAMLDecoder() Decoder {\n\treturn &goccyYamlDecoder{}\n}\n\nfunc (dec *goccyYamlDecoder) Init(reader io.Reader) error {\n\tdec.cm = yaml.CommentMap{}\n\tdec.decoder = *yaml.NewDecoder(reader, yaml.CommentToMap(dec.cm), yaml.UseOrderedMap())\n\tdec.anchorMap = make(map[string]*CandidateNode)\n\treturn nil\n}\n\nfunc (dec *goccyYamlDecoder) Decode() (*CandidateNode, error) {\n\n\tvar ast ast.Node\n\n\terr := dec.decoder.Decode(&ast)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tcandidateNode := &CandidateNode{}\n\tif err := candidateNode.UnmarshalGoccyYAML(ast, dec.cm, dec.anchorMap); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn candidateNode, nil\n}\n" }, { "path": "pkg/yqlib/decoder_hcl.go", "content": "//go:build !yq_nohcl\n\npackage yqlib\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"math/big\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/hashicorp/hcl/v2\"\n\t\"github.com/hashicorp/hcl/v2/hclsyntax\"\n\t\"github.com/zclconf/go-cty/cty\"\n)\n\ntype hclDecoder struct {\n\tfile *hcl.File\n\tfileBytes []byte\n\treadAnything bool\n\tdocumentIndex uint\n}\n\nfunc NewHclDecoder() Decoder {\n\treturn &hclDecoder{}\n}\n\n// sortedAttributes returns attributes in declaration order by source position\nfunc sortedAttributes(attrs hclsyntax.Attributes) []*attributeWithName {\n\tvar sorted []*attributeWithName\n\tfor name, attr := range attrs {\n\t\tsorted = append(sorted, &attributeWithName{Name: name, Attr: attr})\n\t}\n\tsort.Slice(sorted, func(i, j int) bool {\n\t\treturn sorted[i].Attr.Range().Start.Byte < sorted[j].Attr.Range().Start.Byte\n\t})\n\treturn sorted\n}\n\ntype attributeWithName struct {\n\tName string\n\tAttr *hclsyntax.Attribute\n}\n\n// extractLineComment extracts any inline comment after the given position\nfunc extractLineComment(src []byte, endPos int) string {\n\t// Look for # comment after the token\n\tfor i := endPos; i < len(src); i++ {\n\t\tif src[i] == '#' {\n\t\t\t// Found comment, extract until end of line\n\t\t\tstart := i\n\t\t\tfor i < len(src) && src[i] != '\\n' {\n\t\t\t\ti++\n\t\t\t}\n\t\t\treturn strings.TrimSpace(string(src[start:i]))\n\t\t}\n\t\tif src[i] == '\\n' {\n\t\t\t// Hit newline before comment\n\t\t\tbreak\n\t\t}\n\t\t// Skip whitespace and other characters\n\t}\n\treturn \"\"\n}\n\n// extractHeadComment extracts comments before a given start position\nfunc extractHeadComment(src []byte, startPos int) string {\n\tvar comments []string\n\n\t// Start just before the token and skip trailing whitespace\n\ti := startPos - 1\n\tfor i >= 0 && (src[i] == ' ' || src[i] == '\\t' || src[i] == '\\n' || src[i] == '\\r') {\n\t\ti--\n\t}\n\n\tfor i >= 0 {\n\t\t// Find line boundaries\n\t\tlineEnd := i\n\t\tfor i >= 0 && src[i] != '\\n' {\n\t\t\ti--\n\t\t}\n\t\tlineStart := i + 1\n\n\t\tline := strings.TrimRight(string(src[lineStart:lineEnd+1]), \" \\t\\r\")\n\t\ttrimmed := strings.TrimSpace(line)\n\n\t\tif trimmed == \"\" {\n\t\t\tbreak\n\t\t}\n\n\t\tif !strings.HasPrefix(trimmed, \"#\") {\n\t\t\tbreak\n\t\t}\n\n\t\tcomments = append([]string{trimmed}, comments...)\n\n\t\t// Move to previous line (skip any whitespace/newlines)\n\t\ti = lineStart - 1\n\t\tfor i >= 0 && (src[i] == ' ' || src[i] == '\\t' || src[i] == '\\n' || src[i] == '\\r') {\n\t\t\ti--\n\t\t}\n\t}\n\n\tif len(comments) > 0 {\n\t\treturn strings.Join(comments, \"\\n\")\n\t}\n\treturn \"\"\n}\n\nfunc (dec *hclDecoder) Init(reader io.Reader) error {\n\tdata, err := io.ReadAll(reader)\n\tif err != nil {\n\t\treturn err\n\t}\n\tfile, diags := hclsyntax.ParseConfig(data, \"input.hcl\", hcl.Pos{Line: 1, Column: 1})\n\tif diags != nil && diags.HasErrors() {\n\t\treturn fmt.Errorf(\"hcl parse error: %w\", diags)\n\t}\n\tdec.file = file\n\tdec.fileBytes = data\n\tdec.readAnything = false\n\tdec.documentIndex = 0\n\treturn nil\n}\n\nfunc (dec *hclDecoder) Decode() (*CandidateNode, error) {\n\tif dec.readAnything {\n\t\treturn nil, io.EOF\n\t}\n\tdec.readAnything = true\n\n\tif dec.file == nil {\n\t\treturn nil, fmt.Errorf(\"no hcl file parsed\")\n\t}\n\n\troot := &CandidateNode{Kind: MappingNode}\n\n\t// process attributes in declaration order\n\tbody := dec.file.Body.(*hclsyntax.Body)\n\tfirstAttr := true\n\tfor _, attrWithName := range sortedAttributes(body.Attributes) {\n\t\tkeyNode := createStringScalarNode(attrWithName.Name)\n\t\tvalNode := convertHclExprToNode(attrWithName.Attr.Expr, dec.fileBytes)\n\n\t\t// Attach comments if any\n\t\tattrRange := attrWithName.Attr.Range()\n\t\theadComment := extractHeadComment(dec.fileBytes, attrRange.Start.Byte)\n\t\tif firstAttr && headComment != \"\" {\n\t\t\t// For the first attribute, apply its head comment to the root\n\t\t\troot.HeadComment = headComment\n\t\t\tfirstAttr = false\n\t\t} else if headComment != \"\" {\n\t\t\tkeyNode.HeadComment = headComment\n\t\t}\n\t\tif lineComment := extractLineComment(dec.fileBytes, attrRange.End.Byte); lineComment != \"\" {\n\t\t\tvalNode.LineComment = lineComment\n\t\t}\n\n\t\troot.AddKeyValueChild(keyNode, valNode)\n\t}\n\n\t// process blocks\n\t// Count blocks by type at THIS level to detect multiple separate blocks\n\tblocksByType := make(map[string]int)\n\tfor _, block := range body.Blocks {\n\t\tblocksByType[block.Type]++\n\t}\n\n\tfor _, block := range body.Blocks {\n\t\taddBlockToMapping(root, block, dec.fileBytes, blocksByType[block.Type] > 1)\n\t}\n\n\tdec.documentIndex++\n\troot.document = dec.documentIndex - 1\n\treturn root, nil\n}\n\nfunc hclBodyToNode(body *hclsyntax.Body, src []byte) *CandidateNode {\n\tnode := &CandidateNode{Kind: MappingNode}\n\tfor _, attrWithName := range sortedAttributes(body.Attributes) {\n\t\tkey := createStringScalarNode(attrWithName.Name)\n\t\tval := convertHclExprToNode(attrWithName.Attr.Expr, src)\n\n\t\t// Attach comments if any\n\t\tattrRange := attrWithName.Attr.Range()\n\t\tif headComment := extractHeadComment(src, attrRange.Start.Byte); headComment != \"\" {\n\t\t\tkey.HeadComment = headComment\n\t\t}\n\t\tif lineComment := extractLineComment(src, attrRange.End.Byte); lineComment != \"\" {\n\t\t\tval.LineComment = lineComment\n\t\t}\n\n\t\tnode.AddKeyValueChild(key, val)\n\t}\n\n\t// Process nested blocks, counting blocks by type at THIS level\n\t// to detect which block types appear multiple times\n\tblocksByType := make(map[string]int)\n\tfor _, block := range body.Blocks {\n\t\tblocksByType[block.Type]++\n\t}\n\n\tfor _, block := range body.Blocks {\n\t\taddBlockToMapping(node, block, src, blocksByType[block.Type] > 1)\n\t}\n\treturn node\n}\n\n// addBlockToMapping nests block type and labels into the parent mapping, merging children.\n// isMultipleBlocksOfType indicates if there are multiple blocks of this type at THIS level\nfunc addBlockToMapping(parent *CandidateNode, block *hclsyntax.Block, src []byte, isMultipleBlocksOfType bool) {\n\tbodyNode := hclBodyToNode(block.Body, src)\n\tcurrent := parent\n\n\t// ensure block type mapping exists\n\tvar typeNode *CandidateNode\n\tfor i := 0; i < len(current.Content); i += 2 {\n\t\tif current.Content[i].Value == block.Type {\n\t\t\ttypeNode = current.Content[i+1]\n\t\t\tbreak\n\t\t}\n\t}\n\tif typeNode == nil {\n\t\t_, typeNode = current.AddKeyValueChild(createStringScalarNode(block.Type), &CandidateNode{Kind: MappingNode})\n\t\t// Mark the type node if there are multiple blocks of this type at this level\n\t\t// This tells the encoder to emit them as separate blocks rather than consolidating them\n\t\tif isMultipleBlocksOfType {\n\t\t\ttypeNode.EncodeSeparate = true\n\t\t}\n\t}\n\tcurrent = typeNode\n\n\t// walk labels, creating/merging mappings\n\tfor _, label := range block.Labels {\n\t\tvar next *CandidateNode\n\t\tfor i := 0; i < len(current.Content); i += 2 {\n\t\t\tif current.Content[i].Value == label {\n\t\t\t\tnext = current.Content[i+1]\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif next == nil {\n\t\t\t_, next = current.AddKeyValueChild(createStringScalarNode(label), &CandidateNode{Kind: MappingNode})\n\t\t}\n\t\tcurrent = next\n\t}\n\n\t// merge body attributes/blocks into the final mapping\n\tfor i := 0; i < len(bodyNode.Content); i += 2 {\n\t\tcurrent.AddKeyValueChild(bodyNode.Content[i], bodyNode.Content[i+1])\n\t}\n}\n\nfunc convertHclExprToNode(expr hclsyntax.Expression, src []byte) *CandidateNode {\n\t// handle literal values directly\n\tswitch e := expr.(type) {\n\tcase *hclsyntax.LiteralValueExpr:\n\t\tv := e.Val\n\t\tif v.IsNull() {\n\t\t\treturn createScalarNode(nil, \"\")\n\t\t}\n\t\tswitch {\n\t\tcase v.Type().Equals(cty.String):\n\t\t\t// prefer to extract exact source (to avoid extra quoting) when available\n\t\t\t// Prefer the actual cty string value\n\t\t\ts := v.AsString()\n\t\t\tnode := createScalarNode(s, s)\n\t\t\t// Don't set style for regular quoted strings - let YAML handle naturally\n\t\t\treturn node\n\t\tcase v.Type().Equals(cty.Bool):\n\t\t\tb := v.True()\n\t\t\treturn createScalarNode(b, strconv.FormatBool(b))\n\t\tcase v.Type() == cty.Number:\n\t\t\t// prefer integers when the numeric value is integral\n\t\t\tbf := v.AsBigFloat()\n\t\t\tif bf == nil {\n\t\t\t\t// fallback to string\n\t\t\t\treturn createStringScalarNode(v.GoString())\n\t\t\t}\n\t\t\t// check if bf represents an exact integer\n\t\t\tif intVal, acc := bf.Int(nil); acc == big.Exact {\n\t\t\t\ts := intVal.String()\n\t\t\t\treturn createScalarNode(intVal.Int64(), s)\n\t\t\t}\n\t\t\ts := bf.Text('g', -1)\n\t\t\treturn createScalarNode(0.0, s)\n\t\tcase v.Type().IsTupleType() || v.Type().IsListType() || v.Type().IsSetType():\n\t\t\tseq := &CandidateNode{Kind: SequenceNode}\n\t\t\tit := v.ElementIterator()\n\t\t\tfor it.Next() {\n\t\t\t\t_, val := it.Element()\n\t\t\t\t// convert cty.Value to a node by wrapping in literal expr via string representation\n\t\t\t\tchild := convertCtyValueToNode(val)\n\t\t\t\tseq.AddChild(child)\n\t\t\t}\n\t\t\treturn seq\n\t\tcase v.Type().IsMapType() || v.Type().IsObjectType():\n\t\t\tm := &CandidateNode{Kind: MappingNode}\n\t\t\tit := v.ElementIterator()\n\t\t\tfor it.Next() {\n\t\t\t\tkey, val := it.Element()\n\t\t\t\tkeyStr := key.AsString()\n\t\t\t\tkeyNode := createStringScalarNode(keyStr)\n\t\t\t\tvalNode := convertCtyValueToNode(val)\n\t\t\t\tm.AddKeyValueChild(keyNode, valNode)\n\t\t\t}\n\t\t\treturn m\n\t\tdefault:\n\t\t\t// fallback to string\n\t\t\ts := v.GoString()\n\t\t\treturn createStringScalarNode(s)\n\t\t}\n\tcase *hclsyntax.TupleConsExpr:\n\t\t// parse tuple/list into YAML sequence\n\t\tseq := &CandidateNode{Kind: SequenceNode}\n\t\tfor _, exprVal := range e.Exprs {\n\t\t\tchild := convertHclExprToNode(exprVal, src)\n\t\t\tseq.AddChild(child)\n\t\t}\n\t\treturn seq\n\tcase *hclsyntax.ObjectConsExpr:\n\t\t// parse object into YAML mapping\n\t\tm := &CandidateNode{Kind: MappingNode}\n\t\tm.Style = FlowStyle // Mark as inline object (flow style) for encoder\n\t\tfor _, item := range e.Items {\n\t\t\t// evaluate key expression to get the key string\n\t\t\tkeyVal, keyDiags := item.KeyExpr.Value(nil)\n\t\t\tif keyDiags != nil && keyDiags.HasErrors() {\n\t\t\t\t// fallback: try to extract key from source\n\t\t\t\tr := item.KeyExpr.Range()\n\t\t\t\tstart := r.Start.Byte\n\t\t\t\tend := r.End.Byte\n\t\t\t\tif start >= 0 && end >= start && end <= len(src) {\n\t\t\t\t\tkeyNode := createStringScalarNode(strings.TrimSpace(string(src[start:end])))\n\t\t\t\t\tvalNode := convertHclExprToNode(item.ValueExpr, src)\n\t\t\t\t\tm.AddKeyValueChild(keyNode, valNode)\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tkeyStr := keyVal.AsString()\n\t\t\tkeyNode := createStringScalarNode(keyStr)\n\t\t\tvalNode := convertHclExprToNode(item.ValueExpr, src)\n\t\t\tm.AddKeyValueChild(keyNode, valNode)\n\t\t}\n\t\treturn m\n\tcase *hclsyntax.TemplateExpr:\n\t\t// Reconstruct template string, preserving ${} syntax for interpolations\n\t\tvar parts []string\n\t\tfor _, p := range e.Parts {\n\t\t\tswitch lp := p.(type) {\n\t\t\tcase *hclsyntax.LiteralValueExpr:\n\t\t\t\tif lp.Val.Type().Equals(cty.String) {\n\t\t\t\t\tparts = append(parts, lp.Val.AsString())\n\t\t\t\t} else {\n\t\t\t\t\tparts = append(parts, lp.Val.GoString())\n\t\t\t\t}\n\t\t\tdefault:\n\t\t\t\t// Non-literal expression - reconstruct with ${} wrapper\n\t\t\t\tr := p.Range()\n\t\t\t\tstart := r.Start.Byte\n\t\t\t\tend := r.End.Byte\n\t\t\t\tif start >= 0 && end >= start && end <= len(src) {\n\t\t\t\t\texprText := string(src[start:end])\n\t\t\t\t\tparts = append(parts, \"${\"+exprText+\"}\")\n\t\t\t\t} else {\n\t\t\t\t\tparts = append(parts, fmt.Sprintf(\"${%v}\", p))\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tcombined := strings.Join(parts, \"\")\n\t\tnode := createScalarNode(combined, combined)\n\t\t// Set DoubleQuotedStyle for all templates (which includes all quoted strings in HCL)\n\t\t// This ensures HCL roundtrips preserve quotes, and YAML properly quotes strings with ${}\n\t\tnode.Style = DoubleQuotedStyle\n\t\treturn node\n\tcase *hclsyntax.ScopeTraversalExpr:\n\t\t// Simple identifier/traversal (e.g. unquoted string literal in HCL)\n\t\tr := e.Range()\n\t\tstart := r.Start.Byte\n\t\tend := r.End.Byte\n\t\tif start >= 0 && end >= start && end <= len(src) {\n\t\t\ttext := strings.TrimSpace(string(src[start:end]))\n\t\t\treturn createStringScalarNode(text)\n\t\t}\n\t\t// Fallback to root name if source unavailable\n\t\tif len(e.Traversal) > 0 {\n\t\t\tif root, ok := e.Traversal[0].(hcl.TraverseRoot); ok {\n\t\t\t\treturn createStringScalarNode(root.Name)\n\t\t\t}\n\t\t}\n\t\treturn createStringScalarNode(\"\")\n\tcase *hclsyntax.FunctionCallExpr:\n\t\t// Preserve function calls as raw expressions for roundtrip\n\t\tr := e.Range()\n\t\tstart := r.Start.Byte\n\t\tend := r.End.Byte\n\t\tif start >= 0 && end >= start && end <= len(src) {\n\t\t\ttext := strings.TrimSpace(string(src[start:end]))\n\t\t\tnode := createStringScalarNode(text)\n\t\t\tnode.Style = 0\n\t\t\treturn node\n\t\t}\n\t\tnode := createStringScalarNode(e.Name)\n\t\tnode.Style = 0\n\t\treturn node\n\tdefault:\n\t\t// try to evaluate the expression (handles unary, binary ops, etc.)\n\t\tval, diags := expr.Value(nil)\n\t\tif diags == nil || !diags.HasErrors() {\n\t\t\t// successfully evaluated, convert cty.Value to node\n\t\t\treturn convertCtyValueToNode(val)\n\t\t}\n\t\t// fallback: extract source text for the expression\n\t\tr := expr.Range()\n\t\tstart := r.Start.Byte\n\t\tend := r.End.Byte\n\t\tif start >= 0 && end >= start && end <= len(src) {\n\t\t\ttext := string(src[start:end])\n\t\t\t// Mark as unquoted expression so encoder emits without quoting\n\t\t\tnode := createStringScalarNode(text)\n\t\t\tnode.Style = 0\n\t\t\treturn node\n\t\t}\n\t\treturn createStringScalarNode(fmt.Sprintf(\"%v\", expr))\n\t}\n}\n\nfunc convertCtyValueToNode(v cty.Value) *CandidateNode {\n\tif v.IsNull() {\n\t\treturn createScalarNode(nil, \"\")\n\t}\n\tswitch {\n\tcase v.Type().Equals(cty.String):\n\t\treturn createScalarNode(\"\", v.AsString())\n\tcase v.Type().Equals(cty.Bool):\n\t\tb := v.True()\n\t\treturn createScalarNode(b, strconv.FormatBool(b))\n\tcase v.Type() == cty.Number:\n\t\tbf := v.AsBigFloat()\n\t\tif bf == nil {\n\t\t\treturn createStringScalarNode(v.GoString())\n\t\t}\n\t\tif intVal, acc := bf.Int(nil); acc == big.Exact {\n\t\t\ts := intVal.String()\n\t\t\treturn createScalarNode(intVal.Int64(), s)\n\t\t}\n\t\ts := bf.Text('g', -1)\n\t\treturn createScalarNode(0.0, s)\n\tcase v.Type().IsTupleType() || v.Type().IsListType() || v.Type().IsSetType():\n\t\tseq := &CandidateNode{Kind: SequenceNode}\n\t\tit := v.ElementIterator()\n\t\tfor it.Next() {\n\t\t\t_, val := it.Element()\n\t\t\tseq.AddChild(convertCtyValueToNode(val))\n\t\t}\n\t\treturn seq\n\tcase v.Type().IsMapType() || v.Type().IsObjectType():\n\t\tm := &CandidateNode{Kind: MappingNode}\n\t\tit := v.ElementIterator()\n\t\tfor it.Next() {\n\t\t\tkey, val := it.Element()\n\t\t\tkeyNode := createStringScalarNode(key.AsString())\n\t\t\tvalNode := convertCtyValueToNode(val)\n\t\t\tm.AddKeyValueChild(keyNode, valNode)\n\t\t}\n\t\treturn m\n\tdefault:\n\t\treturn createStringScalarNode(v.GoString())\n\t}\n}\n" }, { "path": "pkg/yqlib/decoder_ini.go", "content": "//go:build !yq_noini\n\npackage yqlib\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\n\t\"github.com/go-ini/ini\"\n)\n\ntype iniDecoder struct {\n\treader io.Reader\n\tfinished bool // Flag to signal completion of processing\n}\n\nfunc NewINIDecoder() Decoder {\n\treturn &iniDecoder{\n\t\tfinished: false, // Initialise the flag as false\n\t}\n}\n\nfunc (dec *iniDecoder) Init(reader io.Reader) error {\n\t// Store the reader for use in Decode\n\tdec.reader = reader\n\treturn nil\n}\n\nfunc (dec *iniDecoder) Decode() (*CandidateNode, error) {\n\t// If processing is already finished, return io.EOF\n\tif dec.finished {\n\t\treturn nil, io.EOF\n\t}\n\n\t// Read all content from the stored reader\n\tcontent, err := io.ReadAll(dec.reader)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to read INI content: %w\", err)\n\t}\n\n\t// Parse the INI content\n\tcfg, err := ini.Load(content)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to parse INI content: %w\", err)\n\t}\n\n\t// Create a root CandidateNode as a MappingNode (since INI is key-value based)\n\troot := &CandidateNode{\n\t\tKind: MappingNode,\n\t\tTag: \"!!map\",\n\t\tValue: \"\",\n\t}\n\n\t// Process each section in the INI file\n\tfor _, section := range cfg.Sections() {\n\t\tsectionName := section.Name()\n\n\t\tif sectionName == ini.DefaultSection {\n\t\t\t// For the default section, add key-value pairs directly to the root node\n\t\t\tfor _, key := range section.Keys() {\n\t\t\t\tkeyName := key.Name()\n\t\t\t\tkeyValue := key.String()\n\n\t\t\t\t// Create a key node (scalar for the key name)\n\t\t\t\tkeyNode := createStringScalarNode(keyName)\n\t\t\t\t// Create a value node (scalar for the value)\n\t\t\t\tvalueNode := createStringScalarNode(keyValue)\n\n\t\t\t\t// Add key-value pair to the root node\n\t\t\t\troot.AddKeyValueChild(keyNode, valueNode)\n\t\t\t}\n\t\t} else {\n\t\t\t// For named sections, create a nested map\n\t\t\tsectionNode := &CandidateNode{\n\t\t\t\tKind: MappingNode,\n\t\t\t\tTag: \"!!map\",\n\t\t\t\tValue: \"\",\n\t\t\t}\n\n\t\t\t// Add key-value pairs to the section node\n\t\t\tfor _, key := range section.Keys() {\n\t\t\t\tkeyName := key.Name()\n\t\t\t\tkeyValue := key.String()\n\n\t\t\t\t// Create a key node (scalar for the key name)\n\t\t\t\tkeyNode := createStringScalarNode(keyName)\n\t\t\t\t// Create a value node (scalar for the value)\n\t\t\t\tvalueNode := createStringScalarNode(keyValue)\n\n\t\t\t\t// Add key-value pair to the section node\n\t\t\t\tsectionNode.AddKeyValueChild(keyNode, valueNode)\n\t\t\t}\n\n\t\t\t// Create a key node for the section name\n\t\t\tsectionKeyNode := createStringScalarNode(sectionName)\n\t\t\t// Add the section as a nested map to the root node\n\t\t\troot.AddKeyValueChild(sectionKeyNode, sectionNode)\n\t\t}\n\t}\n\n\t// Set the finished flag to true to prevent further Decode calls\n\tdec.finished = true\n\n\t// Return the root node\n\treturn root, nil\n}\n" }, { "path": "pkg/yqlib/decoder_json.go", "content": "//go:build !yq_nojson\n\npackage yqlib\n\nimport (\n\t\"io\"\n\n\t\"github.com/goccy/go-json\"\n)\n\ntype jsonDecoder struct {\n\tdecoder json.Decoder\n}\n\nfunc NewJSONDecoder() Decoder {\n\treturn &jsonDecoder{}\n}\n\nfunc (dec *jsonDecoder) Init(reader io.Reader) error {\n\tdec.decoder = *json.NewDecoder(reader)\n\treturn nil\n}\n\nfunc (dec *jsonDecoder) Decode() (*CandidateNode, error) {\n\n\tvar dataBucket CandidateNode\n\terr := dec.decoder.Decode(&dataBucket)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &dataBucket, nil\n}\n" }, { "path": "pkg/yqlib/decoder_lua.go", "content": "//go:build !yq_nolua\n\npackage yqlib\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\n\tlua \"github.com/yuin/gopher-lua\"\n)\n\ntype luaDecoder struct {\n\treader io.Reader\n\tfinished bool\n\tprefs LuaPreferences\n}\n\nfunc NewLuaDecoder(prefs LuaPreferences) Decoder {\n\treturn &luaDecoder{\n\t\tprefs: prefs,\n\t}\n}\n\nfunc (dec *luaDecoder) Init(reader io.Reader) error {\n\tdec.reader = reader\n\treturn nil\n}\n\nfunc (dec *luaDecoder) convertToYamlNode(ls *lua.LState, lv lua.LValue) *CandidateNode {\n\tswitch lv.Type() {\n\tcase lua.LTNil:\n\t\treturn &CandidateNode{\n\t\t\tKind: ScalarNode,\n\t\t\tTag: \"!!null\",\n\t\t\tValue: \"\",\n\t\t}\n\tcase lua.LTBool:\n\t\treturn &CandidateNode{\n\t\t\tKind: ScalarNode,\n\t\t\tTag: \"!!bool\",\n\t\t\tValue: lv.String(),\n\t\t}\n\tcase lua.LTNumber:\n\t\tn := float64(lua.LVAsNumber(lv))\n\t\t// various special case floats\n\t\tif math.IsNaN(n) {\n\t\t\treturn &CandidateNode{\n\t\t\t\tKind: ScalarNode,\n\t\t\t\tTag: \"!!float\",\n\t\t\t\tValue: \".nan\",\n\t\t\t}\n\t\t}\n\t\tif math.IsInf(n, 1) {\n\t\t\treturn &CandidateNode{\n\t\t\t\tKind: ScalarNode,\n\t\t\t\tTag: \"!!float\",\n\t\t\t\tValue: \".inf\",\n\t\t\t}\n\t\t}\n\t\tif math.IsInf(n, -1) {\n\t\t\treturn &CandidateNode{\n\t\t\t\tKind: ScalarNode,\n\t\t\t\tTag: \"!!float\",\n\t\t\t\tValue: \"-.inf\",\n\t\t\t}\n\t\t}\n\n\t\t// does it look like an integer?\n\t\tif n == float64(int(n)) {\n\t\t\treturn &CandidateNode{\n\t\t\t\tKind: ScalarNode,\n\t\t\t\tTag: \"!!int\",\n\t\t\t\tValue: lv.String(),\n\t\t\t}\n\t\t}\n\n\t\treturn &CandidateNode{\n\t\t\tKind: ScalarNode,\n\t\t\tTag: \"!!float\",\n\t\t\tValue: lv.String(),\n\t\t}\n\tcase lua.LTString:\n\t\treturn &CandidateNode{\n\t\t\tKind: ScalarNode,\n\t\t\tTag: \"!!str\",\n\t\t\tValue: lv.String(),\n\t\t}\n\tcase lua.LTFunction:\n\t\treturn &CandidateNode{\n\t\t\tKind: ScalarNode,\n\t\t\tTag: \"tag:lua.org,2006,function\",\n\t\t\tValue: lv.String(),\n\t\t}\n\tcase lua.LTTable:\n\t\t// Simultaneously create a sequence and a map, pick which one to return\n\t\t// based on whether all keys were consecutive integers\n\t\ti := 1\n\t\tyaml_sequence := &CandidateNode{\n\t\t\tKind: SequenceNode,\n\t\t\tTag: \"!!seq\",\n\t\t}\n\t\tyaml_map := &CandidateNode{\n\t\t\tKind: MappingNode,\n\t\t\tTag: \"!!map\",\n\t\t}\n\t\tt := lv.(*lua.LTable)\n\t\tk, v := ls.Next(t, lua.LNil)\n\t\tfor k != lua.LNil {\n\t\t\tif ki, ok := k.(lua.LNumber); i != 0 && ok && math.Mod(float64(ki), 1) == 0 && int(ki) == i {\n\t\t\t\ti++\n\t\t\t} else {\n\t\t\t\ti = 0\n\t\t\t}\n\t\t\tnewKey := dec.convertToYamlNode(ls, k)\n\n\t\t\tyv := dec.convertToYamlNode(ls, v)\n\t\t\tyaml_map.AddKeyValueChild(newKey, yv)\n\n\t\t\tif i != 0 {\n\t\t\t\tyaml_sequence.AddChild(yv)\n\t\t\t}\n\t\t\tk, v = ls.Next(t, k)\n\t\t}\n\t\tif i != 0 {\n\t\t\treturn yaml_sequence\n\t\t}\n\t\treturn yaml_map\n\tdefault:\n\t\treturn &CandidateNode{\n\t\t\tKind: ScalarNode,\n\t\t\tLineComment: fmt.Sprintf(\"Unhandled Lua type: %s\", lv.Type().String()),\n\t\t\tTag: \"!!null\",\n\t\t\tValue: lv.String(),\n\t\t}\n\t}\n}\n\nfunc (dec *luaDecoder) decideTopLevelNode(ls *lua.LState) *CandidateNode {\n\tif ls.GetTop() == 0 {\n\t\t// no items were explicitly returned, encode the globals table instead\n\t\treturn dec.convertToYamlNode(ls, ls.Get(lua.GlobalsIndex))\n\t}\n\treturn dec.convertToYamlNode(ls, ls.Get(1))\n}\n\nfunc (dec *luaDecoder) Decode() (*CandidateNode, error) {\n\tif dec.finished {\n\t\treturn nil, io.EOF\n\t}\n\tls := lua.NewState(lua.Options{SkipOpenLibs: true})\n\tdefer ls.Close()\n\tfn, err := ls.Load(dec.reader, \"@input\")\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tls.Push(fn)\n\terr = ls.PCall(0, lua.MultRet, nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tfirstNode := dec.decideTopLevelNode(ls)\n\tdec.finished = true\n\treturn firstNode, nil\n}\n" }, { "path": "pkg/yqlib/decoder_properties.go", "content": "//go:build !yq_noprops\n\npackage yqlib\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"io\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/magiconair/properties\"\n)\n\ntype propertiesDecoder struct {\n\treader io.Reader\n\tfinished bool\n\td DataTreeNavigator\n}\n\nfunc NewPropertiesDecoder() Decoder {\n\treturn &propertiesDecoder{d: NewDataTreeNavigator(), finished: false}\n}\n\nfunc (dec *propertiesDecoder) Init(reader io.Reader) error {\n\tdec.reader = reader\n\tdec.finished = false\n\treturn nil\n}\n\nfunc parsePropKey(key string) []interface{} {\n\tpathStrArray := strings.Split(key, \".\")\n\tpath := make([]interface{}, len(pathStrArray))\n\tfor i, pathStr := range pathStrArray {\n\t\tnum, err := strconv.ParseInt(pathStr, 10, 32)\n\t\tif err == nil {\n\t\t\tpath[i] = num\n\t\t} else {\n\t\t\tpath[i] = pathStr\n\t\t}\n\t}\n\treturn path\n}\n\nfunc (dec *propertiesDecoder) processComment(c string) string {\n\tif c == \"\" {\n\t\treturn \"\"\n\t}\n\treturn \"# \" + c\n}\n\nfunc (dec *propertiesDecoder) applyPropertyComments(context Context, path []interface{}, comments []string) error {\n\tassignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}\n\n\trhsCandidateNode := &CandidateNode{\n\t\tTag: \"!!str\",\n\t\tValue: fmt.Sprintf(\"%v\", path[len(path)-1]),\n\t\tHeadComment: dec.processComment(strings.Join(comments, \"\\n\")),\n\t\tKind: ScalarNode,\n\t}\n\n\trhsCandidateNode.Tag = rhsCandidateNode.guessTagFromCustomType()\n\n\trhsOp := &Operation{OperationType: referenceOpType, CandidateNode: rhsCandidateNode}\n\n\tassignmentOpNode := &ExpressionNode{\n\t\tOperation: assignmentOp,\n\t\tLHS: createTraversalTree(path, traversePreferences{}, true),\n\t\tRHS: &ExpressionNode{Operation: rhsOp},\n\t}\n\n\t_, err := dec.d.GetMatchingNodes(context, assignmentOpNode)\n\treturn err\n}\n\nfunc (dec *propertiesDecoder) applyProperty(context Context, properties *properties.Properties, key string) error {\n\tvalue, _ := properties.Get(key)\n\tpath := parsePropKey(key)\n\n\tpropertyComments := properties.GetComments(key)\n\tif len(propertyComments) > 0 {\n\t\terr := dec.applyPropertyComments(context, path, propertyComments)\n\t\tif err != nil {\n\t\t\treturn nil\n\t\t}\n\t}\n\n\trhsNode := createStringScalarNode(value)\n\trhsNode.Tag = rhsNode.guessTagFromCustomType()\n\n\treturn dec.d.DeeplyAssign(context, path, rhsNode)\n}\n\nfunc (dec *propertiesDecoder) Decode() (*CandidateNode, error) {\n\tif dec.finished {\n\t\treturn nil, io.EOF\n\t}\n\tbuf := new(bytes.Buffer)\n\n\tif _, err := buf.ReadFrom(dec.reader); err != nil {\n\t\treturn nil, err\n\t}\n\tif buf.Len() == 0 {\n\t\tdec.finished = true\n\t\treturn nil, io.EOF\n\t}\n\tproperties, err := properties.LoadString(buf.String())\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tproperties.DisableExpansion = true\n\n\trootMap := &CandidateNode{\n\t\tKind: MappingNode,\n\t\tTag: \"!!map\",\n\t}\n\n\tcontext := Context{}\n\tcontext = context.SingleChildContext(rootMap)\n\n\tfor _, key := range properties.Keys() {\n\t\tif err := dec.applyProperty(context, properties, key); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t}\n\tdec.finished = true\n\n\treturn rootMap, nil\n\n}\n" }, { "path": "pkg/yqlib/decoder_test.go", "content": "package yqlib\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"fmt\"\n\t\"strings\"\n)\n\ntype formatScenario struct {\n\tinput string\n\tindent int\n\texpression string\n\texpected string\n\tdescription string\n\tsubdescription string\n\tskipDoc bool\n\tscenarioType string\n\texpectedError string\n}\n\nfunc processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) (string, error) {\n\tvar output bytes.Buffer\n\twriter := bufio.NewWriter(&output)\n\n\tif decoder == nil {\n\t\tdecoder = NewYamlDecoder(ConfiguredYamlPreferences)\n\t}\n\n\tlog.Debugf(\"reading docs\")\n\tinputs, err := readDocuments(strings.NewReader(s.input), \"sample.yml\", 0, decoder)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tlog.Debugf(\"done reading the documents\")\n\n\texpression := s.expression\n\tif expression == \"\" {\n\t\texpression = \".\"\n\t}\n\n\texp, err := getExpressionParser().ParseExpression(expression)\n\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tcontext, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: inputs}, exp)\n\n\tlog.Debugf(\"Going to print: %v\", NodesToString(context.MatchingNodes))\n\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\tprinter := NewPrinter(encoder, NewSinglePrinterWriter(writer))\n\terr = printer.PrintResults(context.MatchingNodes)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\twriter.Flush()\n\n\treturn output.String(), nil\n}\n\nfunc mustProcessFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) string {\n\n\tresult, err := processFormatScenario(s, decoder, encoder)\n\tif err != nil {\n\t\tlog.Error(\"Bad scenario %v: %w\", s.description, err)\n\t\treturn fmt.Sprintf(\"Bad scenario %v: %v\", s.description, err.Error())\n\t}\n\treturn result\n\n}\n" }, { "path": "pkg/yqlib/decoder_toml.go", "content": "//go:build !yq_notoml\n\npackage yqlib\n\nimport (\n\t\"bytes\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\ttoml \"github.com/pelletier/go-toml/v2/unstable\"\n)\n\ntype tomlDecoder struct {\n\tparser toml.Parser\n\tfinished bool\n\td DataTreeNavigator\n\trootMap *CandidateNode\n\tpendingComments []string // Head comments collected from Comment nodes\n\tfirstContentSeen bool // Track if we've processed the first non-comment node\n}\n\nfunc NewTomlDecoder() Decoder {\n\treturn &tomlDecoder{\n\t\tfinished: false,\n\t\td: NewDataTreeNavigator(),\n\t}\n}\n\nfunc (dec *tomlDecoder) Init(reader io.Reader) error {\n\tdec.parser = toml.Parser{KeepComments: true}\n\tbuf := new(bytes.Buffer)\n\t_, err := buf.ReadFrom(reader)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdec.parser.Reset(buf.Bytes())\n\tdec.rootMap = &CandidateNode{\n\t\tKind: MappingNode,\n\t\tTag: \"!!map\",\n\t}\n\tdec.pendingComments = make([]string, 0)\n\tdec.firstContentSeen = false\n\treturn nil\n}\n\nfunc (dec *tomlDecoder) attachOrphanedCommentsToNode(tableNodeValue *CandidateNode) {\n\tif len(dec.pendingComments) > 0 {\n\t\tcomments := strings.Join(dec.pendingComments, \"\\n\")\n\t\tif tableNodeValue.HeadComment == \"\" {\n\t\t\ttableNodeValue.HeadComment = comments\n\t\t} else {\n\t\t\ttableNodeValue.HeadComment = tableNodeValue.HeadComment + \"\\n\" + comments\n\t\t}\n\t\tdec.pendingComments = make([]string, 0)\n\t}\n}\n\nfunc (dec *tomlDecoder) getFullPath(tomlNode *toml.Node) []interface{} {\n\tpath := make([]interface{}, 0)\n\tfor {\n\t\tpath = append(path, string(tomlNode.Data))\n\t\ttomlNode = tomlNode.Next()\n\t\tif tomlNode == nil {\n\t\t\treturn path\n\t\t}\n\t}\n}\n\nfunc (dec *tomlDecoder) processKeyValueIntoMap(rootMap *CandidateNode, tomlNode *toml.Node) error {\n\tvalue := tomlNode.Value()\n\tpath := dec.getFullPath(value.Next())\n\n\tvalueNode, err := dec.decodeNode(value)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Attach pending head comments\n\tif len(dec.pendingComments) > 0 {\n\t\tvalueNode.HeadComment = strings.Join(dec.pendingComments, \"\\n\")\n\t\tdec.pendingComments = make([]string, 0)\n\t}\n\n\t// Check for inline comment chained to the KeyValue node\n\tnextNode := tomlNode.Next()\n\tif nextNode != nil && nextNode.Kind == toml.Comment {\n\t\tvalueNode.LineComment = string(nextNode.Data)\n\t}\n\n\tcontext := Context{}\n\tcontext = context.SingleChildContext(rootMap)\n\n\treturn dec.d.DeeplyAssign(context, path, valueNode)\n}\n\nfunc (dec *tomlDecoder) decodeKeyValuesIntoMap(rootMap *CandidateNode, tomlNode *toml.Node) (bool, error) {\n\tlog.Debug(\"decodeKeyValuesIntoMap -- processing first (current) entry\")\n\tif err := dec.processKeyValueIntoMap(rootMap, tomlNode); err != nil {\n\t\treturn false, err\n\t}\n\n\tfor dec.parser.NextExpression() {\n\t\tnextItem := dec.parser.Expression()\n\t\tlog.Debug(\"decodeKeyValuesIntoMap -- next exp, its a %v\", nextItem.Kind)\n\n\t\tswitch nextItem.Kind {\n\t\tcase toml.KeyValue:\n\t\t\tif err := dec.processKeyValueIntoMap(rootMap, nextItem); err != nil {\n\t\t\t\treturn false, err\n\t\t\t}\n\t\tcase toml.Comment:\n\t\t\t// Standalone comment - add to pending for next element\n\t\t\tdec.pendingComments = append(dec.pendingComments, string(nextItem.Data))\n\t\tdefault:\n\t\t\t// run out of key values\n\t\t\tlog.Debug(\"done in decodeKeyValuesIntoMap, gota a %v\", nextItem.Kind)\n\t\t\treturn true, nil\n\t\t}\n\t}\n\tlog.Debug(\"no more things to read in\")\n\treturn false, nil\n}\n\nfunc (dec *tomlDecoder) createInlineTableMap(tomlNode *toml.Node) (*CandidateNode, error) {\n\tcontent := make([]*CandidateNode, 0)\n\tlog.Debug(\"createInlineTableMap\")\n\n\titerator := tomlNode.Children()\n\tfor iterator.Next() {\n\t\tchild := iterator.Node()\n\t\tif child.Kind != toml.KeyValue {\n\t\t\treturn nil, fmt.Errorf(\"only keyvalue pairs are supported in inlinetables, got %v instead\", child.Kind)\n\t\t}\n\n\t\tkeyValues := &CandidateNode{\n\t\t\tKind: MappingNode,\n\t\t\tTag: \"!!map\",\n\t\t}\n\n\t\tif err := dec.processKeyValueIntoMap(keyValues, child); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tcontent = append(content, keyValues.Content...)\n\t}\n\n\treturn &CandidateNode{\n\t\tKind: MappingNode,\n\t\tTag: \"!!map\",\n\t\tContent: content,\n\t}, nil\n}\n\nfunc (dec *tomlDecoder) createArray(tomlNode *toml.Node) (*CandidateNode, error) {\n\tcontent := make([]*CandidateNode, 0)\n\tvar pendingArrayComments []string\n\n\titerator := tomlNode.Children()\n\tfor iterator.Next() {\n\t\tchild := iterator.Node()\n\n\t\t// Handle comments within arrays\n\t\tif child.Kind == toml.Comment {\n\t\t\t// Collect comments to attach to the next array element\n\t\t\tpendingArrayComments = append(pendingArrayComments, string(child.Data))\n\t\t\tcontinue\n\t\t}\n\n\t\tyamlNode, err := dec.decodeNode(child)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\t// Attach any pending comments to this array element\n\t\tif len(pendingArrayComments) > 0 {\n\t\t\tyamlNode.HeadComment = strings.Join(pendingArrayComments, \"\\n\")\n\t\t\tpendingArrayComments = make([]string, 0)\n\t\t}\n\n\t\tcontent = append(content, yamlNode)\n\t}\n\n\treturn &CandidateNode{\n\t\tKind: SequenceNode,\n\t\tTag: \"!!seq\",\n\t\tContent: content,\n\t}, nil\n\n}\n\nfunc (dec *tomlDecoder) createStringScalar(tomlNode *toml.Node) (*CandidateNode, error) {\n\tcontent := string(tomlNode.Data)\n\treturn createScalarNode(content, content), nil\n}\n\nfunc (dec *tomlDecoder) createBoolScalar(tomlNode *toml.Node) (*CandidateNode, error) {\n\tcontent := string(tomlNode.Data)\n\treturn createScalarNode(content == \"true\", content), nil\n}\n\nfunc (dec *tomlDecoder) createIntegerScalar(tomlNode *toml.Node) (*CandidateNode, error) {\n\tcontent := string(tomlNode.Data)\n\t_, num, err := parseInt64(content)\n\treturn createScalarNode(num, content), err\n}\n\nfunc (dec *tomlDecoder) createDateTimeScalar(tomlNode *toml.Node) (*CandidateNode, error) {\n\tcontent := string(tomlNode.Data)\n\tval, err := parseDateTime(time.RFC3339, content)\n\treturn createScalarNode(val, content), err\n}\n\nfunc (dec *tomlDecoder) createFloatScalar(tomlNode *toml.Node) (*CandidateNode, error) {\n\tcontent := string(tomlNode.Data)\n\tnum, err := strconv.ParseFloat(content, 64)\n\treturn createScalarNode(num, content), err\n}\n\nfunc (dec *tomlDecoder) decodeNode(tomlNode *toml.Node) (*CandidateNode, error) {\n\tswitch tomlNode.Kind {\n\tcase toml.Key, toml.String:\n\t\treturn dec.createStringScalar(tomlNode)\n\tcase toml.Bool:\n\t\treturn dec.createBoolScalar(tomlNode)\n\tcase toml.Integer:\n\t\treturn dec.createIntegerScalar(tomlNode)\n\tcase toml.DateTime:\n\t\treturn dec.createDateTimeScalar(tomlNode)\n\tcase toml.Float:\n\t\treturn dec.createFloatScalar(tomlNode)\n\tcase toml.Array:\n\t\treturn dec.createArray(tomlNode)\n\tcase toml.InlineTable:\n\t\treturn dec.createInlineTableMap(tomlNode)\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"unsupported type %v\", tomlNode.Kind)\n\t}\n\n}\n\nfunc (dec *tomlDecoder) Decode() (*CandidateNode, error) {\n\tif dec.finished {\n\t\treturn nil, io.EOF\n\t}\n\t//\n\t// toml library likes to panic\n\tvar deferredError error\n\tdefer func() { //catch or finally\n\t\tif r := recover(); r != nil {\n\t\t\tvar ok bool\n\t\t\tdeferredError, ok = r.(error)\n\t\t\tif !ok {\n\t\t\t\tdeferredError = fmt.Errorf(\"pkg: %v\", r)\n\t\t\t}\n\t\t}\n\t}()\n\n\tlog.Debug(\"ok here we go\")\n\tvar runAgainstCurrentExp = false\n\tvar err error\n\tfor runAgainstCurrentExp || dec.parser.NextExpression() {\n\n\t\tif runAgainstCurrentExp {\n\t\t\tlog.Debug(\"running against current exp\")\n\t\t}\n\n\t\tcurrentNode := dec.parser.Expression()\n\n\t\tlog.Debug(\"currentNode: %v \", currentNode.Kind)\n\t\trunAgainstCurrentExp, err = dec.processTopLevelNode(currentNode)\n\t\tif err != nil {\n\t\t\treturn dec.rootMap, err\n\t\t}\n\n\t}\n\n\terr = dec.parser.Error()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t// must have finished\n\tdec.finished = true\n\n\tif len(dec.rootMap.Content) == 0 {\n\t\treturn nil, io.EOF\n\t}\n\n\treturn dec.rootMap, deferredError\n\n}\n\nfunc (dec *tomlDecoder) processTopLevelNode(currentNode *toml.Node) (bool, error) {\n\tvar runAgainstCurrentExp bool\n\tvar err error\n\tlog.Debug(\"processTopLevelNode: Going to process %v state is current %v\", currentNode.Kind, NodeToString(dec.rootMap))\n\tswitch currentNode.Kind {\n\tcase toml.Comment:\n\t\t// Collect comment to attach to next element\n\t\tcommentText := string(currentNode.Data)\n\t\t// If we haven't seen any content yet, accumulate comments for root\n\t\tif !dec.firstContentSeen {\n\t\t\tif dec.rootMap.HeadComment == \"\" {\n\t\t\t\tdec.rootMap.HeadComment = commentText\n\t\t\t} else {\n\t\t\t\tdec.rootMap.HeadComment = dec.rootMap.HeadComment + \"\\n\" + commentText\n\t\t\t}\n\t\t} else {\n\t\t\t// We've seen content, so these comments are for the next element\n\t\t\tdec.pendingComments = append(dec.pendingComments, commentText)\n\t\t}\n\t\treturn false, nil\n\tcase toml.Table:\n\t\tdec.firstContentSeen = true\n\t\trunAgainstCurrentExp, err = dec.processTable(currentNode)\n\tcase toml.ArrayTable:\n\t\tdec.firstContentSeen = true\n\t\trunAgainstCurrentExp, err = dec.processArrayTable(currentNode)\n\tdefault:\n\t\tdec.firstContentSeen = true\n\t\trunAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(dec.rootMap, currentNode)\n\t}\n\n\tlog.Debug(\"processTopLevelNode: DONE Processing state is now %v\", NodeToString(dec.rootMap))\n\treturn runAgainstCurrentExp, err\n}\n\nfunc (dec *tomlDecoder) processTable(currentNode *toml.Node) (bool, error) {\n\tlog.Debug(\"Enter processTable\")\n\tchild := currentNode.Child()\n\tfullPath := dec.getFullPath(child)\n\tlog.Debug(\"fullpath: %v\", fullPath)\n\n\tc := Context{}\n\tc = c.SingleChildContext(dec.rootMap)\n\n\tfullPath, err := getPathToUse(fullPath, dec, c)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\ttableNodeValue := &CandidateNode{\n\t\tKind: MappingNode,\n\t\tTag: \"!!map\",\n\t\tContent: make([]*CandidateNode, 0),\n\t\tEncodeSeparate: true,\n\t}\n\n\t// Attach pending head comments to the table\n\tif len(dec.pendingComments) > 0 {\n\t\ttableNodeValue.HeadComment = strings.Join(dec.pendingComments, \"\\n\")\n\t\tdec.pendingComments = make([]string, 0)\n\t}\n\n\tvar tableValue *toml.Node\n\trunAgainstCurrentExp := false\n\tsawKeyValue := false\n\tfor dec.parser.NextExpression() {\n\t\ttableValue = dec.parser.Expression()\n\t\t// Allow standalone comments inside the table before the first key-value.\n\t\t// These should be associated with the next element in the table (usually the first key-value),\n\t\t// not treated as \"end of table\" (which would cause subsequent key-values to be parsed at root).\n\t\tif tableValue.Kind == toml.Comment {\n\t\t\tdec.pendingComments = append(dec.pendingComments, string(tableValue.Data))\n\t\t\tcontinue\n\t\t}\n\n\t\t// next expression is not table data, so we are done (but we need to re-process it at top-level)\n\t\tif tableValue.Kind != toml.KeyValue {\n\t\t\tlog.Debug(\"got an empty table (or reached next section)\")\n\t\t\t// If the table had only comments, attach them to the table itself so they don't leak to the next node.\n\t\t\tif !sawKeyValue {\n\t\t\t\tdec.attachOrphanedCommentsToNode(tableNodeValue)\n\t\t\t}\n\t\t\trunAgainstCurrentExp = true\n\t\t\tbreak\n\t\t}\n\n\t\tsawKeyValue = true\n\t\trunAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue)\n\t\tif err != nil && !errors.Is(err, io.EOF) {\n\t\t\treturn false, err\n\t\t}\n\t\tbreak\n\t}\n\t// If we hit EOF after only seeing comments inside this table, attach them to the table itself\n\t// so they don't leak to whatever comes next.\n\tif !sawKeyValue {\n\t\tdec.attachOrphanedCommentsToNode(tableNodeValue)\n\t}\n\n\terr = dec.d.DeeplyAssign(c, fullPath, tableNodeValue)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\treturn runAgainstCurrentExp, nil\n}\n\nfunc (dec *tomlDecoder) arrayAppend(context Context, path []interface{}, rhsNode *CandidateNode) error {\n\tlog.Debug(\"arrayAppend to path: %v,%v\", path, NodeToString(rhsNode))\n\trhsCandidateNode := &CandidateNode{\n\t\tKind: SequenceNode,\n\t\tTag: \"!!seq\",\n\t\tContent: []*CandidateNode{rhsNode},\n\t}\n\n\tassignmentOp := &Operation{OperationType: addAssignOpType}\n\n\trhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}\n\n\tassignmentOpNode := &ExpressionNode{\n\t\tOperation: assignmentOp,\n\t\tLHS: createTraversalTree(path, traversePreferences{}, false),\n\t\tRHS: &ExpressionNode{Operation: rhsOp},\n\t}\n\n\t_, err := dec.d.GetMatchingNodes(context, assignmentOpNode)\n\treturn err\n}\n\nfunc (dec *tomlDecoder) processArrayTable(currentNode *toml.Node) (bool, error) {\n\tlog.Debug(\"Enter processArrayTable\")\n\tchild := currentNode.Child()\n\tfullPath := dec.getFullPath(child)\n\tlog.Debug(\"Fullpath: %v\", fullPath)\n\n\tc := Context{}\n\tc = c.SingleChildContext(dec.rootMap)\n\n\tfullPath, err := getPathToUse(fullPath, dec, c)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\t// need to use the array append exp to add another entry to\n\t// this array: fullpath += [ thing ]\n\thasValue := dec.parser.NextExpression()\n\n\ttableNodeValue := &CandidateNode{\n\t\tKind: MappingNode,\n\t\tTag: \"!!map\",\n\t\tEncodeSeparate: true,\n\t}\n\n\t// Attach pending head comments to the array table\n\tif len(dec.pendingComments) > 0 {\n\t\ttableNodeValue.HeadComment = strings.Join(dec.pendingComments, \"\\n\")\n\t\tdec.pendingComments = make([]string, 0)\n\t}\n\n\trunAgainstCurrentExp := false\n\tsawKeyValue := false\n\tif hasValue {\n\t\tfor {\n\t\t\texp := dec.parser.Expression()\n\t\t\t// Allow standalone comments inside array tables before the first key-value.\n\t\t\tif exp.Kind == toml.Comment {\n\t\t\t\tdec.pendingComments = append(dec.pendingComments, string(exp.Data))\n\t\t\t\thasValue = dec.parser.NextExpression()\n\t\t\t\tif !hasValue {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\t// if the next value is a ArrayTable or Table, then its not part of this declaration (not a key value pair)\n\t\t\t// so lets leave that expression for the next round of parsing\n\t\t\tif exp.Kind == toml.ArrayTable || exp.Kind == toml.Table {\n\t\t\t\t// If this array-table entry had only comments, attach them to the entry so they don't leak.\n\t\t\t\tif !sawKeyValue {\n\t\t\t\t\tdec.attachOrphanedCommentsToNode(tableNodeValue)\n\t\t\t\t}\n\t\t\t\trunAgainstCurrentExp = true\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t\tsawKeyValue = true\n\t\t\t// otherwise, if there is a value, it must be some key value pairs of the\n\t\t\t// first object in the array!\n\t\t\trunAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(tableNodeValue, exp)\n\t\t\tif err != nil && !errors.Is(err, io.EOF) {\n\t\t\t\treturn false, err\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t}\n\t// If we hit EOF after only seeing comments inside this array-table entry, attach them to the entry\n\t// so they don't leak to whatever comes next.\n\tif !sawKeyValue && len(dec.pendingComments) > 0 {\n\t\tcomments := strings.Join(dec.pendingComments, \"\\n\")\n\t\tif tableNodeValue.HeadComment == \"\" {\n\t\t\ttableNodeValue.HeadComment = comments\n\t\t} else {\n\t\t\ttableNodeValue.HeadComment = tableNodeValue.HeadComment + \"\\n\" + comments\n\t\t}\n\t\tdec.pendingComments = make([]string, 0)\n\t}\n\n\t// += function\n\terr = dec.arrayAppend(c, fullPath, tableNodeValue)\n\n\treturn runAgainstCurrentExp, err\n}\n\n// if fullPath points to an array of maps rather than a map\n// then it should set this element into the _last_ element of that array.\n// Because TOML. So we'll inject the last index into the path.\n\nfunc getPathToUse(fullPath []interface{}, dec *tomlDecoder, c Context) ([]interface{}, error) {\n\t// We need to check the entire path (except the last element), not just the immediate parent,\n\t// because we may have nested array tables like [[array.subarray.subsubarray]]\n\t// where both 'array' and 'subarray' are arrays that already exist.\n\n\tif len(fullPath) == 0 {\n\t\treturn fullPath, nil\n\t}\n\n\tresultPath := make([]interface{}, 0, len(fullPath)*2) // preallocate with extra space for indices\n\n\t// Process all segments except the last one\n\tfor i := 0; i < len(fullPath)-1; i++ {\n\t\tresultPath = append(resultPath, fullPath[i])\n\n\t\t// Check if the current path segment points to an array\n\t\treadOp := createTraversalTree(resultPath, traversePreferences{DontAutoCreate: true}, false)\n\t\tresultContext, err := dec.d.GetMatchingNodes(c, readOp)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tif resultContext.MatchingNodes.Len() >= 1 {\n\t\t\tmatch := resultContext.MatchingNodes.Front().Value.(*CandidateNode)\n\t\t\t// If this segment points to an array, we need to add the last index\n\t\t\t// before continuing with the rest of the path\n\t\t\tif match.Kind == SequenceNode && len(match.Content) > 0 {\n\t\t\t\tlastIndex := len(match.Content) - 1\n\t\t\t\tresultPath = append(resultPath, lastIndex)\n\t\t\t\tlog.Debugf(\"Path segment %v is an array, injecting index %d\", resultPath[:len(resultPath)-1], lastIndex)\n\t\t\t}\n\t\t}\n\t}\n\n\t// Add the last segment\n\tresultPath = append(resultPath, fullPath[len(fullPath)-1])\n\n\tlog.Debugf(\"getPathToUse: original path %v -> result path %v\", fullPath, resultPath)\n\treturn resultPath, nil\n}\n" }, { "path": "pkg/yqlib/decoder_uri.go", "content": "//go:build !yq_nouri\n\npackage yqlib\n\nimport (\n\t\"bytes\"\n\t\"io\"\n\t\"net/url\"\n)\n\ntype uriDecoder struct {\n\treader io.Reader\n\tfinished bool\n\treadAnything bool\n}\n\nfunc NewUriDecoder() Decoder {\n\treturn &uriDecoder{finished: false}\n}\n\nfunc (dec *uriDecoder) Init(reader io.Reader) error {\n\tdec.reader = reader\n\tdec.readAnything = false\n\tdec.finished = false\n\treturn nil\n}\n\nfunc (dec *uriDecoder) Decode() (*CandidateNode, error) {\n\tif dec.finished {\n\t\treturn nil, io.EOF\n\t}\n\n\tbuf := new(bytes.Buffer)\n\n\tif _, err := buf.ReadFrom(dec.reader); err != nil {\n\t\treturn nil, err\n\t}\n\tif buf.Len() == 0 {\n\t\tdec.finished = true\n\n\t\t// if we've read _only_ an empty string, lets return that\n\t\t// otherwise if we've already read some bytes, and now we get\n\t\t// an empty string, then we are done.\n\t\tif dec.readAnything {\n\t\t\treturn nil, io.EOF\n\t\t}\n\t}\n\tnewValue, err := url.QueryUnescape(buf.String())\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdec.readAnything = true\n\treturn createStringScalarNode(newValue), nil\n}\n" }, { "path": "pkg/yqlib/decoder_uri_test.go", "content": "//go:build !yq_nouri\n\npackage yqlib\n\nimport (\n\t\"io\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nfunc TestUriDecoder_Init(t *testing.T) {\n\tdecoder := NewUriDecoder()\n\treader := strings.NewReader(\"test\")\n\terr := decoder.Init(reader)\n\ttest.AssertResult(t, nil, err)\n}\n\nfunc TestUriDecoder_DecodeSimpleString(t *testing.T) {\n\tdecoder := NewUriDecoder()\n\treader := strings.NewReader(\"hello%20world\")\n\terr := decoder.Init(reader)\n\ttest.AssertResult(t, nil, err)\n\n\tnode, err := decoder.Decode()\n\ttest.AssertResult(t, nil, err)\n\ttest.AssertResult(t, \"!!str\", node.Tag)\n\ttest.AssertResult(t, \"hello world\", node.Value)\n}\n\nfunc TestUriDecoder_DecodeSpecialCharacters(t *testing.T) {\n\tdecoder := NewUriDecoder()\n\treader := strings.NewReader(\"hello%21%40%23%24%25\")\n\terr := decoder.Init(reader)\n\ttest.AssertResult(t, nil, err)\n\n\tnode, err := decoder.Decode()\n\ttest.AssertResult(t, nil, err)\n\ttest.AssertResult(t, \"hello!@#$%\", node.Value)\n}\n\nfunc TestUriDecoder_DecodeUTF8(t *testing.T) {\n\tdecoder := NewUriDecoder()\n\treader := strings.NewReader(\"%E2%9C%93%20check\")\n\terr := decoder.Init(reader)\n\ttest.AssertResult(t, nil, err)\n\n\tnode, err := decoder.Decode()\n\ttest.AssertResult(t, nil, err)\n\ttest.AssertResult(t, \"✓ check\", node.Value)\n}\n\nfunc TestUriDecoder_DecodePlusSign(t *testing.T) {\n\tdecoder := NewUriDecoder()\n\treader := strings.NewReader(\"a+b\")\n\terr := decoder.Init(reader)\n\ttest.AssertResult(t, nil, err)\n\n\tnode, err := decoder.Decode()\n\ttest.AssertResult(t, nil, err)\n\t// Note: url.QueryUnescape does NOT convert + to space\n\t// That's only for form encoding (url.ParseQuery)\n\ttest.AssertResult(t, \"a b\", node.Value)\n}\n\nfunc TestUriDecoder_DecodeEmptyString(t *testing.T) {\n\tdecoder := NewUriDecoder()\n\treader := strings.NewReader(\"\")\n\terr := decoder.Init(reader)\n\ttest.AssertResult(t, nil, err)\n\n\tnode, err := decoder.Decode()\n\ttest.AssertResult(t, nil, err)\n\ttest.AssertResult(t, \"\", node.Value)\n\n\t// Second decode should return EOF\n\tnode, err = decoder.Decode()\n\ttest.AssertResult(t, io.EOF, err)\n\ttest.AssertResult(t, (*CandidateNode)(nil), node)\n}\n\nfunc TestUriDecoder_DecodeMultipleCalls(t *testing.T) {\n\tdecoder := NewUriDecoder()\n\treader := strings.NewReader(\"test\")\n\terr := decoder.Init(reader)\n\ttest.AssertResult(t, nil, err)\n\n\t// First decode\n\tnode, err := decoder.Decode()\n\ttest.AssertResult(t, nil, err)\n\ttest.AssertResult(t, \"test\", node.Value)\n\n\t// Second decode should return EOF since we've consumed all input\n\tnode, err = decoder.Decode()\n\ttest.AssertResult(t, io.EOF, err)\n\ttest.AssertResult(t, (*CandidateNode)(nil), node)\n}\n\nfunc TestUriDecoder_DecodeInvalidEscape(t *testing.T) {\n\tdecoder := NewUriDecoder()\n\treader := strings.NewReader(\"test%ZZ\")\n\terr := decoder.Init(reader)\n\ttest.AssertResult(t, nil, err)\n\n\t_, err = decoder.Decode()\n\t// Should return an error for invalid escape sequence\n\tif err == nil {\n\t\tt.Error(\"Expected error for invalid escape sequence, got nil\")\n\t}\n}\n\nfunc TestUriDecoder_DecodeSlashAndQuery(t *testing.T) {\n\tdecoder := NewUriDecoder()\n\treader := strings.NewReader(\"path%2Fto%2Ffile%3Fquery%3Dvalue\")\n\terr := decoder.Init(reader)\n\ttest.AssertResult(t, nil, err)\n\n\tnode, err := decoder.Decode()\n\ttest.AssertResult(t, nil, err)\n\ttest.AssertResult(t, \"path/to/file?query=value\", node.Value)\n}\n\nfunc TestUriDecoder_DecodePercent(t *testing.T) {\n\tdecoder := NewUriDecoder()\n\treader := strings.NewReader(\"100%25\")\n\terr := decoder.Init(reader)\n\ttest.AssertResult(t, nil, err)\n\n\tnode, err := decoder.Decode()\n\ttest.AssertResult(t, nil, err)\n\ttest.AssertResult(t, \"100%\", node.Value)\n}\n\nfunc TestUriDecoder_DecodeNoEscaping(t *testing.T) {\n\tdecoder := NewUriDecoder()\n\treader := strings.NewReader(\"simple_text-123\")\n\terr := decoder.Init(reader)\n\ttest.AssertResult(t, nil, err)\n\n\tnode, err := decoder.Decode()\n\ttest.AssertResult(t, nil, err)\n\ttest.AssertResult(t, \"simple_text-123\", node.Value)\n}\n\n// Mock reader that returns an error\ntype errorReader struct{}\n\nfunc (e *errorReader) Read(_ []byte) (n int, err error) {\n\treturn 0, io.ErrUnexpectedEOF\n}\n\nfunc TestUriDecoder_DecodeReadError(t *testing.T) {\n\tdecoder := NewUriDecoder()\n\terr := decoder.Init(&errorReader{})\n\ttest.AssertResult(t, nil, err)\n\n\t_, err = decoder.Decode()\n\ttest.AssertResult(t, io.ErrUnexpectedEOF, err)\n}\n" }, { "path": "pkg/yqlib/decoder_xml.go", "content": "//go:build !yq_noxml\n\npackage yqlib\n\nimport (\n\t\"encoding/xml\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"regexp\"\n\t\"strings\"\n\t\"unicode\"\n\n\t\"golang.org/x/net/html/charset\"\n)\n\ntype xmlDecoder struct {\n\treader io.Reader\n\treadAnything bool\n\tfinished bool\n\tprefs XmlPreferences\n}\n\nfunc NewXMLDecoder(prefs XmlPreferences) Decoder {\n\treturn &xmlDecoder{\n\t\tfinished: false,\n\t\tprefs: prefs,\n\t}\n}\n\nfunc (dec *xmlDecoder) Init(reader io.Reader) error {\n\tdec.reader = reader\n\tdec.readAnything = false\n\tdec.finished = false\n\treturn nil\n}\n\nfunc (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*CandidateNode, error) {\n\tyamlNode := &CandidateNode{Kind: SequenceNode, Tag: \"!!seq\"}\n\tfor _, child := range nodes {\n\t\tyamlChild, err := dec.convertToYamlNode(child)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tyamlNode.AddChild(yamlChild)\n\t}\n\n\treturn yamlNode, nil\n}\n\nvar decoderCommentPrefix = regexp.MustCompile(`(^|\\n)([[:alpha:]])`)\n\nfunc (dec *xmlDecoder) processComment(c string) string {\n\tif c == \"\" {\n\t\treturn \"\"\n\t}\n\t//need to replace \"cat \" with \"# cat\"\n\t// \"\\ncat\\n\" with \"\\n cat\\n\"\n\t// ensure non-empty comments starting with newline have a space in front\n\n\treplacement := decoderCommentPrefix.ReplaceAllString(c, \"$1 $2\")\n\treplacement = \"#\" + strings.ReplaceAll(strings.TrimRight(replacement, \" \"), \"\\n\", \"\\n#\")\n\treturn replacement\n}\n\nfunc (dec *xmlDecoder) createMap(n *xmlNode) (*CandidateNode, error) {\n\tlog.Debug(\"createMap: headC: %v, lineC: %v, footC: %v\", n.HeadComment, n.LineComment, n.FootComment)\n\tyamlNode := &CandidateNode{Kind: MappingNode, Tag: \"!!map\"}\n\n\tif len(n.Data) > 0 {\n\t\tlog.Debugf(\"creating content node for map: %v\", dec.prefs.ContentName)\n\t\tlabel := dec.prefs.ContentName\n\t\tlabelNode := createScalarNode(label, label)\n\t\tlabelNode.HeadComment = dec.processComment(n.HeadComment)\n\t\tlabelNode.LineComment = dec.processComment(n.LineComment)\n\t\tlabelNode.FootComment = dec.processComment(n.FootComment)\n\t\tyamlNode.AddKeyValueChild(labelNode, dec.createValueNodeFromData(n.Data))\n\t}\n\n\tfor i, keyValuePair := range n.Children {\n\t\tlabel := keyValuePair.K\n\t\tchildren := keyValuePair.V\n\t\tlabelNode := createScalarNode(label, label)\n\t\tvar valueNode *CandidateNode\n\t\tvar err error\n\n\t\tif i == 0 {\n\t\t\tlog.Debugf(\"head comment here\")\n\t\t\tlabelNode.HeadComment = dec.processComment(n.HeadComment)\n\n\t\t}\n\t\tlog.Debugf(\"label=%v, i=%v, keyValuePair.FootComment: %v\", label, i, keyValuePair.FootComment)\n\t\tlabelNode.FootComment = dec.processComment(keyValuePair.FootComment)\n\n\t\tlog.Debug(\"len of children in %v is %v\", label, len(children))\n\t\tif len(children) > 1 {\n\t\t\tvalueNode, err = dec.createSequence(children)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t} else {\n\t\t\t// comment hack for maps of scalars\n\t\t\t// if the value is a scalar, the head comment of the scalar needs to go on the key?\n\t\t\t// add tests for as well as multiple of inputXmlWithComments > yaml\n\t\t\tif len(children[0].Children) == 0 && children[0].HeadComment != \"\" {\n\t\t\t\tif len(children[0].Data) > 0 {\n\n\t\t\t\t\tlog.Debug(\"scalar comment hack, currentlabel [%v]\", labelNode.HeadComment)\n\t\t\t\t\tlabelNode.HeadComment = joinComments([]string{labelNode.HeadComment, strings.TrimSpace(children[0].HeadComment)}, \"\\n\")\n\t\t\t\t\tchildren[0].HeadComment = \"\"\n\t\t\t\t} else {\n\t\t\t\t\t// child is null, put the headComment as a linecomment for reasons\n\t\t\t\t\tchildren[0].LineComment = children[0].HeadComment\n\t\t\t\t\tchildren[0].HeadComment = \"\"\n\t\t\t\t}\n\t\t\t}\n\t\t\tvalueNode, err = dec.convertToYamlNode(children[0])\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t}\n\t\tyamlNode.AddKeyValueChild(labelNode, valueNode)\n\t}\n\n\treturn yamlNode, nil\n}\n\nfunc (dec *xmlDecoder) createValueNodeFromData(values []string) *CandidateNode {\n\tswitch len(values) {\n\tcase 0:\n\t\treturn createScalarNode(nil, \"\")\n\tcase 1:\n\t\treturn createScalarNode(values[0], values[0])\n\tdefault:\n\t\tcontent := make([]*CandidateNode, 0)\n\t\tfor _, value := range values {\n\t\t\tcontent = append(content, createScalarNode(value, value))\n\t\t}\n\t\treturn &CandidateNode{\n\t\t\tKind: SequenceNode,\n\t\t\tTag: \"!!seq\",\n\t\t\tContent: content,\n\t\t}\n\t}\n}\n\nfunc (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*CandidateNode, error) {\n\tif len(n.Children) > 0 {\n\t\treturn dec.createMap(n)\n\t}\n\n\tscalar := dec.createValueNodeFromData(n.Data)\n\n\tlog.Debug(\"scalar (%v), headC: %v, lineC: %v, footC: %v\", scalar.Tag, n.HeadComment, n.LineComment, n.FootComment)\n\tscalar.HeadComment = dec.processComment(n.HeadComment)\n\tscalar.LineComment = dec.processComment(n.LineComment)\n\tif scalar.Tag == \"!!seq\" {\n\t\tscalar.Content[0].HeadComment = scalar.LineComment\n\t\tscalar.LineComment = \"\"\n\t}\n\n\tscalar.FootComment = dec.processComment(n.FootComment)\n\n\treturn scalar, nil\n}\n\nfunc (dec *xmlDecoder) Decode() (*CandidateNode, error) {\n\tif dec.finished {\n\t\treturn nil, io.EOF\n\t}\n\troot := &xmlNode{}\n\t// cant use xj - it doesn't keep map order.\n\terr := dec.decodeXML(root)\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tfirstNode, err := dec.convertToYamlNode(root)\n\n\tif err != nil {\n\t\treturn nil, err\n\t} else if firstNode.Tag == \"!!null\" {\n\t\tdec.finished = true\n\t\tif dec.readAnything {\n\t\t\treturn nil, io.EOF\n\t\t}\n\t}\n\tdec.readAnything = true\n\tdec.finished = true\n\n\treturn firstNode, nil\n}\n\ntype xmlNode struct {\n\tChildren []*xmlChildrenKv\n\tHeadComment string\n\tFootComment string\n\tLineComment string\n\tData []string\n}\n\ntype xmlChildrenKv struct {\n\tK string\n\tV []*xmlNode\n\tFootComment string\n}\n\n// AddChild appends a node to the list of children\nfunc (n *xmlNode) AddChild(s string, c *xmlNode) {\n\n\tif n.Children == nil {\n\t\tn.Children = make([]*xmlChildrenKv, 0)\n\t}\n\tlog.Debug(\"looking for %s\", s)\n\t// see if we can find an existing entry to add to\n\tfor _, childEntry := range n.Children {\n\t\tif childEntry.K == s {\n\t\t\tlog.Debug(\"found it, appending an entry%s\", s)\n\t\t\tchildEntry.V = append(childEntry.V, c)\n\t\t\tlog.Debug(\"yay len of children in %v is %v\", s, len(childEntry.V))\n\t\t\treturn\n\t\t}\n\t}\n\tlog.Debug(\"not there, making a new one %s\", s)\n\tn.Children = append(n.Children, &xmlChildrenKv{K: s, V: []*xmlNode{c}})\n}\n\ntype element struct {\n\tparent *element\n\tn *xmlNode\n\tlabel string\n\tstate string\n}\n\n// this code is heavily based on https://github.com/basgys/goxml2json\n// main changes are to decode into a structure that preserves the original order\n// of the map keys.\nfunc (dec *xmlDecoder) decodeXML(root *xmlNode) error {\n\txmlDec := xml.NewDecoder(dec.reader)\n\txmlDec.Strict = dec.prefs.StrictMode\n\t// That will convert the charset if the provided XML is non-UTF-8\n\txmlDec.CharsetReader = charset.NewReaderLabel\n\n\tstarted := false\n\n\t// Create first element from the root node\n\telem := &element{\n\t\tparent: nil,\n\t\tn: root,\n\t}\n\n\tgetToken := func() (xml.Token, error) {\n\t\tif dec.prefs.UseRawToken {\n\t\t\treturn xmlDec.RawToken()\n\t\t}\n\t\treturn xmlDec.Token()\n\t}\n\n\tfor {\n\t\tt, e := getToken()\n\t\tif e != nil && !errors.Is(e, io.EOF) {\n\t\t\treturn e\n\t\t}\n\t\tif t == nil {\n\t\t\tbreak\n\t\t}\n\n\t\tswitch se := t.(type) {\n\t\tcase xml.StartElement:\n\t\t\tlog.Debug(\"start element %v\", se.Name.Local)\n\t\t\telem.state = \"started\"\n\t\t\t// Build new a new current element and link it to its parent\n\t\t\tvar label = se.Name.Local\n\t\t\tif dec.prefs.KeepNamespace {\n\t\t\t\tif se.Name.Space != \"\" {\n\t\t\t\t\tlabel = se.Name.Space + \":\" + se.Name.Local\n\t\t\t\t}\n\t\t\t}\n\t\t\telem = &element{\n\t\t\t\tparent: elem,\n\t\t\t\tn: &xmlNode{},\n\t\t\t\tlabel: label,\n\t\t\t}\n\n\t\t\t// Extract attributes as children\n\t\t\tfor _, a := range se.Attr {\n\t\t\t\tif dec.prefs.KeepNamespace {\n\t\t\t\t\tif a.Name.Space != \"\" {\n\t\t\t\t\t\ta.Name.Local = a.Name.Space + \":\" + a.Name.Local\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\telem.n.AddChild(dec.prefs.AttributePrefix+a.Name.Local, &xmlNode{Data: []string{a.Value}})\n\t\t\t}\n\t\tcase xml.CharData:\n\n\t\t\t// Extract XML data (if any)\n\t\t\tnewBit := trimNonGraphic(string(se))\n\t\t\tif !started && len(newBit) > 0 {\n\t\t\t\treturn fmt.Errorf(\"invalid XML: Encountered chardata [%v] outside of XML node\", newBit)\n\t\t\t}\n\n\t\t\tif len(newBit) > 0 {\n\t\t\t\telem.n.Data = append(elem.n.Data, newBit)\n\t\t\t\telem.state = \"chardata\"\n\t\t\t\tlog.Debug(\"chardata [%v] for %v\", elem.n.Data, elem.label)\n\t\t\t}\n\t\tcase xml.EndElement:\n\t\t\tif elem == nil {\n\t\t\t\tlog.Debug(\"no element, probably bad xml\")\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tlog.Debug(\"end element %v\", elem.label)\n\t\t\telem.state = \"finished\"\n\t\t\t// And add it to its parent list\n\t\t\tif elem.parent != nil {\n\t\t\t\telem.parent.n.AddChild(elem.label, elem.n)\n\t\t\t}\n\n\t\t\t// Then change the current element to its parent\n\t\t\telem = elem.parent\n\t\tcase xml.Comment:\n\n\t\t\tcommentStr := string(xml.CharData(se))\n\t\t\tswitch elem.state {\n\t\t\tcase \"started\":\n\t\t\t\tapplyFootComment(elem, commentStr)\n\n\t\t\tcase \"chardata\":\n\t\t\t\tlog.Debug(\"got a line comment for (%v) %v: [%v]\", elem.state, elem.label, commentStr)\n\t\t\t\telem.n.LineComment = joinComments([]string{elem.n.LineComment, commentStr}, \" \")\n\t\t\tdefault:\n\t\t\t\tlog.Debug(\"got a head comment for (%v) %v: [%v]\", elem.state, elem.label, commentStr)\n\t\t\t\telem.n.HeadComment = joinComments([]string{elem.n.HeadComment, commentStr}, \" \")\n\t\t\t}\n\n\t\tcase xml.ProcInst:\n\t\t\tif !dec.prefs.SkipProcInst {\n\t\t\t\telem.n.AddChild(dec.prefs.ProcInstPrefix+se.Target, &xmlNode{Data: []string{string(se.Inst)}})\n\t\t\t}\n\t\tcase xml.Directive:\n\t\t\tif !dec.prefs.SkipDirectives {\n\t\t\t\telem.n.AddChild(dec.prefs.DirectiveName, &xmlNode{Data: []string{string(se)}})\n\t\t\t}\n\t\t}\n\t\tstarted = true\n\t}\n\n\treturn nil\n}\n\nfunc applyFootComment(elem *element, commentStr string) {\n\n\t// first lets try to put the comment on the last child\n\tif len(elem.n.Children) > 0 {\n\t\tlastChildIndex := len(elem.n.Children) - 1\n\t\tchildKv := elem.n.Children[lastChildIndex]\n\t\tlog.Debug(\"got a foot comment, putting on last child for %v: [%v]\", childKv.K, commentStr)\n\t\t// if it's an array of scalars, put the foot comment on the scalar itself\n\t\tif len(childKv.V) > 0 && len(childKv.V[0].Children) == 0 {\n\t\t\tnodeToUpdate := childKv.V[len(childKv.V)-1]\n\t\t\tnodeToUpdate.FootComment = joinComments([]string{nodeToUpdate.FootComment, commentStr}, \" \")\n\t\t} else {\n\t\t\tchildKv.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, \" \")\n\t\t}\n\t} else {\n\t\tlog.Debug(\"got a foot comment for %v: [%v]\", elem.label, commentStr)\n\t\telem.n.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, \" \")\n\t}\n}\n\nfunc joinComments(rawStrings []string, joinStr string) string {\n\tstringsToJoin := make([]string, 0)\n\tfor _, str := range rawStrings {\n\t\tif str != \"\" {\n\t\t\tstringsToJoin = append(stringsToJoin, str)\n\t\t}\n\t}\n\treturn strings.Join(stringsToJoin, joinStr)\n}\n\n// trimNonGraphic returns a slice of the string s, with all leading and trailing\n// non graphic characters and spaces removed.\n//\n// Graphic characters include letters, marks, numbers, punctuation, symbols,\n// and spaces, from categories L, M, N, P, S, Zs.\n// Spacing characters are set by category Z and property Pattern_White_Space.\nfunc trimNonGraphic(s string) string {\n\tif s == \"\" {\n\t\treturn s\n\t}\n\n\tvar first *int\n\tvar last int\n\tfor i, r := range []rune(s) {\n\t\tif !unicode.IsGraphic(r) || unicode.IsSpace(r) {\n\t\t\tcontinue\n\t\t}\n\n\t\tif first == nil {\n\t\t\tf := i // copy i\n\t\t\tfirst = &f\n\t\t\tlast = i\n\t\t} else {\n\t\t\tlast = i\n\t\t}\n\t}\n\n\t// If first is nil, it means there are no graphic characters\n\tif first == nil {\n\t\treturn \"\"\n\t}\n\n\treturn string([]rune(s)[*first : last+1])\n}\n" }, { "path": "pkg/yqlib/decoder_yaml.go", "content": "package yqlib\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"errors\"\n\t\"io\"\n\t\"regexp\"\n\t\"strings\"\n\n\tyaml \"go.yaml.in/yaml/v4\"\n)\n\nvar (\n\tcommentLineRe = regexp.MustCompile(`^\\s*#`)\n\tyamlDirectiveLineRe = regexp.MustCompile(`^\\s*%YAML`)\n\tseparatorLineRe = regexp.MustCompile(`^\\s*---\\s*$`)\n\tseparatorPrefixRe = regexp.MustCompile(`^\\s*---\\s+`)\n)\n\ntype yamlDecoder struct {\n\tdecoder yaml.Decoder\n\n\tprefs YamlPreferences\n\n\t// work around of various parsing issues by yaml.v3 with document headers\n\tleadingContent string\n\tbufferRead bytes.Buffer\n\n\t// anchor map persists over multiple documents for convenience.\n\tanchorMap map[string]*CandidateNode\n\n\treadAnything bool\n\tfirstFile bool\n\tdocumentIndex uint\n}\n\nfunc NewYamlDecoder(prefs YamlPreferences) Decoder {\n\treturn &yamlDecoder{prefs: prefs, firstFile: true}\n}\n\nfunc (dec *yamlDecoder) processReadStream(reader *bufio.Reader) (io.Reader, string, error) {\n\tvar sb strings.Builder\n\n\tfor {\n\t\tline, err := reader.ReadString('\\n')\n\t\tif errors.Is(err, io.EOF) && line == \"\" {\n\t\t\t// no more data\n\t\t\treturn reader, sb.String(), nil\n\t\t}\n\t\tif err != nil && !errors.Is(err, io.EOF) {\n\t\t\treturn reader, sb.String(), err\n\t\t}\n\n\t\t// Determine newline style and strip it for inspection\n\t\tnewline := \"\"\n\t\tif strings.HasSuffix(line, \"\\r\\n\") {\n\t\t\tnewline = \"\\r\\n\"\n\t\t\tline = strings.TrimSuffix(line, \"\\r\\n\")\n\t\t} else if strings.HasSuffix(line, \"\\n\") {\n\t\t\tnewline = \"\\n\"\n\t\t\tline = strings.TrimSuffix(line, \"\\n\")\n\t\t}\n\n\t\ttrimmed := strings.TrimSpace(line)\n\n\t\t// Document separator: exact line '---' or a '--- ' prefix followed by content\n\t\tif separatorLineRe.MatchString(trimmed) {\n\t\t\tsb.WriteString(\"$yqDocSeparator$\")\n\t\t\tsb.WriteString(newline)\n\t\t\tif errors.Is(err, io.EOF) {\n\t\t\t\treturn reader, sb.String(), nil\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\t\t// Handle lines that start with '--- ' followed by more content (e.g. '--- cat')\n\t\tif separatorPrefixRe.MatchString(line) {\n\t\t\tmatch := separatorPrefixRe.FindString(line)\n\t\t\tremainder := line[len(match):]\n\t\t\t// normalise separator newline: if original had none, default to LF\n\t\t\tsepNewline := newline\n\t\t\tif sepNewline == \"\" {\n\t\t\t\tsepNewline = \"\\n\"\n\t\t\t}\n\t\t\tsb.WriteString(\"$yqDocSeparator$\")\n\t\t\tsb.WriteString(sepNewline)\n\t\t\t// push the remainder back onto the reader and continue processing\n\t\t\treader = bufio.NewReader(io.MultiReader(strings.NewReader(remainder), reader))\n\t\t\tif errors.Is(err, io.EOF) && remainder == \"\" {\n\t\t\t\treturn reader, sb.String(), nil\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\t\t// Comments, YAML directives, and blank lines are leading content\n\t\tif commentLineRe.MatchString(line) || yamlDirectiveLineRe.MatchString(line) || trimmed == \"\" {\n\t\t\tsb.WriteString(line)\n\t\t\tsb.WriteString(newline)\n\t\t\tif errors.Is(err, io.EOF) {\n\t\t\t\treturn reader, sb.String(), nil\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\t\t// First non-leading line: push it back onto a reader and return\n\t\toriginalLine := line + newline\n\t\treturn io.MultiReader(strings.NewReader(originalLine), reader), sb.String(), nil\n\t}\n}\n\nfunc (dec *yamlDecoder) Init(reader io.Reader) error {\n\treaderToUse := reader\n\tleadingContent := \"\"\n\tdec.bufferRead = bytes.Buffer{}\n\tvar err error\n\t// if we 'evaluating together' - we only process the leading content\n\t// of the first file - this ensures comments from subsequent files are\n\t// merged together correctly.\n\tif dec.prefs.LeadingContentPreProcessing && (!dec.prefs.EvaluateTogether || dec.firstFile) {\n\t\treaderToUse, leadingContent, err = dec.processReadStream(bufio.NewReader(reader))\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t} else if !dec.prefs.LeadingContentPreProcessing {\n\t\t// if we're not process the leading content\n\t\t// keep a copy of what we've read. This is incase its a\n\t\t// doc with only comments - the decoder will return nothing\n\t\t// then we can read the comments from bufferRead\n\t\treaderToUse = io.TeeReader(reader, &dec.bufferRead)\n\t}\n\tdec.leadingContent = leadingContent\n\tdec.readAnything = false\n\tdec.decoder = *yaml.NewDecoder(readerToUse)\n\tdec.firstFile = false\n\tdec.documentIndex = 0\n\tdec.anchorMap = make(map[string]*CandidateNode)\n\treturn nil\n}\n\nfunc (dec *yamlDecoder) Decode() (*CandidateNode, error) {\n\tvar yamlNode yaml.Node\n\terr := dec.decoder.Decode(&yamlNode)\n\n\tif errors.Is(err, io.EOF) && dec.leadingContent != \"\" && !dec.readAnything {\n\t\t// force returning an empty node with a comment.\n\t\tdec.readAnything = true\n\t\treturn dec.blankNodeWithComment(), nil\n\t} else if errors.Is(err, io.EOF) && !dec.prefs.LeadingContentPreProcessing && !dec.readAnything {\n\t\t// didn't find any yaml,\n\t\t// check the tee buffer, maybe there were comments\n\t\tdec.readAnything = true\n\t\tdec.leadingContent = dec.bufferRead.String()\n\t\tif dec.leadingContent != \"\" {\n\t\t\treturn dec.blankNodeWithComment(), nil\n\t\t}\n\t\treturn nil, err\n\t} else if err != nil {\n\t\treturn nil, err\n\t} else if len(yamlNode.Content) == 0 {\n\t\treturn nil, errors.New(\"yaml node has no content\")\n\t}\n\n\tcandidateNode := CandidateNode{document: dec.documentIndex}\n\t// don't bother with the DocumentNode\n\terr = candidateNode.UnmarshalYAML(yamlNode.Content[0], dec.anchorMap)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tcandidateNode.HeadComment = yamlNode.HeadComment + candidateNode.HeadComment\n\tcandidateNode.FootComment = yamlNode.FootComment + candidateNode.FootComment\n\n\tif dec.leadingContent != \"\" {\n\t\tcandidateNode.LeadingContent = dec.leadingContent\n\t\tdec.leadingContent = \"\"\n\t}\n\tdec.readAnything = true\n\tdec.documentIndex++\n\treturn &candidateNode, nil\n}\n\nfunc (dec *yamlDecoder) blankNodeWithComment() *CandidateNode {\n\tnode := createScalarNode(nil, \"\")\n\tnode.LeadingContent = dec.leadingContent\n\treturn node\n}\n" }, { "path": "pkg/yqlib/doc/.gitignore", "content": "*.zip\n" }, { "path": "pkg/yqlib/doc/notification-snippet.md", "content": "" }, { "path": "pkg/yqlib/doc/operators/add.md", "content": "# Add\n\nAdd behaves differently according to the type of the LHS:\n* arrays: concatenate\n* number scalars: arithmetic addition\n* string scalars: concatenate\n* maps: shallow merge (use the multiply operator (`*`) to deeply merge)\n\nUse `+=` as a relative append assign for things like increment. Note that `.a += .x` is equivalent to running `.a = .a + .x`.\n\n\n## Concatenate arrays\nGiven a sample.yml file of:\n```yaml\na:\n - 1\n - 2\nb:\n - 3\n - 4\n```\nthen\n```bash\nyq '.a + .b' sample.yml\n```\nwill output\n```yaml\n- 1\n- 2\n- 3\n- 4\n```\n\n## Concatenate to existing array\nNote that the styling of `a` is kept.\n\nGiven a sample.yml file of:\n```yaml\na: [1,2]\nb:\n - 3\n - 4\n```\nthen\n```bash\nyq '.a += .b' sample.yml\n```\nwill output\n```yaml\na: [1, 2, 3, 4]\nb:\n - 3\n - 4\n```\n\n## Concatenate null to array\nGiven a sample.yml file of:\n```yaml\na:\n - 1\n - 2\n```\nthen\n```bash\nyq '.a + null' sample.yml\n```\nwill output\n```yaml\n- 1\n- 2\n```\n\n## Append to existing array\nNote that the styling is copied from existing array elements\n\nGiven a sample.yml file of:\n```yaml\na: ['dog']\n```\nthen\n```bash\nyq '.a += \"cat\"' sample.yml\n```\nwill output\n```yaml\na: ['dog', 'cat']\n```\n\n## Prepend to existing array\nGiven a sample.yml file of:\n```yaml\na:\n - dog\n```\nthen\n```bash\nyq '.a = [\"cat\"] + .a' sample.yml\n```\nwill output\n```yaml\na:\n - cat\n - dog\n```\n\n## Add new object to array\nGiven a sample.yml file of:\n```yaml\na:\n - dog: woof\n```\nthen\n```bash\nyq '.a + {\"cat\": \"meow\"}' sample.yml\n```\nwill output\n```yaml\n- dog: woof\n- cat: meow\n```\n\n## Relative append\nGiven a sample.yml file of:\n```yaml\na:\n a1:\n b:\n - cat\n a2:\n b:\n - dog\n a3: {}\n```\nthen\n```bash\nyq '.a[].b += [\"mouse\"]' sample.yml\n```\nwill output\n```yaml\na:\n a1:\n b:\n - cat\n - mouse\n a2:\n b:\n - dog\n - mouse\n a3:\n b:\n - mouse\n```\n\n## String concatenation\nGiven a sample.yml file of:\n```yaml\na: cat\nb: meow\n```\nthen\n```bash\nyq '.a += .b' sample.yml\n```\nwill output\n```yaml\na: catmeow\nb: meow\n```\n\n## Number addition - float\nIf the lhs or rhs are floats then the expression will be calculated with floats.\n\nGiven a sample.yml file of:\n```yaml\na: 3\nb: 4.9\n```\nthen\n```bash\nyq '.a = .a + .b' sample.yml\n```\nwill output\n```yaml\na: 7.9\nb: 4.9\n```\n\n## Number addition - int\nIf both the lhs and rhs are ints then the expression will be calculated with ints.\n\nGiven a sample.yml file of:\n```yaml\na: 3\nb: 4\n```\nthen\n```bash\nyq '.a = .a + .b' sample.yml\n```\nwill output\n```yaml\na: 7\nb: 4\n```\n\n## Increment numbers\nGiven a sample.yml file of:\n```yaml\na: 3\nb: 5\n```\nthen\n```bash\nyq '.[] += 1' sample.yml\n```\nwill output\n```yaml\na: 4\nb: 6\n```\n\n## Date addition\nYou can add durations to dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.\n\nGiven a sample.yml file of:\n```yaml\na: 2021-01-01T00:00:00Z\n```\nthen\n```bash\nyq '.a += \"3h10m\"' sample.yml\n```\nwill output\n```yaml\na: 2021-01-01T03:10:00Z\n```\n\n## Date addition - custom format\nYou can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.\n\nGiven a sample.yml file of:\n```yaml\na: Saturday, 15-Dec-01 at 2:59AM GMT\n```\nthen\n```bash\nyq 'with_dtf(\"Monday, 02-Jan-06 at 3:04PM MST\", .a += \"3h1m\")' sample.yml\n```\nwill output\n```yaml\na: Saturday, 15-Dec-01 at 6:00AM GMT\n```\n\n## Add to null\nAdding to null simply returns the rhs\n\nRunning\n```bash\nyq --null-input 'null + \"cat\"'\n```\nwill output\n```yaml\ncat\n```\n\n## Add maps to shallow merge\nAdding objects together shallow merges them. Use `*` to deeply merge.\n\nGiven a sample.yml file of:\n```yaml\na:\n thing:\n name: Astuff\n value: x\n a1: cool\nb:\n thing:\n name: Bstuff\n legs: 3\n b1: neat\n```\nthen\n```bash\nyq '.a += .b' sample.yml\n```\nwill output\n```yaml\na:\n thing:\n name: Bstuff\n legs: 3\n a1: cool\n b1: neat\nb:\n thing:\n name: Bstuff\n legs: 3\n b1: neat\n```\n\n## Custom types: that are really strings\nWhen custom tags are encountered, yq will try to decode the underlying type.\n\nGiven a sample.yml file of:\n```yaml\na: !horse cat\nb: !goat _meow\n```\nthen\n```bash\nyq '.a += .b' sample.yml\n```\nwill output\n```yaml\na: !horse cat_meow\nb: !goat _meow\n```\n\n## Custom types: that are really numbers\nWhen custom tags are encountered, yq will try to decode the underlying type.\n\nGiven a sample.yml file of:\n```yaml\na: !horse 1.2\nb: !goat 2.3\n```\nthen\n```bash\nyq '.a += .b' sample.yml\n```\nwill output\n```yaml\na: !horse 3.5\nb: !goat 2.3\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/alternative-default-value.md", "content": "# Alternative (Default value)\n\nThis operator is used to provide alternative (or default) values when a particular expression is either null or false.\n\n## LHS is defined\nGiven a sample.yml file of:\n```yaml\na: bridge\n```\nthen\n```bash\nyq '.a // \"hello\"' sample.yml\n```\nwill output\n```yaml\nbridge\n```\n\n## LHS is not defined\nGiven a sample.yml file of:\n```yaml\n{}\n```\nthen\n```bash\nyq '.a // \"hello\"' sample.yml\n```\nwill output\n```yaml\nhello\n```\n\n## LHS is null\nGiven a sample.yml file of:\n```yaml\na: ~\n```\nthen\n```bash\nyq '.a // \"hello\"' sample.yml\n```\nwill output\n```yaml\nhello\n```\n\n## LHS is false\nGiven a sample.yml file of:\n```yaml\na: false\n```\nthen\n```bash\nyq '.a // \"hello\"' sample.yml\n```\nwill output\n```yaml\nhello\n```\n\n## RHS is an expression\nGiven a sample.yml file of:\n```yaml\na: false\nb: cat\n```\nthen\n```bash\nyq '.a // .b' sample.yml\n```\nwill output\n```yaml\ncat\n```\n\n## Update or create - entity exists\nThis initialises `a` if it's not present\n\nGiven a sample.yml file of:\n```yaml\na: 1\n```\nthen\n```bash\nyq '(.a // (.a = 0)) += 1' sample.yml\n```\nwill output\n```yaml\na: 2\n```\n\n## Update or create - entity does not exist\nThis initialises `a` if it's not present\n\nGiven a sample.yml file of:\n```yaml\nb: camel\n```\nthen\n```bash\nyq '(.a // (.a = 0)) += 1' sample.yml\n```\nwill output\n```yaml\nb: camel\na: 1\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/anchor-and-alias-operators.md", "content": "# Anchor and Alias Operators\n\nUse the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference (or expands) aliases and remove anchor names).\n\n`yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.\n\n\n## NOTE --yaml-fix-merge-anchor-to-spec flag\n`yq` doesn't merge anchors `<<:` to spec, in some circumstances it incorrectly overrides existing keys when the spec documents not to do that.\n\nTo minimise disruption while still fixing the issue, a flag has been added to toggle this behaviour. This will first default to false; and log warnings to users. Then it will default to true (and still allow users to specify false if needed).\n\nThis flag also enables advanced merging, like inline maps, as well as fixes to ensure when exploding a particular path, neighbours are not affect ed.\n\nLong story short, you should be setting this flag to true.\n\nSee examples of the flag differences below, where LEGACY is with the flag off; and FIXED is with the flag on.\n\n\n## Merge one map\nsee https://yaml.org/type/merge.html\n\nGiven a sample.yml file of:\n```yaml\n- &CENTRE\n x: 1\n y: 2\n- &LEFT\n x: 0\n y: 2\n- &BIG\n r: 10\n- &SMALL\n r: 1\n- !!merge <<: *CENTRE\n r: 10\n```\nthen\n```bash\nyq '.[4] | explode(.)' sample.yml\n```\nwill output\n```yaml\nx: 1\ny: 2\nr: 10\n```\n\n## Get anchor\nGiven a sample.yml file of:\n```yaml\na: &billyBob cat\n```\nthen\n```bash\nyq '.a | anchor' sample.yml\n```\nwill output\n```yaml\nbillyBob\n```\n\n## Set anchor\nGiven a sample.yml file of:\n```yaml\na: cat\n```\nthen\n```bash\nyq '.a anchor = \"foobar\"' sample.yml\n```\nwill output\n```yaml\na: &foobar cat\n```\n\n## Set anchor relatively using assign-update\nGiven a sample.yml file of:\n```yaml\na:\n b: cat\n```\nthen\n```bash\nyq '.a anchor |= .b' sample.yml\n```\nwill output\n```yaml\na: &cat\n b: cat\n```\n\n## Get alias\nGiven a sample.yml file of:\n```yaml\nb: &billyBob meow\na: *billyBob\n```\nthen\n```bash\nyq '.a | alias' sample.yml\n```\nwill output\n```yaml\nbillyBob\n```\n\n## Set alias\nGiven a sample.yml file of:\n```yaml\nb: &meow purr\na: cat\n```\nthen\n```bash\nyq '.a alias = \"meow\"' sample.yml\n```\nwill output\n```yaml\nb: &meow purr\na: *meow\n```\n\n## Set alias to blank does nothing\nGiven a sample.yml file of:\n```yaml\nb: &meow purr\na: cat\n```\nthen\n```bash\nyq '.a alias = \"\"' sample.yml\n```\nwill output\n```yaml\nb: &meow purr\na: cat\n```\n\n## Set alias relatively using assign-update\nGiven a sample.yml file of:\n```yaml\nb: &meow purr\na:\n f: meow\n```\nthen\n```bash\nyq '.a alias |= .f' sample.yml\n```\nwill output\n```yaml\nb: &meow purr\na: *meow\n```\n\n## Explode alias and anchor\nGiven a sample.yml file of:\n```yaml\nf:\n a: &a cat\n b: *a\n```\nthen\n```bash\nyq 'explode(.f)' sample.yml\n```\nwill output\n```yaml\nf:\n a: cat\n b: cat\n```\n\n## Explode with no aliases or anchors\nGiven a sample.yml file of:\n```yaml\na: mike\n```\nthen\n```bash\nyq 'explode(.a)' sample.yml\n```\nwill output\n```yaml\na: mike\n```\n\n## Explode with alias keys\nGiven a sample.yml file of:\n```yaml\nf:\n a: &a cat\n *a : b\n```\nthen\n```bash\nyq 'explode(.f)' sample.yml\n```\nwill output\n```yaml\nf:\n a: cat\n cat: b\n```\n\n## Dereference and update a field\nUse explode with multiply to dereference an object\n\nGiven a sample.yml file of:\n```yaml\nitem_value: &item_value\n value: true\nthingOne:\n name: item_1\n !!merge <<: *item_value\nthingTwo:\n name: item_2\n !!merge <<: *item_value\n```\nthen\n```bash\nyq '.thingOne |= (explode(.) | sort_keys(.)) * {\"value\": false}' sample.yml\n```\nwill output\n```yaml\nitem_value: &item_value\n value: true\nthingOne:\n name: item_1\n value: false\nthingTwo:\n name: item_2\n !!merge <<: *item_value\n```\n\n## LEGACY: Explode with merge anchors\nCaution: this is for when --yaml-fix-merge-anchor-to-spec=false; it's not to YAML spec because the merge anchors incorrectly override the object values (foobarList.b is set to bar_b when it should still be foobarList_b). Flag will default to true in late 2025\n\nGiven a sample.yml file of:\n```yaml\nfoo: &foo\n a: foo_a\n thing: foo_thing\n c: foo_c\nbar: &bar\n b: bar_b\n thing: bar_thing\n c: bar_c\nfoobarList:\n b: foobarList_b\n !!merge <<:\n - *foo\n - *bar\n c: foobarList_c\nfoobar:\n c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n```\nthen\n```bash\nyq 'explode(.)' sample.yml\n```\nwill output\n```yaml\nfoo:\n a: foo_a\n thing: foo_thing\n c: foo_c\nbar:\n b: bar_b\n thing: bar_thing\n c: bar_c\nfoobarList:\n b: bar_b\n thing: foo_thing\n c: foobarList_c\n a: foo_a\nfoobar:\n c: foo_c\n a: foo_a\n thing: foobar_thing\n```\n\n## LEGACY: Merge multiple maps\nsee https://yaml.org/type/merge.html. This has the correct data, but the wrong key order; set --yaml-fix-merge-anchor-to-spec=true to fix the key order.\n\nGiven a sample.yml file of:\n```yaml\n- &CENTRE\n x: 1\n y: 2\n- &LEFT\n x: 0\n y: 2\n- &BIG\n r: 10\n- &SMALL\n r: 1\n- !!merge <<:\n - *CENTRE\n - *BIG\n```\nthen\n```bash\nyq '.[4] | explode(.)' sample.yml\n```\nwill output\n```yaml\nr: 10\nx: 1\ny: 2\n```\n\n## LEGACY: Override\nsee https://yaml.org/type/merge.html. This has the correct data, but the wrong key order; set --yaml-fix-merge-anchor-to-spec=true to fix the key order.\n\nGiven a sample.yml file of:\n```yaml\n- &CENTRE\n x: 1\n y: 2\n- &LEFT\n x: 0\n y: 2\n- &BIG\n r: 10\n- &SMALL\n r: 1\n- !!merge <<:\n - *BIG\n - *LEFT\n - *SMALL\n x: 1\n```\nthen\n```bash\nyq '.[4] | explode(.)' sample.yml\n```\nwill output\n```yaml\nr: 10\nx: 1\ny: 2\n```\n\n## FIXED: Explode with merge anchors\nSet `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour (flag will default to true in late 2025).\nObserve that foobarList.b property is still foobarList_b.\n\nGiven a sample.yml file of:\n```yaml\nfoo: &foo\n a: foo_a\n thing: foo_thing\n c: foo_c\nbar: &bar\n b: bar_b\n thing: bar_thing\n c: bar_c\nfoobarList:\n b: foobarList_b\n !!merge <<:\n - *foo\n - *bar\n c: foobarList_c\nfoobar:\n c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n```\nthen\n```bash\nyq 'explode(.)' sample.yml\n```\nwill output\n```yaml\nfoo:\n a: foo_a\n thing: foo_thing\n c: foo_c\nbar:\n b: bar_b\n thing: bar_thing\n c: bar_c\nfoobarList:\n b: foobarList_b\n a: foo_a\n thing: foo_thing\n c: foobarList_c\nfoobar:\n c: foobar_c\n a: foo_a\n thing: foobar_thing\n```\n\n## FIXED: Merge multiple maps\nSet `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour (flag will default to true in late 2025).\nTaken from https://yaml.org/type/merge.html. Same values as legacy, but with the correct key order.\n\nGiven a sample.yml file of:\n```yaml\n- &CENTRE\n x: 1\n y: 2\n- &LEFT\n x: 0\n y: 2\n- &BIG\n r: 10\n- &SMALL\n r: 1\n- !!merge <<:\n - *CENTRE\n - *BIG\n```\nthen\n```bash\nyq '.[4] | explode(.)' sample.yml\n```\nwill output\n```yaml\nx: 1\ny: 2\nr: 10\n```\n\n## FIXED: Override\nSet `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour (flag will default to true in late 2025).\nTaken from https://yaml.org/type/merge.html. Same values as legacy, but with the correct key order.\n\nGiven a sample.yml file of:\n```yaml\n- &CENTRE\n x: 1\n y: 2\n- &LEFT\n x: 0\n y: 2\n- &BIG\n r: 10\n- &SMALL\n r: 1\n- !!merge <<:\n - *BIG\n - *LEFT\n - *SMALL\n x: 1\n```\nthen\n```bash\nyq '.[4] | explode(.)' sample.yml\n```\nwill output\n```yaml\nr: 10\ny: 2\nx: 1\n```\n\n## Exploding inline merge anchor\nSet `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour (flag will default to true in late 2025).\n\n\nGiven a sample.yml file of:\n```yaml\na:\n b: &b 42\n!!merge <<:\n c: *b\n```\nthen\n```bash\nyq 'explode(.) | sort_keys(.)' sample.yml\n```\nwill output\n```yaml\na:\n b: 42\nc: 42\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/array-to-map.md", "content": "# Array to Map\n\nUse this operator to convert an array to..a map. The indices are used as map keys, null values in the array are skipped over.\n\nBehind the scenes, this is implemented using reduce:\n\n```\n(.[] | select(. != null) ) as $i ireduce({}; .[$i | key] = $i)\n```\n\n## Simple example\nGiven a sample.yml file of:\n```yaml\ncool:\n - null\n - null\n - hello\n```\nthen\n```bash\nyq '.cool |= array_to_map' sample.yml\n```\nwill output\n```yaml\ncool:\n 2: hello\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/assign-update.md", "content": "# Assign (Update)\n\nThis operator is used to update node values. It can be used in either the:\n\n### plain form: `=`\nWhich will set the LHS node values equal to the RHS node values. The RHS expression is run against the matching nodes in the pipeline.\n\n### relative form: `|=`\nThis will do a similar thing to the plain form, but the RHS expression is run with _each LHS node as context_. This is useful for updating values based on old values, e.g. increment.\n\n\n### Flags\n- `c` clobber custom tags\n\n## Create yaml file\nRunning\n```bash\nyq --null-input '.a.b = \"cat\" | .x = \"frog\"'\n```\nwill output\n```yaml\na:\n b: cat\nx: frog\n```\n\n## Update node to be the child value\nGiven a sample.yml file of:\n```yaml\na:\n b:\n g: foof\n```\nthen\n```bash\nyq '.a |= .b' sample.yml\n```\nwill output\n```yaml\na:\n g: foof\n```\n\n## Double elements in an array\nGiven a sample.yml file of:\n```yaml\n- 1\n- 2\n- 3\n```\nthen\n```bash\nyq '.[] |= . * 2' sample.yml\n```\nwill output\n```yaml\n- 2\n- 4\n- 6\n```\n\n## Update node from another file\nNote this will also work when the second file is a scalar (string/number)\n\nGiven a sample.yml file of:\n```yaml\na: apples\n```\nAnd another sample another.yml file of:\n```yaml\nb: bob\n```\nthen\n```bash\nyq eval-all 'select(fileIndex==0).a = select(fileIndex==1) | select(fileIndex==0)' sample.yml another.yml\n```\nwill output\n```yaml\na:\n b: bob\n```\n\n## Update node to be the sibling value\nGiven a sample.yml file of:\n```yaml\na:\n b: child\nb: sibling\n```\nthen\n```bash\nyq '.a = .b' sample.yml\n```\nwill output\n```yaml\na: sibling\nb: sibling\n```\n\n## Updated multiple paths\nGiven a sample.yml file of:\n```yaml\na: fieldA\nb: fieldB\nc: fieldC\n```\nthen\n```bash\nyq '(.a, .c) = \"potato\"' sample.yml\n```\nwill output\n```yaml\na: potato\nb: fieldB\nc: potato\n```\n\n## Update string value\nGiven a sample.yml file of:\n```yaml\na:\n b: apple\n```\nthen\n```bash\nyq '.a.b = \"frog\"' sample.yml\n```\nwill output\n```yaml\na:\n b: frog\n```\n\n## Update string value via |=\nNote there is no difference between `=` and `|=` when the RHS is a scalar\n\nGiven a sample.yml file of:\n```yaml\na:\n b: apple\n```\nthen\n```bash\nyq '.a.b |= \"frog\"' sample.yml\n```\nwill output\n```yaml\na:\n b: frog\n```\n\n## Update deeply selected results\nNote that the LHS is wrapped in brackets! This is to ensure we don't first filter out the yaml and then update the snippet.\n\nGiven a sample.yml file of:\n```yaml\na:\n b: apple\n c: cactus\n```\nthen\n```bash\nyq '(.a[] | select(. == \"apple\")) = \"frog\"' sample.yml\n```\nwill output\n```yaml\na:\n b: frog\n c: cactus\n```\n\n## Update array values\nGiven a sample.yml file of:\n```yaml\n- candy\n- apple\n- sandy\n```\nthen\n```bash\nyq '(.[] | select(. == \"*andy\")) = \"bogs\"' sample.yml\n```\nwill output\n```yaml\n- bogs\n- apple\n- bogs\n```\n\n## Update empty object\nGiven a sample.yml file of:\n```yaml\n{}\n```\nthen\n```bash\nyq '.a.b |= \"bogs\"' sample.yml\n```\nwill output\n```yaml\na:\n b: bogs\n```\n\n## Update node value that has an anchor\nAnchor will remain\n\nGiven a sample.yml file of:\n```yaml\na: &cool cat\n```\nthen\n```bash\nyq '.a = \"dog\"' sample.yml\n```\nwill output\n```yaml\na: &cool dog\n```\n\n## Update empty object and array\nGiven a sample.yml file of:\n```yaml\n{}\n```\nthen\n```bash\nyq '.a.b.[0] |= \"bogs\"' sample.yml\n```\nwill output\n```yaml\na:\n b:\n - bogs\n```\n\n## Custom types are maintained by default\nGiven a sample.yml file of:\n```yaml\na: !cat meow\nb: !dog woof\n```\nthen\n```bash\nyq '.a = .b' sample.yml\n```\nwill output\n```yaml\na: !cat woof\nb: !dog woof\n```\n\n## Custom types: clobber\nUse the `c` option to clobber custom tags\n\nGiven a sample.yml file of:\n```yaml\na: !cat meow\nb: !dog woof\n```\nthen\n```bash\nyq '.a =c .b' sample.yml\n```\nwill output\n```yaml\na: !dog woof\nb: !dog woof\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/boolean-operators.md", "content": "# Boolean Operators\n\nThe `or` and `and` operators take two parameters and return a boolean result. \n\n`not` flips a boolean from true to false, or vice versa. \n\n`any` will return `true` if there are any `true` values in an array sequence, and `all` will return true if _all_ elements in an array are true.\n\n`any_c(condition)` and `all_c(condition)` are like `any` and `all` but they take a condition expression that is used against each element to determine if it's `true`. Note: in `jq` you can simply pass a condition to `any` or `all` and it simply works - `yq` isn't that clever..yet\n\nThese are most commonly used with the `select` operator to filter particular nodes.\n\n## Related Operators\n\n- equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals)\n- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare)\n- select operator [here](https://mikefarah.gitbook.io/yq/operators/select)\n\n## `or` example\nRunning\n```bash\nyq --null-input 'true or false'\n```\nwill output\n```yaml\ntrue\n```\n\n## \"yes\" and \"no\" are strings\nIn the yaml 1.2 standard, support for yes/no as booleans was dropped - they are now considered strings. See '10.2.1.2. Boolean' in https://yaml.org/spec/1.2.2/\n\nGiven a sample.yml file of:\n```yaml\n- yes\n- no\n```\nthen\n```bash\nyq '.[] | tag' sample.yml\n```\nwill output\n```yaml\n!!str\n!!str\n```\n\n## `and` example\nRunning\n```bash\nyq --null-input 'true and false'\n```\nwill output\n```yaml\nfalse\n```\n\n## Matching nodes with select, equals and or\nGiven a sample.yml file of:\n```yaml\n- a: bird\n b: dog\n- a: frog\n b: bird\n- a: cat\n b: fly\n```\nthen\n```bash\nyq '[.[] | select(.a == \"cat\" or .b == \"dog\")]' sample.yml\n```\nwill output\n```yaml\n- a: bird\n b: dog\n- a: cat\n b: fly\n```\n\n## `any` returns true if any boolean in a given array is true\nGiven a sample.yml file of:\n```yaml\n- false\n- true\n```\nthen\n```bash\nyq 'any' sample.yml\n```\nwill output\n```yaml\ntrue\n```\n\n## `any` returns false for an empty array\nGiven a sample.yml file of:\n```yaml\n[]\n```\nthen\n```bash\nyq 'any' sample.yml\n```\nwill output\n```yaml\nfalse\n```\n\n## `any_c` returns true if any element in the array is true for the given condition.\nGiven a sample.yml file of:\n```yaml\na:\n - rad\n - awesome\nb:\n - meh\n - whatever\n```\nthen\n```bash\nyq '.[] |= any_c(. == \"awesome\")' sample.yml\n```\nwill output\n```yaml\na: true\nb: false\n```\n\n## `all` returns true if all booleans in a given array are true\nGiven a sample.yml file of:\n```yaml\n- true\n- true\n```\nthen\n```bash\nyq 'all' sample.yml\n```\nwill output\n```yaml\ntrue\n```\n\n## `all` returns true for an empty array\nGiven a sample.yml file of:\n```yaml\n[]\n```\nthen\n```bash\nyq 'all' sample.yml\n```\nwill output\n```yaml\ntrue\n```\n\n## `all_c` returns true if all elements in the array are true for the given condition.\nGiven a sample.yml file of:\n```yaml\na:\n - rad\n - awesome\nb:\n - meh\n - 12\n```\nthen\n```bash\nyq '.[] |= all_c(tag == \"!!str\")' sample.yml\n```\nwill output\n```yaml\na: true\nb: false\n```\n\n## Not true is false\nRunning\n```bash\nyq --null-input 'true | not'\n```\nwill output\n```yaml\nfalse\n```\n\n## Not false is true\nRunning\n```bash\nyq --null-input 'false | not'\n```\nwill output\n```yaml\ntrue\n```\n\n## String values considered to be true\nRunning\n```bash\nyq --null-input '\"cat\" | not'\n```\nwill output\n```yaml\nfalse\n```\n\n## Empty string value considered to be true\nRunning\n```bash\nyq --null-input '\"\" | not'\n```\nwill output\n```yaml\nfalse\n```\n\n## Numbers are considered to be true\nRunning\n```bash\nyq --null-input '1 | not'\n```\nwill output\n```yaml\nfalse\n```\n\n## Zero is considered to be true\nRunning\n```bash\nyq --null-input '0 | not'\n```\nwill output\n```yaml\nfalse\n```\n\n## Null is considered to be false\nRunning\n```bash\nyq --null-input '~ | not'\n```\nwill output\n```yaml\ntrue\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/collect-into-array.md", "content": "# Collect into Array\n\nThis creates an array using the expression between the square brackets.\n\n\n## Collect empty\nRunning\n```bash\nyq --null-input '[]'\n```\nwill output\n```yaml\n[]\n```\n\n## Collect single\nRunning\n```bash\nyq --null-input '[\"cat\"]'\n```\nwill output\n```yaml\n- cat\n```\n\n## Collect many\nGiven a sample.yml file of:\n```yaml\na: cat\nb: dog\n```\nthen\n```bash\nyq '[.a, .b]' sample.yml\n```\nwill output\n```yaml\n- cat\n- dog\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/column.md", "content": "# Column\n\nReturns the column of the matching node. Starts from 1, 0 indicates there was no column data.\n\nColumn is the number of characters that precede that node on the line it starts.\n\n## Returns column of _value_ node\nGiven a sample.yml file of:\n```yaml\na: cat\nb: bob\n```\nthen\n```bash\nyq '.b | column' sample.yml\n```\nwill output\n```yaml\n4\n```\n\n## Returns column of _key_ node\nPipe through the key operator to get the column of the key\n\nGiven a sample.yml file of:\n```yaml\na: cat\nb: bob\n```\nthen\n```bash\nyq '.b | key | column' sample.yml\n```\nwill output\n```yaml\n1\n```\n\n## First column is 1\nGiven a sample.yml file of:\n```yaml\na: cat\n```\nthen\n```bash\nyq '.a | key | column' sample.yml\n```\nwill output\n```yaml\n1\n```\n\n## No column data is 0\nRunning\n```bash\nyq --null-input '{\"a\": \"new entry\"} | column'\n```\nwill output\n```yaml\n0\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/comment-operators.md", "content": "# Comment Operators\n\nUse these comment operators to set or retrieve comments. Note that line comments on maps/arrays are actually set on the _key_ node as opposed to the _value_ (map/array). See below for examples.\n\nLike the `=` and `|=` assign operators, the same syntax applies when updating comments:\n\n### plain form: `=`\nThis will set the LHS nodes' comments equal to the expression on the RHS. The RHS is run against the matching nodes in the pipeline\n\n### relative form: `|=` \nThis is similar to the plain form, but it evaluates the RHS with _each matching LHS node as context_. This is useful if you want to set the comments as a relative expression of the node, for instance its value or path.\n\n## Set line comment\nSet the comment on the key node for more reliability (see below).\n\nGiven a sample.yml file of:\n```yaml\na: cat\n```\nthen\n```bash\nyq '.a line_comment=\"single\"' sample.yml\n```\nwill output\n```yaml\na: cat # single\n```\n\n## Set line comment of a maps/arrays\nFor maps and arrays, you need to set the line comment on the _key_ node. This will also work for scalars.\n\nGiven a sample.yml file of:\n```yaml\na:\n b: things\n```\nthen\n```bash\nyq '(.a | key) line_comment=\"single\"' sample.yml\n```\nwill output\n```yaml\na: # single\n b: things\n```\n\n## Use update assign to perform relative updates\nGiven a sample.yml file of:\n```yaml\na: cat\nb: dog\n```\nthen\n```bash\nyq '.. line_comment |= .' sample.yml\n```\nwill output\n```yaml\na: cat # cat\nb: dog # dog\n```\n\n## Where is the comment - map key example\nThe underlying yaml parser can assign comments in a document to surprising nodes. Use an expression like this to find where you comment is. 'p' indicates the path, 'isKey' is if the node is a map key (as opposed to a map value).\nFrom this, you can see the 'hello-world-comment' is actually on the 'hello' key\n\nGiven a sample.yml file of:\n```yaml\nhello: # hello-world-comment\n message: world\n```\nthen\n```bash\nyq '[... | {\"p\": path | join(\".\"), \"isKey\": is_key, \"hc\": headComment, \"lc\": lineComment, \"fc\": footComment}]' sample.yml\n```\nwill output\n```yaml\n- p: \"\"\n isKey: false\n hc: \"\"\n lc: \"\"\n fc: \"\"\n- p: hello\n isKey: true\n hc: \"\"\n lc: hello-world-comment\n fc: \"\"\n- p: hello\n isKey: false\n hc: \"\"\n lc: \"\"\n fc: \"\"\n- p: hello.message\n isKey: true\n hc: \"\"\n lc: \"\"\n fc: \"\"\n- p: hello.message\n isKey: false\n hc: \"\"\n lc: \"\"\n fc: \"\"\n```\n\n## Retrieve comment - map key example\nFrom the previous example, we know that the comment is on the 'hello' _key_ as a lineComment\n\nGiven a sample.yml file of:\n```yaml\nhello: # hello-world-comment\n message: world\n```\nthen\n```bash\nyq '.hello | key | line_comment' sample.yml\n```\nwill output\n```yaml\nhello-world-comment\n```\n\n## Where is the comment - array example\nThe underlying yaml parser can assign comments in a document to surprising nodes. Use an expression like this to find where you comment is. 'p' indicates the path, 'isKey' is if the node is a map key (as opposed to a map value).\nFrom this, you can see the 'under-name-comment' is actually on the first child\n\nGiven a sample.yml file of:\n```yaml\nname:\n # under-name-comment\n - first-array-child\n```\nthen\n```bash\nyq '[... | {\"p\": path | join(\".\"), \"isKey\": is_key, \"hc\": headComment, \"lc\": lineComment, \"fc\": footComment}]' sample.yml\n```\nwill output\n```yaml\n- p: \"\"\n isKey: false\n hc: \"\"\n lc: \"\"\n fc: \"\"\n- p: name\n isKey: true\n hc: \"\"\n lc: \"\"\n fc: \"\"\n- p: name\n isKey: false\n hc: \"\"\n lc: \"\"\n fc: \"\"\n- p: name.0\n isKey: false\n hc: under-name-comment\n lc: \"\"\n fc: \"\"\n```\n\n## Retrieve comment - array example\nFrom the previous example, we know that the comment is on the first child as a headComment\n\nGiven a sample.yml file of:\n```yaml\nname:\n # under-name-comment\n - first-array-child\n```\nthen\n```bash\nyq '.name[0] | headComment' sample.yml\n```\nwill output\n```yaml\nunder-name-comment\n```\n\n## Set head comment\nGiven a sample.yml file of:\n```yaml\na: cat\n```\nthen\n```bash\nyq '. head_comment=\"single\"' sample.yml\n```\nwill output\n```yaml\n# single\na: cat\n```\n\n## Set head comment of a map entry\nGiven a sample.yml file of:\n```yaml\nf: foo\na:\n b: cat\n```\nthen\n```bash\nyq '(.a | key) head_comment=\"single\"' sample.yml\n```\nwill output\n```yaml\nf: foo\n# single\na:\n b: cat\n```\n\n## Set foot comment, using an expression\nGiven a sample.yml file of:\n```yaml\na: cat\n```\nthen\n```bash\nyq '. foot_comment=.a' sample.yml\n```\nwill output\n```yaml\na: cat\n# cat\n```\n\n## Remove comment\nGiven a sample.yml file of:\n```yaml\na: cat # comment\nb: dog # leave this\n```\nthen\n```bash\nyq '.a line_comment=\"\"' sample.yml\n```\nwill output\n```yaml\na: cat\nb: dog # leave this\n```\n\n## Remove (strip) all comments\nNote the use of `...` to ensure key nodes are included.\n\nGiven a sample.yml file of:\n```yaml\n# hi\n\na: cat # comment\n# great\nb: # key comment\n```\nthen\n```bash\nyq '... comments=\"\"' sample.yml\n```\nwill output\n```yaml\na: cat\nb:\n```\n\n## Get line comment\nGiven a sample.yml file of:\n```yaml\n# welcome!\n\na: cat # meow\n# have a great day\n```\nthen\n```bash\nyq '.a | line_comment' sample.yml\n```\nwill output\n```yaml\nmeow\n```\n\n## Get head comment\nGiven a sample.yml file of:\n```yaml\n# welcome!\n\na: cat # meow\n\n# have a great day\n```\nthen\n```bash\nyq '. | head_comment' sample.yml\n```\nwill output\n```yaml\nwelcome!\n\n```\n\n## Head comment with document split\nGiven a sample.yml file of:\n```yaml\n# welcome!\n---\n# bob\na: cat # meow\n\n# have a great day\n```\nthen\n```bash\nyq 'head_comment' sample.yml\n```\nwill output\n```yaml\nwelcome!\nbob\n```\n\n## Get foot comment\nGiven a sample.yml file of:\n```yaml\n# welcome!\n\na: cat # meow\n\n# have a great day\n# no really\n```\nthen\n```bash\nyq '. | foot_comment' sample.yml\n```\nwill output\n```yaml\nhave a great day\nno really\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/compare.md", "content": "# Compare Operators\n\nComparison operators (`>`, `>=`, `<`, `<=`) can be used for comparing scalar values of the same time.\n\nThe following types are currently supported:\n\n- numbers\n- strings\n- datetimes\n\n## Related Operators\n\n- equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals)\n- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators)\n- select operator [here](https://mikefarah.gitbook.io/yq/operators/select)\n\n## Compare numbers (>)\nGiven a sample.yml file of:\n```yaml\na: 5\nb: 4\n```\nthen\n```bash\nyq '.a > .b' sample.yml\n```\nwill output\n```yaml\ntrue\n```\n\n## Compare equal numbers (>=)\nGiven a sample.yml file of:\n```yaml\na: 5\nb: 5\n```\nthen\n```bash\nyq '.a >= .b' sample.yml\n```\nwill output\n```yaml\ntrue\n```\n\n## Compare strings\nCompares strings by their bytecode.\n\nGiven a sample.yml file of:\n```yaml\na: zoo\nb: apple\n```\nthen\n```bash\nyq '.a > .b' sample.yml\n```\nwill output\n```yaml\ntrue\n```\n\n## Compare date times\nYou can compare date times. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.\n\nGiven a sample.yml file of:\n```yaml\na: 2021-01-01T03:10:00Z\nb: 2020-01-01T03:10:00Z\n```\nthen\n```bash\nyq '.a > .b' sample.yml\n```\nwill output\n```yaml\ntrue\n```\n\n## Both sides are null: > is false\nRunning\n```bash\nyq --null-input '.a > .b'\n```\nwill output\n```yaml\nfalse\n```\n\n## Both sides are null: >= is true\nRunning\n```bash\nyq --null-input '.a >= .b'\n```\nwill output\n```yaml\ntrue\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/contains.md", "content": "# Contains\n\nThis returns `true` if the context contains the passed in parameter, and false otherwise. For arrays, this will return true if the passed in array is contained within the array. For strings, it will return true if the string is a substring.\n\n{% hint style=\"warning\" %}\n\n_Note_ that, just like jq, when checking if an array of strings `contains` another, this will use `contains` and _not_ equals to check each string. This means an expression like `contains([\"cat\"])` will return true for an array `[\"cats\"]`.\n\nSee the \"Array has a subset array\" example below on how to check for a subset.\n\n{% endhint %}\n\n## Array contains array\nArray is equal or subset of\n\nGiven a sample.yml file of:\n```yaml\n- foobar\n- foobaz\n- blarp\n```\nthen\n```bash\nyq 'contains([\"baz\", \"bar\"])' sample.yml\n```\nwill output\n```yaml\ntrue\n```\n\n## Array has a subset array\nSubtract the superset array from the subset, if there's anything left, it's not a subset\n\nGiven a sample.yml file of:\n```yaml\n- foobar\n- foobaz\n- blarp\n```\nthen\n```bash\nyq '[\"baz\", \"bar\"] - . | length == 0' sample.yml\n```\nwill output\n```yaml\nfalse\n```\n\n## Object included in array\nGiven a sample.yml file of:\n```yaml\n\"foo\": 12\n\"bar\":\n - 1\n - 2\n - \"barp\": 12\n \"blip\": 13\n```\nthen\n```bash\nyq 'contains({\"bar\": [{\"barp\": 12}]})' sample.yml\n```\nwill output\n```yaml\ntrue\n```\n\n## Object not included in array\nGiven a sample.yml file of:\n```yaml\n\"foo\": 12\n\"bar\":\n - 1\n - 2\n - \"barp\": 12\n \"blip\": 13\n```\nthen\n```bash\nyq 'contains({\"foo\": 12, \"bar\": [{\"barp\": 15}]})' sample.yml\n```\nwill output\n```yaml\nfalse\n```\n\n## String contains substring\nGiven a sample.yml file of:\n```yaml\nfoobar\n```\nthen\n```bash\nyq 'contains(\"bar\")' sample.yml\n```\nwill output\n```yaml\ntrue\n```\n\n## String equals string\nGiven a sample.yml file of:\n```yaml\nmeow\n```\nthen\n```bash\nyq 'contains(\"meow\")' sample.yml\n```\nwill output\n```yaml\ntrue\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/create-collect-into-object.md", "content": "# Create, Collect into Object\n\nThis is used to construct objects (or maps). This can be used against existing yaml, or to create fresh yaml documents.\n\n## Collect empty object\nRunning\n```bash\nyq --null-input '{}'\n```\nwill output\n```yaml\n{}\n```\n\n## Wrap (prefix) existing object\nGiven a sample.yml file of:\n```yaml\nname: Mike\n```\nthen\n```bash\nyq '{\"wrap\": .}' sample.yml\n```\nwill output\n```yaml\nwrap:\n name: Mike\n```\n\n## Using splat to create multiple objects\nGiven a sample.yml file of:\n```yaml\nname: Mike\npets:\n - cat\n - dog\n```\nthen\n```bash\nyq '{.name: .pets.[]}' sample.yml\n```\nwill output\n```yaml\nMike: cat\nMike: dog\n```\n\n## Working with multiple documents\nGiven a sample.yml file of:\n```yaml\nname: Mike\npets:\n - cat\n - dog\n---\nname: Rosey\npets:\n - monkey\n - sheep\n```\nthen\n```bash\nyq '{.name: .pets.[]}' sample.yml\n```\nwill output\n```yaml\nMike: cat\nMike: dog\n---\nRosey: monkey\nRosey: sheep\n```\n\n## Creating yaml from scratch\nRunning\n```bash\nyq --null-input '{\"wrap\": \"frog\"}'\n```\nwill output\n```yaml\nwrap: frog\n```\n\n## Creating yaml from scratch with multiple objects\nRunning\n```bash\nyq --null-input '(.a.b = \"foo\") | (.d.e = \"bar\")'\n```\nwill output\n```yaml\na:\n b: foo\nd:\n e: bar\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/datetime.md", "content": "# Date Time\n\nVarious operators for parsing and manipulating dates. \n\n## Date time formatting\nThis uses Golang's built in time library for parsing and formatting date times.\n\nWhen not specified, the RFC3339 standard is assumed `2006-01-02T15:04:05Z07:00` for parsing.\n\nTo specify a custom parsing format, use the `with_dtf` operator. The first parameter sets the datetime parsing format for the expression in the second parameter. The expression can be any valid `yq` expression tree.\n\n```bash\nyq 'with_dtf(\"myformat\"; .a + \"3h\" | tz(\"Australia/Melbourne\"))'\n```\n\nSee the [library docs](https://pkg.go.dev/time#pkg-constants) for examples of formatting options.\n\n\n## Timezones\nThis uses Golang's built in LoadLocation function to parse timezones strings. See the [library docs](https://pkg.go.dev/time#LoadLocation) for more details.\n\n\n## Durations\nDurations are parsed using Golang's built in [ParseDuration](https://pkg.go.dev/time#ParseDuration) function.\n\nYou can add durations to time using the `+` operator.\n\n## Format: from standard RFC3339 format\nProviding a single parameter assumes a standard RFC3339 datetime format. If the target format is not a valid yaml datetime format, the result will be a string tagged node.\n\nGiven a sample.yml file of:\n```yaml\na: 2001-12-15T02:59:43.1Z\n```\nthen\n```bash\nyq '.a |= format_datetime(\"Monday, 02-Jan-06 at 3:04PM\")' sample.yml\n```\nwill output\n```yaml\na: Saturday, 15-Dec-01 at 2:59AM\n```\n\n## Format: from custom date time\nUse with_dtf to set a custom datetime format for parsing.\n\nGiven a sample.yml file of:\n```yaml\na: Saturday, 15-Dec-01 at 2:59AM\n```\nthen\n```bash\nyq '.a |= with_dtf(\"Monday, 02-Jan-06 at 3:04PM\"; format_datetime(\"2006-01-02\"))' sample.yml\n```\nwill output\n```yaml\na: 2001-12-15\n```\n\n## Format: get the day of the week\nGiven a sample.yml file of:\n```yaml\na: 2001-12-15\n```\nthen\n```bash\nyq '.a | format_datetime(\"Monday\")' sample.yml\n```\nwill output\n```yaml\nSaturday\n```\n\n## Now\nGiven a sample.yml file of:\n```yaml\na: cool\n```\nthen\n```bash\nyq '.updated = now' sample.yml\n```\nwill output\n```yaml\na: cool\nupdated: 2021-05-19T01:02:03Z\n```\n\n## From Unix\nConverts from unix time. Note, you don't have to pipe through the tz operator :)\n\nRunning\n```bash\nyq --null-input '1675301929 | from_unix | tz(\"UTC\")'\n```\nwill output\n```yaml\n2023-02-02T01:38:49Z\n```\n\n## To Unix\nConverts to unix time\n\nRunning\n```bash\nyq --null-input 'now | to_unix'\n```\nwill output\n```yaml\n1621386123\n```\n\n## Timezone: from standard RFC3339 format\nReturns a new datetime in the specified timezone. Specify standard IANA Time Zone format or 'utc', 'local'. When given a single parameter, this assumes the datetime is in RFC3339 format.\n\nGiven a sample.yml file of:\n```yaml\na: cool\n```\nthen\n```bash\nyq '.updated = (now | tz(\"Australia/Sydney\"))' sample.yml\n```\nwill output\n```yaml\na: cool\nupdated: 2021-05-19T11:02:03+10:00\n```\n\n## Timezone: with custom format\nSpecify standard IANA Time Zone format or 'utc', 'local'\n\nGiven a sample.yml file of:\n```yaml\na: Saturday, 15-Dec-01 at 2:59AM GMT\n```\nthen\n```bash\nyq '.a |= with_dtf(\"Monday, 02-Jan-06 at 3:04PM MST\"; tz(\"Australia/Sydney\"))' sample.yml\n```\nwill output\n```yaml\na: Saturday, 15-Dec-01 at 1:59PM AEDT\n```\n\n## Add and tz custom format\nSpecify standard IANA Time Zone format or 'utc', 'local'\n\nGiven a sample.yml file of:\n```yaml\na: Saturday, 15-Dec-01 at 2:59AM GMT\n```\nthen\n```bash\nyq '.a |= with_dtf(\"Monday, 02-Jan-06 at 3:04PM MST\"; tz(\"Australia/Sydney\"))' sample.yml\n```\nwill output\n```yaml\na: Saturday, 15-Dec-01 at 1:59PM AEDT\n```\n\n## Date addition\nGiven a sample.yml file of:\n```yaml\na: 2021-01-01T00:00:00Z\n```\nthen\n```bash\nyq '.a += \"3h10m\"' sample.yml\n```\nwill output\n```yaml\na: 2021-01-01T03:10:00Z\n```\n\n## Date subtraction\nYou can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/datetime#date-time-formattings) for more information.\n\nGiven a sample.yml file of:\n```yaml\na: 2021-01-01T03:10:00Z\n```\nthen\n```bash\nyq '.a -= \"3h10m\"' sample.yml\n```\nwill output\n```yaml\na: 2021-01-01T00:00:00Z\n```\n\n## Date addition - custom format\nGiven a sample.yml file of:\n```yaml\na: Saturday, 15-Dec-01 at 2:59AM GMT\n```\nthen\n```bash\nyq 'with_dtf(\"Monday, 02-Jan-06 at 3:04PM MST\"; .a += \"3h1m\")' sample.yml\n```\nwill output\n```yaml\na: Saturday, 15-Dec-01 at 6:00AM GMT\n```\n\n## Date script with custom format\nYou can embed full expressions in with_dtf if needed.\n\nGiven a sample.yml file of:\n```yaml\na: Saturday, 15-Dec-01 at 2:59AM GMT\n```\nthen\n```bash\nyq 'with_dtf(\"Monday, 02-Jan-06 at 3:04PM MST\"; .a = (.a + \"3h1m\" | tz(\"Australia/Perth\")))' sample.yml\n```\nwill output\n```yaml\na: Saturday, 15-Dec-01 at 2:00PM AWST\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/delete.md", "content": "# Delete\n\nDeletes matching entries in maps or arrays.\n\n## Delete entry in map\nGiven a sample.yml file of:\n```yaml\na: cat\nb: dog\n```\nthen\n```bash\nyq 'del(.b)' sample.yml\n```\nwill output\n```yaml\na: cat\n```\n\n## Delete nested entry in map\nGiven a sample.yml file of:\n```yaml\na:\n a1: fred\n a2: frood\n```\nthen\n```bash\nyq 'del(.a.a1)' sample.yml\n```\nwill output\n```yaml\na:\n a2: frood\n```\n\n## Delete entry in array\nGiven a sample.yml file of:\n```yaml\n- 1\n- 2\n- 3\n```\nthen\n```bash\nyq 'del(.[1])' sample.yml\n```\nwill output\n```yaml\n- 1\n- 3\n```\n\n## Delete nested entry in array\nGiven a sample.yml file of:\n```yaml\n- a: cat\n b: dog\n```\nthen\n```bash\nyq 'del(.[0].a)' sample.yml\n```\nwill output\n```yaml\n- b: dog\n```\n\n## Delete no matches\nGiven a sample.yml file of:\n```yaml\na: cat\nb: dog\n```\nthen\n```bash\nyq 'del(.c)' sample.yml\n```\nwill output\n```yaml\na: cat\nb: dog\n```\n\n## Delete matching entries\nGiven a sample.yml file of:\n```yaml\na: cat\nb: dog\nc: bat\n```\nthen\n```bash\nyq 'del( .[] | select(. == \"*at\") )' sample.yml\n```\nwill output\n```yaml\nb: dog\n```\n\n## Recursively delete matching keys\nGiven a sample.yml file of:\n```yaml\na:\n name: frog\n b:\n name: blog\n age: 12\n```\nthen\n```bash\nyq 'del(.. | select(has(\"name\")).name)' sample.yml\n```\nwill output\n```yaml\na:\n b:\n age: 12\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/divide.md", "content": "# Divide\n\nDivide behaves differently according to the type of the LHS:\n* strings: split by the divider\n* number: arithmetic division\n\n## String split\nGiven a sample.yml file of:\n```yaml\na: cat_meow\nb: _\n```\nthen\n```bash\nyq '.c = .a / .b' sample.yml\n```\nwill output\n```yaml\na: cat_meow\nb: _\nc:\n - cat\n - meow\n```\n\n## Number division\nThe result during division is calculated as a float\n\nGiven a sample.yml file of:\n```yaml\na: 12\nb: 2.5\n```\nthen\n```bash\nyq '.a = .a / .b' sample.yml\n```\nwill output\n```yaml\na: 4.8\nb: 2.5\n```\n\n## Number division by zero\nDividing by zero results in +Inf or -Inf\n\nGiven a sample.yml file of:\n```yaml\na: 1\nb: -1\n```\nthen\n```bash\nyq '.a = .a / 0 | .b = .b / 0' sample.yml\n```\nwill output\n```yaml\na: !!float +Inf\nb: !!float -Inf\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/document-index.md", "content": "# Document Index\n\nUse the `documentIndex` operator (or the `di` shorthand) to select nodes of a particular document.\n\n## Retrieve a document index\nGiven a sample.yml file of:\n```yaml\na: cat\n---\na: frog\n```\nthen\n```bash\nyq '.a | document_index' sample.yml\n```\nwill output\n```yaml\n0\n---\n1\n```\n\n## Retrieve a document index, shorthand\nGiven a sample.yml file of:\n```yaml\na: cat\n---\na: frog\n```\nthen\n```bash\nyq '.a | di' sample.yml\n```\nwill output\n```yaml\n0\n---\n1\n```\n\n## Filter by document index\nGiven a sample.yml file of:\n```yaml\na: cat\n---\na: frog\n```\nthen\n```bash\nyq 'select(document_index == 1)' sample.yml\n```\nwill output\n```yaml\na: frog\n```\n\n## Filter by document index shorthand\nGiven a sample.yml file of:\n```yaml\na: cat\n---\na: frog\n```\nthen\n```bash\nyq 'select(di == 1)' sample.yml\n```\nwill output\n```yaml\na: frog\n```\n\n## Print Document Index with matches\nGiven a sample.yml file of:\n```yaml\na: cat\n---\na: frog\n```\nthen\n```bash\nyq '.a | ({\"match\": ., \"doc\": document_index})' sample.yml\n```\nwill output\n```yaml\nmatch: cat\ndoc: 0\n---\nmatch: frog\ndoc: 1\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/encode-decode.md", "content": "# Encoder / Decoder\n\nEncode operators will take the piped in object structure and encode it as a string in the desired format. The decode operators do the opposite, they take a formatted string and decode it into the relevant object structure.\n\nNote that you can optionally pass an indent value to the encode functions (see below).\n\nThese operators are useful to process yaml documents that have stringified embedded yaml/json/props in them.\n\n\n| Format | Decode (from string) | Encode (to string) |\n| --- | -- | --|\n| Yaml | from_yaml/@yamld | to_yaml(i)/@yaml |\n| JSON | from_json/@jsond | to_json(i)/@json |\n| Properties | from_props/@propsd | to_props/@props |\n| CSV | from_csv/@csvd | to_csv/@csv |\n| TSV | from_tsv/@tsvd | to_tsv/@tsv |\n| XML | from_xml/@xmld | to_xml(i)/@xml |\n| Base64 | @base64d | @base64 |\n| URI | @urid | @uri |\n| Shell | | @sh |\n\n\nSee CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) for accepted formats.\n\nXML uses the `--xml-attribute-prefix` and `xml-content-name` flags to identify attributes and content fields.\n\n\nBase64 assumes [rfc4648](https://rfc-editor.org/rfc/rfc4648.html) encoding. Encoding and decoding both assume that the content is a utf-8 string and not binary content.\n\n## Encode value as json string\nGiven a sample.yml file of:\n```yaml\na:\n cool: thing\n```\nthen\n```bash\nyq '.b = (.a | to_json)' sample.yml\n```\nwill output\n```yaml\na:\n cool: thing\nb: |\n {\n \"cool\": \"thing\"\n }\n```\n\n## Encode value as json string, on one line\nPass in a 0 indent to print json on a single line.\n\nGiven a sample.yml file of:\n```yaml\na:\n cool: thing\n```\nthen\n```bash\nyq '.b = (.a | to_json(0))' sample.yml\n```\nwill output\n```yaml\na:\n cool: thing\nb: '{\"cool\":\"thing\"}'\n```\n\n## Encode value as json string, on one line shorthand\nPass in a 0 indent to print json on a single line.\n\nGiven a sample.yml file of:\n```yaml\na:\n cool: thing\n```\nthen\n```bash\nyq '.b = (.a | @json)' sample.yml\n```\nwill output\n```yaml\na:\n cool: thing\nb: '{\"cool\":\"thing\"}'\n```\n\n## Decode a json encoded string\nKeep in mind JSON is a subset of YAML. If you want idiomatic yaml, pipe through the style operator to clear out the JSON styling.\n\nGiven a sample.yml file of:\n```yaml\na: '{\"cool\":\"thing\"}'\n```\nthen\n```bash\nyq '.a | from_json | ... style=\"\"' sample.yml\n```\nwill output\n```yaml\ncool: thing\n```\n\n## Encode value as props string\nGiven a sample.yml file of:\n```yaml\na:\n cool: thing\n```\nthen\n```bash\nyq '.b = (.a | @props)' sample.yml\n```\nwill output\n```yaml\na:\n cool: thing\nb: |\n cool = thing\n```\n\n## Decode props encoded string\nGiven a sample.yml file of:\n```yaml\na: |-\n cats=great\n dogs=cool as well\n```\nthen\n```bash\nyq '.a |= @propsd' sample.yml\n```\nwill output\n```yaml\na:\n cats: great\n dogs: cool as well\n```\n\n## Decode csv encoded string\nGiven a sample.yml file of:\n```yaml\na: |-\n cats,dogs\n great,cool as well\n```\nthen\n```bash\nyq '.a |= @csvd' sample.yml\n```\nwill output\n```yaml\na:\n - cats: great\n dogs: cool as well\n```\n\n## Decode tsv encoded string\nGiven a sample.yml file of:\n```yaml\na: |-\n cats\tdogs\n great\tcool as well\n```\nthen\n```bash\nyq '.a |= @tsvd' sample.yml\n```\nwill output\n```yaml\na:\n - cats: great\n dogs: cool as well\n```\n\n## Encode value as yaml string\nIndent defaults to 2\n\nGiven a sample.yml file of:\n```yaml\na:\n cool:\n bob: dylan\n```\nthen\n```bash\nyq '.b = (.a | to_yaml)' sample.yml\n```\nwill output\n```yaml\na:\n cool:\n bob: dylan\nb: |\n cool:\n bob: dylan\n```\n\n## Encode value as yaml string, with custom indentation\nYou can specify the indentation level as the first parameter.\n\nGiven a sample.yml file of:\n```yaml\na:\n cool:\n bob: dylan\n```\nthen\n```bash\nyq '.b = (.a | to_yaml(8))' sample.yml\n```\nwill output\n```yaml\na:\n cool:\n bob: dylan\nb: |\n cool:\n bob: dylan\n```\n\n## Decode a yaml encoded string\nGiven a sample.yml file of:\n```yaml\na: 'foo: bar'\n```\nthen\n```bash\nyq '.b = (.a | from_yaml)' sample.yml\n```\nwill output\n```yaml\na: 'foo: bar'\nb:\n foo: bar\n```\n\n## Update a multiline encoded yaml string\nGiven a sample.yml file of:\n```yaml\na: |\n foo: bar\n baz: dog\n\n```\nthen\n```bash\nyq '.a |= (from_yaml | .foo = \"cat\" | to_yaml)' sample.yml\n```\nwill output\n```yaml\na: |\n foo: cat\n baz: dog\n```\n\n## Update a single line encoded yaml string\nGiven a sample.yml file of:\n```yaml\na: 'foo: bar'\n```\nthen\n```bash\nyq '.a |= (from_yaml | .foo = \"cat\" | to_yaml)' sample.yml\n```\nwill output\n```yaml\na: 'foo: cat'\n```\n\n## Encode array of scalars as csv string\nScalars are strings, numbers and booleans.\n\nGiven a sample.yml file of:\n```yaml\n- cat\n- thing1,thing2\n- true\n- 3.40\n```\nthen\n```bash\nyq '@csv' sample.yml\n```\nwill output\n```yaml\ncat,\"thing1,thing2\",true,3.40\n```\n\n## Encode array of arrays as csv string\nGiven a sample.yml file of:\n```yaml\n- - cat\n - thing1,thing2\n - true\n - 3.40\n- - dog\n - thing3\n - false\n - 12\n```\nthen\n```bash\nyq '@csv' sample.yml\n```\nwill output\n```yaml\ncat,\"thing1,thing2\",true,3.40\ndog,thing3,false,12\n```\n\n## Encode array of arrays as tsv string\nScalars are strings, numbers and booleans.\n\nGiven a sample.yml file of:\n```yaml\n- - cat\n - thing1,thing2\n - true\n - 3.40\n- - dog\n - thing3\n - false\n - 12\n```\nthen\n```bash\nyq '@tsv' sample.yml\n```\nwill output\n```yaml\ncat\tthing1,thing2\ttrue\t3.40\ndog\tthing3\tfalse\t12\n```\n\n## Encode value as xml string\nGiven a sample.yml file of:\n```yaml\na:\n cool:\n foo: bar\n +@id: hi\n```\nthen\n```bash\nyq '.a | to_xml' sample.yml\n```\nwill output\n```yaml\n\n bar\n\n\n```\n\n## Encode value as xml string on a single line\nGiven a sample.yml file of:\n```yaml\na:\n cool:\n foo: bar\n +@id: hi\n```\nthen\n```bash\nyq '.a | @xml' sample.yml\n```\nwill output\n```yaml\nbar\n\n```\n\n## Encode value as xml string with custom indentation\nGiven a sample.yml file of:\n```yaml\na:\n cool:\n foo: bar\n +@id: hi\n```\nthen\n```bash\nyq '{\"cat\": .a | to_xml(1)}' sample.yml\n```\nwill output\n```yaml\ncat: |\n \n bar\n \n```\n\n## Decode a xml encoded string\nGiven a sample.yml file of:\n```yaml\na: bar\n```\nthen\n```bash\nyq '.b = (.a | from_xml)' sample.yml\n```\nwill output\n```yaml\na: bar\nb:\n foo: bar\n```\n\n## Encode a string to base64\nGiven a sample.yml file of:\n```yaml\ncoolData: a special string\n```\nthen\n```bash\nyq '.coolData | @base64' sample.yml\n```\nwill output\n```yaml\nYSBzcGVjaWFsIHN0cmluZw==\n```\n\n## Encode a yaml document to base64\nPipe through @yaml first to convert to a string, then use @base64 to encode it.\n\nGiven a sample.yml file of:\n```yaml\na: apple\n```\nthen\n```bash\nyq '@yaml | @base64' sample.yml\n```\nwill output\n```yaml\nYTogYXBwbGUK\n```\n\n## Encode a string to uri\nGiven a sample.yml file of:\n```yaml\ncoolData: this has & special () characters *\n```\nthen\n```bash\nyq '.coolData | @uri' sample.yml\n```\nwill output\n```yaml\nthis+has+%26+special+%28%29+characters+%2A\n```\n\n## Decode a URI to a string\nGiven a sample.yml file of:\n```yaml\nthis+has+%26+special+%28%29+characters+%2A\n```\nthen\n```bash\nyq '@urid' sample.yml\n```\nwill output\n```yaml\nthis has & special () characters *\n```\n\n## Encode a string to sh\nSh/Bash friendly string\n\nGiven a sample.yml file of:\n```yaml\ncoolData: strings with spaces and a 'quote'\n```\nthen\n```bash\nyq '.coolData | @sh' sample.yml\n```\nwill output\n```yaml\nstrings' with spaces and a '\\'quote\\'\n```\n\n## Decode a base64 encoded string\nDecoded data is assumed to be a string.\n\nGiven a sample.yml file of:\n```yaml\ncoolData: V29ya3Mgd2l0aCBVVEYtMTYg8J+Yig==\n```\nthen\n```bash\nyq '.coolData | @base64d' sample.yml\n```\nwill output\n```yaml\nWorks with UTF-16 😊\n```\n\n## Decode a base64 encoded yaml document\nPipe through `from_yaml` to parse the decoded base64 string as a yaml document.\n\nGiven a sample.yml file of:\n```yaml\ncoolData: YTogYXBwbGUK\n```\nthen\n```bash\nyq '.coolData |= (@base64d | from_yaml)' sample.yml\n```\nwill output\n```yaml\ncoolData:\n a: apple\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/entries.md", "content": "# Entries\n\nSimilar to the same named functions in `jq` these functions convert to/from an object and an array of key-value pairs. This is most useful for performing operations on keys of maps.\n\nUse `with_entries(op)` as a syntactic sugar for doing `to_entries | op | from_entries`.\n\n## to_entries Map\nGiven a sample.yml file of:\n```yaml\na: 1\nb: 2\n```\nthen\n```bash\nyq 'to_entries' sample.yml\n```\nwill output\n```yaml\n- key: a\n value: 1\n- key: b\n value: 2\n```\n\n## to_entries Array\nGiven a sample.yml file of:\n```yaml\n- a\n- b\n```\nthen\n```bash\nyq 'to_entries' sample.yml\n```\nwill output\n```yaml\n- key: 0\n value: a\n- key: 1\n value: b\n```\n\n## to_entries null\nGiven a sample.yml file of:\n```yaml\nnull\n```\nthen\n```bash\nyq 'to_entries' sample.yml\n```\nwill output\n```yaml\n```\n\n## from_entries map\nGiven a sample.yml file of:\n```yaml\na: 1\nb: 2\n```\nthen\n```bash\nyq 'to_entries | from_entries' sample.yml\n```\nwill output\n```yaml\na: 1\nb: 2\n```\n\n## from_entries with numeric key indices\nfrom_entries always creates a map, even for numeric keys\n\nGiven a sample.yml file of:\n```yaml\n- a\n- b\n```\nthen\n```bash\nyq 'to_entries | from_entries' sample.yml\n```\nwill output\n```yaml\n0: a\n1: b\n```\n\n## Use with_entries to update keys\nGiven a sample.yml file of:\n```yaml\na: 1\nb: 2\n```\nthen\n```bash\nyq 'with_entries(.key |= \"KEY_\" + .)' sample.yml\n```\nwill output\n```yaml\nKEY_a: 1\nKEY_b: 2\n```\n\n## Use with_entries to update keys recursively\nWe use (.. | select(tag=\"map\")) to find all the maps in the doc, then |= to update each one of those maps. In the update, with_entries is used.\n\nGiven a sample.yml file of:\n```yaml\na: 1\nb:\n b_a: nested\n b_b: thing\n```\nthen\n```bash\nyq '(.. | select(tag==\"!!map\")) |= with_entries(.key |= \"KEY_\" + .)' sample.yml\n```\nwill output\n```yaml\nKEY_a: 1\nKEY_b:\n KEY_b_a: nested\n KEY_b_b: thing\n```\n\n## Custom sort map keys\nUse to_entries to convert to an array of key/value pairs, sort the array using sort/sort_by/etc, and convert it back.\n\nGiven a sample.yml file of:\n```yaml\na: 1\nc: 3\nb: 2\n```\nthen\n```bash\nyq 'to_entries | sort_by(.key) | reverse | from_entries' sample.yml\n```\nwill output\n```yaml\nc: 3\nb: 2\na: 1\n```\n\n## Use with_entries to filter the map\nGiven a sample.yml file of:\n```yaml\na:\n b: bird\nc:\n d: dog\n```\nthen\n```bash\nyq 'with_entries(select(.value | has(\"b\")))' sample.yml\n```\nwill output\n```yaml\na:\n b: bird\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/env-variable-operators.md", "content": "# Env Variable Operators\n\nThese operators are used to handle environment variables usage in expressions and documents. While environment variables can, of course, be passed in via your CLI with string interpolation, this often comes with complex quote escaping and can be tricky to write and read. \n\nThere are three operators:\n\n- `env` which takes a single environment variable name and parse the variable as a yaml node (be it a map, array, string, number of boolean) \n- `strenv` which also takes a single environment variable name, and always parses the variable as a string.\n- `envsubst` which you pipe strings into and it interpolates environment variables in strings using [envsubst](https://github.com/a8m/envsubst). \n\n\n## EnvSubst Options\nYou can optionally pass envsubst any of the following options:\n\n - nu: NoUnset, this will fail if there are any referenced variables that are not set\n - ne: NoEmpty, this will fail if there are any referenced variables that are empty\n - ff: FailFast, this will abort on the first failure (rather than collect all the errors)\n\nE.g:\n`envsubst(ne, ff)` will fail on the first empty variable.\n\nSee [Imposing Restrictions](https://github.com/a8m/envsubst#imposing-restrictions) in the `envsubst` documentation for more information, and below for examples.\n\n## Tip\nTo replace environment variables across all values in a document, `envsubst` can be used with the recursive descent operator\nas follows:\n\n```bash\nyq '(.. | select(tag == \"!!str\")) |= envsubst' file.yaml\n```\n\n## Disabling env operators\nIf required, you can use the `--security-disable-env-ops` to disable env operations.\n\n\n## Read string environment variable\nRunning\n```bash\nmyenv=\"cat meow\" yq --null-input '.a = env(myenv)'\n```\nwill output\n```yaml\na: cat meow\n```\n\n## Read boolean environment variable\nRunning\n```bash\nmyenv=\"true\" yq --null-input '.a = env(myenv)'\n```\nwill output\n```yaml\na: true\n```\n\n## Read numeric environment variable\nRunning\n```bash\nmyenv=\"12\" yq --null-input '.a = env(myenv)'\n```\nwill output\n```yaml\na: 12\n```\n\n## Read yaml environment variable\nRunning\n```bash\nmyenv=\"{b: fish}\" yq --null-input '.a = env(myenv)'\n```\nwill output\n```yaml\na: {b: fish}\n```\n\n## Read boolean environment variable as a string\nRunning\n```bash\nmyenv=\"true\" yq --null-input '.a = strenv(myenv)'\n```\nwill output\n```yaml\na: \"true\"\n```\n\n## Read numeric environment variable as a string\nRunning\n```bash\nmyenv=\"12\" yq --null-input '.a = strenv(myenv)'\n```\nwill output\n```yaml\na: \"12\"\n```\n\n## Dynamically update a path from an environment variable\nThe env variable can be any valid yq expression.\n\nGiven a sample.yml file of:\n```yaml\na:\n b:\n - name: dog\n - name: cat\n```\nthen\n```bash\npathEnv=\".a.b[0].name\" valueEnv=\"moo\" yq 'eval(strenv(pathEnv)) = strenv(valueEnv)' sample.yml\n```\nwill output\n```yaml\na:\n b:\n - name: moo\n - name: cat\n```\n\n## Dynamic key lookup with environment variable\nGiven a sample.yml file of:\n```yaml\ncat: meow\ndog: woof\n```\nthen\n```bash\nmyenv=\"cat\" yq '.[env(myenv)]' sample.yml\n```\nwill output\n```yaml\nmeow\n```\n\n## Replace strings with envsubst\nRunning\n```bash\nmyenv=\"cat\" yq --null-input '\"the ${myenv} meows\" | envsubst'\n```\nwill output\n```yaml\nthe cat meows\n```\n\n## Replace strings with envsubst, missing variables\nRunning\n```bash\nyq --null-input '\"the ${myenvnonexisting} meows\" | envsubst'\n```\nwill output\n```yaml\nthe meows\n```\n\n## Replace strings with envsubst(nu), missing variables\n(nu) not unset, will fail if there are unset (missing) variables\n\nRunning\n```bash\nyq --null-input '\"the ${myenvnonexisting} meows\" | envsubst(nu)'\n```\nwill output\n```bash\nError: variable ${myenvnonexisting} not set\n```\n\n## Replace strings with envsubst(ne), missing variables\n(ne) not empty, only validates set variables\n\nRunning\n```bash\nyq --null-input '\"the ${myenvnonexisting} meows\" | envsubst(ne)'\n```\nwill output\n```yaml\nthe meows\n```\n\n## Replace strings with envsubst(ne), empty variable\n(ne) not empty, will fail if a references variable is empty\n\nRunning\n```bash\nmyenv=\"\" yq --null-input '\"the ${myenv} meows\" | envsubst(ne)'\n```\nwill output\n```bash\nError: variable ${myenv} set but empty\n```\n\n## Replace strings with envsubst, missing variables with defaults\nRunning\n```bash\nyq --null-input '\"the ${myenvnonexisting-dog} meows\" | envsubst'\n```\nwill output\n```yaml\nthe dog meows\n```\n\n## Replace strings with envsubst(nu), missing variables with defaults\nHaving a default specified skips over the missing variable.\n\nRunning\n```bash\nyq --null-input '\"the ${myenvnonexisting-dog} meows\" | envsubst(nu)'\n```\nwill output\n```yaml\nthe dog meows\n```\n\n## Replace strings with envsubst(ne), missing variables with defaults\nFails, because the variable is explicitly set to blank.\n\nRunning\n```bash\nmyEmptyEnv=\"\" yq --null-input '\"the ${myEmptyEnv-dog} meows\" | envsubst(ne)'\n```\nwill output\n```bash\nError: variable ${myEmptyEnv} set but empty\n```\n\n## Replace string environment variable in document\nGiven a sample.yml file of:\n```yaml\nv: ${myenv}\n```\nthen\n```bash\nmyenv=\"cat meow\" yq '.v |= envsubst' sample.yml\n```\nwill output\n```yaml\nv: cat meow\n```\n\n## (Default) Return all envsubst errors\nBy default, all errors are returned at once.\n\nRunning\n```bash\nyq --null-input '\"the ${notThere} ${alsoNotThere}\" | envsubst(nu)'\n```\nwill output\n```bash\nError: variable ${notThere} not set\nvariable ${alsoNotThere} not set\n```\n\n## Fail fast, return the first envsubst error (and abort)\nRunning\n```bash\nyq --null-input '\"the ${notThere} ${alsoNotThere}\" | envsubst(nu,ff)'\n```\nwill output\n```bash\nError: variable ${notThere} not set\n```\n\n## env() operation fails when security is enabled\nUse `--security-disable-env-ops` to disable env operations for security.\n\nRunning\n```bash\nyq --null-input 'env(\"MYENV\")'\n```\nwill output\n```bash\nError: env operations have been disabled\n```\n\n## strenv() operation fails when security is enabled\nUse `--security-disable-env-ops` to disable env operations for security.\n\nRunning\n```bash\nyq --null-input 'strenv(\"MYENV\")'\n```\nwill output\n```bash\nError: env operations have been disabled\n```\n\n## envsubst() operation fails when security is enabled\nUse `--security-disable-env-ops` to disable env operations for security.\n\nRunning\n```bash\nyq --null-input '\"value: ${MYENV}\" | envsubst'\n```\nwill output\n```bash\nError: env operations have been disabled\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/equals.md", "content": "# Equals / Not Equals\n\nThis is a boolean operator that will return `true` if the LHS is equal to the RHS and `false` otherwise.\n\n```\n.a == .b\n```\n\nIt is most often used with the select operator to find particular nodes:\n\n```\nselect(.a == .b)\n```\n\nThe not equals `!=` operator returns `false` if the LHS is equal to the RHS.\n\n## Related Operators\n\n- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare)\n- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators)\n- select operator [here](https://mikefarah.gitbook.io/yq/operators/select)\n\n\n## Match string\nGiven a sample.yml file of:\n```yaml\n- cat\n- goat\n- dog\n```\nthen\n```bash\nyq '.[] | (. == \"*at\")' sample.yml\n```\nwill output\n```yaml\ntrue\ntrue\nfalse\n```\n\n## Don't match string\nGiven a sample.yml file of:\n```yaml\n- cat\n- goat\n- dog\n```\nthen\n```bash\nyq '.[] | (. != \"*at\")' sample.yml\n```\nwill output\n```yaml\nfalse\nfalse\ntrue\n```\n\n## Match number\nGiven a sample.yml file of:\n```yaml\n- 3\n- 4\n- 5\n```\nthen\n```bash\nyq '.[] | (. == 4)' sample.yml\n```\nwill output\n```yaml\nfalse\ntrue\nfalse\n```\n\n## Don't match number\nGiven a sample.yml file of:\n```yaml\n- 3\n- 4\n- 5\n```\nthen\n```bash\nyq '.[] | (. != 4)' sample.yml\n```\nwill output\n```yaml\ntrue\nfalse\ntrue\n```\n\n## Match nulls\nRunning\n```bash\nyq --null-input 'null == ~'\n```\nwill output\n```yaml\ntrue\n```\n\n## Non existent key doesn't equal a value\nGiven a sample.yml file of:\n```yaml\na: frog\n```\nthen\n```bash\nyq 'select(.b != \"thing\")' sample.yml\n```\nwill output\n```yaml\na: frog\n```\n\n## Two non existent keys are equal\nGiven a sample.yml file of:\n```yaml\na: frog\n```\nthen\n```bash\nyq 'select(.b == .c)' sample.yml\n```\nwill output\n```yaml\na: frog\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/error.md", "content": "# Error\n\nUse this operation to short-circuit expressions. Useful for validation.\n\n## Validate a particular value\nGiven a sample.yml file of:\n```yaml\na: hello\n```\nthen\n```bash\nyq 'select(.a == \"howdy\") or error(\".a [\" + .a + \"] is not howdy!\")' sample.yml\n```\nwill output\n```bash\nError: .a [hello] is not howdy!\n```\n\n## Validate the environment variable is a number - invalid\nRunning\n```bash\nnumberOfCats=\"please\" yq --null-input 'env(numberOfCats) | select(tag == \"!!int\") or error(\"numberOfCats is not a number :(\")'\n```\nwill output\n```bash\nError: numberOfCats is not a number :(\n```\n\n## Validate the environment variable is a number - valid\n`with` can be a convenient way of encapsulating validation.\n\nGiven a sample.yml file of:\n```yaml\nname: Bob\nfavouriteAnimal: cat\n```\nthen\n```bash\nnumberOfCats=\"3\" yq '\n\twith(env(numberOfCats); select(tag == \"!!int\") or error(\"numberOfCats is not a number :(\")) | \n\t.numPets = env(numberOfCats)\n' sample.yml\n```\nwill output\n```yaml\nname: Bob\nfavouriteAnimal: cat\nnumPets: 3\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/eval.md", "content": "# Eval\n\nUse `eval` to dynamically process an expression - for instance from an environment variable.\n\n`eval` takes a single argument, and evaluates that as a `yq` expression. Any valid expression can be used, be it a path `.a.b.c | select(. == \"cat\")`, or an update `.a.b.c = \"gogo\"`.\n\nTip: This can be a useful way to parameterise complex scripts.\n\n## Dynamically evaluate a path\nGiven a sample.yml file of:\n```yaml\npathExp: .a.b[] | select(.name == \"cat\")\na:\n b:\n - name: dog\n - name: cat\n```\nthen\n```bash\nyq 'eval(.pathExp)' sample.yml\n```\nwill output\n```yaml\nname: cat\n```\n\n## Dynamically update a path from an environment variable\nThe env variable can be any valid yq expression.\n\nGiven a sample.yml file of:\n```yaml\na:\n b:\n - name: dog\n - name: cat\n```\nthen\n```bash\npathEnv=\".a.b[0].name\" valueEnv=\"moo\" yq 'eval(strenv(pathEnv)) = strenv(valueEnv)' sample.yml\n```\nwill output\n```yaml\na:\n b:\n - name: moo\n - name: cat\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/file-operators.md", "content": "# File Operators\n\nFile operators are most often used with merge when needing to merge specific files together. Note that when doing this, you will need to use `eval-all` to ensure all yaml documents are loaded into memory before performing the merge (as opposed to `eval` which runs the expression once per document).\n\nNote that the `fileIndex` operator has a short alias of `fi`.\n\n## Merging files\nNote the use of eval-all to ensure all documents are loaded into memory.\n```bash\nyq eval-all 'select(fi == 0) * select(filename == \"file2.yaml\")' file1.yaml file2.yaml\n```\n\n## Get filename\nGiven a sample.yml file of:\n```yaml\na: cat\n```\nthen\n```bash\nyq 'filename' sample.yml\n```\nwill output\n```yaml\nsample.yml\n```\n\n## Get file index\nGiven a sample.yml file of:\n```yaml\na: cat\n```\nthen\n```bash\nyq 'file_index' sample.yml\n```\nwill output\n```yaml\n0\n```\n\n## Get file indices of multiple documents\nGiven a sample.yml file of:\n```yaml\na: cat\n```\nAnd another sample another.yml file of:\n```yaml\na: cat\n```\nthen\n```bash\nyq eval-all 'file_index' sample.yml another.yml\n```\nwill output\n```yaml\n0\n1\n```\n\n## Get file index alias\nGiven a sample.yml file of:\n```yaml\na: cat\n```\nthen\n```bash\nyq 'fi' sample.yml\n```\nwill output\n```yaml\n0\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/filter.md", "content": "# Filter\n\nFilters an array (or map values) by the expression given. Equivalent to doing `map(select(exp))`.\n\n\n## Filter array\nGiven a sample.yml file of:\n```yaml\n- 1\n- 2\n- 3\n```\nthen\n```bash\nyq 'filter(. < 3)' sample.yml\n```\nwill output\n```yaml\n- 1\n- 2\n```\n\n## Filter map values\nGiven a sample.yml file of:\n```yaml\nc:\n things: cool\n frog: yes\nd:\n things: hot\n frog: false\n```\nthen\n```bash\nyq 'filter(.things == \"cool\")' sample.yml\n```\nwill output\n```yaml\n- things: cool\n frog: yes\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/first.md", "content": "# First\n\nReturns the first matching element in an array, or first matching value in a map.\n\nCan be given an expression to match with, otherwise will just return the first.\n\n## First matching element from array\nGiven a sample.yml file of:\n```yaml\n- a: banana\n- a: cat\n- a: apple\n```\nthen\n```bash\nyq 'first(.a == \"cat\")' sample.yml\n```\nwill output\n```yaml\na: cat\n```\n\n## First matching element from array with multiple matches\nGiven a sample.yml file of:\n```yaml\n- a: banana\n- a: cat\n b: firstCat\n- a: apple\n- a: cat\n b: secondCat\n```\nthen\n```bash\nyq 'first(.a == \"cat\")' sample.yml\n```\nwill output\n```yaml\na: cat\nb: firstCat\n```\n\n## First matching element from array with numeric condition\nGiven a sample.yml file of:\n```yaml\n- a: 10\n- a: 100\n- a: 1\n- a: 101\n```\nthen\n```bash\nyq 'first(.a > 50)' sample.yml\n```\nwill output\n```yaml\na: 100\n```\n\n## First matching element from array with boolean condition\nGiven a sample.yml file of:\n```yaml\n- a: false\n- a: true\n b: firstTrue\n- a: false\n- a: true\n b: secondTrue\n```\nthen\n```bash\nyq 'first(.a == true)' sample.yml\n```\nwill output\n```yaml\na: true\nb: firstTrue\n```\n\n## First matching element from array with null values\nGiven a sample.yml file of:\n```yaml\n- a: null\n- a: cat\n- a: apple\n```\nthen\n```bash\nyq 'first(.a != null)' sample.yml\n```\nwill output\n```yaml\na: cat\n```\n\n## First matching element from array with complex condition\nGiven a sample.yml file of:\n```yaml\n- a: dog\n b: 7\n- a: cat\n b: 3\n- a: apple\n b: 5\n```\nthen\n```bash\nyq 'first(.b > 4 and .b < 6)' sample.yml\n```\nwill output\n```yaml\na: apple\nb: 5\n```\n\n## First matching element from map\nGiven a sample.yml file of:\n```yaml\nx:\n a: banana\ny:\n a: cat\nz:\n a: apple\n```\nthen\n```bash\nyq 'first(.a == \"cat\")' sample.yml\n```\nwill output\n```yaml\na: cat\n```\n\n## First matching element from map with numeric condition\nGiven a sample.yml file of:\n```yaml\nx:\n a: 10\ny:\n a: 100\nz:\n a: 101\n```\nthen\n```bash\nyq 'first(.a > 50)' sample.yml\n```\nwill output\n```yaml\na: 100\n```\n\n## First matching element from nested structure\nGiven a sample.yml file of:\n```yaml\nitems:\n - a: banana\n - a: cat\n - a: apple\n```\nthen\n```bash\nyq '.items | first(.a == \"cat\")' sample.yml\n```\nwill output\n```yaml\na: cat\n```\n\n## First matching element with no matches\nGiven a sample.yml file of:\n```yaml\n- a: banana\n- a: cat\n- a: apple\n```\nthen\n```bash\nyq 'first(.a == \"dog\")' sample.yml\n```\nwill output\n```yaml\n```\n\n## First matching element from empty array\nGiven a sample.yml file of:\n```yaml\n[]\n```\nthen\n```bash\nyq 'first(.a == \"cat\")' sample.yml\n```\nwill output\n```yaml\n```\n\n## First matching element from scalar node\nGiven a sample.yml file of:\n```yaml\nhello\n```\nthen\n```bash\nyq 'first(. == \"hello\")' sample.yml\n```\nwill output\n```yaml\n```\n\n## First matching element from null node\nGiven a sample.yml file of:\n```yaml\nnull\n```\nthen\n```bash\nyq 'first(. == \"hello\")' sample.yml\n```\nwill output\n```yaml\n```\n\n## First matching element with string condition\nGiven a sample.yml file of:\n```yaml\n- a: banana\n- a: cat\n- a: apple\n```\nthen\n```bash\nyq 'first(.a | test(\"^c\"))' sample.yml\n```\nwill output\n```yaml\na: cat\n```\n\n## First matching element with length condition\nGiven a sample.yml file of:\n```yaml\n- a: hi\n- a: hello\n- a: world\n```\nthen\n```bash\nyq 'first(.a | length > 4)' sample.yml\n```\nwill output\n```yaml\na: hello\n```\n\n## First matching element from array of strings\nGiven a sample.yml file of:\n```yaml\n- banana\n- cat\n- apple\n```\nthen\n```bash\nyq 'first(. == \"cat\")' sample.yml\n```\nwill output\n```yaml\ncat\n```\n\n## First matching element from array of numbers\nGiven a sample.yml file of:\n```yaml\n- 10\n- 100\n- 1\n```\nthen\n```bash\nyq 'first(. > 50)' sample.yml\n```\nwill output\n```yaml\n100\n```\n\n## First element with no filter from array\nGiven a sample.yml file of:\n```yaml\n- 10\n- 100\n- 1\n```\nthen\n```bash\nyq 'first' sample.yml\n```\nwill output\n```yaml\n10\n```\n\n## First element with no filter from array of maps\nGiven a sample.yml file of:\n```yaml\n- a: 10\n- a: 100\n```\nthen\n```bash\nyq 'first' sample.yml\n```\nwill output\n```yaml\na: 10\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/flatten.md", "content": "# Flatten\nThis recursively flattens arrays.\n\n## Flatten\nRecursively flattens all arrays\n\nGiven a sample.yml file of:\n```yaml\n- 1\n- - 2\n- - - 3\n```\nthen\n```bash\nyq 'flatten' sample.yml\n```\nwill output\n```yaml\n- 1\n- 2\n- 3\n```\n\n## Flatten with depth of one\nGiven a sample.yml file of:\n```yaml\n- 1\n- - 2\n- - - 3\n```\nthen\n```bash\nyq 'flatten(1)' sample.yml\n```\nwill output\n```yaml\n- 1\n- 2\n- - 3\n```\n\n## Flatten empty array\nGiven a sample.yml file of:\n```yaml\n- []\n```\nthen\n```bash\nyq 'flatten' sample.yml\n```\nwill output\n```yaml\n[]\n```\n\n## Flatten array of objects\nGiven a sample.yml file of:\n```yaml\n- foo: bar\n- - foo: baz\n```\nthen\n```bash\nyq 'flatten' sample.yml\n```\nwill output\n```yaml\n- foo: bar\n- foo: baz\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/group-by.md", "content": "# Group By\n\nThis is used to group items in an array by an expression.\n\n## Group by field\nGiven a sample.yml file of:\n```yaml\n- foo: 1\n bar: 10\n- foo: 3\n bar: 100\n- foo: 1\n bar: 1\n```\nthen\n```bash\nyq 'group_by(.foo)' sample.yml\n```\nwill output\n```yaml\n- - foo: 1\n bar: 10\n - foo: 1\n bar: 1\n- - foo: 3\n bar: 100\n```\n\n## Group by field, with nulls\nGiven a sample.yml file of:\n```yaml\n- cat: dog\n- foo: 1\n bar: 10\n- foo: 3\n bar: 100\n- no: foo for you\n- foo: 1\n bar: 1\n```\nthen\n```bash\nyq 'group_by(.foo)' sample.yml\n```\nwill output\n```yaml\n- - cat: dog\n - no: foo for you\n- - foo: 1\n bar: 10\n - foo: 1\n bar: 1\n- - foo: 3\n bar: 100\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/has.md", "content": "# Has\n\nThis operation returns true if the key exists in a map (or index in an array), false otherwise.\n\n## Has map key\nGiven a sample.yml file of:\n```yaml\n- a: yes\n- a: ~\n- a:\n- b: nope\n```\nthen\n```bash\nyq '.[] | has(\"a\")' sample.yml\n```\nwill output\n```yaml\ntrue\ntrue\ntrue\nfalse\n```\n\n## Select, checking for existence of deep paths\nSimply pipe in parent expressions into `has`\n\nGiven a sample.yml file of:\n```yaml\n- a:\n b:\n c: cat\n- a:\n b:\n d: dog\n```\nthen\n```bash\nyq '.[] | select(.a.b | has(\"c\"))' sample.yml\n```\nwill output\n```yaml\na:\n b:\n c: cat\n```\n\n## Has array index\nGiven a sample.yml file of:\n```yaml\n- []\n- [1]\n- [1, 2]\n- [1, null]\n- [1, 2, 3]\n\n```\nthen\n```bash\nyq '.[] | has(1)' sample.yml\n```\nwill output\n```yaml\nfalse\nfalse\ntrue\ntrue\ntrue\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/headers/Main.md", "content": "# NAME\n *yq* is a portable command-line data file processor\n\n# SYNOPSIS \n\nyq [eval/eval-all] [expression] files..\n\neval/e - (default) Apply the expression to each document in each yaml file in sequence\n\neval-all/ea - Loads all yaml documents of all yaml files and runs expression once\n\n# DESCRIPTION\n\na lightweight and portable command-line data file processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml, json, xml, csv, properties and TOML files. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.\n\nThis documentation is also available at https://mikefarah.gitbook.io/yq/\n# QUICK GUIDE \n\n## Read a value:\n```bash\nyq '.a.b[0].c' file.yaml\n```\n\n## Pipe from STDIN:\n```bash\ncat file.yaml | yq '.a.b[0].c'\n```\n\n## Update a yaml file, in place\n```bash\nyq -i '.a.b[0].c = \"cool\"' file.yaml\n```\n\n## Update using environment variables\n```bash\nNAME=mike yq -i '.a.b[0].c = strenv(NAME)' file.yaml\n```\n\n## Merge multiple files\n```\nyq ea '. as $item ireduce ({}; . * $item )' path/to/*.yml\n```\nNote the use of `ea` to evaluate all files at once (instead of in sequence.)\n\n## Multiple updates to a yaml file\n```bash\nyq -i '\n .a.b[0].c = \"cool\" |\n .x.y.z = \"foobar\" |\n .person.name = strenv(NAME)\n' file.yaml\n```\n\nSee the [documentation](https://mikefarah.gitbook.io/yq/) for more.\n\n# KNOWN ISSUES / MISSING FEATURES\n- `yq` attempts to preserve comment positions and whitespace as much as possible, but it does not handle all scenarios (see https://github.com/go-yaml/yaml/tree/v3 for details)\n- Powershell has its own...opinions: https://mikefarah.gitbook.io/yq/usage/tips-and-tricks#quotes-in-windows-powershell\n\n# BUGS / ISSUES / FEATURE REQUESTS\n\nPlease visit the GitHub page https://github.com/mikefarah/yq/.\n\n" }, { "path": "pkg/yqlib/doc/operators/headers/add.md", "content": "# Add\n\nAdd behaves differently according to the type of the LHS:\n* arrays: concatenate\n* number scalars: arithmetic addition\n* string scalars: concatenate\n* maps: shallow merge (use the multiply operator (`*`) to deeply merge)\n\nUse `+=` as a relative append assign for things like increment. Note that `.a += .x` is equivalent to running `.a = .a + .x`.\n\n" }, { "path": "pkg/yqlib/doc/operators/headers/alternative-default-value.md", "content": "# Alternative (Default value)\n\nThis operator is used to provide alternative (or default) values when a particular expression is either null or false.\n" }, { "path": "pkg/yqlib/doc/operators/headers/anchor-and-alias-operators.md", "content": "# Anchor and Alias Operators\n\nUse the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference (or expands) aliases and remove anchor names).\n\n`yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.\n\n\n## NOTE --yaml-fix-merge-anchor-to-spec flag\n`yq` doesn't merge anchors `<<:` to spec, in some circumstances it incorrectly overrides existing keys when the spec documents not to do that.\n\nTo minimise disruption while still fixing the issue, a flag has been added to toggle this behaviour. This will first default to false; and log warnings to users. Then it will default to true (and still allow users to specify false if needed).\n\nThis flag also enables advanced merging, like inline maps, as well as fixes to ensure when exploding a particular path, neighbours are not affect ed.\n\nLong story short, you should be setting this flag to true.\n\nSee examples of the flag differences below, where LEGACY is with the flag off; and FIXED is with the flag on.\n\n" }, { "path": "pkg/yqlib/doc/operators/headers/array-to-map.md", "content": "# Array to Map\n\nUse this operator to convert an array to..a map. The indices are used as map keys, null values in the array are skipped over.\n\nBehind the scenes, this is implemented using reduce:\n\n```\n(.[] | select(. != null) ) as $i ireduce({}; .[$i | key] = $i)\n```\n" }, { "path": "pkg/yqlib/doc/operators/headers/assign-update.md", "content": "# Assign (Update)\n\nThis operator is used to update node values. It can be used in either the:\n\n### plain form: `=`\nWhich will set the LHS node values equal to the RHS node values. The RHS expression is run against the matching nodes in the pipeline.\n\n### relative form: `|=`\nThis will do a similar thing to the plain form, but the RHS expression is run with _each LHS node as context_. This is useful for updating values based on old values, e.g. increment.\n\n\n### Flags\n- `c` clobber custom tags\n" }, { "path": "pkg/yqlib/doc/operators/headers/boolean-operators.md", "content": "# Boolean Operators\n\nThe `or` and `and` operators take two parameters and return a boolean result. \n\n`not` flips a boolean from true to false, or vice versa. \n\n`any` will return `true` if there are any `true` values in an array sequence, and `all` will return true if _all_ elements in an array are true.\n\n`any_c(condition)` and `all_c(condition)` are like `any` and `all` but they take a condition expression that is used against each element to determine if it's `true`. Note: in `jq` you can simply pass a condition to `any` or `all` and it simply works - `yq` isn't that clever..yet\n\nThese are most commonly used with the `select` operator to filter particular nodes.\n\n## Related Operators\n\n- equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals)\n- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare)\n- select operator [here](https://mikefarah.gitbook.io/yq/operators/select)\n" }, { "path": "pkg/yqlib/doc/operators/headers/collect-into-array.md", "content": "# Collect into Array\n\nThis creates an array using the expression between the square brackets.\n\n" }, { "path": "pkg/yqlib/doc/operators/headers/column.md", "content": "# Column\n\nReturns the column of the matching node. Starts from 1, 0 indicates there was no column data.\n\nColumn is the number of characters that precede that node on the line it starts.\n" }, { "path": "pkg/yqlib/doc/operators/headers/comment-operators.md", "content": "# Comment Operators\n\nUse these comment operators to set or retrieve comments. Note that line comments on maps/arrays are actually set on the _key_ node as opposed to the _value_ (map/array). See below for examples.\n\nLike the `=` and `|=` assign operators, the same syntax applies when updating comments:\n\n### plain form: `=`\nThis will set the LHS nodes' comments equal to the expression on the RHS. The RHS is run against the matching nodes in the pipeline\n\n### relative form: `|=` \nThis is similar to the plain form, but it evaluates the RHS with _each matching LHS node as context_. This is useful if you want to set the comments as a relative expression of the node, for instance its value or path.\n" }, { "path": "pkg/yqlib/doc/operators/headers/compare.md", "content": "# Compare Operators\n\nComparison operators (`>`, `>=`, `<`, `<=`) can be used for comparing scalar values of the same time.\n\nThe following types are currently supported:\n\n- numbers\n- strings\n- datetimes\n\n## Related Operators\n\n- equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals)\n- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators)\n- select operator [here](https://mikefarah.gitbook.io/yq/operators/select)\n" }, { "path": "pkg/yqlib/doc/operators/headers/contains.md", "content": "# Contains\n\nThis returns `true` if the context contains the passed in parameter, and false otherwise. For arrays, this will return true if the passed in array is contained within the array. For strings, it will return true if the string is a substring.\n\n{% hint style=\"warning\" %}\n\n_Note_ that, just like jq, when checking if an array of strings `contains` another, this will use `contains` and _not_ equals to check each string. This means an expression like `contains([\"cat\"])` will return true for an array `[\"cats\"]`.\n\nSee the \"Array has a subset array\" example below on how to check for a subset.\n\n{% endhint %}\n" }, { "path": "pkg/yqlib/doc/operators/headers/create-collect-into-object.md", "content": "# Create, Collect into Object\n\nThis is used to construct objects (or maps). This can be used against existing yaml, or to create fresh yaml documents.\n" }, { "path": "pkg/yqlib/doc/operators/headers/datetime.md", "content": "# Date Time\n\nVarious operators for parsing and manipulating dates. \n\n## Date time formatting\nThis uses Golang's built in time library for parsing and formatting date times.\n\nWhen not specified, the RFC3339 standard is assumed `2006-01-02T15:04:05Z07:00` for parsing.\n\nTo specify a custom parsing format, use the `with_dtf` operator. The first parameter sets the datetime parsing format for the expression in the second parameter. The expression can be any valid `yq` expression tree.\n\n```bash\nyq 'with_dtf(\"myformat\"; .a + \"3h\" | tz(\"Australia/Melbourne\"))'\n```\n\nSee the [library docs](https://pkg.go.dev/time#pkg-constants) for examples of formatting options.\n\n\n## Timezones\nThis uses Golang's built in LoadLocation function to parse timezones strings. See the [library docs](https://pkg.go.dev/time#LoadLocation) for more details.\n\n\n## Durations\nDurations are parsed using Golang's built in [ParseDuration](https://pkg.go.dev/time#ParseDuration) function.\n\nYou can add durations to time using the `+` operator.\n" }, { "path": "pkg/yqlib/doc/operators/headers/delete.md", "content": "# Delete\n\nDeletes matching entries in maps or arrays.\n" }, { "path": "pkg/yqlib/doc/operators/headers/divide.md", "content": "# Divide\n\nDivide behaves differently according to the type of the LHS:\n* strings: split by the divider\n* number: arithmetic division\n" }, { "path": "pkg/yqlib/doc/operators/headers/document-index.md", "content": "# Document Index\n\nUse the `documentIndex` operator (or the `di` shorthand) to select nodes of a particular document.\n" }, { "path": "pkg/yqlib/doc/operators/headers/encode-decode.md", "content": "# Encoder / Decoder\n\nEncode operators will take the piped in object structure and encode it as a string in the desired format. The decode operators do the opposite, they take a formatted string and decode it into the relevant object structure.\n\nNote that you can optionally pass an indent value to the encode functions (see below).\n\nThese operators are useful to process yaml documents that have stringified embedded yaml/json/props in them.\n\n\n| Format | Decode (from string) | Encode (to string) |\n| --- | -- | --|\n| Yaml | from_yaml/@yamld | to_yaml(i)/@yaml |\n| JSON | from_json/@jsond | to_json(i)/@json |\n| Properties | from_props/@propsd | to_props/@props |\n| CSV | from_csv/@csvd | to_csv/@csv |\n| TSV | from_tsv/@tsvd | to_tsv/@tsv |\n| XML | from_xml/@xmld | to_xml(i)/@xml |\n| Base64 | @base64d | @base64 |\n| URI | @urid | @uri |\n| Shell | | @sh |\n\n\nSee CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) for accepted formats.\n\nXML uses the `--xml-attribute-prefix` and `xml-content-name` flags to identify attributes and content fields.\n\n\nBase64 assumes [rfc4648](https://rfc-editor.org/rfc/rfc4648.html) encoding. Encoding and decoding both assume that the content is a utf-8 string and not binary content.\n" }, { "path": "pkg/yqlib/doc/operators/headers/entries.md", "content": "# Entries\n\nSimilar to the same named functions in `jq` these functions convert to/from an object and an array of key-value pairs. This is most useful for performing operations on keys of maps.\n\nUse `with_entries(op)` as a syntactic sugar for doing `to_entries | op | from_entries`.\n" }, { "path": "pkg/yqlib/doc/operators/headers/env-variable-operators.md", "content": "# Env Variable Operators\n\nThese operators are used to handle environment variables usage in expressions and documents. While environment variables can, of course, be passed in via your CLI with string interpolation, this often comes with complex quote escaping and can be tricky to write and read. \n\nThere are three operators:\n\n- `env` which takes a single environment variable name and parse the variable as a yaml node (be it a map, array, string, number of boolean) \n- `strenv` which also takes a single environment variable name, and always parses the variable as a string.\n- `envsubst` which you pipe strings into and it interpolates environment variables in strings using [envsubst](https://github.com/a8m/envsubst). \n\n\n## EnvSubst Options\nYou can optionally pass envsubst any of the following options:\n\n - nu: NoUnset, this will fail if there are any referenced variables that are not set\n - ne: NoEmpty, this will fail if there are any referenced variables that are empty\n - ff: FailFast, this will abort on the first failure (rather than collect all the errors)\n\nE.g:\n`envsubst(ne, ff)` will fail on the first empty variable.\n\nSee [Imposing Restrictions](https://github.com/a8m/envsubst#imposing-restrictions) in the `envsubst` documentation for more information, and below for examples.\n\n## Tip\nTo replace environment variables across all values in a document, `envsubst` can be used with the recursive descent operator\nas follows:\n\n```bash\nyq '(.. | select(tag == \"!!str\")) |= envsubst' file.yaml\n```\n\n## Disabling env operators\nIf required, you can use the `--security-disable-env-ops` to disable env operations.\n\n" }, { "path": "pkg/yqlib/doc/operators/headers/equals.md", "content": "# Equals / Not Equals\n\nThis is a boolean operator that will return `true` if the LHS is equal to the RHS and `false` otherwise.\n\n```\n.a == .b\n```\n\nIt is most often used with the select operator to find particular nodes:\n\n```\nselect(.a == .b)\n```\n\nThe not equals `!=` operator returns `false` if the LHS is equal to the RHS.\n\n## Related Operators\n\n- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare)\n- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators)\n- select operator [here](https://mikefarah.gitbook.io/yq/operators/select)\n\n" }, { "path": "pkg/yqlib/doc/operators/headers/error.md", "content": "# Error\n\nUse this operation to short-circuit expressions. Useful for validation.\n" }, { "path": "pkg/yqlib/doc/operators/headers/eval.md", "content": "# Eval\n\nUse `eval` to dynamically process an expression - for instance from an environment variable.\n\n`eval` takes a single argument, and evaluates that as a `yq` expression. Any valid expression can be used, be it a path `.a.b.c | select(. == \"cat\")`, or an update `.a.b.c = \"gogo\"`.\n\nTip: This can be a useful way to parameterise complex scripts.\n" }, { "path": "pkg/yqlib/doc/operators/headers/file-operators.md", "content": "# File Operators\n\nFile operators are most often used with merge when needing to merge specific files together. Note that when doing this, you will need to use `eval-all` to ensure all yaml documents are loaded into memory before performing the merge (as opposed to `eval` which runs the expression once per document).\n\nNote that the `fileIndex` operator has a short alias of `fi`.\n\n## Merging files\nNote the use of eval-all to ensure all documents are loaded into memory.\n```bash\nyq eval-all 'select(fi == 0) * select(filename == \"file2.yaml\")' file1.yaml file2.yaml\n```\n" }, { "path": "pkg/yqlib/doc/operators/headers/filter.md", "content": "# Filter\n\nFilters an array (or map values) by the expression given. Equivalent to doing `map(select(exp))`.\n\n" }, { "path": "pkg/yqlib/doc/operators/headers/first.md", "content": "# First\n\nReturns the first matching element in an array, or first matching value in a map.\n\nCan be given an expression to match with, otherwise will just return the first.\n" }, { "path": "pkg/yqlib/doc/operators/headers/flatten.md", "content": "# Flatten\nThis recursively flattens arrays.\n" }, { "path": "pkg/yqlib/doc/operators/headers/group-by.md", "content": "# Group By\n\nThis is used to group items in an array by an expression.\n" }, { "path": "pkg/yqlib/doc/operators/headers/has.md", "content": "# Has\n\nThis operation returns true if the key exists in a map (or index in an array), false otherwise.\n" }, { "path": "pkg/yqlib/doc/operators/headers/keys.md", "content": "# Keys\n\nUse the `keys` operator to return map keys or array indices. \n" }, { "path": "pkg/yqlib/doc/operators/headers/kind.md", "content": "# Kind\n\nThe `kind` operator identifies the type of a node as either `scalar`, `map`, or `seq`.\n\nThis can be used for filtering or transforming nodes based on their type.\n\nNote that `null` values are treated as `scalar`.\n" }, { "path": "pkg/yqlib/doc/operators/headers/length.md", "content": "# Length\n\nReturns the lengths of the nodes. Length is defined according to the type of the node.\n" }, { "path": "pkg/yqlib/doc/operators/headers/line.md", "content": "# Line\n\nReturns the line of the matching node. Starts from 1, 0 indicates there was no line data.\n" }, { "path": "pkg/yqlib/doc/operators/headers/load.md", "content": "# Load\n\nThe load operators allows you to load in content from another file.\n\nNote that you can use string operators like `+` and `sub` to modify the value in the yaml file to a path that exists in your system.\n\nYou can load files of the following supported types:\n\n|Format | Load Operator |\n| --- | --- |\n| Yaml | load |\n| XML | load_xml |\n| Properties | load_props |\n| Plain String | load_str |\n| Base64 | load_base64 |\n\nNote that load_base64 only works for base64 encoded utf-8 strings.\n\n## Samples files for tests:\n\n### yaml\n\n`../../examples/thing.yml`:\n\n```yaml\na: apple is included\nb: cool\n```\n\n### xml\n`small.xml`:\n\n```xml\nis some xml\n```\n\n### properties\n`small.properties`:\n\n```properties\nthis.is = a properties file\n```\n\n### base64\n`base64.txt`:\n```\nbXkgc2VjcmV0IGNoaWxsaSByZWNpcGUgaXMuLi4u\n```\n\n## Disabling file operators\nIf required, you can use the `--security-disable-file-ops` to disable file operations.\n\n" }, { "path": "pkg/yqlib/doc/operators/headers/map.md", "content": "# Map\n\nMaps values of an array. Use `map_values` to map values of an object.\n" }, { "path": "pkg/yqlib/doc/operators/headers/max.md", "content": "# Max\n\nComputes the maximum among an incoming sequence of scalar values.\n" }, { "path": "pkg/yqlib/doc/operators/headers/min.md", "content": "# Min\n\nComputes the minimum among an incoming sequence of scalar values.\n" }, { "path": "pkg/yqlib/doc/operators/headers/modulo.md", "content": "# Modulo\n\nArithmetic modulo operator, returns the remainder from dividing two numbers.\n" }, { "path": "pkg/yqlib/doc/operators/headers/multiply-merge.md", "content": "# Multiply (Merge)\n\nLike the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently numbers, arrays and objects are supported.\n\n## Objects and arrays - merging\nObjects are merged _deeply_ matching on matching keys. By default, array values override and are not deeply merged.\n\nYou can use the add operator `+`, to shallow merge objects, see more info [here](https://mikefarah.gitbook.io/yq/operators/add).\n\nNote that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.\n\n### Merge Flags\nYou can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`. See examples below\n\n- `+` append arrays\n- `d` deeply merge arrays\n- `?` only merge _existing_ fields\n- `n` only merge _new_ fields\n- `c` clobber custom tags\n\nTo perform a shallow merge only, use the add operator `+`, see more info [here](https://mikefarah.gitbook.io/yq/operators/add).\n\n### Merge two files together\nThis uses the load operator to merge file2 into file1.\n```bash\nyq '. *= load(\"file2.yml\")' file1.yml\n```\n\n### Merging all files\nNote the use of `eval-all` to ensure all documents are loaded into memory.\n\n```bash\nyq eval-all '. as $item ireduce ({}; . * $item )' *.yml\n```\n\n# Merging complex arrays together by a key field\nBy default - `yq` merge is naive. It merges maps when they match the key name, and arrays are merged either by appending them together, or merging the entries by their position in the array.\n\nFor more complex array merging (e.g. merging items that match on a certain key) please see the example [here](https://mikefarah.gitbook.io/yq/operators/multiply-merge#merge-arrays-of-objects-together-matching-on-a-key)\n\n" }, { "path": "pkg/yqlib/doc/operators/headers/omit.md", "content": "# Omit\n\nWorks like `pick`, but instead you specify the keys/indices that you _don't_ want included.\n" }, { "path": "pkg/yqlib/doc/operators/headers/parent.md", "content": "# Parent\n\nParent simply returns the parent nodes of the matching nodes.\n" }, { "path": "pkg/yqlib/doc/operators/headers/path.md", "content": "# Path\n\nThe `path` operator can be used to get the traversal paths of matching nodes in an expression. The path is returned as an array, which if traversed in order will lead to the matching node.\n\nYou can get the key/index of matching nodes by using the `path` operator to return the path array then piping that through `.[-1]` to get the last element of that array, the key.\n\nUse `setpath` to set a value to the path array returned by `path`, and similarly `delpaths` for an array of path arrays.\n\n" }, { "path": "pkg/yqlib/doc/operators/headers/pick.md", "content": "# Pick\n\nFilter a map by the specified list of keys. Map is returned with the key in the order of the pick list.\n\nSimilarly, filter an array by the specified list of indices.\n" }, { "path": "pkg/yqlib/doc/operators/headers/pipe.md", "content": "# Pipe\n\nPipe the results of an expression into another. Like the bash operator.\n" }, { "path": "pkg/yqlib/doc/operators/headers/pivot.md", "content": "# Pivot\n\nEmulates the `PIVOT` function supported by several popular RDBMS systems.\n" }, { "path": "pkg/yqlib/doc/operators/headers/recursive-descent-glob.md", "content": "# Recursive Descent (Glob)\n\nThis operator recursively matches (or globs) all children nodes given of a particular element, including that node itself. This is most often used to apply a filter recursively against all matches.\n\n## match values form `..`\nThis will, like the `jq` equivalent, recursively match all _value_ nodes. Use it to find/manipulate particular values.\n\nFor instance to set the `style` of all _value_ nodes in a yaml doc, excluding map keys:\n\n```bash\nyq '.. style= \"flow\"' file.yaml\n```\n\n## match values and map keys form `...`\nThe also includes map keys in the results set. This is particularly useful in YAML as unlike JSON, map keys can have their own styling and tags and also use anchors and aliases.\n\nFor instance to set the `style` of all nodes in a yaml doc, including the map keys:\n\n```bash\nyq '... style= \"flow\"' file.yaml\n```" }, { "path": "pkg/yqlib/doc/operators/headers/reduce.md", "content": "# Reduce\n\nReduce is a powerful way to process a collection of data into a new form.\n\n```\n as $ ireduce (; )\n```\n\ne.g.\n\n```\n.[] as $item ireduce (0; . + $item)\n```\n\nOn the LHS we are configuring the collection of items that will be reduced `` as well as what each element will be called `$`. Note that the array has been splatted into its individual elements.\n\nOn the RHS there is ``, the starting value of the accumulator and ``, the expression that will update the accumulator for each element in the collection. Note that within the block expression, `.` will evaluate to the current value of the accumulator. \n\n## yq vs jq syntax\nReduce syntax in `yq` is a little different from `jq` - as `yq` (currently) isn't as sophisticated as `jq` and its only supports infix notation (e.g. a + b, where the operator is in the middle of the two parameters) - where as `jq` uses a mix of infix notation with _prefix_ notation (e.g. `reduce a b` is like writing `+ a b`).\n\nTo that end, the reduce operator is called `ireduce` for backwards compatibility if a `jq` like prefix version of `reduce` is ever added.\n" }, { "path": "pkg/yqlib/doc/operators/headers/reverse.md", "content": "# Reverse\n\nReverses the order of the items in an array \n" }, { "path": "pkg/yqlib/doc/operators/headers/select.md", "content": "# Select\n\nSelect is used to filter arrays and maps by a boolean expression.\n\n## Related Operators\n\n- equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals)\n- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare)\n- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators)\n" }, { "path": "pkg/yqlib/doc/operators/headers/shuffle.md", "content": "# Shuffle\n\nShuffles an array. Note that this command does _not_ use a cryptographically secure random number generator to randomise the array order.\n\n" }, { "path": "pkg/yqlib/doc/operators/headers/slice-array.md", "content": "# Slice/Splice Array\n\nThe slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array.\n\nYou may leave out the first or second number, which will refer to the start or end of the array respectively.\n" }, { "path": "pkg/yqlib/doc/operators/headers/sort-keys.md", "content": "# Sort Keys\n\nThe Sort Keys operator sorts maps by their keys (based on their string value). This operator does not do anything to arrays or scalars (so you can easily recursively apply it to all maps).\n\nSort is particularly useful for diffing two different yaml documents:\n\n```bash\nyq -i -P 'sort_keys(..)' file1.yml\nyq -i -P 'sort_keys(..)' file2.yml\ndiff file1.yml file2.yml\n```\n\nNote that `yq` does not yet consider anchors when sorting by keys - this may result in invalid yaml documents if you are using merge anchors.\n\nFor more advanced sorting, you can use the [sort_by](https://mikefarah.gitbook.io/yq/operators/sort) function on a map, and give it a custom function like `sort_by(key | downcase)`.\n\n" }, { "path": "pkg/yqlib/doc/operators/headers/sort.md", "content": "# Sort\n\nSorts an array. Use `sort` to sort an array as is, or `sort_by(exp)` to sort by a particular expression (e.g. subfield).\n\nTo sort by descending order, pipe the results through the `reverse` operator after sorting.\n\nNote that at this stage, `yq` only sorts scalar fields.\n\n" }, { "path": "pkg/yqlib/doc/operators/headers/split-into-documents.md", "content": "# Split into Documents\n\nThis operator splits all matches into separate documents\n" }, { "path": "pkg/yqlib/doc/operators/headers/string-operators.md", "content": "# String Operators\n\n## RegEx\nThis uses Golang's native regex functions under the hood - See their [docs](https://github.com/google/re2/wiki/Syntax) for the supported syntax.\n\nCase insensitive tip: prefix the regex with `(?i)` - e.g. `test(\"(?i)cats\")`.\n\n### match(regEx)\nThis operator returns the substring match details of the given regEx.\n\n### capture(regEx)\nCapture returns named RegEx capture groups in a map. Can be more convenient than `match` depending on what you are doing.\n\n## test(regEx)\nReturns true if the string matches the RegEx, false otherwise.\n\n## sub(regEx, replacement)\nSubstitutes matched substrings. The first parameter is the regEx to match substrings within the original string. The second parameter specifies what to replace those matches with. This can refer to capture groups from the first RegEx.\n\n## String blocks, bash and newlines\nBash is notorious for chomping on precious trailing newline characters, making it tricky to set strings with newlines properly. In particular, the `$( exp )` _will trim trailing newlines_.\n\nFor instance to get this yaml:\n\n```\na: |\n cat\n```\n\nUsing `$( exp )` wont work, as it will trim the trailing newline.\n\n```\nm=$(echo \"cat\\n\") yq -n '.a = strenv(m)'\na: cat\n```\n\nHowever, using printf works:\n```\nprintf -v m \"cat\\n\" ; m=\"$m\" yq -n '.a = strenv(m)'\na: |\n cat\n```\n\nAs well as having multiline expressions:\n```\nm=\"cat\n\" yq -n '.a = strenv(m)'\na: |\n cat\n```\n\nSimilarly, if you're trying to set the content from a file, and want a trailing newline:\n\n```\nIFS= read -rd '' output < <(cat my_file)\noutput=$output ./yq '.data.values = strenv(output)' first.yml\n```\n" }, { "path": "pkg/yqlib/doc/operators/headers/style.md", "content": "# Style\n\nThe style operator can be used to get or set the style of nodes (e.g. string style, yaml style).\nUse this to control the formatting of the document in yaml.\n\n" }, { "path": "pkg/yqlib/doc/operators/headers/subtract.md", "content": "# Subtract\n\nYou can use subtract to subtract numbers as well as remove elements from an array.\n" }, { "path": "pkg/yqlib/doc/operators/headers/tag.md", "content": "# Tag\n\nThe tag operator can be used to get or set the tag of nodes (e.g. `!!str`, `!!int`, `!!bool`).\n" }, { "path": "pkg/yqlib/doc/operators/headers/to_number.md", "content": "# To Number\nParses the input as a number. yq will try to parse values as an int first, failing that it will try float. Values that already ints or floats will be left alone.\n" }, { "path": "pkg/yqlib/doc/operators/headers/traverse-read.md", "content": "# Traverse (Read)\n\nThis is the simplest (and perhaps most used) operator. It is used to navigate deeply into yaml structures.\n\n\n## NOTE --yaml-fix-merge-anchor-to-spec flag\n`yq` doesn't merge anchors `<<:` to spec, in some circumstances it incorrectly overrides existing keys when the spec documents not to do that.\n\nTo minimise disruption while still fixing the issue, a flag has been added to toggle this behaviour. This will first default to false; and log warnings to users. Then it will default to true (and still allow users to specify false if needed)\n\nSee examples of the flag differences below, where LEGACY is the flag off; and FIXED is with the flag on.\n\n" }, { "path": "pkg/yqlib/doc/operators/headers/union.md", "content": "# Union\n\nThis operator is used to combine different results together.\n" }, { "path": "pkg/yqlib/doc/operators/headers/unique.md", "content": "# Unique\n\nThis is used to filter out duplicated items in an array. Note that the original order of the array is maintained.\n\n" }, { "path": "pkg/yqlib/doc/operators/headers/variable-operators.md", "content": "# Variable Operators\n\nLike the `jq` equivalents, variables are sometimes required for the more complex expressions (or swapping values between fields).\n\nNote that there is also an additional `ref` operator that holds a reference (instead of a copy) of the path, allowing you to make multiple changes to the same path.\n" }, { "path": "pkg/yqlib/doc/operators/headers/with.md", "content": "# With\n\nUse the `with` operator to conveniently make multiple updates to a deeply nested path, or to update array elements relatively to each other. The first argument expression sets the root context, and the second expression runs against that root context.\n" }, { "path": "pkg/yqlib/doc/operators/keys.md", "content": "# Keys\n\nUse the `keys` operator to return map keys or array indices. \n\n## Map keys\nGiven a sample.yml file of:\n```yaml\ndog: woof\ncat: meow\n```\nthen\n```bash\nyq 'keys' sample.yml\n```\nwill output\n```yaml\n- dog\n- cat\n```\n\n## Array keys\nGiven a sample.yml file of:\n```yaml\n- apple\n- banana\n```\nthen\n```bash\nyq 'keys' sample.yml\n```\nwill output\n```yaml\n- 0\n- 1\n```\n\n## Retrieve array key\nGiven a sample.yml file of:\n```yaml\n- 1\n- 2\n- 3\n```\nthen\n```bash\nyq '.[1] | key' sample.yml\n```\nwill output\n```yaml\n1\n```\n\n## Retrieve map key\nGiven a sample.yml file of:\n```yaml\na: thing\n```\nthen\n```bash\nyq '.a | key' sample.yml\n```\nwill output\n```yaml\na\n```\n\n## No key\nGiven a sample.yml file of:\n```yaml\n{}\n```\nthen\n```bash\nyq 'key' sample.yml\n```\nwill output\n```yaml\n```\n\n## Update map key\nGiven a sample.yml file of:\n```yaml\na:\n x: 3\n y: 4\n```\nthen\n```bash\nyq '(.a.x | key) = \"meow\"' sample.yml\n```\nwill output\n```yaml\na:\n meow: 3\n y: 4\n```\n\n## Get comment from map key\nGiven a sample.yml file of:\n```yaml\na:\n # comment on key\n x: 3\n y: 4\n```\nthen\n```bash\nyq '.a.x | key | headComment' sample.yml\n```\nwill output\n```yaml\ncomment on key\n```\n\n## Check node is a key\nGiven a sample.yml file of:\n```yaml\na:\n b:\n - cat\n c: frog\n```\nthen\n```bash\nyq '[... | { \"p\": path | join(\".\"), \"isKey\": is_key, \"tag\": tag }]' sample.yml\n```\nwill output\n```yaml\n- p: \"\"\n isKey: false\n tag: '!!map'\n- p: a\n isKey: true\n tag: '!!str'\n- p: a\n isKey: false\n tag: '!!map'\n- p: a.b\n isKey: true\n tag: '!!str'\n- p: a.b\n isKey: false\n tag: '!!seq'\n- p: a.b.0\n isKey: false\n tag: '!!str'\n- p: a.c\n isKey: true\n tag: '!!str'\n- p: a.c\n isKey: false\n tag: '!!str'\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/kind.md", "content": "# Kind\n\nThe `kind` operator identifies the type of a node as either `scalar`, `map`, or `seq`.\n\nThis can be used for filtering or transforming nodes based on their type.\n\nNote that `null` values are treated as `scalar`.\n\n## Get kind\nGiven a sample.yml file of:\n```yaml\na: cat\nb: 5\nc: 3.2\ne: true\nf: []\ng: {}\nh: null\n```\nthen\n```bash\nyq '.. | kind' sample.yml\n```\nwill output\n```yaml\nmap\nscalar\nscalar\nscalar\nscalar\nseq\nmap\nscalar\n```\n\n## Get kind, ignores custom tags\nUnlike tag, kind is not affected by custom tags.\n\nGiven a sample.yml file of:\n```yaml\na: !!thing cat\nb: !!foo {}\nc: !!bar []\n```\nthen\n```bash\nyq '.. | kind' sample.yml\n```\nwill output\n```yaml\nmap\nscalar\nmap\nseq\n```\n\n## Add comments only to scalars\nAn example of how you can use kind\n\nGiven a sample.yml file of:\n```yaml\na:\n b: 5\n c: 3.2\ne: true\nf: []\ng: {}\nh: null\n```\nthen\n```bash\nyq '(.. | select(kind == \"scalar\")) line_comment = \"this is a scalar\"' sample.yml\n```\nwill output\n```yaml\na:\n b: 5 # this is a scalar\n c: 3.2 # this is a scalar\ne: true # this is a scalar\nf: []\ng: {}\nh: null # this is a scalar\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/length.md", "content": "# Length\n\nReturns the lengths of the nodes. Length is defined according to the type of the node.\n\n## String length\nreturns length of string\n\nGiven a sample.yml file of:\n```yaml\na: cat\n```\nthen\n```bash\nyq '.a | length' sample.yml\n```\nwill output\n```yaml\n3\n```\n\n## null length\nGiven a sample.yml file of:\n```yaml\na: null\n```\nthen\n```bash\nyq '.a | length' sample.yml\n```\nwill output\n```yaml\n0\n```\n\n## Map length\nreturns number of entries\n\nGiven a sample.yml file of:\n```yaml\na: cat\nc: dog\n```\nthen\n```bash\nyq 'length' sample.yml\n```\nwill output\n```yaml\n2\n```\n\n## Array length\nreturns number of elements\n\nGiven a sample.yml file of:\n```yaml\n- 2\n- 4\n- 6\n- 8\n```\nthen\n```bash\nyq 'length' sample.yml\n```\nwill output\n```yaml\n4\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/line.md", "content": "# Line\n\nReturns the line of the matching node. Starts from 1, 0 indicates there was no line data.\n\n## Returns line of _value_ node\nGiven a sample.yml file of:\n```yaml\na: cat\nb:\n c: cat\n```\nthen\n```bash\nyq '.b | line' sample.yml\n```\nwill output\n```yaml\n3\n```\n\n## Returns line of _key_ node\nPipe through the key operator to get the line of the key\n\nGiven a sample.yml file of:\n```yaml\na: cat\nb:\n c: cat\n```\nthen\n```bash\nyq '.b | key | line' sample.yml\n```\nwill output\n```yaml\n2\n```\n\n## First line is 1\nGiven a sample.yml file of:\n```yaml\na: cat\n```\nthen\n```bash\nyq '.a | line' sample.yml\n```\nwill output\n```yaml\n1\n```\n\n## No line data is 0\nRunning\n```bash\nyq --null-input '{\"a\": \"new entry\"} | line'\n```\nwill output\n```yaml\n0\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/load.md", "content": "# Load\n\nThe load operators allows you to load in content from another file.\n\nNote that you can use string operators like `+` and `sub` to modify the value in the yaml file to a path that exists in your system.\n\nYou can load files of the following supported types:\n\n|Format | Load Operator |\n| --- | --- |\n| Yaml | load |\n| XML | load_xml |\n| Properties | load_props |\n| Plain String | load_str |\n| Base64 | load_base64 |\n\nNote that load_base64 only works for base64 encoded utf-8 strings.\n\n## Samples files for tests:\n\n### yaml\n\n`../../examples/thing.yml`:\n\n```yaml\na: apple is included\nb: cool\n```\n\n### xml\n`small.xml`:\n\n```xml\nis some xml\n```\n\n### properties\n`small.properties`:\n\n```properties\nthis.is = a properties file\n```\n\n### base64\n`base64.txt`:\n```\nbXkgc2VjcmV0IGNoaWxsaSByZWNpcGUgaXMuLi4u\n```\n\n## Disabling file operators\nIf required, you can use the `--security-disable-file-ops` to disable file operations.\n\n\n## Simple example\nGiven a sample.yml file of:\n```yaml\nmyFile: ../../examples/thing.yml\n```\nthen\n```bash\nyq 'load(.myFile)' sample.yml\n```\nwill output\n```yaml\na: apple is included\nb: cool.\n```\n\n## Replace node with referenced file\nNote that you can modify the filename in the load operator if needed.\n\nGiven a sample.yml file of:\n```yaml\nsomething:\n file: thing.yml\n```\nthen\n```bash\nyq '.something |= load(\"../../examples/\" + .file)' sample.yml\n```\nwill output\n```yaml\nsomething:\n a: apple is included\n b: cool.\n```\n\n## Replace _all_ nodes with referenced file\nRecursively match all the nodes (`..`) and then filter the ones that have a 'file' attribute. \n\nGiven a sample.yml file of:\n```yaml\nsomething:\n file: thing.yml\nover:\n here:\n - file: thing.yml\n```\nthen\n```bash\nyq '(.. | select(has(\"file\"))) |= load(\"../../examples/\" + .file)' sample.yml\n```\nwill output\n```yaml\nsomething:\n a: apple is included\n b: cool.\nover:\n here:\n - a: apple is included\n b: cool.\n```\n\n## Replace node with referenced file as string\nThis will work for any text based file\n\nGiven a sample.yml file of:\n```yaml\nsomething:\n file: thing.yml\n```\nthen\n```bash\nyq '.something |= load_str(\"../../examples/\" + .file)' sample.yml\n```\nwill output\n```yaml\nsomething: |-\n a: apple is included\n b: cool.\n```\n\n## Load from XML\nGiven a sample.yml file of:\n```yaml\ncool: things\n```\nthen\n```bash\nyq '.more_stuff = load_xml(\"../../examples/small.xml\")' sample.yml\n```\nwill output\n```yaml\ncool: things\nmore_stuff:\n this: is some xml\n```\n\n## Load from Properties\nGiven a sample.yml file of:\n```yaml\ncool: things\n```\nthen\n```bash\nyq '.more_stuff = load_props(\"../../examples/small.properties\")' sample.yml\n```\nwill output\n```yaml\ncool: things\nmore_stuff:\n this:\n is: a properties file\n```\n\n## Merge from properties\nThis can be used as a convenient way to update a yaml document\n\nGiven a sample.yml file of:\n```yaml\nthis:\n is: from yaml\n cool: ay\n```\nthen\n```bash\nyq '. *= load_props(\"../../examples/small.properties\")' sample.yml\n```\nwill output\n```yaml\nthis:\n is: a properties file\n cool: ay\n```\n\n## Load from base64 encoded file\nGiven a sample.yml file of:\n```yaml\ncool: things\n```\nthen\n```bash\nyq '.more_stuff = load_base64(\"../../examples/base64.txt\")' sample.yml\n```\nwill output\n```yaml\ncool: things\nmore_stuff: my secret chilli recipe is....\n```\n\n## load() operation fails when security is enabled\nUse `--security-disable-file-ops` to disable file operations for security.\n\nRunning\n```bash\nyq --null-input 'load(\"../../examples/thing.yml\")'\n```\nwill output\n```bash\nError: file operations have been disabled\n```\n\n## load_str() operation fails when security is enabled\nUse `--security-disable-file-ops` to disable file operations for security.\n\nRunning\n```bash\nyq --null-input 'load_str(\"../../examples/thing.yml\")'\n```\nwill output\n```bash\nError: file operations have been disabled\n```\n\n## load_xml() operation fails when security is enabled\nUse `--security-disable-file-ops` to disable file operations for security.\n\nRunning\n```bash\nyq --null-input 'load_xml(\"../../examples/small.xml\")'\n```\nwill output\n```bash\nError: file operations have been disabled\n```\n\n## load_props() operation fails when security is enabled\nUse `--security-disable-file-ops` to disable file operations for security.\n\nRunning\n```bash\nyq --null-input 'load_props(\"../../examples/small.properties\")'\n```\nwill output\n```bash\nError: file operations have been disabled\n```\n\n## load_base64() operation fails when security is enabled\nUse `--security-disable-file-ops` to disable file operations for security.\n\nRunning\n```bash\nyq --null-input 'load_base64(\"../../examples/base64.txt\")'\n```\nwill output\n```bash\nError: file operations have been disabled\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/map.md", "content": "# Map\n\nMaps values of an array. Use `map_values` to map values of an object.\n\n## Map array\nGiven a sample.yml file of:\n```yaml\n- 1\n- 2\n- 3\n```\nthen\n```bash\nyq 'map(. + 1)' sample.yml\n```\nwill output\n```yaml\n- 2\n- 3\n- 4\n```\n\n## Map object values\nGiven a sample.yml file of:\n```yaml\na: 1\nb: 2\nc: 3\n```\nthen\n```bash\nyq 'map_values(. + 1)' sample.yml\n```\nwill output\n```yaml\na: 2\nb: 3\nc: 4\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/max.md", "content": "# Max\n\nComputes the maximum among an incoming sequence of scalar values.\n\n## Maximum int\nGiven a sample.yml file of:\n```yaml\n- 99\n- 16\n- 12\n- 6\n- 66\n```\nthen\n```bash\nyq 'max' sample.yml\n```\nwill output\n```yaml\n99\n```\n\n## Maximum string\nGiven a sample.yml file of:\n```yaml\n- foo\n- bar\n- baz\n```\nthen\n```bash\nyq 'max' sample.yml\n```\nwill output\n```yaml\nfoo\n```\n\n## Maximum of empty\nGiven a sample.yml file of:\n```yaml\n[]\n```\nthen\n```bash\nyq 'max' sample.yml\n```\nwill output\n```yaml\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/min.md", "content": "# Min\n\nComputes the minimum among an incoming sequence of scalar values.\n\n## Minimum int\nGiven a sample.yml file of:\n```yaml\n- 99\n- 16\n- 12\n- 6\n- 66\n```\nthen\n```bash\nyq 'min' sample.yml\n```\nwill output\n```yaml\n6\n```\n\n## Minimum string\nGiven a sample.yml file of:\n```yaml\n- foo\n- bar\n- baz\n```\nthen\n```bash\nyq 'min' sample.yml\n```\nwill output\n```yaml\nbar\n```\n\n## Minimum of empty\nGiven a sample.yml file of:\n```yaml\n[]\n```\nthen\n```bash\nyq 'min' sample.yml\n```\nwill output\n```yaml\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/modulo.md", "content": "# Modulo\n\nArithmetic modulo operator, returns the remainder from dividing two numbers.\n\n## Number modulo - int\nIf the lhs and rhs are ints then the expression will be calculated with ints.\n\nGiven a sample.yml file of:\n```yaml\na: 13\nb: 2\n```\nthen\n```bash\nyq '.a = .a % .b' sample.yml\n```\nwill output\n```yaml\na: 1\nb: 2\n```\n\n## Number modulo - float\nIf the lhs or rhs are floats then the expression will be calculated with floats.\n\nGiven a sample.yml file of:\n```yaml\na: 12\nb: 2.5\n```\nthen\n```bash\nyq '.a = .a % .b' sample.yml\n```\nwill output\n```yaml\na: !!float 2\nb: 2.5\n```\n\n## Number modulo - int by zero\nIf the lhs is an int and rhs is a 0 the result is an error.\n\nGiven a sample.yml file of:\n```yaml\na: 1\nb: 0\n```\nthen\n```bash\nyq '.a = .a % .b' sample.yml\n```\nwill output\n```bash\nError: cannot modulo by 0\n```\n\n## Number modulo - float by zero\nIf the lhs is a float and rhs is a 0 the result is NaN.\n\nGiven a sample.yml file of:\n```yaml\na: 1.1\nb: 0\n```\nthen\n```bash\nyq '.a = .a % .b' sample.yml\n```\nwill output\n```yaml\na: !!float NaN\nb: 0\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/multiply-merge.md", "content": "# Multiply (Merge)\n\nLike the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently numbers, arrays and objects are supported.\n\n## Objects and arrays - merging\nObjects are merged _deeply_ matching on matching keys. By default, array values override and are not deeply merged.\n\nYou can use the add operator `+`, to shallow merge objects, see more info [here](https://mikefarah.gitbook.io/yq/operators/add).\n\nNote that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.\n\n### Merge Flags\nYou can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`. See examples below\n\n- `+` append arrays\n- `d` deeply merge arrays\n- `?` only merge _existing_ fields\n- `n` only merge _new_ fields\n- `c` clobber custom tags\n\nTo perform a shallow merge only, use the add operator `+`, see more info [here](https://mikefarah.gitbook.io/yq/operators/add).\n\n### Merge two files together\nThis uses the load operator to merge file2 into file1.\n```bash\nyq '. *= load(\"file2.yml\")' file1.yml\n```\n\n### Merging all files\nNote the use of `eval-all` to ensure all documents are loaded into memory.\n\n```bash\nyq eval-all '. as $item ireduce ({}; . * $item )' *.yml\n```\n\n# Merging complex arrays together by a key field\nBy default - `yq` merge is naive. It merges maps when they match the key name, and arrays are merged either by appending them together, or merging the entries by their position in the array.\n\nFor more complex array merging (e.g. merging items that match on a certain key) please see the example [here](https://mikefarah.gitbook.io/yq/operators/multiply-merge#merge-arrays-of-objects-together-matching-on-a-key)\n\n\n## Multiply integers\nGiven a sample.yml file of:\n```yaml\na: 3\nb: 4\n```\nthen\n```bash\nyq '.a *= .b' sample.yml\n```\nwill output\n```yaml\na: 12\nb: 4\n```\n\n## Multiply string node X int\nGiven a sample.yml file of:\n```yaml\nb: banana\n```\nthen\n```bash\nyq '.b * 4' sample.yml\n```\nwill output\n```yaml\nbananabananabananabanana\n```\n\n## Multiply int X string node\nGiven a sample.yml file of:\n```yaml\nb: banana\n```\nthen\n```bash\nyq '4 * .b' sample.yml\n```\nwill output\n```yaml\nbananabananabananabanana\n```\n\n## Multiply string X int node\nGiven a sample.yml file of:\n```yaml\nn: 4\n```\nthen\n```bash\nyq '\"banana\" * .n' sample.yml\n```\nwill output\n```yaml\nbananabananabananabanana\n```\n\n## Multiply int node X string\nGiven a sample.yml file of:\n```yaml\nn: 4\n```\nthen\n```bash\nyq '.n * \"banana\"' sample.yml\n```\nwill output\n```yaml\nbananabananabananabanana\n```\n\n## Merge objects together, returning merged result only\nGiven a sample.yml file of:\n```yaml\na:\n field: me\n fieldA: cat\nb:\n field:\n g: wizz\n fieldB: dog\n```\nthen\n```bash\nyq '.a * .b' sample.yml\n```\nwill output\n```yaml\nfield:\n g: wizz\nfieldA: cat\nfieldB: dog\n```\n\n## Merge objects together, returning parent object\nGiven a sample.yml file of:\n```yaml\na:\n field: me\n fieldA: cat\nb:\n field:\n g: wizz\n fieldB: dog\n```\nthen\n```bash\nyq '. * {\"a\":.b}' sample.yml\n```\nwill output\n```yaml\na:\n field:\n g: wizz\n fieldA: cat\n fieldB: dog\nb:\n field:\n g: wizz\n fieldB: dog\n```\n\n## Merge keeps style of LHS\nGiven a sample.yml file of:\n```yaml\na: {things: great}\nb:\n also: \"me\"\n```\nthen\n```bash\nyq '. * {\"a\":.b}' sample.yml\n```\nwill output\n```yaml\na: {things: great, also: \"me\"}\nb:\n also: \"me\"\n```\n\n## Merge arrays\nGiven a sample.yml file of:\n```yaml\na:\n - 1\n - 2\n - 3\nb:\n - 3\n - 4\n - 5\n```\nthen\n```bash\nyq '. * {\"a\":.b}' sample.yml\n```\nwill output\n```yaml\na:\n - 3\n - 4\n - 5\nb:\n - 3\n - 4\n - 5\n```\n\n## Merge, only existing fields\nGiven a sample.yml file of:\n```yaml\na:\n thing: one\n cat: frog\nb:\n missing: two\n thing: two\n```\nthen\n```bash\nyq '.a *? .b' sample.yml\n```\nwill output\n```yaml\nthing: two\ncat: frog\n```\n\n## Merge, only new fields\nGiven a sample.yml file of:\n```yaml\na:\n thing: one\n cat: frog\nb:\n missing: two\n thing: two\n```\nthen\n```bash\nyq '.a *n .b' sample.yml\n```\nwill output\n```yaml\nthing: one\ncat: frog\nmissing: two\n```\n\n## Merge, appending arrays\nGiven a sample.yml file of:\n```yaml\na:\n array:\n - 1\n - 2\n - animal: dog\n value: coconut\nb:\n array:\n - 3\n - 4\n - animal: cat\n value: banana\n```\nthen\n```bash\nyq '.a *+ .b' sample.yml\n```\nwill output\n```yaml\narray:\n - 1\n - 2\n - animal: dog\n - 3\n - 4\n - animal: cat\nvalue: banana\n```\n\n## Merge, only existing fields, appending arrays\nGiven a sample.yml file of:\n```yaml\na:\n thing:\n - 1\n - 2\nb:\n thing:\n - 3\n - 4\n another:\n - 1\n```\nthen\n```bash\nyq '.a *?+ .b' sample.yml\n```\nwill output\n```yaml\nthing:\n - 1\n - 2\n - 3\n - 4\n```\n\n## Merge, deeply merging arrays\nMerging arrays deeply means arrays are merged like objects, with indices as their key. In this case, we merge the first item in the array and do nothing with the second.\n\nGiven a sample.yml file of:\n```yaml\na:\n - name: fred\n age: 12\n - name: bob\n age: 32\nb:\n - name: fred\n age: 34\n```\nthen\n```bash\nyq '.a *d .b' sample.yml\n```\nwill output\n```yaml\n- name: fred\n age: 34\n- name: bob\n age: 32\n```\n\n## Merge arrays of objects together, matching on a key\n\nThis is a fairly complex expression - you can use it as is by providing the environment variables as seen in the example below.\n\nIt merges in the array provided in the second file into the first - matching on equal keys.\n\nExplanation:\n\nThe approach, at a high level, is to reduce into a merged map (keyed by the unique key)\nand then convert that back into an array.\n\nFirst the expression will create a map from the arrays keyed by the idPath, the unique field we want to merge by.\nThe reduce operator is merging '({}; . * $item )', so array elements with the matching key will be merged together.\n\nNext, we convert the map back to an array, using reduce again, concatenating all the map values together.\n\nFinally, we set the result of the merged array back into the first doc.\n\nThanks Kev from [stackoverflow](https://stackoverflow.com/a/70109529/1168223)\n\n\nGiven a sample.yml file of:\n```yaml\nmyArray:\n - a: apple\n b: appleB\n - a: kiwi\n b: kiwiB\n - a: banana\n b: bananaB\nsomething: else\n```\nAnd another sample another.yml file of:\n```yaml\nnewArray:\n - a: banana\n c: bananaC\n - a: apple\n b: appleB2\n - a: dingo\n c: dingoC\n```\nthen\n```bash\nidPath=\".a\" originalPath=\".myArray\" otherPath=\".newArray\" yq eval-all '\n(\n (( (eval(strenv(originalPath)) + eval(strenv(otherPath))) | .[] | {(eval(strenv(idPath))): .}) as $item ireduce ({}; . * $item )) as $uniqueMap\n | ( $uniqueMap | to_entries | .[]) as $item ireduce([]; . + $item.value)\n) as $mergedArray\n| select(fi == 0) | (eval(strenv(originalPath))) = $mergedArray\n' sample.yml another.yml\n```\nwill output\n```yaml\nmyArray:\n - a: apple\n b: appleB2\n - a: kiwi\n b: kiwiB\n - a: banana\n b: bananaB\n c: bananaC\n - a: dingo\n c: dingoC\nsomething: else\n```\n\n## Merge to prefix an element\nGiven a sample.yml file of:\n```yaml\na: cat\nb: dog\n```\nthen\n```bash\nyq '. * {\"a\": {\"c\": .a}}' sample.yml\n```\nwill output\n```yaml\na:\n c: cat\nb: dog\n```\n\n## Merge with simple aliases\nGiven a sample.yml file of:\n```yaml\na: &cat\n c: frog\nb:\n f: *cat\nc:\n g: thongs\n```\nthen\n```bash\nyq '.c * .b' sample.yml\n```\nwill output\n```yaml\ng: thongs\nf: *cat\n```\n\n## Merge copies anchor names\nGiven a sample.yml file of:\n```yaml\na:\n c: &cat frog\nb:\n f: *cat\nc:\n g: thongs\n```\nthen\n```bash\nyq '.c * .a' sample.yml\n```\nwill output\n```yaml\ng: thongs\nc: &cat frog\n```\n\n## Merge with merge anchors\nGiven a sample.yml file of:\n```yaml\nfoo: &foo\n a: foo_a\n thing: foo_thing\n c: foo_c\nbar: &bar\n b: bar_b\n thing: bar_thing\n c: bar_c\nfoobarList:\n b: foobarList_b\n !!merge <<:\n - *foo\n - *bar\n c: foobarList_c\nfoobar:\n c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n```\nthen\n```bash\nyq '.foobar * .foobarList' sample.yml\n```\nwill output\n```yaml\nc: foobarList_c\n!!merge <<:\n - *foo\n - *bar\nthing: foobar_thing\nb: foobarList_b\n```\n\n## Custom types: that are really numbers\nWhen custom tags are encountered, yq will try to decode the underlying type.\n\nGiven a sample.yml file of:\n```yaml\na: !horse 2\nb: !goat 3\n```\nthen\n```bash\nyq '.a = .a * .b' sample.yml\n```\nwill output\n```yaml\na: !horse 6\nb: !goat 3\n```\n\n## Custom types: that are really maps\nCustom tags will be maintained.\n\nGiven a sample.yml file of:\n```yaml\na: !horse\n cat: meow\nb: !goat\n dog: woof\n```\nthen\n```bash\nyq '.a = .a * .b' sample.yml\n```\nwill output\n```yaml\na: !horse\n cat: meow\n dog: woof\nb: !goat\n dog: woof\n```\n\n## Custom types: clobber tags\nUse the `c` option to clobber custom tags. Note that the second tag is now used.\n\nGiven a sample.yml file of:\n```yaml\na: !horse\n cat: meow\nb: !goat\n dog: woof\n```\nthen\n```bash\nyq '.a *=c .b' sample.yml\n```\nwill output\n```yaml\na: !goat\n cat: meow\n dog: woof\nb: !goat\n dog: woof\n```\n\n## Merging a null with a map\nRunning\n```bash\nyq --null-input 'null * {\"some\": \"thing\"}'\n```\nwill output\n```yaml\nsome: thing\n```\n\n## Merging a map with null\nRunning\n```bash\nyq --null-input '{\"some\": \"thing\"} * null'\n```\nwill output\n```yaml\nsome: thing\n```\n\n## Merging a null with an array\nRunning\n```bash\nyq --null-input 'null * [\"some\"]'\n```\nwill output\n```yaml\n- some\n```\n\n## Merging an array with null\nRunning\n```bash\nyq --null-input '[\"some\"] * null'\n```\nwill output\n```yaml\n- some\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/omit.md", "content": "# Omit\n\nWorks like `pick`, but instead you specify the keys/indices that you _don't_ want included.\n\n## Omit keys from map\nNote that non existent keys are skipped.\n\nGiven a sample.yml file of:\n```yaml\nmyMap:\n cat: meow\n dog: bark\n thing: hamster\n hamster: squeak\n```\nthen\n```bash\nyq '.myMap |= omit([\"hamster\", \"cat\", \"goat\"])' sample.yml\n```\nwill output\n```yaml\nmyMap:\n dog: bark\n thing: hamster\n```\n\n## Omit indices from array\nNote that non existent indices are skipped.\n\nGiven a sample.yml file of:\n```yaml\n- cat\n- leopard\n- lion\n```\nthen\n```bash\nyq 'omit([2, 0, 734, -5])' sample.yml\n```\nwill output\n```yaml\n- leopard\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/parent.md", "content": "# Parent\n\nParent simply returns the parent nodes of the matching nodes.\n\n## Simple example\nGiven a sample.yml file of:\n```yaml\na:\n nested: cat\n```\nthen\n```bash\nyq '.a.nested | parent' sample.yml\n```\nwill output\n```yaml\nnested: cat\n```\n\n## Parent of nested matches\nGiven a sample.yml file of:\n```yaml\na:\n fruit: apple\n name: bob\nb:\n fruit: banana\n name: sam\n```\nthen\n```bash\nyq '.. | select(. == \"banana\") | parent' sample.yml\n```\nwill output\n```yaml\nfruit: banana\nname: sam\n```\n\n## Get parent attribute\nGiven a sample.yml file of:\n```yaml\na:\n fruit: apple\n name: bob\nb:\n fruit: banana\n name: sam\n```\nthen\n```bash\nyq '.. | select(. == \"banana\") | parent.name' sample.yml\n```\nwill output\n```yaml\nsam\n```\n\n## Get parents\nMatch all parents\n\nGiven a sample.yml file of:\n```yaml\na:\n b:\n c: cat\n```\nthen\n```bash\nyq '.a.b.c | parents' sample.yml\n```\nwill output\n```yaml\n- c: cat\n- b:\n c: cat\n- a:\n b:\n c: cat\n```\n\n## Get the top (root) parent\nUse negative numbers to get the top parents. You can think of this as indexing into the 'parents' array above\n\nGiven a sample.yml file of:\n```yaml\na:\n b:\n c: cat\n```\nthen\n```bash\nyq '.a.b.c | parent(-1)' sample.yml\n```\nwill output\n```yaml\na:\n b:\n c: cat\n```\n\n## Root\nAlias for parent(-1), returns the top level parent. This is usually the document node.\n\nGiven a sample.yml file of:\n```yaml\na:\n b:\n c: cat\n```\nthen\n```bash\nyq '.a.b.c | root' sample.yml\n```\nwill output\n```yaml\na:\n b:\n c: cat\n```\n\n## N-th parent\nYou can optionally supply the number of levels to go up for the parent, the default being 1.\n\nGiven a sample.yml file of:\n```yaml\na:\n b:\n c: cat\n```\nthen\n```bash\nyq '.a.b.c | parent(2)' sample.yml\n```\nwill output\n```yaml\nb:\n c: cat\n```\n\n## N-th parent - another level\nGiven a sample.yml file of:\n```yaml\na:\n b:\n c: cat\n```\nthen\n```bash\nyq '.a.b.c | parent(3)' sample.yml\n```\nwill output\n```yaml\na:\n b:\n c: cat\n```\n\n## N-th negative\nSimilarly, use negative numbers to index backwards from the parents array\n\nGiven a sample.yml file of:\n```yaml\na:\n b:\n c: cat\n```\nthen\n```bash\nyq '.a.b.c | parent(-2)' sample.yml\n```\nwill output\n```yaml\nb:\n c: cat\n```\n\n## No parent\nGiven a sample.yml file of:\n```yaml\n{}\n```\nthen\n```bash\nyq 'parent' sample.yml\n```\nwill output\n```yaml\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/path.md", "content": "# Path\n\nThe `path` operator can be used to get the traversal paths of matching nodes in an expression. The path is returned as an array, which if traversed in order will lead to the matching node.\n\nYou can get the key/index of matching nodes by using the `path` operator to return the path array then piping that through `.[-1]` to get the last element of that array, the key.\n\nUse `setpath` to set a value to the path array returned by `path`, and similarly `delpaths` for an array of path arrays.\n\n\n## Map path\nGiven a sample.yml file of:\n```yaml\na:\n b: cat\n```\nthen\n```bash\nyq '.a.b | path' sample.yml\n```\nwill output\n```yaml\n- a\n- b\n```\n\n## Get map key\nGiven a sample.yml file of:\n```yaml\na:\n b: cat\n```\nthen\n```bash\nyq '.a.b | path | .[-1]' sample.yml\n```\nwill output\n```yaml\nb\n```\n\n## Array path\nGiven a sample.yml file of:\n```yaml\na:\n - cat\n - dog\n```\nthen\n```bash\nyq '.a.[] | select(. == \"dog\") | path' sample.yml\n```\nwill output\n```yaml\n- a\n- 1\n```\n\n## Get array index\nGiven a sample.yml file of:\n```yaml\na:\n - cat\n - dog\n```\nthen\n```bash\nyq '.a.[] | select(. == \"dog\") | path | .[-1]' sample.yml\n```\nwill output\n```yaml\n1\n```\n\n## Print path and value\nGiven a sample.yml file of:\n```yaml\na:\n - cat\n - dog\n - frog\n```\nthen\n```bash\nyq '.a[] | select(. == \"*og\") | [{\"path\":path, \"value\":.}]' sample.yml\n```\nwill output\n```yaml\n- path:\n - a\n - 1\n value: dog\n- path:\n - a\n - 2\n value: frog\n```\n\n## Set path\nGiven a sample.yml file of:\n```yaml\na:\n b: cat\n```\nthen\n```bash\nyq 'setpath([\"a\", \"b\"]; \"things\")' sample.yml\n```\nwill output\n```yaml\na:\n b: things\n```\n\n## Set on empty document\nRunning\n```bash\nyq --null-input 'setpath([\"a\", \"b\"]; \"things\")'\n```\nwill output\n```yaml\na:\n b: things\n```\n\n## Set path to prune deep paths\nLike pick but recursive. This uses `ireduce` to deeply set the selected paths into an empty object.\n\nGiven a sample.yml file of:\n```yaml\n\nparentA: bob\nparentB:\n child1: i am child1\n child2: i am child2\nparentC:\n child1: me child1\n child2: me child2\n```\nthen\n```bash\nyq '(.parentB.child2, .parentC.child1) as $i\n ireduce({}; setpath($i | path; $i))' sample.yml\n```\nwill output\n```yaml\nparentB:\n child2: i am child2\nparentC:\n child1: me child1\n```\n\n## Set array path\nGiven a sample.yml file of:\n```yaml\na:\n - cat\n - frog\n```\nthen\n```bash\nyq 'setpath([\"a\", 0]; \"things\")' sample.yml\n```\nwill output\n```yaml\na:\n - things\n - frog\n```\n\n## Set array path empty\nRunning\n```bash\nyq --null-input 'setpath([\"a\", 0]; \"things\")'\n```\nwill output\n```yaml\na:\n - things\n```\n\n## Delete path\nNotice delpaths takes an _array_ of paths.\n\nGiven a sample.yml file of:\n```yaml\na:\n b: cat\n c: dog\n d: frog\n```\nthen\n```bash\nyq 'delpaths([[\"a\", \"c\"], [\"a\", \"d\"]])' sample.yml\n```\nwill output\n```yaml\na:\n b: cat\n```\n\n## Delete array path\nGiven a sample.yml file of:\n```yaml\na:\n - cat\n - frog\n```\nthen\n```bash\nyq 'delpaths([[\"a\", 0]])' sample.yml\n```\nwill output\n```yaml\na:\n - frog\n```\n\n## Delete - wrong parameter\ndelpaths does not work with a single path array\n\nGiven a sample.yml file of:\n```yaml\na:\n - cat\n - frog\n```\nthen\n```bash\nyq 'delpaths([\"a\", 0])' sample.yml\n```\nwill output\n```bash\nError: DELPATHS: expected entry [0] to be a sequence, but its a !!str. Note that delpaths takes an array of path arrays, e.g. [[\"a\", \"b\"]]\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/pick.md", "content": "# Pick\n\nFilter a map by the specified list of keys. Map is returned with the key in the order of the pick list.\n\nSimilarly, filter an array by the specified list of indices.\n\n## Pick keys from map\nNote that the order of the keys matches the pick order and non existent keys are skipped.\n\nGiven a sample.yml file of:\n```yaml\nmyMap:\n cat: meow\n dog: bark\n thing: hamster\n hamster: squeak\n```\nthen\n```bash\nyq '.myMap |= pick([\"hamster\", \"cat\", \"goat\"])' sample.yml\n```\nwill output\n```yaml\nmyMap:\n hamster: squeak\n cat: meow\n```\n\n## Pick keys from map, included all the keys\nWe create a map of the picked keys plus all the current keys, and run that through unique\n\nGiven a sample.yml file of:\n```yaml\nmyMap:\n cat: meow\n dog: bark\n thing: hamster\n hamster: squeak\n```\nthen\n```bash\nyq '.myMap |= pick( ([\"thing\"] + keys) | unique)' sample.yml\n```\nwill output\n```yaml\nmyMap:\n thing: hamster\n cat: meow\n dog: bark\n hamster: squeak\n```\n\n## Pick indices from array\nNote that the order of the indices matches the pick order and non existent indices are skipped.\n\nGiven a sample.yml file of:\n```yaml\n- cat\n- leopard\n- lion\n```\nthen\n```bash\nyq 'pick([2, 0, 734, -5])' sample.yml\n```\nwill output\n```yaml\n- lion\n- cat\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/pipe.md", "content": "# Pipe\n\nPipe the results of an expression into another. Like the bash operator.\n\n## Simple Pipe\nGiven a sample.yml file of:\n```yaml\na:\n b: cat\n```\nthen\n```bash\nyq '.a | .b' sample.yml\n```\nwill output\n```yaml\ncat\n```\n\n## Multiple updates\nGiven a sample.yml file of:\n```yaml\na: cow\nb: sheep\nc: same\n```\nthen\n```bash\nyq '.a = \"cat\" | .b = \"dog\"' sample.yml\n```\nwill output\n```yaml\na: cat\nb: dog\nc: same\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/pivot.md", "content": "# Pivot\n\nEmulates the `PIVOT` function supported by several popular RDBMS systems.\n\n## Pivot a sequence of sequences\nGiven a sample.yml file of:\n```yaml\n- - foo\n - bar\n - baz\n- - sis\n - boom\n - bah\n```\nthen\n```bash\nyq 'pivot' sample.yml\n```\nwill output\n```yaml\n- - foo\n - sis\n- - bar\n - boom\n- - baz\n - bah\n```\n\n## Pivot sequence of heterogeneous sequences\nMissing values are \"padded\" to null.\n\nGiven a sample.yml file of:\n```yaml\n- - foo\n - bar\n - baz\n- - sis\n - boom\n - bah\n - blah\n```\nthen\n```bash\nyq 'pivot' sample.yml\n```\nwill output\n```yaml\n- - foo\n - sis\n- - bar\n - boom\n- - baz\n - bah\n- -\n - blah\n```\n\n## Pivot sequence of maps\nGiven a sample.yml file of:\n```yaml\n- foo: a\n bar: b\n baz: c\n- foo: x\n bar: y\n baz: z\n```\nthen\n```bash\nyq 'pivot' sample.yml\n```\nwill output\n```yaml\nfoo:\n - a\n - x\nbar:\n - b\n - y\nbaz:\n - c\n - z\n```\n\n## Pivot sequence of heterogeneous maps\nMissing values are \"padded\" to null.\n\nGiven a sample.yml file of:\n```yaml\n- foo: a\n bar: b\n baz: c\n- foo: x\n bar: y\n baz: z\n what: ever\n```\nthen\n```bash\nyq 'pivot' sample.yml\n```\nwill output\n```yaml\nfoo:\n - a\n - x\nbar:\n - b\n - y\nbaz:\n - c\n - z\nwhat:\n -\n - ever\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/recursive-descent-glob.md", "content": "# Recursive Descent (Glob)\n\nThis operator recursively matches (or globs) all children nodes given of a particular element, including that node itself. This is most often used to apply a filter recursively against all matches.\n\n## match values form `..`\nThis will, like the `jq` equivalent, recursively match all _value_ nodes. Use it to find/manipulate particular values.\n\nFor instance to set the `style` of all _value_ nodes in a yaml doc, excluding map keys:\n\n```bash\nyq '.. style= \"flow\"' file.yaml\n```\n\n## match values and map keys form `...`\nThe also includes map keys in the results set. This is particularly useful in YAML as unlike JSON, map keys can have their own styling and tags and also use anchors and aliases.\n\nFor instance to set the `style` of all nodes in a yaml doc, including the map keys:\n\n```bash\nyq '... style= \"flow\"' file.yaml\n```\n## Recurse map (values only)\nGiven a sample.yml file of:\n```yaml\na: frog\n```\nthen\n```bash\nyq '..' sample.yml\n```\nwill output\n```yaml\na: frog\nfrog\n```\n\n## Recursively find nodes with keys\nNote that this example has wrapped the expression in `[]` to show that there are two matches returned. You do not have to wrap in `[]` in your path expression.\n\nGiven a sample.yml file of:\n```yaml\na:\n name: frog\n b:\n name: blog\n age: 12\n```\nthen\n```bash\nyq '[.. | select(has(\"name\"))]' sample.yml\n```\nwill output\n```yaml\n- name: frog\n b:\n name: blog\n age: 12\n- name: blog\n age: 12\n```\n\n## Recursively find nodes with values\nGiven a sample.yml file of:\n```yaml\na:\n nameA: frog\n b:\n nameB: frog\n age: 12\n```\nthen\n```bash\nyq '.. | select(. == \"frog\")' sample.yml\n```\nwill output\n```yaml\nfrog\nfrog\n```\n\n## Recurse map (values and keys)\nNote that the map key appears in the results\n\nGiven a sample.yml file of:\n```yaml\na: frog\n```\nthen\n```bash\nyq '...' sample.yml\n```\nwill output\n```yaml\na: frog\na\nfrog\n```\n\n## Aliases are not traversed\nGiven a sample.yml file of:\n```yaml\na: &cat\n c: frog\nb: *cat\n```\nthen\n```bash\nyq '[..]' sample.yml\n```\nwill output\n```yaml\n- a: &cat\n c: frog\n b: *cat\n- &cat\n c: frog\n- frog\n- *cat\n```\n\n## Merge docs are not traversed\nGiven a sample.yml file of:\n```yaml\nfoo: &foo\n a: foo_a\n thing: foo_thing\n c: foo_c\nbar: &bar\n b: bar_b\n thing: bar_thing\n c: bar_c\nfoobarList:\n b: foobarList_b\n !!merge <<:\n - *foo\n - *bar\n c: foobarList_c\nfoobar:\n c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n```\nthen\n```bash\nyq '.foobar | [..]' sample.yml\n```\nwill output\n```yaml\n- c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n- foobar_c\n- *foo\n- foobar_thing\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/reduce.md", "content": "# Reduce\n\nReduce is a powerful way to process a collection of data into a new form.\n\n```\n as $ ireduce (; )\n```\n\ne.g.\n\n```\n.[] as $item ireduce (0; . + $item)\n```\n\nOn the LHS we are configuring the collection of items that will be reduced `` as well as what each element will be called `$`. Note that the array has been splatted into its individual elements.\n\nOn the RHS there is ``, the starting value of the accumulator and ``, the expression that will update the accumulator for each element in the collection. Note that within the block expression, `.` will evaluate to the current value of the accumulator. \n\n## yq vs jq syntax\nReduce syntax in `yq` is a little different from `jq` - as `yq` (currently) isn't as sophisticated as `jq` and its only supports infix notation (e.g. a + b, where the operator is in the middle of the two parameters) - where as `jq` uses a mix of infix notation with _prefix_ notation (e.g. `reduce a b` is like writing `+ a b`).\n\nTo that end, the reduce operator is called `ireduce` for backwards compatibility if a `jq` like prefix version of `reduce` is ever added.\n\n## Sum numbers\nGiven a sample.yml file of:\n```yaml\n- 10\n- 2\n- 5\n- 3\n```\nthen\n```bash\nyq '.[] as $item ireduce (0; . + $item)' sample.yml\n```\nwill output\n```yaml\n20\n```\n\n## Merge all yaml files together\nGiven a sample.yml file of:\n```yaml\na: cat\n```\nAnd another sample another.yml file of:\n```yaml\nb: dog\n```\nthen\n```bash\nyq eval-all '. as $item ireduce ({}; . * $item )' sample.yml another.yml\n```\nwill output\n```yaml\na: cat\nb: dog\n```\n\n## Convert an array to an object\nGiven a sample.yml file of:\n```yaml\n- name: Cathy\n has: apples\n- name: Bob\n has: bananas\n```\nthen\n```bash\nyq '.[] as $item ireduce ({}; .[$item | .name] = ($item | .has) )' sample.yml\n```\nwill output\n```yaml\nCathy: apples\nBob: bananas\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/reverse.md", "content": "# Reverse\n\nReverses the order of the items in an array \n\n## Reverse\nGiven a sample.yml file of:\n```yaml\n- 1\n- 2\n- 3\n```\nthen\n```bash\nyq 'reverse' sample.yml\n```\nwill output\n```yaml\n- 3\n- 2\n- 1\n```\n\n## Sort descending by string field\nUse sort with reverse to sort in descending order.\n\nGiven a sample.yml file of:\n```yaml\n- a: banana\n- a: cat\n- a: apple\n```\nthen\n```bash\nyq 'sort_by(.a) | reverse' sample.yml\n```\nwill output\n```yaml\n- a: cat\n- a: banana\n- a: apple\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/select.md", "content": "# Select\n\nSelect is used to filter arrays and maps by a boolean expression.\n\n## Related Operators\n\n- equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals)\n- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare)\n- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators)\n\n## Select elements from array using wildcard prefix\nGiven a sample.yml file of:\n```yaml\n- cat\n- goat\n- dog\n```\nthen\n```bash\nyq '.[] | select(. == \"*at\")' sample.yml\n```\nwill output\n```yaml\ncat\ngoat\n```\n\n## Select elements from array using wildcard suffix\nGiven a sample.yml file of:\n```yaml\n- go-kart\n- goat\n- dog\n```\nthen\n```bash\nyq '.[] | select(. == \"go*\")' sample.yml\n```\nwill output\n```yaml\ngo-kart\ngoat\n```\n\n## Select elements from array using wildcard prefix and suffix\nGiven a sample.yml file of:\n```yaml\n- ago\n- go\n- meow\n- going\n```\nthen\n```bash\nyq '.[] | select(. == \"*go*\")' sample.yml\n```\nwill output\n```yaml\nago\ngo\ngoing\n```\n\n## Select elements from array with regular expression\nSee more regular expression examples under the [`string` operator docs](https://mikefarah.gitbook.io/yq/operators/string-operators).\n\nGiven a sample.yml file of:\n```yaml\n- this_0\n- not_this\n- nor_0_this\n- thisTo_4\n```\nthen\n```bash\nyq '.[] | select(test(\"[a-zA-Z]+_[0-9]$\"))' sample.yml\n```\nwill output\n```yaml\nthis_0\nthisTo_4\n```\n\n## Select items from a map\nGiven a sample.yml file of:\n```yaml\nthings: cat\nbob: goat\nhorse: dog\n```\nthen\n```bash\nyq '.[] | select(. == \"cat\" or test(\"og$\"))' sample.yml\n```\nwill output\n```yaml\ncat\ndog\n```\n\n## Use select and with_entries to filter map keys\nGiven a sample.yml file of:\n```yaml\nname: bob\nlegs: 2\ngame: poker\n```\nthen\n```bash\nyq 'with_entries(select(.key | test(\"ame$\")))' sample.yml\n```\nwill output\n```yaml\nname: bob\ngame: poker\n```\n\n## Select multiple items in a map and update\nNote the brackets around the entire LHS.\n\nGiven a sample.yml file of:\n```yaml\na:\n things: cat\n bob: goat\n horse: dog\n```\nthen\n```bash\nyq '(.a.[] | select(. == \"cat\" or . == \"goat\")) |= \"rabbit\"' sample.yml\n```\nwill output\n```yaml\na:\n things: rabbit\n bob: rabbit\n horse: dog\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/shuffle.md", "content": "# Shuffle\n\nShuffles an array. Note that this command does _not_ use a cryptographically secure random number generator to randomise the array order.\n\n\n## Shuffle array\nGiven a sample.yml file of:\n```yaml\n- 1\n- 2\n- 3\n- 4\n- 5\n```\nthen\n```bash\nyq 'shuffle' sample.yml\n```\nwill output\n```yaml\n- 5\n- 2\n- 4\n- 1\n- 3\n```\n\n## Shuffle array in place\nGiven a sample.yml file of:\n```yaml\ncool:\n - 1\n - 2\n - 3\n - 4\n - 5\n```\nthen\n```bash\nyq '.cool |= shuffle' sample.yml\n```\nwill output\n```yaml\ncool:\n - 5\n - 2\n - 4\n - 1\n - 3\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/slice-array.md", "content": "# Slice/Splice Array\n\nThe slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array.\n\nYou may leave out the first or second number, which will refer to the start or end of the array respectively.\n\n## Slicing arrays\nGiven a sample.yml file of:\n```yaml\n- cat\n- dog\n- frog\n- cow\n```\nthen\n```bash\nyq '.[1:3]' sample.yml\n```\nwill output\n```yaml\n- dog\n- frog\n```\n\n## Slicing arrays - without the first number\nStarts from the start of the array\n\nGiven a sample.yml file of:\n```yaml\n- cat\n- dog\n- frog\n- cow\n```\nthen\n```bash\nyq '.[:2]' sample.yml\n```\nwill output\n```yaml\n- cat\n- dog\n```\n\n## Slicing arrays - without the second number\nFinishes at the end of the array\n\nGiven a sample.yml file of:\n```yaml\n- cat\n- dog\n- frog\n- cow\n```\nthen\n```bash\nyq '.[2:]' sample.yml\n```\nwill output\n```yaml\n- frog\n- cow\n```\n\n## Slicing arrays - use negative numbers to count backwards from the end\nGiven a sample.yml file of:\n```yaml\n- cat\n- dog\n- frog\n- cow\n```\nthen\n```bash\nyq '.[1:-1]' sample.yml\n```\nwill output\n```yaml\n- dog\n- frog\n```\n\n## Inserting into the middle of an array\nusing an expression to find the index\n\nGiven a sample.yml file of:\n```yaml\n- cat\n- dog\n- frog\n- cow\n```\nthen\n```bash\nyq '(.[] | select(. == \"dog\") | key + 1) as $pos | .[0:($pos)] + [\"rabbit\"] + .[$pos:]' sample.yml\n```\nwill output\n```yaml\n- cat\n- dog\n- rabbit\n- frog\n- cow\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/sort-keys.md", "content": "# Sort Keys\n\nThe Sort Keys operator sorts maps by their keys (based on their string value). This operator does not do anything to arrays or scalars (so you can easily recursively apply it to all maps).\n\nSort is particularly useful for diffing two different yaml documents:\n\n```bash\nyq -i -P 'sort_keys(..)' file1.yml\nyq -i -P 'sort_keys(..)' file2.yml\ndiff file1.yml file2.yml\n```\n\nNote that `yq` does not yet consider anchors when sorting by keys - this may result in invalid yaml documents if you are using merge anchors.\n\nFor more advanced sorting, you can use the [sort_by](https://mikefarah.gitbook.io/yq/operators/sort) function on a map, and give it a custom function like `sort_by(key | downcase)`.\n\n\n## Sort keys of map\nGiven a sample.yml file of:\n```yaml\nc: frog\na: blah\nb: bing\n```\nthen\n```bash\nyq 'sort_keys(.)' sample.yml\n```\nwill output\n```yaml\na: blah\nb: bing\nc: frog\n```\n\n## Sort keys recursively\nNote the array elements are left unsorted, but maps inside arrays are sorted\n\nGiven a sample.yml file of:\n```yaml\nbParent:\n c: dog\n array:\n - 3\n - 1\n - 2\naParent:\n z: donkey\n x:\n - c: yum\n b: delish\n - b: ew\n a: apple\n```\nthen\n```bash\nyq 'sort_keys(..)' sample.yml\n```\nwill output\n```yaml\naParent:\n x:\n - b: delish\n c: yum\n - a: apple\n b: ew\n z: donkey\nbParent:\n array:\n - 3\n - 1\n - 2\n c: dog\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/sort.md", "content": "# Sort\n\nSorts an array. Use `sort` to sort an array as is, or `sort_by(exp)` to sort by a particular expression (e.g. subfield).\n\nTo sort by descending order, pipe the results through the `reverse` operator after sorting.\n\nNote that at this stage, `yq` only sorts scalar fields.\n\n\n## Sort by string field\nGiven a sample.yml file of:\n```yaml\n- a: banana\n- a: cat\n- a: apple\n```\nthen\n```bash\nyq 'sort_by(.a)' sample.yml\n```\nwill output\n```yaml\n- a: apple\n- a: banana\n- a: cat\n```\n\n## Sort by multiple fields\nGiven a sample.yml file of:\n```yaml\n- a: dog\n- a: cat\n b: banana\n- a: cat\n b: apple\n```\nthen\n```bash\nyq 'sort_by(.a, .b)' sample.yml\n```\nwill output\n```yaml\n- a: cat\n b: apple\n- a: cat\n b: banana\n- a: dog\n```\n\n## Sort descending by string field\nUse sort with reverse to sort in descending order.\n\nGiven a sample.yml file of:\n```yaml\n- a: banana\n- a: cat\n- a: apple\n```\nthen\n```bash\nyq 'sort_by(.a) | reverse' sample.yml\n```\nwill output\n```yaml\n- a: cat\n- a: banana\n- a: apple\n```\n\n## Sort array in place\nGiven a sample.yml file of:\n```yaml\ncool:\n - a: banana\n - a: cat\n - a: apple\n```\nthen\n```bash\nyq '.cool |= sort_by(.a)' sample.yml\n```\nwill output\n```yaml\ncool:\n - a: apple\n - a: banana\n - a: cat\n```\n\n## Sort array of objects by key\nNote that you can give sort_by complex expressions, not just paths\n\nGiven a sample.yml file of:\n```yaml\ncool:\n - b: banana\n - a: banana\n - c: banana\n```\nthen\n```bash\nyq '.cool |= sort_by(keys | .[0])' sample.yml\n```\nwill output\n```yaml\ncool:\n - a: banana\n - b: banana\n - c: banana\n```\n\n## Sort a map\nSorting a map, by default this will sort by the values\n\nGiven a sample.yml file of:\n```yaml\ny: b\nz: a\nx: c\n```\nthen\n```bash\nyq 'sort' sample.yml\n```\nwill output\n```yaml\nz: a\ny: b\nx: c\n```\n\n## Sort a map by keys\nUse sort_by to sort a map using a custom function\n\nGiven a sample.yml file of:\n```yaml\nY: b\nz: a\nx: c\n```\nthen\n```bash\nyq 'sort_by(key | downcase)' sample.yml\n```\nwill output\n```yaml\nx: c\nY: b\nz: a\n```\n\n## Sort is stable\nNote the order of the elements in unchanged when equal in sorting.\n\nGiven a sample.yml file of:\n```yaml\n- a: banana\n b: 1\n- a: banana\n b: 2\n- a: banana\n b: 3\n- a: banana\n b: 4\n```\nthen\n```bash\nyq 'sort_by(.a)' sample.yml\n```\nwill output\n```yaml\n- a: banana\n b: 1\n- a: banana\n b: 2\n- a: banana\n b: 3\n- a: banana\n b: 4\n```\n\n## Sort by numeric field\nGiven a sample.yml file of:\n```yaml\n- a: 10\n- a: 100\n- a: 1\n```\nthen\n```bash\nyq 'sort_by(.a)' sample.yml\n```\nwill output\n```yaml\n- a: 1\n- a: 10\n- a: 100\n```\n\n## Sort by custom date field\nGiven a sample.yml file of:\n```yaml\n- a: 12-Jun-2011\n- a: 23-Dec-2010\n- a: 10-Aug-2011\n```\nthen\n```bash\nyq 'with_dtf(\"02-Jan-2006\"; sort_by(.a))' sample.yml\n```\nwill output\n```yaml\n- a: 23-Dec-2010\n- a: 12-Jun-2011\n- a: 10-Aug-2011\n```\n\n## Sort, nulls come first\nGiven a sample.yml file of:\n```yaml\n- 8\n- 3\n- null\n- 6\n- true\n- false\n- cat\n```\nthen\n```bash\nyq 'sort' sample.yml\n```\nwill output\n```yaml\n- null\n- false\n- true\n- 3\n- 6\n- 8\n- cat\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/split-into-documents.md", "content": "# Split into Documents\n\nThis operator splits all matches into separate documents\n\n## Split empty\nRunning\n```bash\nyq --null-input 'split_doc'\n```\nwill output\n```yaml\n\n```\n\n## Split array\nGiven a sample.yml file of:\n```yaml\n- a: cat\n- b: dog\n```\nthen\n```bash\nyq '.[] | split_doc' sample.yml\n```\nwill output\n```yaml\na: cat\n---\nb: dog\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/string-operators.md", "content": "# String Operators\n\n## RegEx\nThis uses Golang's native regex functions under the hood - See their [docs](https://github.com/google/re2/wiki/Syntax) for the supported syntax.\n\nCase insensitive tip: prefix the regex with `(?i)` - e.g. `test(\"(?i)cats\")`.\n\n### match(regEx)\nThis operator returns the substring match details of the given regEx.\n\n### capture(regEx)\nCapture returns named RegEx capture groups in a map. Can be more convenient than `match` depending on what you are doing.\n\n## test(regEx)\nReturns true if the string matches the RegEx, false otherwise.\n\n## sub(regEx, replacement)\nSubstitutes matched substrings. The first parameter is the regEx to match substrings within the original string. The second parameter specifies what to replace those matches with. This can refer to capture groups from the first RegEx.\n\n## String blocks, bash and newlines\nBash is notorious for chomping on precious trailing newline characters, making it tricky to set strings with newlines properly. In particular, the `$( exp )` _will trim trailing newlines_.\n\nFor instance to get this yaml:\n\n```\na: |\n cat\n```\n\nUsing `$( exp )` wont work, as it will trim the trailing newline.\n\n```\nm=$(echo \"cat\\n\") yq -n '.a = strenv(m)'\na: cat\n```\n\nHowever, using printf works:\n```\nprintf -v m \"cat\\n\" ; m=\"$m\" yq -n '.a = strenv(m)'\na: |\n cat\n```\n\nAs well as having multiline expressions:\n```\nm=\"cat\n\" yq -n '.a = strenv(m)'\na: |\n cat\n```\n\nSimilarly, if you're trying to set the content from a file, and want a trailing newline:\n\n```\nIFS= read -rd '' output < <(cat my_file)\noutput=$output ./yq '.data.values = strenv(output)' first.yml\n```\n\n## Interpolation\nGiven a sample.yml file of:\n```yaml\nvalue: things\nanother: stuff\n```\nthen\n```bash\nyq '.message = \"I like \\(.value) and \\(.another)\"' sample.yml\n```\nwill output\n```yaml\nvalue: things\nanother: stuff\nmessage: I like things and stuff\n```\n\n## Interpolation - not a string\nGiven a sample.yml file of:\n```yaml\nvalue:\n an: apple\n```\nthen\n```bash\nyq '.message = \"I like \\(.value)\"' sample.yml\n```\nwill output\n```yaml\nvalue:\n an: apple\nmessage: 'I like an: apple'\n```\n\n## To up (upper) case\nWorks with unicode characters\n\nGiven a sample.yml file of:\n```yaml\nágua\n```\nthen\n```bash\nyq 'upcase' sample.yml\n```\nwill output\n```yaml\nÁGUA\n```\n\n## To down (lower) case\nWorks with unicode characters\n\nGiven a sample.yml file of:\n```yaml\nÁgUA\n```\nthen\n```bash\nyq 'downcase' sample.yml\n```\nwill output\n```yaml\nágua\n```\n\n## Join strings\nGiven a sample.yml file of:\n```yaml\n- cat\n- meow\n- 1\n- null\n- true\n```\nthen\n```bash\nyq 'join(\"; \")' sample.yml\n```\nwill output\n```yaml\ncat; meow; 1; ; true\n```\n\n## Trim strings\nGiven a sample.yml file of:\n```yaml\n- ' cat'\n- 'dog '\n- ' cow cow '\n- horse\n```\nthen\n```bash\nyq '.[] | trim' sample.yml\n```\nwill output\n```yaml\ncat\ndog\ncow cow\nhorse\n```\n\n## Match string\nGiven a sample.yml file of:\n```yaml\nfoo bar foo\n```\nthen\n```bash\nyq 'match(\"foo\")' sample.yml\n```\nwill output\n```yaml\nstring: foo\noffset: 0\nlength: 3\ncaptures: []\n```\n\n## Match string, case insensitive\nGiven a sample.yml file of:\n```yaml\nfoo bar FOO\n```\nthen\n```bash\nyq '[match(\"(?i)foo\"; \"g\")]' sample.yml\n```\nwill output\n```yaml\n- string: foo\n offset: 0\n length: 3\n captures: []\n- string: FOO\n offset: 8\n length: 3\n captures: []\n```\n\n## Match with global capture group\nGiven a sample.yml file of:\n```yaml\nabc abc\n```\nthen\n```bash\nyq '[match(\"(ab)(c)\"; \"g\")]' sample.yml\n```\nwill output\n```yaml\n- string: abc\n offset: 0\n length: 3\n captures:\n - string: ab\n offset: 0\n length: 2\n - string: c\n offset: 2\n length: 1\n- string: abc\n offset: 4\n length: 3\n captures:\n - string: ab\n offset: 4\n length: 2\n - string: c\n offset: 6\n length: 1\n```\n\n## Match with named capture groups\nGiven a sample.yml file of:\n```yaml\nfoo bar foo foo foo\n```\nthen\n```bash\nyq '[match(\"foo (?Pbar)? foo\"; \"g\")]' sample.yml\n```\nwill output\n```yaml\n- string: foo bar foo\n offset: 0\n length: 11\n captures:\n - string: bar\n offset: 4\n length: 3\n name: bar123\n- string: foo foo\n offset: 12\n length: 8\n captures:\n - string: null\n offset: -1\n length: 0\n name: bar123\n```\n\n## Capture named groups into a map\nGiven a sample.yml file of:\n```yaml\nxyzzy-14\n```\nthen\n```bash\nyq 'capture(\"(?P[a-z]+)-(?P[0-9]+)\")' sample.yml\n```\nwill output\n```yaml\na: xyzzy\nn: \"14\"\n```\n\n## Match without global flag\nGiven a sample.yml file of:\n```yaml\ncat cat\n```\nthen\n```bash\nyq 'match(\"cat\")' sample.yml\n```\nwill output\n```yaml\nstring: cat\noffset: 0\nlength: 3\ncaptures: []\n```\n\n## Match with global flag\nGiven a sample.yml file of:\n```yaml\ncat cat\n```\nthen\n```bash\nyq '[match(\"cat\"; \"g\")]' sample.yml\n```\nwill output\n```yaml\n- string: cat\n offset: 0\n length: 3\n captures: []\n- string: cat\n offset: 4\n length: 3\n captures: []\n```\n\n## Test using regex\nLike jq's equivalent, this works like match but only returns true/false instead of full match details\n\nGiven a sample.yml file of:\n```yaml\n- cat\n- dog\n```\nthen\n```bash\nyq '.[] | test(\"at\")' sample.yml\n```\nwill output\n```yaml\ntrue\nfalse\n```\n\n## Substitute / Replace string\nThis uses Golang's regex, described [here](https://github.com/google/re2/wiki/Syntax).\nNote the use of `|=` to run in context of the current string value.\n\nGiven a sample.yml file of:\n```yaml\na: dogs are great\n```\nthen\n```bash\nyq '.a |= sub(\"dogs\", \"cats\")' sample.yml\n```\nwill output\n```yaml\na: cats are great\n```\n\n## Substitute / Replace string with regex\nThis uses Golang's regex, described [here](https://github.com/google/re2/wiki/Syntax).\nNote the use of `|=` to run in context of the current string value.\n\nGiven a sample.yml file of:\n```yaml\na: cat\nb: heat\n```\nthen\n```bash\nyq '.[] |= sub(\"(a)\", \"${1}r\")' sample.yml\n```\nwill output\n```yaml\na: cart\nb: heart\n```\n\n## Custom types: that are really strings\nWhen custom tags are encountered, yq will try to decode the underlying type.\n\nGiven a sample.yml file of:\n```yaml\na: !horse cat\nb: !goat heat\n```\nthen\n```bash\nyq '.[] |= sub(\"(a)\", \"${1}r\")' sample.yml\n```\nwill output\n```yaml\na: !horse cart\nb: !goat heart\n```\n\n## Split strings\nGiven a sample.yml file of:\n```yaml\ncat; meow; 1; ; true\n```\nthen\n```bash\nyq 'split(\"; \")' sample.yml\n```\nwill output\n```yaml\n- cat\n- meow\n- \"1\"\n- \"\"\n- \"true\"\n```\n\n## Split strings one match\nGiven a sample.yml file of:\n```yaml\nword\n```\nthen\n```bash\nyq 'split(\"; \")' sample.yml\n```\nwill output\n```yaml\n- word\n```\n\n## To string\nNote that you may want to force `yq` to leave scalar values wrapped by passing in `--unwrapScalar=false` or `-r=f`\n\nGiven a sample.yml file of:\n```yaml\n- 1\n- true\n- null\n- ~\n- cat\n- an: object\n- - array\n - 2\n```\nthen\n```bash\nyq '.[] |= to_string' sample.yml\n```\nwill output\n```yaml\n- \"1\"\n- \"true\"\n- \"null\"\n- \"~\"\n- cat\n- \"an: object\"\n- \"- array\\n- 2\"\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/style.md", "content": "# Style\n\nThe style operator can be used to get or set the style of nodes (e.g. string style, yaml style).\nUse this to control the formatting of the document in yaml.\n\n\n## Update and set style of a particular node (simple)\nGiven a sample.yml file of:\n```yaml\na:\n b: thing\n c: something\n```\nthen\n```bash\nyq '.a.b = \"new\" | .a.b style=\"double\"' sample.yml\n```\nwill output\n```yaml\na:\n b: \"new\"\n c: something\n```\n\n## Update and set style of a particular node using path variables\nGiven a sample.yml file of:\n```yaml\na:\n b: thing\n c: something\n```\nthen\n```bash\nyq 'with(.a.b ; . = \"new\" | . style=\"double\")' sample.yml\n```\nwill output\n```yaml\na:\n b: \"new\"\n c: something\n```\n\n## Set tagged style\nGiven a sample.yml file of:\n```yaml\na: cat\nb: 5\nc: 3.2\ne: true\nf:\n - 1\n - 2\n - 3\ng:\n something: cool\n```\nthen\n```bash\nyq '.. style=\"tagged\"' sample.yml\n```\nwill output\n```yaml\n!!map\na: !!str cat\nb: !!int 5\nc: !!float 3.2\ne: !!bool true\nf: !!seq\n - !!int 1\n - !!int 2\n - !!int 3\ng: !!map\n something: !!str cool\n```\n\n## Set double quote style\nGiven a sample.yml file of:\n```yaml\na: cat\nb: 5\nc: 3.2\ne: true\nf:\n - 1\n - 2\n - 3\ng:\n something: cool\n```\nthen\n```bash\nyq '.. style=\"double\"' sample.yml\n```\nwill output\n```yaml\na: \"cat\"\nb: \"5\"\nc: \"3.2\"\ne: \"true\"\nf:\n - \"1\"\n - \"2\"\n - \"3\"\ng:\n something: \"cool\"\n```\n\n## Set double quote style on map keys too\nGiven a sample.yml file of:\n```yaml\na: cat\nb: 5\nc: 3.2\ne: true\nf:\n - 1\n - 2\n - 3\ng:\n something: cool\n```\nthen\n```bash\nyq '... style=\"double\"' sample.yml\n```\nwill output\n```yaml\n\"a\": \"cat\"\n\"b\": \"5\"\n\"c\": \"3.2\"\n\"e\": \"true\"\n\"f\":\n - \"1\"\n - \"2\"\n - \"3\"\n\"g\":\n \"something\": \"cool\"\n```\n\n## Set single quote style\nGiven a sample.yml file of:\n```yaml\na: cat\nb: 5\nc: 3.2\ne: true\nf:\n - 1\n - 2\n - 3\ng:\n something: cool\n```\nthen\n```bash\nyq '.. style=\"single\"' sample.yml\n```\nwill output\n```yaml\na: 'cat'\nb: '5'\nc: '3.2'\ne: 'true'\nf:\n - '1'\n - '2'\n - '3'\ng:\n something: 'cool'\n```\n\n## Set literal quote style\nGiven a sample.yml file of:\n```yaml\na: cat\nb: 5\nc: 3.2\ne: true\nf:\n - 1\n - 2\n - 3\ng:\n something: cool\n```\nthen\n```bash\nyq '.. style=\"literal\"' sample.yml\n```\nwill output\n```yaml\na: |-\n cat\nb: |-\n 5\nc: |-\n 3.2\ne: |-\n true\nf:\n - |-\n 1\n - |-\n 2\n - |-\n 3\ng:\n something: |-\n cool\n```\n\n## Set folded quote style\nGiven a sample.yml file of:\n```yaml\na: cat\nb: 5\nc: 3.2\ne: true\nf:\n - 1\n - 2\n - 3\ng:\n something: cool\n```\nthen\n```bash\nyq '.. style=\"folded\"' sample.yml\n```\nwill output\n```yaml\na: >-\n cat\nb: >-\n 5\nc: >-\n 3.2\ne: >-\n true\nf:\n - >-\n 1\n - >-\n 2\n - >-\n 3\ng:\n something: >-\n cool\n```\n\n## Set flow quote style\nGiven a sample.yml file of:\n```yaml\na: cat\nb: 5\nc: 3.2\ne: true\nf:\n - 1\n - 2\n - 3\ng:\n something: cool\n```\nthen\n```bash\nyq '.. style=\"flow\"' sample.yml\n```\nwill output\n```yaml\n{a: cat, b: 5, c: 3.2, e: true, f: [1, 2, 3], g: {something: cool}}\n```\n\n## Reset style - or pretty print\nSet empty (default) quote style, note the usage of `...` to match keys too. Note that there is a `--prettyPrint/-P` short flag for this.\n\nGiven a sample.yml file of:\n```yaml\n{a: cat, \"b\": 5, 'c': 3.2, \"e\": true, f: [1,2,3], \"g\": { something: \"cool\"} }\n```\nthen\n```bash\nyq '... style=\"\"' sample.yml\n```\nwill output\n```yaml\na: cat\nb: 5\nc: 3.2\ne: true\nf:\n - 1\n - 2\n - 3\ng:\n something: cool\n```\n\n## Set style relatively with assign-update\nGiven a sample.yml file of:\n```yaml\na: single\nb: double\n```\nthen\n```bash\nyq '.[] style |= .' sample.yml\n```\nwill output\n```yaml\na: 'single'\nb: \"double\"\n```\n\n## Read style\nGiven a sample.yml file of:\n```yaml\n{a: \"cat\", b: 'thing'}\n```\nthen\n```bash\nyq '.. | style' sample.yml\n```\nwill output\n```yaml\nflow\ndouble\nsingle\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/subtract.md", "content": "# Subtract\n\nYou can use subtract to subtract numbers as well as remove elements from an array.\n\n## Array subtraction\nRunning\n```bash\nyq --null-input '[1,2] - [2,3]'\n```\nwill output\n```yaml\n- 1\n```\n\n## Array subtraction with nested array\nRunning\n```bash\nyq --null-input '[[1], 1, 2] - [[1], 3]'\n```\nwill output\n```yaml\n- 1\n- 2\n```\n\n## Array subtraction with nested object\nNote that order of the keys does not matter\n\nGiven a sample.yml file of:\n```yaml\n- a: b\n c: d\n- a: b\n```\nthen\n```bash\nyq '. - [{\"c\": \"d\", \"a\": \"b\"}]' sample.yml\n```\nwill output\n```yaml\n- a: b\n```\n\n## Number subtraction - float\nIf the lhs or rhs are floats then the expression will be calculated with floats.\n\nGiven a sample.yml file of:\n```yaml\na: 3\nb: 4.5\n```\nthen\n```bash\nyq '.a = .a - .b' sample.yml\n```\nwill output\n```yaml\na: -1.5\nb: 4.5\n```\n\n## Number subtraction - int\nIf both the lhs and rhs are ints then the expression will be calculated with ints.\n\nGiven a sample.yml file of:\n```yaml\na: 3\nb: 4\n```\nthen\n```bash\nyq '.a = .a - .b' sample.yml\n```\nwill output\n```yaml\na: -1\nb: 4\n```\n\n## Decrement numbers\nGiven a sample.yml file of:\n```yaml\na: 3\nb: 5\n```\nthen\n```bash\nyq '.[] -= 1' sample.yml\n```\nwill output\n```yaml\na: 2\nb: 4\n```\n\n## Date subtraction\nYou can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.\n\nGiven a sample.yml file of:\n```yaml\na: 2021-01-01T03:10:00Z\n```\nthen\n```bash\nyq '.a -= \"3h10m\"' sample.yml\n```\nwill output\n```yaml\na: 2021-01-01T00:00:00Z\n```\n\n## Date subtraction - custom format\nUse with_dtf to specify your datetime format. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.\n\nGiven a sample.yml file of:\n```yaml\na: Saturday, 15-Dec-01 at 6:00AM GMT\n```\nthen\n```bash\nyq 'with_dtf(\"Monday, 02-Jan-06 at 3:04PM MST\", .a -= \"3h1m\")' sample.yml\n```\nwill output\n```yaml\na: Saturday, 15-Dec-01 at 2:59AM GMT\n```\n\n## Custom types: that are really numbers\nWhen custom tags are encountered, yq will try to decode the underlying type.\n\nGiven a sample.yml file of:\n```yaml\na: !horse 2\nb: !goat 1\n```\nthen\n```bash\nyq '.a -= .b' sample.yml\n```\nwill output\n```yaml\na: !horse 1\nb: !goat 1\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/tag.md", "content": "# Tag\n\nThe tag operator can be used to get or set the tag of nodes (e.g. `!!str`, `!!int`, `!!bool`).\n\n## Get tag\nGiven a sample.yml file of:\n```yaml\na: cat\nb: 5\nc: 3.2\ne: true\nf: []\n```\nthen\n```bash\nyq '.. | tag' sample.yml\n```\nwill output\n```yaml\n!!map\n!!str\n!!int\n!!float\n!!bool\n!!seq\n```\n\n## type is an alias for tag\nGiven a sample.yml file of:\n```yaml\na: cat\nb: 5\nc: 3.2\ne: true\nf: []\n```\nthen\n```bash\nyq '.. | type' sample.yml\n```\nwill output\n```yaml\n!!map\n!!str\n!!int\n!!float\n!!bool\n!!seq\n```\n\n## Set custom tag\nGiven a sample.yml file of:\n```yaml\na: str\n```\nthen\n```bash\nyq '.a tag = \"!!mikefarah\"' sample.yml\n```\nwill output\n```yaml\na: !!mikefarah str\n```\n\n## Find numbers and convert them to strings\nGiven a sample.yml file of:\n```yaml\na: cat\nb: 5\nc: 3.2\ne: true\n```\nthen\n```bash\nyq '(.. | select(tag == \"!!int\")) tag= \"!!str\"' sample.yml\n```\nwill output\n```yaml\na: cat\nb: \"5\"\nc: 3.2\ne: true\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/to_number.md", "content": "# To Number\nParses the input as a number. yq will try to parse values as an int first, failing that it will try float. Values that already ints or floats will be left alone.\n\n## Converts strings to numbers\nGiven a sample.yml file of:\n```yaml\n- \"3\"\n- \"3.1\"\n- \"-1e3\"\n```\nthen\n```bash\nyq '.[] | to_number' sample.yml\n```\nwill output\n```yaml\n3\n3.1\n-1e3\n```\n\n## Doesn't change numbers\nGiven a sample.yml file of:\n```yaml\n- 3\n- 3.1\n- -1e3\n```\nthen\n```bash\nyq '.[] | to_number' sample.yml\n```\nwill output\n```yaml\n3\n3.1\n-1e3\n```\n\n## Cannot convert null\nRunning\n```bash\nyq --null-input '.a.b | to_number'\n```\nwill output\n```bash\nError: cannot convert node value [null] at path a.b of tag !!null to number\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/traverse-read.md", "content": "# Traverse (Read)\n\nThis is the simplest (and perhaps most used) operator. It is used to navigate deeply into yaml structures.\n\n\n## NOTE --yaml-fix-merge-anchor-to-spec flag\n`yq` doesn't merge anchors `<<:` to spec, in some circumstances it incorrectly overrides existing keys when the spec documents not to do that.\n\nTo minimise disruption while still fixing the issue, a flag has been added to toggle this behaviour. This will first default to false; and log warnings to users. Then it will default to true (and still allow users to specify false if needed)\n\nSee examples of the flag differences below, where LEGACY is the flag off; and FIXED is with the flag on.\n\n\n## Simple map navigation\nGiven a sample.yml file of:\n```yaml\na:\n b: apple\n```\nthen\n```bash\nyq '.a' sample.yml\n```\nwill output\n```yaml\nb: apple\n```\n\n## Splat\nOften used to pipe children into other operators\n\nGiven a sample.yml file of:\n```yaml\n- b: apple\n- c: banana\n```\nthen\n```bash\nyq '.[]' sample.yml\n```\nwill output\n```yaml\nb: apple\nc: banana\n```\n\n## Optional Splat\nJust like splat, but won't error if you run it against scalars\n\nGiven a sample.yml file of:\n```yaml\ncat\n```\nthen\n```bash\nyq '.[]' sample.yml\n```\nwill output\n```yaml\n```\n\n## Special characters\nUse quotes with square brackets around path elements with special characters\n\nGiven a sample.yml file of:\n```yaml\n\"{}\": frog\n```\nthen\n```bash\nyq '.[\"{}\"]' sample.yml\n```\nwill output\n```yaml\nfrog\n```\n\n## Nested special characters\nGiven a sample.yml file of:\n```yaml\na:\n \"key.withdots\":\n \"another.key\": apple\n```\nthen\n```bash\nyq '.a[\"key.withdots\"][\"another.key\"]' sample.yml\n```\nwill output\n```yaml\napple\n```\n\n## Keys with spaces\nUse quotes with square brackets around path elements with special characters\n\nGiven a sample.yml file of:\n```yaml\n\"red rabbit\": frog\n```\nthen\n```bash\nyq '.[\"red rabbit\"]' sample.yml\n```\nwill output\n```yaml\nfrog\n```\n\n## Dynamic keys\nExpressions within [] can be used to dynamically lookup / calculate keys\n\nGiven a sample.yml file of:\n```yaml\nb: apple\napple: crispy yum\nbanana: soft yum\n```\nthen\n```bash\nyq '.[.b]' sample.yml\n```\nwill output\n```yaml\ncrispy yum\n```\n\n## Children don't exist\nNodes are added dynamically while traversing\n\nGiven a sample.yml file of:\n```yaml\nc: banana\n```\nthen\n```bash\nyq '.a.b' sample.yml\n```\nwill output\n```yaml\nnull\n```\n\n## Optional identifier\nLike jq, does not output an error when the yaml is not an array or object as expected\n\nGiven a sample.yml file of:\n```yaml\n- 1\n- 2\n- 3\n```\nthen\n```bash\nyq '.a?' sample.yml\n```\nwill output\n```yaml\n```\n\n## Wildcard matching\nGiven a sample.yml file of:\n```yaml\na:\n cat: apple\n mad: things\n```\nthen\n```bash\nyq '.a.\"*a*\"' sample.yml\n```\nwill output\n```yaml\napple\nthings\n```\n\n## Aliases\nGiven a sample.yml file of:\n```yaml\na: &cat\n c: frog\nb: *cat\n```\nthen\n```bash\nyq '.b' sample.yml\n```\nwill output\n```yaml\n*cat\n```\n\n## Traversing aliases with splat\nGiven a sample.yml file of:\n```yaml\na: &cat\n c: frog\nb: *cat\n```\nthen\n```bash\nyq '.b[]' sample.yml\n```\nwill output\n```yaml\nfrog\n```\n\n## Traversing aliases explicitly\nGiven a sample.yml file of:\n```yaml\na: &cat\n c: frog\nb: *cat\n```\nthen\n```bash\nyq '.b.c' sample.yml\n```\nwill output\n```yaml\nfrog\n```\n\n## Traversing arrays by index\nGiven a sample.yml file of:\n```yaml\n- 1\n- 2\n- 3\n```\nthen\n```bash\nyq '.[0]' sample.yml\n```\nwill output\n```yaml\n1\n```\n\n## Traversing nested arrays by index\nGiven a sample.yml file of:\n```yaml\n[[], [cat]]\n```\nthen\n```bash\nyq '.[1][0]' sample.yml\n```\nwill output\n```yaml\ncat\n```\n\n## Maps with numeric keys\nGiven a sample.yml file of:\n```yaml\n2: cat\n```\nthen\n```bash\nyq '.[2]' sample.yml\n```\nwill output\n```yaml\ncat\n```\n\n## Maps with non existing numeric keys\nGiven a sample.yml file of:\n```yaml\na: b\n```\nthen\n```bash\nyq '.[0]' sample.yml\n```\nwill output\n```yaml\nnull\n```\n\n## Traversing merge anchors\nGiven a sample.yml file of:\n```yaml\nfoo: &foo\n a: foo_a\n thing: foo_thing\n c: foo_c\nbar: &bar\n b: bar_b\n thing: bar_thing\n c: bar_c\nfoobarList:\n b: foobarList_b\n !!merge <<:\n - *foo\n - *bar\n c: foobarList_c\nfoobar:\n c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n```\nthen\n```bash\nyq '.foobar.a' sample.yml\n```\nwill output\n```yaml\nfoo_a\n```\n\n## Traversing merge anchors with local override\nGiven a sample.yml file of:\n```yaml\nfoo: &foo\n a: foo_a\n thing: foo_thing\n c: foo_c\nbar: &bar\n b: bar_b\n thing: bar_thing\n c: bar_c\nfoobarList:\n b: foobarList_b\n !!merge <<:\n - *foo\n - *bar\n c: foobarList_c\nfoobar:\n c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n```\nthen\n```bash\nyq '.foobar.thing' sample.yml\n```\nwill output\n```yaml\nfoobar_thing\n```\n\n## Select multiple indices\nGiven a sample.yml file of:\n```yaml\na:\n - a\n - b\n - c\n```\nthen\n```bash\nyq '.a[0, 2]' sample.yml\n```\nwill output\n```yaml\na\nc\n```\n\n## LEGACY: Traversing merge anchors with override\nThis is legacy behaviour, see --yaml-fix-merge-anchor-to-spec\n\nGiven a sample.yml file of:\n```yaml\nfoo: &foo\n a: foo_a\n thing: foo_thing\n c: foo_c\nbar: &bar\n b: bar_b\n thing: bar_thing\n c: bar_c\nfoobarList:\n b: foobarList_b\n !!merge <<:\n - *foo\n - *bar\n c: foobarList_c\nfoobar:\n c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n```\nthen\n```bash\nyq '.foobar.c' sample.yml\n```\nwill output\n```yaml\nfoo_c\n```\n\n## LEGACY: Traversing merge anchor lists\nNote that the later merge anchors override previous, but this is legacy behaviour, see --yaml-fix-merge-anchor-to-spec\n\nGiven a sample.yml file of:\n```yaml\nfoo: &foo\n a: foo_a\n thing: foo_thing\n c: foo_c\nbar: &bar\n b: bar_b\n thing: bar_thing\n c: bar_c\nfoobarList:\n b: foobarList_b\n !!merge <<:\n - *foo\n - *bar\n c: foobarList_c\nfoobar:\n c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n```\nthen\n```bash\nyq '.foobarList.thing' sample.yml\n```\nwill output\n```yaml\nbar_thing\n```\n\n## LEGACY: Splatting merge anchors\nWith legacy override behaviour, see --yaml-fix-merge-anchor-to-spec\n\nGiven a sample.yml file of:\n```yaml\nfoo: &foo\n a: foo_a\n thing: foo_thing\n c: foo_c\nbar: &bar\n b: bar_b\n thing: bar_thing\n c: bar_c\nfoobarList:\n b: foobarList_b\n !!merge <<:\n - *foo\n - *bar\n c: foobarList_c\nfoobar:\n c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n```\nthen\n```bash\nyq '.foobar[]' sample.yml\n```\nwill output\n```yaml\nfoo_c\nfoo_a\nfoobar_thing\n```\n\n## LEGACY: Splatting merge anchor lists\nWith legacy override behaviour, see --yaml-fix-merge-anchor-to-spec\n\nGiven a sample.yml file of:\n```yaml\nfoo: &foo\n a: foo_a\n thing: foo_thing\n c: foo_c\nbar: &bar\n b: bar_b\n thing: bar_thing\n c: bar_c\nfoobarList:\n b: foobarList_b\n !!merge <<:\n - *foo\n - *bar\n c: foobarList_c\nfoobar:\n c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n```\nthen\n```bash\nyq '.foobarList[]' sample.yml\n```\nwill output\n```yaml\nbar_b\nfoo_a\nbar_thing\nfoobarList_c\n```\n\n## FIXED: Traversing merge anchors with override\nSet `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour.\n\nGiven a sample.yml file of:\n```yaml\nfoo: &foo\n a: foo_a\n thing: foo_thing\n c: foo_c\nbar: &bar\n b: bar_b\n thing: bar_thing\n c: bar_c\nfoobarList:\n b: foobarList_b\n !!merge <<:\n - *foo\n - *bar\n c: foobarList_c\nfoobar:\n c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n```\nthen\n```bash\nyq '.foobar.c' sample.yml\n```\nwill output\n```yaml\nfoobar_c\n```\n\n## FIXED: Traversing merge anchor lists\nSet `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Note that the keys earlier in the merge anchors sequence override later ones\n\nGiven a sample.yml file of:\n```yaml\nfoo: &foo\n a: foo_a\n thing: foo_thing\n c: foo_c\nbar: &bar\n b: bar_b\n thing: bar_thing\n c: bar_c\nfoobarList:\n b: foobarList_b\n !!merge <<:\n - *foo\n - *bar\n c: foobarList_c\nfoobar:\n c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n```\nthen\n```bash\nyq '.foobarList.thing' sample.yml\n```\nwill output\n```yaml\nfoo_thing\n```\n\n## FIXED: Splatting merge anchors\nSet `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Note that the keys earlier in the merge anchors sequence override later ones\n\nGiven a sample.yml file of:\n```yaml\nfoo: &foo\n a: foo_a\n thing: foo_thing\n c: foo_c\nbar: &bar\n b: bar_b\n thing: bar_thing\n c: bar_c\nfoobarList:\n b: foobarList_b\n !!merge <<:\n - *foo\n - *bar\n c: foobarList_c\nfoobar:\n c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n```\nthen\n```bash\nyq '.foobar[]' sample.yml\n```\nwill output\n```yaml\nfoo_a\nfoobar_thing\nfoobar_c\n```\n\n## FIXED: Splatting merge anchor lists\nSet `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Note that the keys earlier in the merge anchors sequence override later ones\n\nGiven a sample.yml file of:\n```yaml\nfoo: &foo\n a: foo_a\n thing: foo_thing\n c: foo_c\nbar: &bar\n b: bar_b\n thing: bar_thing\n c: bar_c\nfoobarList:\n b: foobarList_b\n !!merge <<:\n - *foo\n - *bar\n c: foobarList_c\nfoobar:\n c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n```\nthen\n```bash\nyq '.foobarList[]' sample.yml\n```\nwill output\n```yaml\nfoobarList_b\nfoo_thing\nfoobarList_c\nfoo_a\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/union.md", "content": "# Union\n\nThis operator is used to combine different results together.\n\n## Combine scalars\nRunning\n```bash\nyq --null-input '1, true, \"cat\"'\n```\nwill output\n```yaml\n1\ntrue\ncat\n```\n\n## Combine selected paths\nGiven a sample.yml file of:\n```yaml\na: fieldA\nb: fieldB\nc: fieldC\n```\nthen\n```bash\nyq '.a, .c' sample.yml\n```\nwill output\n```yaml\nfieldA\nfieldC\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/unique.md", "content": "# Unique\n\nThis is used to filter out duplicated items in an array. Note that the original order of the array is maintained.\n\n\n## Unique array of scalars (string/numbers)\nNote that unique maintains the original order of the array.\n\nGiven a sample.yml file of:\n```yaml\n- 2\n- 1\n- 3\n- 2\n```\nthen\n```bash\nyq 'unique' sample.yml\n```\nwill output\n```yaml\n- 2\n- 1\n- 3\n```\n\n## Unique nulls\nUnique works on the node value, so it considers different representations of nulls to be different\n\nGiven a sample.yml file of:\n```yaml\n- ~\n- null\n- ~\n- null\n```\nthen\n```bash\nyq 'unique' sample.yml\n```\nwill output\n```yaml\n- ~\n- null\n```\n\n## Unique all nulls\nRun against the node tag to unique all the nulls\n\nGiven a sample.yml file of:\n```yaml\n- ~\n- null\n- ~\n- null\n```\nthen\n```bash\nyq 'unique_by(tag)' sample.yml\n```\nwill output\n```yaml\n- ~\n```\n\n## Unique array objects\nGiven a sample.yml file of:\n```yaml\n- name: harry\n pet: cat\n- name: billy\n pet: dog\n- name: harry\n pet: cat\n```\nthen\n```bash\nyq 'unique' sample.yml\n```\nwill output\n```yaml\n- name: harry\n pet: cat\n- name: billy\n pet: dog\n```\n\n## Unique array of objects by a field\nGiven a sample.yml file of:\n```yaml\n- name: harry\n pet: cat\n- name: billy\n pet: dog\n- name: harry\n pet: dog\n```\nthen\n```bash\nyq 'unique_by(.name)' sample.yml\n```\nwill output\n```yaml\n- name: harry\n pet: cat\n- name: billy\n pet: dog\n```\n\n## Unique array of arrays\nGiven a sample.yml file of:\n```yaml\n- - cat\n - dog\n- - cat\n - sheep\n- - cat\n - dog\n```\nthen\n```bash\nyq 'unique' sample.yml\n```\nwill output\n```yaml\n- - cat\n - dog\n- - cat\n - sheep\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/variable-operators.md", "content": "# Variable Operators\n\nLike the `jq` equivalents, variables are sometimes required for the more complex expressions (or swapping values between fields).\n\nNote that there is also an additional `ref` operator that holds a reference (instead of a copy) of the path, allowing you to make multiple changes to the same path.\n\n## Single value variable\nGiven a sample.yml file of:\n```yaml\na: cat\n```\nthen\n```bash\nyq '.a as $foo | $foo' sample.yml\n```\nwill output\n```yaml\ncat\n```\n\n## Multi value variable\nGiven a sample.yml file of:\n```yaml\n- cat\n- dog\n```\nthen\n```bash\nyq '.[] as $foo | $foo' sample.yml\n```\nwill output\n```yaml\ncat\ndog\n```\n\n## Using variables as a lookup\nExample taken from [jq](https://stedolan.github.io/jq/manual/#Variable/SymbolicBindingOperator:...as$identifier|...)\n\nGiven a sample.yml file of:\n```yaml\n\"posts\":\n - \"title\": First post\n \"author\": anon\n - \"title\": A well-written article\n \"author\": person1\n\"realnames\":\n \"anon\": Anonymous Coward\n \"person1\": Person McPherson\n```\nthen\n```bash\nyq '.realnames as $names | .posts[] | {\"title\":.title, \"author\": $names[.author]}' sample.yml\n```\nwill output\n```yaml\ntitle: First post\nauthor: Anonymous Coward\ntitle: A well-written article\nauthor: Person McPherson\n```\n\n## Using variables to swap values\nGiven a sample.yml file of:\n```yaml\na: a_value\nb: b_value\n```\nthen\n```bash\nyq '.a as $x | .b as $y | .b = $x | .a = $y' sample.yml\n```\nwill output\n```yaml\na: b_value\nb: a_value\n```\n\n## Use ref to reference a path repeatedly\nNote: You may find the `with` operator more useful.\n\nGiven a sample.yml file of:\n```yaml\na:\n b: thing\n c: something\n```\nthen\n```bash\nyq '.a.b ref $x | $x = \"new\" | $x style=\"double\"' sample.yml\n```\nwill output\n```yaml\na:\n b: \"new\"\n c: something\n```\n\n" }, { "path": "pkg/yqlib/doc/operators/with.md", "content": "# With\n\nUse the `with` operator to conveniently make multiple updates to a deeply nested path, or to update array elements relatively to each other. The first argument expression sets the root context, and the second expression runs against that root context.\n\n## Update and style\nGiven a sample.yml file of:\n```yaml\na:\n deeply:\n nested: value\n```\nthen\n```bash\nyq 'with(.a.deeply.nested; . = \"newValue\" | . style=\"single\")' sample.yml\n```\nwill output\n```yaml\na:\n deeply:\n nested: 'newValue'\n```\n\n## Update multiple deeply nested properties\nGiven a sample.yml file of:\n```yaml\na:\n deeply:\n nested: value\n other: thing\n```\nthen\n```bash\nyq 'with(.a.deeply; .nested = \"newValue\" | .other= \"newThing\")' sample.yml\n```\nwill output\n```yaml\na:\n deeply:\n nested: newValue\n other: newThing\n```\n\n## Update array elements relatively\nThe second expression runs with each element of the array as it's contextual root. This allows you to make updates relative to the element.\n\nGiven a sample.yml file of:\n```yaml\nmyArray:\n - a: apple\n - a: banana\n```\nthen\n```bash\nyq 'with(.myArray[]; .b = .a + \" yum\")' sample.yml\n```\nwill output\n```yaml\nmyArray:\n - a: apple\n b: apple yum\n - a: banana\n b: banana yum\n```\n\n" }, { "path": "pkg/yqlib/doc/usage/base64.md", "content": "# Base64\n\nEncode and decode to and from Base64.\n\nBase64 assumes [RFC4648](https://rfc-editor.org/rfc/rfc4648.html) encoding. Encoding and decoding both assume that the content is a UTF-8 string and not binary content.\n\n\nSee below for examples\n\n\n## Decode base64: simple\nDecoded data is assumed to be a string.\n\nGiven a sample.txt file of:\n```\nYSBzcGVjaWFsIHN0cmluZw==\n```\nthen\n```bash\nyq -p=base64 -oy '.' sample.txt\n```\nwill output\n```yaml\na special string\n```\n\n## Decode base64: UTF-8\nBase64 decoding supports UTF-8 encoded strings.\n\nGiven a sample.txt file of:\n```\nV29ya3Mgd2l0aCBVVEYtMTYg8J+Yig==\n```\nthen\n```bash\nyq -p=base64 -oy '.' sample.txt\n```\nwill output\n```yaml\nWorks with UTF-16 😊\n```\n\n## Decode with extra spaces\nExtra leading/trailing whitespace is stripped\n\nGiven a sample.txt file of:\n```\n\n YSBzcGVjaWFsIHN0cmluZw== \n\n```\nthen\n```bash\nyq -p=base64 -oy '.' sample.txt\n```\nwill output\n```yaml\na special string\n```\n\n## Encode base64: string\nGiven a sample.yml file of:\n```yaml\n\"a special string\"\n```\nthen\n```bash\nyq -o=base64 '.' sample.yml\n```\nwill output\n```\nYSBzcGVjaWFsIHN0cmluZw==```\n\n## Encode base64: string from document\nExtract a string field and encode it to base64.\n\nGiven a sample.yml file of:\n```yaml\ncoolData: \"a special string\"\n```\nthen\n```bash\nyq -o=base64 '.coolData' sample.yml\n```\nwill output\n```\nYSBzcGVjaWFsIHN0cmluZw==```\n\n" }, { "path": "pkg/yqlib/doc/usage/convert.md", "content": "# JSON\n\nEncode and decode to and from JSON. Supports multiple JSON documents in a single file (e.g. NDJSON).\n\nNote that YAML is a superset of (single document) JSON - so you don't have to use the JSON parser to read JSON when there is only one JSON document in the input. You will probably want to pretty print the result in this case, to get idiomatic YAML styling.\n\n\n## Parse json: simple\nJSON is a subset of yaml, so all you need to do is prettify the output\n\nGiven a sample.json file of:\n```json\n{\"cat\": \"meow\"}\n```\nthen\n```bash\nyq -p=json sample.json\n```\nwill output\n```yaml\ncat: meow\n```\n\n## Parse json: complex\nJSON is a subset of yaml, so all you need to do is prettify the output\n\nGiven a sample.json file of:\n```json\n{\"a\":\"Easy! as one two three\",\"b\":{\"c\":2,\"d\":[3,4]}}\n```\nthen\n```bash\nyq -p=json sample.json\n```\nwill output\n```yaml\na: Easy! as one two three\nb:\n c: 2\n d:\n - 3\n - 4\n```\n\n## Encode json: simple\nGiven a sample.yml file of:\n```yaml\ncat: meow\n```\nthen\n```bash\nyq -o=json '.' sample.yml\n```\nwill output\n```json\n{\n \"cat\": \"meow\"\n}\n```\n\n## Encode json: simple - in one line\nGiven a sample.yml file of:\n```yaml\ncat: meow # this is a comment, and it will be dropped.\n```\nthen\n```bash\nyq -o=json -I=0 '.' sample.yml\n```\nwill output\n```json\n{\"cat\":\"meow\"}\n```\n\n## Encode json: comments\nGiven a sample.yml file of:\n```yaml\ncat: meow # this is a comment, and it will be dropped.\n```\nthen\n```bash\nyq -o=json '.' sample.yml\n```\nwill output\n```json\n{\n \"cat\": \"meow\"\n}\n```\n\n## Encode json: anchors\nAnchors are dereferenced\n\nGiven a sample.yml file of:\n```yaml\ncat: &ref meow\nanotherCat: *ref\n```\nthen\n```bash\nyq -o=json '.' sample.yml\n```\nwill output\n```json\n{\n \"cat\": \"meow\",\n \"anotherCat\": \"meow\"\n}\n```\n\n## Encode json: multiple results\nEach matching node is converted into a json doc. This is best used with 0 indent (json document per line)\n\nGiven a sample.yml file of:\n```yaml\nthings: [{stuff: cool}, {whatever: cat}]\n```\nthen\n```bash\nyq -o=json -I=0 '.things[]' sample.yml\n```\nwill output\n```json\n{\"stuff\":\"cool\"}\n{\"whatever\":\"cat\"}\n```\n\n## Roundtrip JSON Lines / NDJSON\nGiven a sample.json file of:\n```json\n{\"this\": \"is a multidoc json file\"}\n{\"each\": [\"line is a valid json document\"]}\n{\"a number\": 4}\n\n```\nthen\n```bash\nyq -p=json -o=json -I=0 sample.json\n```\nwill output\n```yaml\n{\"this\":\"is a multidoc json file\"}\n{\"each\":[\"line is a valid json document\"]}\n{\"a number\":4}\n```\n\n## Roundtrip multi-document JSON\nThe parser can also handle multiple multi-line json documents in a single file (despite this not being in the JSON Lines / NDJSON spec). Typically you would have one entire JSON document per line, but the parser also supports multiple multi-line json documents\n\nGiven a sample.json file of:\n```json\n{\n\t\"this\": \"is a multidoc json file\"\n}\n{\n\t\"it\": [\n\t\t\"has\",\n\t\t\"consecutive\",\n\t\t\"json documents\"\n\t]\n}\n{\n\t\"a number\": 4\n}\n\n```\nthen\n```bash\nyq -p=json -o=json -I=2 sample.json\n```\nwill output\n```yaml\n{\n \"this\": \"is a multidoc json file\"\n}\n{\n \"it\": [\n \"has\",\n \"consecutive\",\n \"json documents\"\n ]\n}\n{\n \"a number\": 4\n}\n```\n\n## Update a specific document in a multi-document json\nDocuments are indexed by the `documentIndex` or `di` operator.\n\nGiven a sample.json file of:\n```json\n{\"this\": \"is a multidoc json file\"}\n{\"each\": [\"line is a valid json document\"]}\n{\"a number\": 4}\n\n```\nthen\n```bash\nyq -p=json -o=json -I=0 '(select(di == 1) | .each ) += \"cool\"' sample.json\n```\nwill output\n```yaml\n{\"this\":\"is a multidoc json file\"}\n{\"each\":[\"line is a valid json document\",\"cool\"]}\n{\"a number\":4}\n```\n\n## Find and update a specific document in a multi-document json\nUse expressions as you normally would.\n\nGiven a sample.json file of:\n```json\n{\"this\": \"is a multidoc json file\"}\n{\"each\": [\"line is a valid json document\"]}\n{\"a number\": 4}\n\n```\nthen\n```bash\nyq -p=json -o=json -I=0 '(select(has(\"each\")) | .each ) += \"cool\"' sample.json\n```\nwill output\n```yaml\n{\"this\":\"is a multidoc json file\"}\n{\"each\":[\"line is a valid json document\",\"cool\"]}\n{\"a number\":4}\n```\n\n## Decode JSON Lines / NDJSON\nGiven a sample.json file of:\n```json\n{\"this\": \"is a multidoc json file\"}\n{\"each\": [\"line is a valid json document\"]}\n{\"a number\": 4}\n\n```\nthen\n```bash\nyq -p=json sample.json\n```\nwill output\n```yaml\nthis: is a multidoc json file\n---\neach:\n - line is a valid json document\n---\na number: 4\n```\n\n" }, { "path": "pkg/yqlib/doc/usage/csv-tsv.md", "content": "# CSV\nEncode/Decode/Roundtrip CSV and TSV files.\n\n## Encode \nCurrently supports arrays of homogeneous flat objects, that is: no nesting and it assumes the _first_ object has all the keys required:\n\n```yaml\n- name: Bobo\n type: dog\n- name: Fifi\n type: cat\n```\n\nAs well as arrays of arrays of scalars (strings/numbers/booleans):\n\n```yaml\n- [Bobo, dog]\n- [Fifi, cat]\n```\n\n## Decode\nDecode assumes the first CSV/TSV row is the header row, and all rows beneath are the entries.\nThe data will be coded into an array of objects, using the header rows as keys.\n\n```csv\nname,type\nBobo,dog\nFifi,cat\n```\n\n\n## Encode CSV simple\nGiven a sample.yml file of:\n```yaml\n- [i, like, csv]\n- [because, excel, is, cool]\n```\nthen\n```bash\nyq -o=csv sample.yml\n```\nwill output\n```csv\ni,like,csv\nbecause,excel,is,cool\n```\n\n## Encode TSV simple\nGiven a sample.yml file of:\n```yaml\n- [i, like, csv]\n- [because, excel, is, cool]\n```\nthen\n```bash\nyq -o=tsv sample.yml\n```\nwill output\n```tsv\ni\tlike\tcsv\nbecause\texcel\tis\tcool\n```\n\n## Encode array of objects to csv\nGiven a sample.yml file of:\n```yaml\n- name: Gary\n numberOfCats: 1\n likesApples: true\n height: 168.8\n- name: Samantha's Rabbit\n numberOfCats: 2\n likesApples: false\n height: -188.8\n\n```\nthen\n```bash\nyq -o=csv sample.yml\n```\nwill output\n```csv\nname,numberOfCats,likesApples,height\nGary,1,true,168.8\nSamantha's Rabbit,2,false,-188.8\n```\n\n## Encode array of objects to custom csv format\nAdd the header row manually, then the we convert each object into an array of values - resulting in an array of arrays. Pick the columns and call the header whatever you like.\n\nGiven a sample.yml file of:\n```yaml\n- name: Gary\n numberOfCats: 1\n likesApples: true\n height: 168.8\n- name: Samantha's Rabbit\n numberOfCats: 2\n likesApples: false\n height: -188.8\n\n```\nthen\n```bash\nyq -o=csv '[[\"Name\", \"Number of Cats\"]] + [.[] | [.name, .numberOfCats ]]' sample.yml\n```\nwill output\n```csv\nName,Number of Cats\nGary,1\nSamantha's Rabbit,2\n```\n\n## Encode array of objects to csv - missing fields behaviour\nFirst entry is used to determine the headers, and it is missing 'likesApples', so it is not included in the csv. Second entry does not have 'numberOfCats' so that is blank\n\nGiven a sample.yml file of:\n```yaml\n- name: Gary\n numberOfCats: 1\n height: 168.8\n- name: Samantha's Rabbit\n height: -188.8\n likesApples: false\n\n```\nthen\n```bash\nyq -o=csv sample.yml\n```\nwill output\n```csv\nname,numberOfCats,height\nGary,1,168.8\nSamantha's Rabbit,,-188.8\n```\n\n## Parse CSV into an array of objects\nFirst row is assumed to be the header row. By default, entries with YAML/JSON formatting will be parsed!\n\nGiven a sample.csv file of:\n```csv\nname,numberOfCats,likesApples,height,facts\nGary,1,true,168.8,cool: true\nSamantha's Rabbit,2,false,-188.8,tall: indeed\n\n```\nthen\n```bash\nyq -p=csv sample.csv\n```\nwill output\n```yaml\n- name: Gary\n numberOfCats: 1\n likesApples: true\n height: 168.8\n facts:\n cool: true\n- name: Samantha's Rabbit\n numberOfCats: 2\n likesApples: false\n height: -188.8\n facts:\n tall: indeed\n```\n\n## Parse CSV into an array of objects, no auto-parsing\nFirst row is assumed to be the header row. Entries with YAML/JSON will be left as strings.\n\nGiven a sample.csv file of:\n```csv\nname,numberOfCats,likesApples,height,facts\nGary,1,true,168.8,cool: true\nSamantha's Rabbit,2,false,-188.8,tall: indeed\n\n```\nthen\n```bash\nyq -p=csv --csv-auto-parse=f sample.csv\n```\nwill output\n```yaml\n- name: Gary\n numberOfCats: 1\n likesApples: true\n height: 168.8\n facts: 'cool: true'\n- name: Samantha's Rabbit\n numberOfCats: 2\n likesApples: false\n height: -188.8\n facts: 'tall: indeed'\n```\n\n## Parse TSV into an array of objects\nFirst row is assumed to be the header row.\n\nGiven a sample.tsv file of:\n```tsv\nname\tnumberOfCats\tlikesApples\theight\nGary\t1\ttrue\t168.8\nSamantha's Rabbit\t2\tfalse\t-188.8\n\n```\nthen\n```bash\nyq -p=tsv sample.tsv\n```\nwill output\n```yaml\n- name: Gary\n numberOfCats: 1\n likesApples: true\n height: 168.8\n- name: Samantha's Rabbit\n numberOfCats: 2\n likesApples: false\n height: -188.8\n```\n\n## Round trip\nGiven a sample.csv file of:\n```csv\nname,numberOfCats,likesApples,height\nGary,1,true,168.8\nSamantha's Rabbit,2,false,-188.8\n\n```\nthen\n```bash\nyq -p=csv -o=csv '(.[] | select(.name == \"Gary\") | .numberOfCats) = 3' sample.csv\n```\nwill output\n```csv\nname,numberOfCats,likesApples,height\nGary,3,true,168.8\nSamantha's Rabbit,2,false,-188.8\n```\n\n" }, { "path": "pkg/yqlib/doc/usage/formatting-expressions.md", "content": "# Formatting Expressions\n\n`From version v4.41+`\n\nYou can put expressions into `.yq` files, use whitespace and comments to break up complex expressions and explain what's going on.\n\n## Using expression files and comments\nNote that you can execute the file directly - but make sure you make the expression file executable.\n\nGiven a sample.yaml file of:\n```yaml\na:\n b: old\n```\nAnd an 'update.yq' expression file of:\n```bash\n#! yq\n\n# This is a yq expression that updates the map\n# for several great reasons outlined here.\n\n.a.b = \"new\" # line comment here\n| .a.c = \"frog\"\n\n# Now good things will happen.\n```\nthen\n```bash\n./update.yq sample.yaml\n```\nwill output\n```yaml\na:\n b: new\n c: frog\n```\n\n## Flags in expression files\nYou can specify flags on the shebang line, this only works when executing the file directly.\n\nGiven a sample.yaml file of:\n```yaml\na:\n b: old\n```\nAnd an 'update.yq' expression file of:\n```bash\n#! yq -oj\n\n# This is a yq expression that updates the map\n# for several great reasons outlined here.\n\n.a.b = \"new\" # line comment here\n| .a.c = \"frog\"\n\n# Now good things will happen.\n```\nthen\n```bash\n./update.yq sample.yaml\n```\nwill output\n```yaml\n{\n \"a\": {\n \"b\": \"new\",\n \"c\": \"frog\"\n }\n}\n```\n\n## Commenting out yq expressions\nNote that `c` is no longer set to 'frog'. In this example we're calling yq directly and passing the expression file into `--from-file`, this is no different from executing the expression file directly.\n\nGiven a sample.yaml file of:\n```yaml\na:\n b: old\n```\nAnd an 'update.yq' expression file of:\n```bash\n#! yq\n# This is a yq expression that updates the map\n# for several great reasons outlined here.\n\n.a.b = \"new\" # line comment here\n# | .a.c = \"frog\"\n\n# Now good things will happen.\n```\nthen\n```bash\nyq --from-file update.yq sample.yml\n```\nwill output\n```yaml\na:\n b: new\n```\n\n" }, { "path": "pkg/yqlib/doc/usage/hcl.md", "content": "# HCL\n\nEncode and decode to and from [HashiCorp Configuration Language (HCL)](https://github.com/hashicorp/hcl).\n\nHCL is commonly used in HashiCorp tools like Terraform for configuration files. The yq HCL encoder and decoder support:\n- Blocks and attributes\n- String interpolation and expressions (preserved without quotes)\n- Comments (leading, head, and line comments)\n- Nested structures (maps and lists)\n- Syntax colorisation when enabled\n\n\n## Parse HCL\nGiven a sample.hcl file of:\n```hcl\nio_mode = \"async\"\n```\nthen\n```bash\nyq -oy sample.hcl\n```\nwill output\n```yaml\nio_mode: \"async\"\n```\n\n## Roundtrip: Sample Doc\nGiven a sample.hcl file of:\n```hcl\nservice \"cat\" {\n process \"main\" {\n command = [\"/usr/local/bin/awesome-app\", \"server\"]\n }\n\n process \"management\" {\n command = [\"/usr/local/bin/awesome-app\", \"management\"]\n }\n}\n\n```\nthen\n```bash\nyq sample.hcl\n```\nwill output\n```hcl\nservice \"cat\" {\n process \"main\" {\n command = [\"/usr/local/bin/awesome-app\", \"server\"]\n }\n process \"management\" {\n command = [\"/usr/local/bin/awesome-app\", \"management\"]\n }\n}\n```\n\n## Roundtrip: With an update\nGiven a sample.hcl file of:\n```hcl\nservice \"cat\" {\n process \"main\" {\n command = [\"/usr/local/bin/awesome-app\", \"server\"]\n }\n\n process \"management\" {\n command = [\"/usr/local/bin/awesome-app\", \"management\"]\n }\n}\n\n```\nthen\n```bash\nyq '.service.cat.process.main.command += \"meow\"' sample.hcl\n```\nwill output\n```hcl\nservice \"cat\" {\n process \"main\" {\n command = [\"/usr/local/bin/awesome-app\", \"server\", \"meow\"]\n }\n process \"management\" {\n command = [\"/usr/local/bin/awesome-app\", \"management\"]\n }\n}\n```\n\n## Parse HCL: Sample Doc\nGiven a sample.hcl file of:\n```hcl\nservice \"cat\" {\n process \"main\" {\n command = [\"/usr/local/bin/awesome-app\", \"server\"]\n }\n\n process \"management\" {\n command = [\"/usr/local/bin/awesome-app\", \"management\"]\n }\n}\n\n```\nthen\n```bash\nyq -oy sample.hcl\n```\nwill output\n```yaml\nservice:\n cat:\n process:\n main:\n command:\n - \"/usr/local/bin/awesome-app\"\n - \"server\"\n management:\n command:\n - \"/usr/local/bin/awesome-app\"\n - \"management\"\n```\n\n## Parse HCL: with comments\nGiven a sample.hcl file of:\n```hcl\n# Configuration\nport = 8080 # server port\n```\nthen\n```bash\nyq -oy sample.hcl\n```\nwill output\n```yaml\n# Configuration\nport: 8080 # server port\n```\n\n## Roundtrip: with comments\nGiven a sample.hcl file of:\n```hcl\n# Configuration\nport = 8080\n```\nthen\n```bash\nyq sample.hcl\n```\nwill output\n```hcl\n# Configuration\nport = 8080\n```\n\n## Roundtrip: With templates, functions and arithmetic\nGiven a sample.hcl file of:\n```hcl\n# Arithmetic with literals and application-provided variables\nsum = 1 + addend\n\n# String interpolation and templates\nmessage = \"Hello, ${name}!\"\n\n# Application-provided functions\nshouty_message = upper(message)\n```\nthen\n```bash\nyq sample.hcl\n```\nwill output\n```hcl\n# Arithmetic with literals and application-provided variables\nsum = 1 + addend\n# String interpolation and templates\nmessage = \"Hello, ${name}!\"\n# Application-provided functions\nshouty_message = upper(message)\n```\n\n## Roundtrip: Separate blocks with same name.\nGiven a sample.hcl file of:\n```hcl\nresource \"aws_instance\" \"web\" {\n ami = \"ami-12345\"\n}\nresource \"aws_instance\" \"db\" {\n ami = \"ami-67890\"\n}\n```\nthen\n```bash\nyq sample.hcl\n```\nwill output\n```hcl\nresource \"aws_instance\" \"web\" {\n ami = \"ami-12345\"\n}\nresource \"aws_instance\" \"db\" {\n ami = \"ami-67890\"\n}\n```\n\n" }, { "path": "pkg/yqlib/doc/usage/headers/base64.md", "content": "# Base64\n\nEncode and decode to and from Base64.\n\nBase64 assumes [RFC4648](https://rfc-editor.org/rfc/rfc4648.html) encoding. Encoding and decoding both assume that the content is a UTF-8 string and not binary content.\n\n\nSee below for examples\n\n" }, { "path": "pkg/yqlib/doc/usage/headers/convert.md", "content": "# JSON\n\nEncode and decode to and from JSON. Supports multiple JSON documents in a single file (e.g. NDJSON).\n\nNote that YAML is a superset of (single document) JSON - so you don't have to use the JSON parser to read JSON when there is only one JSON document in the input. You will probably want to pretty print the result in this case, to get idiomatic YAML styling.\n\n" }, { "path": "pkg/yqlib/doc/usage/headers/csv-tsv.md", "content": "# CSV\nEncode/Decode/Roundtrip CSV and TSV files.\n\n## Encode \nCurrently supports arrays of homogeneous flat objects, that is: no nesting and it assumes the _first_ object has all the keys required:\n\n```yaml\n- name: Bobo\n type: dog\n- name: Fifi\n type: cat\n```\n\nAs well as arrays of arrays of scalars (strings/numbers/booleans):\n\n```yaml\n- [Bobo, dog]\n- [Fifi, cat]\n```\n\n## Decode\nDecode assumes the first CSV/TSV row is the header row, and all rows beneath are the entries.\nThe data will be coded into an array of objects, using the header rows as keys.\n\n```csv\nname,type\nBobo,dog\nFifi,cat\n```\n\n" }, { "path": "pkg/yqlib/doc/usage/headers/formatting-expressions.md", "content": "# Formatting Expressions\n\n`From version v4.41+`\n\nYou can put expressions into `.yq` files, use whitespace and comments to break up complex expressions and explain what's going on.\n" }, { "path": "pkg/yqlib/doc/usage/headers/hcl.md", "content": "# HCL\n\nEncode and decode to and from [HashiCorp Configuration Language (HCL)](https://github.com/hashicorp/hcl).\n\nHCL is commonly used in HashiCorp tools like Terraform for configuration files. The yq HCL encoder and decoder support:\n- Blocks and attributes\n- String interpolation and expressions (preserved without quotes)\n- Comments (leading, head, and line comments)\n- Nested structures (maps and lists)\n- Syntax colorisation when enabled\n\n" }, { "path": "pkg/yqlib/doc/usage/headers/kyaml.md", "content": "# KYaml\n\nEncode and decode to and from KYaml (a restricted subset of YAML that uses flow-style collections).\n\nKYaml is useful when you want YAML data rendered in a compact, JSON-like form while still supporting YAML features like comments.\n\nNotes:\n- Strings are always double-quoted in KYaml output.\n- Anchors and aliases are expanded (KYaml output does not emit them).\n" }, { "path": "pkg/yqlib/doc/usage/headers/properties.md", "content": "# Properties\n\nEncode/Decode/Roundtrip to/from a property file. Line comments on value nodes will be copied across.\n\nBy default, empty maps and arrays are not encoded - see below for an example on how to encode a value for these.\n" }, { "path": "pkg/yqlib/doc/usage/headers/recipes.md", "content": "# Recipes\n\nThese examples are intended to show how you can use multiple operators together so you get an idea of how you can perform complex data manipulation.\n\nPlease see the details [operator docs](https://mikefarah.gitbook.io/yq/operators) for details on each individual operator.\n" }, { "path": "pkg/yqlib/doc/usage/headers/toml.md", "content": "# TOML\n\nDecode from TOML. Note that `yq` does not yet support outputting in TOML format (and therefore it cannot roundtrip)\n\n" }, { "path": "pkg/yqlib/doc/usage/headers/xml.md", "content": "# XML\n\nEncode and decode to and from XML. Whitespace is not conserved for round trips - but the order of the fields are.\n\nConsecutive xml nodes with the same name are assumed to be arrays.\n\nXML content data, attributes processing instructions and directives are all created as plain fields. \n\nThis can be controlled by:\n\n| Flag | Default |Sample XML | \n| -- | -- | -- |\n | `--xml-attribute-prefix` | `+` (changing to `+@` soon) | Legs in `````` | \n | `--xml-content-name` | `+content` | Meow in ```Meow true``` |\n | `--xml-directive-name` | `+directive` | `````` |\n | `--xml-proc-inst-prefix` | `+p_` | `````` |\n\n\n{% hint style=\"warning\" %}\nDefault Attribute Prefix will be changing in v4.30!\nIn order to avoid name conflicts (e.g. having an attribute named \"content\" will create a field that clashes with the default content name of \"+content\") the attribute prefix will be changing to \"+@\".\n\nThis will affect users that have not set their own prefix and are not roundtripping XML changes.\n\n{% endhint %}\n\n## Encoder / Decoder flag options\n\nIn addition to the above flags, there are the following xml encoder/decoder options controlled by flags:\n\n| Flag | Default | Description |\n| -- | -- | -- |\n| `--xml-strict-mode` | false | Strict mode enforces the requirements of the XML specification. When switched off the parser allows input containing common mistakes. See [the Golang xml decoder ](https://pkg.go.dev/encoding/xml#Decoder) for more details.| \n| `--xml-keep-namespace` | true | Keeps the namespace of attributes |\n| `--xml-raw-token` | true | Does not verify that start and end elements match and does not translate name space prefixes to their corresponding URLs. |\n| `--xml-skip-proc-inst` | false | Skips over processing instructions, e.g. `` |\n| `--xml-skip-directives` | false | Skips over directives, e.g. `````` |\n\n\nSee below for examples\n" }, { "path": "pkg/yqlib/doc/usage/kyaml.md", "content": "# KYaml\n\nEncode and decode to and from KYaml (a restricted subset of YAML that uses flow-style collections).\n\nKYaml is useful when you want YAML data rendered in a compact, JSON-like form while still supporting YAML features like comments.\n\nNotes:\n- Strings are always double-quoted in KYaml output.\n- Anchors and aliases are expanded (KYaml output does not emit them).\n\n## Encode kyaml: plain string scalar\nStrings are always double-quoted in KYaml output.\n\nGiven a sample.yml file of:\n```yaml\ncat\n\n```\nthen\n```bash\nyq -o=kyaml '.' sample.yml\n```\nwill output\n```yaml\n\"cat\"\n```\n\n## encode flow mapping and sequence\nGiven a sample.yml file of:\n```yaml\na: b\nc:\n - d\n\n```\nthen\n```bash\nyq -o=kyaml '.' sample.yml\n```\nwill output\n```yaml\n{\n a: \"b\",\n c: [\n \"d\",\n ],\n}\n```\n\n## encode non-string scalars\nGiven a sample.yml file of:\n```yaml\na: 12\nb: true\nc: null\nd: \"true\"\n\n```\nthen\n```bash\nyq -o=kyaml '.' sample.yml\n```\nwill output\n```yaml\n{\n a: 12,\n b: true,\n c: null,\n d: \"true\",\n}\n```\n\n## quote non-identifier keys\nGiven a sample.yml file of:\n```yaml\n\"1a\": b\n\"has space\": c\n\n```\nthen\n```bash\nyq -o=kyaml '.' sample.yml\n```\nwill output\n```yaml\n{\n \"1a\": \"b\",\n \"has space\": \"c\",\n}\n```\n\n## escape quoted strings\nGiven a sample.yml file of:\n```yaml\na: \"line1\\nline2\\t\\\"q\\\"\"\n\n```\nthen\n```bash\nyq -o=kyaml '.' sample.yml\n```\nwill output\n```yaml\n{\n a: \"line1\\nline2\\t\\\"q\\\"\",\n}\n```\n\n## preserve comments when encoding\nGiven a sample.yml file of:\n```yaml\n# leading\na: 1 # a line\n# head b\nb: 2\nc:\n # head d\n - d # d line\n - e\n# trailing\n\n```\nthen\n```bash\nyq -o=kyaml '.' sample.yml\n```\nwill output\n```yaml\n# leading\n{\n a: 1, # a line\n # head b\n b: 2,\n c: [\n # head d\n \"d\", # d line\n \"e\",\n ],\n # trailing\n}\n```\n\n## Encode kyaml: anchors and aliases\nKYaml output does not support anchors/aliases; they are expanded to concrete values.\n\nGiven a sample.yml file of:\n```yaml\nbase: &base\n a: b\ncopy: *base\n\n```\nthen\n```bash\nyq -o=kyaml '.' sample.yml\n```\nwill output\n```yaml\n{\n base: {\n a: \"b\",\n },\n copy: {\n a: \"b\",\n },\n}\n```\n\n## Encode kyaml: yaml to kyaml shows formatting differences\nKYaml uses flow-style collections (braces/brackets) and explicit commas.\n\nGiven a sample.yml file of:\n```yaml\nperson:\n name: John\n pets:\n - cat\n - dog\n\n```\nthen\n```bash\nyq -o=kyaml '.' sample.yml\n```\nwill output\n```yaml\n{\n person: {\n name: \"John\",\n pets: [\n \"cat\",\n \"dog\",\n ],\n },\n}\n```\n\n## Encode kyaml: nested lists of objects\nLists and objects can be nested arbitrarily; KYaml always uses flow-style collections.\n\nGiven a sample.yml file of:\n```yaml\n- name: a\n items:\n - id: 1\n tags:\n - k: x\n v: y\n - k: x2\n v: y2\n - id: 2\n tags:\n - k: z\n v: w\n\n```\nthen\n```bash\nyq -o=kyaml '.' sample.yml\n```\nwill output\n```yaml\n[\n {\n name: \"a\",\n items: [\n {\n id: 1,\n tags: [\n {\n k: \"x\",\n v: \"y\",\n },\n {\n k: \"x2\",\n v: \"y2\",\n },\n ],\n },\n {\n id: 2,\n tags: [\n {\n k: \"z\",\n v: \"w\",\n },\n ],\n },\n ],\n },\n]\n```\n\n" }, { "path": "pkg/yqlib/doc/usage/lua.md", "content": "\n## Basic input example\nGiven a sample.lua file of:\n```lua\nreturn {\n\t[\"country\"] = \"Australia\"; -- this place\n\t[\"cities\"] = {\n\t\t\"Sydney\",\n\t\t\"Melbourne\",\n\t\t\"Brisbane\",\n\t\t\"Perth\",\n\t};\n};\n\n```\nthen\n```bash\nyq -oy '.' sample.lua\n```\nwill output\n```yaml\ncountry: Australia\ncities:\n - Sydney\n - Melbourne\n - Brisbane\n - Perth\n```\n\n## Basic output example\nGiven a sample.yml file of:\n```yaml\n---\ncountry: Australia # this place\ncities:\n- Sydney\n- Melbourne\n- Brisbane\n- Perth\n```\nthen\n```bash\nyq -o=lua '.' sample.yml\n```\nwill output\n```lua\nreturn {\n\t[\"country\"] = \"Australia\"; -- this place\n\t[\"cities\"] = {\n\t\t\"Sydney\",\n\t\t\"Melbourne\",\n\t\t\"Brisbane\",\n\t\t\"Perth\",\n\t};\n};\n```\n\n## Unquoted keys\nUses the `--lua-unquoted` option to produce a nicer-looking output.\n\nGiven a sample.yml file of:\n```yaml\n---\ncountry: Australia # this place\ncities:\n- Sydney\n- Melbourne\n- Brisbane\n- Perth\n```\nthen\n```bash\nyq -o=lua --lua-unquoted '.' sample.yml\n```\nwill output\n```lua\nreturn {\n\tcountry = \"Australia\"; -- this place\n\tcities = {\n\t\t\"Sydney\",\n\t\t\"Melbourne\",\n\t\t\"Brisbane\",\n\t\t\"Perth\",\n\t};\n};\n```\n\n## Globals\nUses the `--lua-globals` option to export the values into the global scope.\n\nGiven a sample.yml file of:\n```yaml\n---\ncountry: Australia # this place\ncities:\n- Sydney\n- Melbourne\n- Brisbane\n- Perth\n```\nthen\n```bash\nyq -o=lua --lua-globals '.' sample.yml\n```\nwill output\n```lua\ncountry = \"Australia\"; -- this place\ncities = {\n\t\"Sydney\",\n\t\"Melbourne\",\n\t\"Brisbane\",\n\t\"Perth\",\n};\n```\n\n## Elaborate example\nGiven a sample.yml file of:\n```yaml\n---\nhello: world\ntables:\n like: this\n keys: values\n ? look: non-string keys\n : True\nnumbers:\n - decimal: 12345\n - hex: 0x7fabc123\n - octal: 0o30\n - float: 123.45\n - infinity: .inf\n plus_infinity: +.inf\n minus_infinity: -.inf\n - not: .nan\n\n```\nthen\n```bash\nyq -o=lua '.' sample.yml\n```\nwill output\n```lua\nreturn {\n\t[\"hello\"] = \"world\";\n\t[\"tables\"] = {\n\t\t[\"like\"] = \"this\";\n\t\t[\"keys\"] = \"values\";\n\t\t[{\n\t\t\t[\"look\"] = \"non-string keys\";\n\t\t}] = true;\n\t};\n\t[\"numbers\"] = {\n\t\t{\n\t\t\t[\"decimal\"] = 12345;\n\t\t},\n\t\t{\n\t\t\t[\"hex\"] = 0x7fabc123;\n\t\t},\n\t\t{\n\t\t\t[\"octal\"] = 24;\n\t\t},\n\t\t{\n\t\t\t[\"float\"] = 123.45;\n\t\t},\n\t\t{\n\t\t\t[\"infinity\"] = (1/0);\n\t\t\t[\"plus_infinity\"] = (1/0);\n\t\t\t[\"minus_infinity\"] = (-1/0);\n\t\t},\n\t\t{\n\t\t\t[\"not\"] = (0/0);\n\t\t},\n\t};\n};\n```\n\n" }, { "path": "pkg/yqlib/doc/usage/properties.md", "content": "# Properties\n\nEncode/Decode/Roundtrip to/from a property file. Line comments on value nodes will be copied across.\n\nBy default, empty maps and arrays are not encoded - see below for an example on how to encode a value for these.\n\n## Encode properties\nNote that empty arrays and maps are not encoded by default.\n\nGiven a sample.yml file of:\n```yaml\n# block comments come through\nperson: # neither do comments on maps\n name: Mike Wazowski # comments on values appear\n pets: \n - cat # comments on array values appear\n - nested:\n - list entry\n food: [pizza] # comments on arrays do not\nemptyArray: []\nemptyMap: []\n\n```\nthen\n```bash\nyq -o=props sample.yml\n```\nwill output\n```properties\n# block comments come through\n# comments on values appear\nperson.name = Mike Wazowski\n\n# comments on array values appear\nperson.pets.0 = cat\nperson.pets.1.nested.0 = list entry\nperson.food.0 = pizza\n```\n\n## Encode properties with array brackets\nDeclare the --properties-array-brackets flag to give array paths in brackets (e.g. SpringBoot).\n\nGiven a sample.yml file of:\n```yaml\n# block comments come through\nperson: # neither do comments on maps\n name: Mike Wazowski # comments on values appear\n pets: \n - cat # comments on array values appear\n - nested:\n - list entry\n food: [pizza] # comments on arrays do not\nemptyArray: []\nemptyMap: []\n\n```\nthen\n```bash\nyq -o=props --properties-array-brackets sample.yml\n```\nwill output\n```properties\n# block comments come through\n# comments on values appear\nperson.name = Mike Wazowski\n\n# comments on array values appear\nperson.pets[0] = cat\nperson.pets[1].nested[0] = list entry\nperson.food[0] = pizza\n```\n\n## Encode properties - custom separator\nUse the --properties-separator flag to specify your own key/value separator.\n\nGiven a sample.yml file of:\n```yaml\n# block comments come through\nperson: # neither do comments on maps\n name: Mike Wazowski # comments on values appear\n pets: \n - cat # comments on array values appear\n - nested:\n - list entry\n food: [pizza] # comments on arrays do not\nemptyArray: []\nemptyMap: []\n\n```\nthen\n```bash\nyq -o=props --properties-separator=\" :@ \" sample.yml\n```\nwill output\n```properties\n# block comments come through\n# comments on values appear\nperson.name :@ Mike Wazowski\n\n# comments on array values appear\nperson.pets.0 :@ cat\nperson.pets.1.nested.0 :@ list entry\nperson.food.0 :@ pizza\n```\n\n## Encode properties: scalar encapsulation\nNote that string values with blank characters in them are encapsulated with double quotes\n\nGiven a sample.yml file of:\n```yaml\n# block comments come through\nperson: # neither do comments on maps\n name: Mike Wazowski # comments on values appear\n pets: \n - cat # comments on array values appear\n - nested:\n - list entry\n food: [pizza] # comments on arrays do not\nemptyArray: []\nemptyMap: []\n\n```\nthen\n```bash\nyq -o=props --unwrapScalar=false sample.yml\n```\nwill output\n```properties\n# block comments come through\n# comments on values appear\nperson.name = \"Mike Wazowski\"\n\n# comments on array values appear\nperson.pets.0 = cat\nperson.pets.1.nested.0 = \"list entry\"\nperson.food.0 = pizza\n```\n\n## Encode properties: no comments\nGiven a sample.yml file of:\n```yaml\n# block comments come through\nperson: # neither do comments on maps\n name: Mike Wazowski # comments on values appear\n pets: \n - cat # comments on array values appear\n - nested:\n - list entry\n food: [pizza] # comments on arrays do not\nemptyArray: []\nemptyMap: []\n\n```\nthen\n```bash\nyq -o=props '... comments = \"\"' sample.yml\n```\nwill output\n```properties\nperson.name = Mike Wazowski\nperson.pets.0 = cat\nperson.pets.1.nested.0 = list entry\nperson.food.0 = pizza\n```\n\n## Encode properties: include empty maps and arrays\nUse a yq expression to set the empty maps and sequences to your desired value.\n\nGiven a sample.yml file of:\n```yaml\n# block comments come through\nperson: # neither do comments on maps\n name: Mike Wazowski # comments on values appear\n pets: \n - cat # comments on array values appear\n - nested:\n - list entry\n food: [pizza] # comments on arrays do not\nemptyArray: []\nemptyMap: []\n\n```\nthen\n```bash\nyq -o=props '(.. | select( (tag == \"!!map\" or tag ==\"!!seq\") and length == 0)) = \"\"' sample.yml\n```\nwill output\n```properties\n# block comments come through\n# comments on values appear\nperson.name = Mike Wazowski\n\n# comments on array values appear\nperson.pets.0 = cat\nperson.pets.1.nested.0 = list entry\nperson.food.0 = pizza\nemptyArray = \nemptyMap = \n```\n\n## Decode properties\nGiven a sample.properties file of:\n```properties\n# block comments come through\n# comments on values appear\nperson.name = Mike Wazowski\n\n# comments on array values appear\nperson.pets.0 = cat\nperson.pets.1.nested.0 = list entry\nperson.food.0 = pizza\n\n```\nthen\n```bash\nyq -p=props sample.properties\n```\nwill output\n```yaml\nperson:\n # block comments come through\n # comments on values appear\n name: Mike Wazowski\n pets:\n # comments on array values appear\n - cat\n - nested:\n - list entry\n food:\n - pizza\n```\n\n## Decode properties: numbers\nAll values are assumed to be strings when parsing properties, but you can use the `from_yaml` operator on all the strings values to autoparse into the correct type.\n\nGiven a sample.properties file of:\n```properties\na.b = 10\n```\nthen\n```bash\nyq -p=props ' (.. | select(tag == \"!!str\")) |= from_yaml' sample.properties\n```\nwill output\n```yaml\na:\n b: 10\n```\n\n## Decode properties - array should be a map\nIf you have a numeric map key in your property files, use array_to_map to convert them to maps.\n\nGiven a sample.properties file of:\n```properties\nthings.10 = mike\n```\nthen\n```bash\nyq -p=props '.things |= array_to_map' sample.properties\n```\nwill output\n```yaml\nthings:\n 10: mike\n```\n\n## Roundtrip\nGiven a sample.properties file of:\n```properties\n# block comments come through\n# comments on values appear\nperson.name = Mike Wazowski\n\n# comments on array values appear\nperson.pets.0 = cat\nperson.pets.1.nested.0 = list entry\nperson.food.0 = pizza\n\n```\nthen\n```bash\nyq -p=props -o=props '.person.pets.0 = \"dog\"' sample.properties\n```\nwill output\n```properties\n# block comments come through\n# comments on values appear\nperson.name = Mike Wazowski\n\n# comments on array values appear\nperson.pets.0 = dog\nperson.pets.1.nested.0 = list entry\nperson.food.0 = pizza\n```\n\n" }, { "path": "pkg/yqlib/doc/usage/recipes.md", "content": "# Recipes\n\nThese examples are intended to show how you can use multiple operators together so you get an idea of how you can perform complex data manipulation.\n\nPlease see the details [operator docs](https://mikefarah.gitbook.io/yq/operators) for details on each individual operator.\n\n## Find items in an array\nWe have an array and we want to find the elements with a particular name.\n\nGiven a sample.yml file of:\n```yaml\n- name: Foo\n numBuckets: 0\n- name: Bar\n numBuckets: 0\n```\nthen\n```bash\nyq '.[] | select(.name == \"Foo\")' sample.yml\n```\nwill output\n```yaml\nname: Foo\nnumBuckets: 0\n```\n\n### Explanation:\n- `.[]` splats the array, and puts all the items in the context.\n- These items are then piped (`|`) into `select(.name == \"Foo\")` which will select all the nodes that have a name property set to 'Foo'.\n- See the [select](https://mikefarah.gitbook.io/yq/operators/select) operator for more information.\n\n## Find and update items in an array\nWe have an array and we want to _update_ the elements with a particular name.\n\nGiven a sample.yml file of:\n```yaml\n- name: Foo\n numBuckets: 0\n- name: Bar\n numBuckets: 0\n```\nthen\n```bash\nyq '(.[] | select(.name == \"Foo\") | .numBuckets) |= . + 1' sample.yml\n```\nwill output\n```yaml\n- name: Foo\n numBuckets: 1\n- name: Bar\n numBuckets: 0\n```\n\n### Explanation:\n- Following from the example above`.[]` splats the array, selects filters the items.\n- We then pipe (`|`) that into `.numBuckets`, which will select that field from all the matching items\n- Splat, select and the field are all in brackets, that whole expression is passed to the `|=` operator as the left hand side expression, with `. + 1` as the right hand side expression.\n- `|=` is the operator that updates fields relative to their own value, which is referenced as dot (`.`).\n- The expression `. + 1` increments the numBuckets counter.\n- See the [assign](https://mikefarah.gitbook.io/yq/operators/assign-update) and [add](https://mikefarah.gitbook.io/yq/operators/add) operators for more information.\n\n## Deeply prune a tree\nSay we are only interested in child1 and child2, and want to filter everything else out.\n\nGiven a sample.yml file of:\n```yaml\nparentA:\n - bob\nparentB:\n child1: i am child1\n child3: hiya\nparentC:\n childX: cool\n child2: me child2\n```\nthen\n```bash\nyq '(\n .. | # recurse through all the nodes\n select(has(\"child1\") or has(\"child2\")) | # match parents that have either child1 or child2\n (.child1, .child2) | # select those children\n select(.) # filter out nulls\n) as $i ireduce({}; # using that set of nodes, create a new result map\n setpath($i | path; $i) # and put in each node, using its original path\n)' sample.yml\n```\nwill output\n```yaml\nparentB:\n child1: i am child1\nparentC:\n child2: me child2\n```\n\n### Explanation:\n- Find all the matching child1 and child2 nodes\n- Using ireduce, create a new map using just those nodes\n- Set each node into the new map using its original path\n\n## Multiple or complex updates to items in an array\nWe have an array and we want to _update_ the elements with a particular name in reference to its type.\n\nGiven a sample.yml file of:\n```yaml\nmyArray:\n - name: Foo\n type: cat\n - name: Bar\n type: dog\n```\nthen\n```bash\nyq 'with(.myArray[]; .name = .name + \" - \" + .type)' sample.yml\n```\nwill output\n```yaml\nmyArray:\n - name: Foo - cat\n type: cat\n - name: Bar - dog\n type: dog\n```\n\n### Explanation:\n- The with operator will effectively loop through each given item in the first given expression, and run the second expression against it.\n- `.myArray[]` splats the array in `myArray`. So `with` will run against each item in that array\n- `.name = .name + \" - \" + .type` this expression is run against every item, updating the name to be a concatenation of the original name as well as the type.\n- See the [with](https://mikefarah.gitbook.io/yq/operators/with) operator for more information and examples.\n\n## Sort an array by a field\nGiven a sample.yml file of:\n```yaml\nmyArray:\n - name: Foo\n numBuckets: 1\n - name: Bar\n numBuckets: 0\n```\nthen\n```bash\nyq '.myArray |= sort_by(.numBuckets)' sample.yml\n```\nwill output\n```yaml\nmyArray:\n - name: Bar\n numBuckets: 0\n - name: Foo\n numBuckets: 1\n```\n\n### Explanation:\n- We want to resort `.myArray`.\n- `sort_by` works by piping an array into it, and it pipes out a sorted array.\n- So, we use `|=` to update `.myArray`. This is the same as doing `.myArray = (.myArray | sort_by(.numBuckets))`\n\n## Filter, flatten, sort and unique\nLets find the unique set of names from the document.\n\nGiven a sample.yml file of:\n```yaml\n- type: foo\n names:\n - Fred\n - Catherine\n- type: bar\n names:\n - Zelda\n- type: foo\n names: Fred\n- type: foo\n names: Ava\n```\nthen\n```bash\nyq '[.[] | select(.type == \"foo\") | .names] | flatten | sort | unique' sample.yml\n```\nwill output\n```yaml\n- Ava\n- Catherine\n- Fred\n```\n\n### Explanation:\n- `.[] | select(.type == \"foo\") | .names` will select the array elements of type \"foo\"\n- Splat `.[]` will unwrap the array and match all the items. We need to do this so we can work on the child items, for instance, filter items out using the `select` operator.\n- But we still want the final results back into an array. So after we're doing working on the children, we wrap everything back into an array using square brackets around the expression. `[.[] | select(.type == \"foo\") | .names]`\n- Now have have an array of all the 'names' values. Which includes arrays of strings as well as strings on their own.\n- Pipe `|` this array through `flatten`. This will flatten nested arrays. So now we have a flat list of all the name value strings\n- Next we pipe `|` that through `sort` and then `unique` to get a sorted, unique list of the names!\n- See the [flatten](https://mikefarah.gitbook.io/yq/operators/flatten), [sort](https://mikefarah.gitbook.io/yq/operators/sort) and [unique](https://mikefarah.gitbook.io/yq/operators/unique) for more information and examples.\n\n## Export as environment variables (script), or any custom format\nGiven a yaml document, lets output a script that will configure environment variables with that data. This same approach can be used for exporting into custom formats.\n\nGiven a sample.yml file of:\n```yaml\nvar0: string0\nvar1: string1\nfruit:\n - apple\n - banana\n - peach\n```\nthen\n```bash\nyq '.[] |(\n\t( select(kind == \"scalar\") | key + \"='\\''\" + . + \"'\\''\"),\n\t( select(kind == \"seq\") | key + \"=(\" + (map(\"'\\''\" + . + \"'\\''\") | join(\",\")) + \")\")\n)' sample.yml\n```\nwill output\n```yaml\nvar0='string0'\nvar1='string1'\nfruit=('apple','banana','peach')\n```\n\n### Explanation:\n- `.[]` matches all top level elements\n- We need a string expression for each of the different types that will produce the bash syntax, we'll use the union operator, to join them together\n- Scalars, we just need the key and quoted value: `( select(kind == \"scalar\") | key + \"='\" + . + \"'\")`\n- Sequences (or arrays) are trickier, we need to quote each value and `join` them with `,`: `map(\"'\" + . + \"'\") | join(\",\")`\n\n## Custom format with nested data\nLike the previous example, but lets handle nested data structures. In this custom example, we're going to join the property paths with _. The important thing to keep in mind is that our expression is not recursive (despite the data structure being so). Instead we match _all_ elements on the tree and operate on them.\n\nGiven a sample.yml file of:\n```yaml\nsimple: string0\nsimpleArray:\n - apple\n - banana\n - peach\ndeep:\n property: value\n array:\n - cat\n```\nthen\n```bash\nyq '.. |(\n\t( select(kind == \"scalar\" and parent | kind != \"seq\") | (path | join(\"_\")) + \"='\\''\" + . + \"'\\''\"),\n\t( select(kind == \"seq\") | (path | join(\"_\")) + \"=(\" + (map(\"'\\''\" + . + \"'\\''\") | join(\",\")) + \")\")\n)' sample.yml\n```\nwill output\n```yaml\nsimple='string0'\ndeep_property='value'\nsimpleArray=('apple','banana','peach')\ndeep_array=('cat')\n```\n\n### Explanation:\n- You'll need to understand how the previous example works to understand this extension.\n- `..` matches _all_ elements, instead of `.[]` from the previous example that just matches top level elements.\n- Like before, we need a string expression for each of the different types that will produce the bash syntax, we'll use the union operator, to join them together\n- This time, however, our expression matches every node in the data structure.\n- We only want to print scalars that are not in arrays (because we handle the separately), so well add `and parent | kind != \"seq\"` to the select operator expression for scalars\n- We don't just want the key any more, we want the full path. So instead of `key` we have `path | join(\"_\")`\n- The expression for sequences follows the same logic\n\n" }, { "path": "pkg/yqlib/doc/usage/shellvariables.md", "content": "\n## Encode shell variables\nNote that comments are dropped and values will be enclosed in single quotes as needed.\n\nGiven a sample.yml file of:\n```yaml\n# comment\nname: Mike Wazowski\neyes:\n color: turquoise\n number: 1\nfriends:\n - James P. Sullivan\n - Celia Mae\n```\nthen\n```bash\nyq -o=shell sample.yml\n```\nwill output\n```sh\nname='Mike Wazowski'\neyes_color=turquoise\neyes_number=1\nfriends_0='James P. Sullivan'\nfriends_1='Celia Mae'\n```\n\n## Encode shell variables: illegal variable names as key.\nKeys that would be illegal as variable keys are adapted.\n\nGiven a sample.yml file of:\n```yaml\nascii_=_symbols: replaced with _\n\"ascii_\t_controls\": dropped (this example uses \\t)\nnonascii_א_characters: dropped\neffort_expeñded_tò_preserve_accented_latin_letters: moderate (via unicode NFKD)\n\n```\nthen\n```bash\nyq -o=shell sample.yml\n```\nwill output\n```sh\nascii___symbols='replaced with _'\nascii__controls='dropped (this example uses \\t)'\nnonascii__characters=dropped\neffort_expended_to_preserve_accented_latin_letters='moderate (via unicode NFKD)'\n```\n\n## Encode shell variables: empty values, arrays and maps\nEmpty values are encoded to empty variables, but empty arrays and maps are skipped.\n\nGiven a sample.yml file of:\n```yaml\nempty:\n value:\n array: []\n map: {}\n```\nthen\n```bash\nyq -o=shell sample.yml\n```\nwill output\n```sh\nempty_value=\n```\n\n## Encode shell variables: single quotes in values\nSingle quotes in values are encoded as '\"'\"' (close single quote, double-quoted single quote, open single quote).\n\nGiven a sample.yml file of:\n```yaml\nname: Miles O'Brien\n```\nthen\n```bash\nyq -o=shell sample.yml\n```\nwill output\n```sh\nname='Miles O'\"'\"'Brien'\n```\n\n## Encode shell variables: custom separator\nUse --shell-key-separator to specify a custom separator between keys. This is useful when the original keys contain underscores.\n\nGiven a sample.yml file of:\n```yaml\nmy_app:\n db_config:\n host: localhost\n port: 5432\n```\nthen\n```bash\nyq -o=shell --shell-key-separator=\"__\" sample.yml\n```\nwill output\n```sh\nmy_app__db_config__host=localhost\nmy_app__db_config__port=5432\n```\n\n" }, { "path": "pkg/yqlib/doc/usage/toml.md", "content": "# TOML\n\nDecode from TOML. Note that `yq` does not yet support outputting in TOML format (and therefore it cannot roundtrip)\n\n\n## Parse: Simple\nGiven a sample.toml file of:\n```toml\nA = \"hello\"\nB = 12\n\n```\nthen\n```bash\nyq -oy '.' sample.toml\n```\nwill output\n```yaml\nA: hello\nB: 12\n```\n\n## Parse: Deep paths\nGiven a sample.toml file of:\n```toml\nperson.name = \"hello\"\nperson.address = \"12 cat st\"\n\n```\nthen\n```bash\nyq -oy '.' sample.toml\n```\nwill output\n```yaml\nperson:\n name: hello\n address: 12 cat st\n```\n\n## Encode: Scalar\nGiven a sample.toml file of:\n```toml\nperson.name = \"hello\"\nperson.address = \"12 cat st\"\n\n```\nthen\n```bash\nyq '.person.name' sample.toml\n```\nwill output\n```yaml\nhello\n```\n\n## Parse: inline table\nGiven a sample.toml file of:\n```toml\nname = { first = \"Tom\", last = \"Preston-Werner\" }\n```\nthen\n```bash\nyq -oy '.' sample.toml\n```\nwill output\n```yaml\nname:\n first: Tom\n last: Preston-Werner\n```\n\n## Parse: Array Table\nGiven a sample.toml file of:\n```toml\n\n[owner.contact]\nname = \"Tom Preston-Werner\"\nage = 36\n\n[[owner.addresses]]\nstreet = \"first street\"\nsuburb = \"ok\"\n\n[[owner.addresses]]\nstreet = \"second street\"\nsuburb = \"nice\"\n\n```\nthen\n```bash\nyq -oy '.' sample.toml\n```\nwill output\n```yaml\nowner:\n contact:\n name: Tom Preston-Werner\n age: 36\n addresses:\n - street: first street\n suburb: ok\n - street: second street\n suburb: nice\n```\n\n## Parse: Array of Array Table\nGiven a sample.toml file of:\n```toml\n\n[[fruits]]\nname = \"apple\"\n[[fruits.varieties]] # nested array of tables\nname = \"red delicious\"\n```\nthen\n```bash\nyq -oy '.' sample.toml\n```\nwill output\n```yaml\nfruits:\n - name: apple\n varieties:\n - name: red delicious\n```\n\n## Parse: Empty Table\nGiven a sample.toml file of:\n```toml\n\n[dependencies]\n\n```\nthen\n```bash\nyq -oy '.' sample.toml\n```\nwill output\n```yaml\ndependencies: {}\n```\n\n## Roundtrip: inline table attribute\nGiven a sample.toml file of:\n```toml\nname = { first = \"Tom\", last = \"Preston-Werner\" }\n\n```\nthen\n```bash\nyq '.' sample.toml\n```\nwill output\n```yaml\nname = { first = \"Tom\", last = \"Preston-Werner\" }\n```\n\n## Roundtrip: table section\nGiven a sample.toml file of:\n```toml\n[owner.contact]\nname = \"Tom\"\nage = 36\n\n```\nthen\n```bash\nyq '.' sample.toml\n```\nwill output\n```yaml\n[owner.contact]\nname = \"Tom\"\nage = 36\n```\n\n## Roundtrip: array of tables\nGiven a sample.toml file of:\n```toml\n[[fruits]]\nname = \"apple\"\n[[fruits.varieties]]\nname = \"red delicious\"\n\n```\nthen\n```bash\nyq '.' sample.toml\n```\nwill output\n```yaml\n[[fruits]]\nname = \"apple\"\n[[fruits.varieties]]\nname = \"red delicious\"\n```\n\n## Roundtrip: arrays and scalars\nGiven a sample.toml file of:\n```toml\nA = [\"hello\", [\"world\", \"again\"]]\nB = 12\n\n```\nthen\n```bash\nyq '.' sample.toml\n```\nwill output\n```yaml\nA = [\"hello\", [\"world\", \"again\"]]\nB = 12\n```\n\n## Roundtrip: simple\nGiven a sample.toml file of:\n```toml\nA = \"hello\"\nB = 12\n\n```\nthen\n```bash\nyq '.' sample.toml\n```\nwill output\n```yaml\nA = \"hello\"\nB = 12\n```\n\n## Roundtrip: deep paths\nGiven a sample.toml file of:\n```toml\n[person]\nname = \"hello\"\naddress = \"12 cat st\"\n\n```\nthen\n```bash\nyq '.' sample.toml\n```\nwill output\n```yaml\n[person]\nname = \"hello\"\naddress = \"12 cat st\"\n```\n\n## Roundtrip: empty array\nGiven a sample.toml file of:\n```toml\nA = []\n\n```\nthen\n```bash\nyq '.' sample.toml\n```\nwill output\n```yaml\nA = []\n```\n\n## Roundtrip: sample table\nGiven a sample.toml file of:\n```toml\nvar = \"x\"\n\n[owner.contact]\nname = \"Tom Preston-Werner\"\nage = 36\n\n```\nthen\n```bash\nyq '.' sample.toml\n```\nwill output\n```yaml\nvar = \"x\"\n\n[owner.contact]\nname = \"Tom Preston-Werner\"\nage = 36\n```\n\n## Roundtrip: empty table\nGiven a sample.toml file of:\n```toml\n[dependencies]\n\n```\nthen\n```bash\nyq '.' sample.toml\n```\nwill output\n```yaml\n[dependencies]\n```\n\n## Roundtrip: comments\nGiven a sample.toml file of:\n```toml\n# This is a comment\nA = \"hello\" # inline comment\nB = 12\n\n# Table comment\n[person]\nname = \"Tom\" # name comment\n\n```\nthen\n```bash\nyq '.' sample.toml\n```\nwill output\n```yaml\n# This is a comment\nA = \"hello\" # inline comment\nB = 12\n\n# Table comment\n[person]\nname = \"Tom\" # name comment\n```\n\n## Roundtrip: sample from web\nGiven a sample.toml file of:\n```toml\n# This is a TOML document\ntitle = \"TOML Example\"\n\n[owner]\nname = \"Tom Preston-Werner\"\ndob = 1979-05-27T07:32:00-08:00\n\n[database]\nenabled = true\nports = [8000, 8001, 8002]\ndata = [[\"delta\", \"phi\"], [3.14]]\ntemp_targets = { cpu = 79.5, case = 72.0 }\n\n# [servers] yq can't do this one yet\n[servers.alpha]\nip = \"10.0.0.1\"\nrole = \"frontend\"\n\n[servers.beta]\nip = \"10.0.0.2\"\nrole = \"backend\"\n\n```\nthen\n```bash\nyq '.' sample.toml\n```\nwill output\n```yaml\n# This is a TOML document\ntitle = \"TOML Example\"\n\n[owner]\nname = \"Tom Preston-Werner\"\ndob = 1979-05-27T07:32:00-08:00\n\n[database]\nenabled = true\nports = [8000, 8001, 8002]\ndata = [[\"delta\", \"phi\"], [3.14]]\ntemp_targets = { cpu = 79.5, case = 72.0 }\n\n# [servers] yq can't do this one yet\n[servers.alpha]\nip = \"10.0.0.1\"\nrole = \"frontend\"\n\n[servers.beta]\nip = \"10.0.0.2\"\nrole = \"backend\"\n```\n\n" }, { "path": "pkg/yqlib/doc/usage/xml.md", "content": "# XML\n\nEncode and decode to and from XML. Whitespace is not conserved for round trips - but the order of the fields are.\n\nConsecutive xml nodes with the same name are assumed to be arrays.\n\nXML content data, attributes processing instructions and directives are all created as plain fields. \n\nThis can be controlled by:\n\n| Flag | Default |Sample XML | \n| -- | -- | -- |\n | `--xml-attribute-prefix` | `+` (changing to `+@` soon) | Legs in `````` | \n | `--xml-content-name` | `+content` | Meow in ```Meow true``` |\n | `--xml-directive-name` | `+directive` | `````` |\n | `--xml-proc-inst-prefix` | `+p_` | `````` |\n\n\n{% hint style=\"warning\" %}\nDefault Attribute Prefix will be changing in v4.30!\nIn order to avoid name conflicts (e.g. having an attribute named \"content\" will create a field that clashes with the default content name of \"+content\") the attribute prefix will be changing to \"+@\".\n\nThis will affect users that have not set their own prefix and are not roundtripping XML changes.\n\n{% endhint %}\n\n## Encoder / Decoder flag options\n\nIn addition to the above flags, there are the following xml encoder/decoder options controlled by flags:\n\n| Flag | Default | Description |\n| -- | -- | -- |\n| `--xml-strict-mode` | false | Strict mode enforces the requirements of the XML specification. When switched off the parser allows input containing common mistakes. See [the Golang xml decoder ](https://pkg.go.dev/encoding/xml#Decoder) for more details.| \n| `--xml-keep-namespace` | true | Keeps the namespace of attributes |\n| `--xml-raw-token` | true | Does not verify that start and end elements match and does not translate name space prefixes to their corresponding URLs. |\n| `--xml-skip-proc-inst` | false | Skips over processing instructions, e.g. `` |\n| `--xml-skip-directives` | false | Skips over directives, e.g. `````` |\n\n\nSee below for examples\n\n## Parse xml: simple\nNotice how all the values are strings, see the next example on how you can fix that.\n\nGiven a sample.xml file of:\n```xml\n\n\n meow\n 4\n true\n\n```\nthen\n```bash\nyq -oy sample.xml\n```\nwill output\n```yaml\n+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n says: meow\n legs: \"4\"\n cute: \"true\"\n```\n\n## Parse xml: number\nAll values are assumed to be strings when parsing XML, but you can use the `from_yaml` operator on all the strings values to autoparse into the correct type.\n\nGiven a sample.xml file of:\n```xml\n\n\n meow\n 4\n true\n\n```\nthen\n```bash\nyq -oy ' (.. | select(tag == \"!!str\")) |= from_yaml' sample.xml\n```\nwill output\n```yaml\n+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n says: meow\n legs: 4\n cute: true\n```\n\n## Parse xml: array\nConsecutive nodes with identical xml names are assumed to be arrays.\n\nGiven a sample.xml file of:\n```xml\n\ncat\ngoat\n```\nthen\n```bash\nyq -oy sample.xml\n```\nwill output\n```yaml\n+p_xml: version=\"1.0\" encoding=\"UTF-8\"\nanimal:\n - cat\n - goat\n```\n\n## Parse xml: force as an array\nIn XML, if your array has a single item, then yq doesn't know its an array. This is how you can consistently force it to be an array. This handles the 3 scenarios of having nothing in the array, having a single item and having multiple.\n\nGiven a sample.xml file of:\n```xml\ncat\n```\nthen\n```bash\nyq -oy '.zoo.animal |= ([] + .)' sample.xml\n```\nwill output\n```yaml\nzoo:\n animal:\n - cat\n```\n\n## Parse xml: force all as an array\nGiven a sample.xml file of:\n```xml\nboing\n```\nthen\n```bash\nyq -oy '.. |= [] + .' sample.xml\n```\nwill output\n```yaml\n- zoo:\n - thing:\n - frog:\n - boing\n```\n\n## Parse xml: attributes\nAttributes are converted to fields, with the default attribute prefix '+'. Use '--xml-attribute-prefix` to set your own.\n\nGiven a sample.xml file of:\n```xml\n\n\n 7\n\n```\nthen\n```bash\nyq -oy sample.xml\n```\nwill output\n```yaml\n+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n +@legs: \"4\"\n legs: \"7\"\n```\n\n## Parse xml: attributes with content\nContent is added as a field, using the default content name of `+content`. Use `--xml-content-name` to set your own.\n\nGiven a sample.xml file of:\n```xml\n\nmeow\n```\nthen\n```bash\nyq -oy sample.xml\n```\nwill output\n```yaml\n+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n +content: meow\n +@legs: \"4\"\n```\n\n## Parse xml: content split between comments/children\nMultiple content texts are collected into a sequence.\n\nGiven a sample.xml file of:\n```xml\n value anotherValue frog cool!\n```\nthen\n```bash\nyq -oy sample.xml\n```\nwill output\n```yaml\nroot:\n +content: # comment\n - value\n - anotherValue\n - cool!\n a: frog\n```\n\n## Parse xml: custom dtd\nDTD entities are processed as directives.\n\nGiven a sample.xml file of:\n```xml\n\n\n\n\n]>\n\n &writer;©right;\n\n```\nthen\n```bash\nyq sample.xml\n```\nwill output\n```xml\n\n\n\n]>\n\n &writer;&copyright;\n\n```\n\n## Parse xml: skip custom dtd\nDTDs are directives, skip over directives to skip DTDs.\n\nGiven a sample.xml file of:\n```xml\n\n\n\n\n]>\n\n &writer;©right;\n\n```\nthen\n```bash\nyq --xml-skip-directives sample.xml\n```\nwill output\n```xml\n\n\n &writer;&copyright;\n\n```\n\n## Parse xml: with comments\nA best attempt is made to preserve comments.\n\nGiven a sample.xml file of:\n```xml\n\n\n\n\t\n\t3\n\t\n\t\n\t\t\n\t\tz\n\t\t\n\t\t\n\t\n\t\n\n\n\n```\nthen\n```bash\nyq -oy sample.xml\n```\nwill output\n```yaml\n# before cat\ncat:\n # in cat before\n x: \"3\" # multi\n # line comment \n # for x\n # before y\n\n y:\n # in y before\n # in d before\n d: z # in d after\n # in y after\n # in_cat_after\n# after cat\n```\n\n## Parse xml: keep attribute namespace\nDefaults to true\n\nGiven a sample.xml file of:\n```xml\n\n\n baz\n foobar\n\n\n```\nthen\n```bash\nyq --xml-keep-namespace=false sample.xml\n```\nwill output\n```xml\n\n\n baz\n foobar\n\n```\n\ninstead of\n```xml\n\n\n baz\n foobar\n\n```\n\n## Parse xml: keep raw attribute namespace\nDefaults to true\n\nGiven a sample.xml file of:\n```xml\n\n\n baz\n foobar\n\n\n```\nthen\n```bash\nyq --xml-raw-token=false sample.xml\n```\nwill output\n```xml\n\n\n baz\n foobar\n\n```\n\ninstead of\n```xml\n\n\n baz\n foobar\n\n```\n\n## Encode xml: simple\nGiven a sample.yml file of:\n```yaml\ncat: purrs\n```\nthen\n```bash\nyq -o=xml sample.yml\n```\nwill output\n```xml\npurrs\n```\n\n## Encode xml: array\nGiven a sample.yml file of:\n```yaml\npets:\n cat:\n - purrs\n - meows\n```\nthen\n```bash\nyq -o=xml sample.yml\n```\nwill output\n```xml\n\n purrs\n meows\n\n```\n\n## Encode xml: attributes\nFields with the matching xml-attribute-prefix are assumed to be attributes.\n\nGiven a sample.yml file of:\n```yaml\ncat:\n +@name: tiger\n meows: true\n\n```\nthen\n```bash\nyq -o=xml sample.yml\n```\nwill output\n```xml\n\n true\n\n```\n\n## Encode xml: attributes with content\nFields with the matching xml-content-name is assumed to be content.\n\nGiven a sample.yml file of:\n```yaml\ncat:\n +@name: tiger\n +content: cool\n\n```\nthen\n```bash\nyq -o=xml sample.yml\n```\nwill output\n```xml\ncool\n```\n\n## Encode xml: comments\nA best attempt is made to copy comments to xml.\n\nGiven a sample.yml file of:\n```yaml\n#\n# header comment\n# above_cat\n#\ncat: # inline_cat\n # above_array\n array: # inline_array\n - val1 # inline_val1\n # above_val2\n - val2 # inline_val2\n# below_cat\n\n```\nthen\n```bash\nyq -o=xml sample.yml\n```\nwill output\n```xml\n\n\n\n val1\n val2\n\n```\n\n## Encode: doctype and xml declaration\nUse the special xml names to add/modify proc instructions and directives.\n\nGiven a sample.yml file of:\n```yaml\n+p_xml: version=\"1.0\"\n+directive: 'DOCTYPE config SYSTEM \"/etc/iwatch/iwatch.dtd\" '\napple:\n +p_coolioo: version=\"1.0\"\n +directive: 'CATYPE meow purr puss '\n b: things\n\n```\nthen\n```bash\nyq -o=xml sample.yml\n```\nwill output\n```xml\n\n\n\n things\n\n```\n\n## Round trip: with comments\nA best effort is made, but comment positions and white space are not preserved perfectly.\n\nGiven a sample.xml file of:\n```xml\n\n\n\n\t\n\t3\n\t\n\t\n\t\t\n\t\tz\n\t\t\n\t\t\n\t\n\t\n\n\n\n```\nthen\n```bash\nyq sample.xml\n```\nwill output\n```xml\n\n\n 3\n \n z\n \n\n```\n\n## Roundtrip: with doctype and declaration\nyq parses XML proc instructions and directives into nodes.\nUnfortunately the underlying XML parser loses whitespace information.\n\nGiven a sample.xml file of:\n```xml\n\n\n\n \n \n things\n\n\n```\nthen\n```bash\nyq sample.xml\n```\nwill output\n```xml\n\n\n\n things\n\n```\n\n" }, { "path": "pkg/yqlib/encoder.go", "content": "package yqlib\n\nimport (\n\t\"bufio\"\n\t\"errors\"\n\t\"io\"\n\t\"strings\"\n\n\t\"github.com/fatih/color\"\n)\n\ntype Encoder interface {\n\tEncode(writer io.Writer, node *CandidateNode) error\n\tPrintDocumentSeparator(writer io.Writer) error\n\tPrintLeadingContent(writer io.Writer, content string) error\n\tCanHandleAliases() bool\n}\n\nfunc mapKeysToStrings(node *CandidateNode) {\n\n\tif node.Kind == MappingNode {\n\t\tfor index, child := range node.Content {\n\t\t\tif index%2 == 0 { // its a map key\n\t\t\t\tchild.Tag = \"!!str\"\n\t\t\t}\n\t\t}\n\t}\n\n\tfor _, child := range node.Content {\n\t\tmapKeysToStrings(child)\n\t}\n}\n\n// Some funcs are shared between encoder_yaml and encoder_kyaml\nfunc PrintYAMLDocumentSeparator(writer io.Writer, PrintDocSeparators bool) error {\n\tif PrintDocSeparators {\n\t\tlog.Debug(\"writing doc sep\")\n\t\tif err := writeString(writer, \"---\\n\"); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\nfunc PrintYAMLLeadingContent(writer io.Writer, content string, PrintDocSeparators bool, ColorsEnabled bool) error {\n\treader := bufio.NewReader(strings.NewReader(content))\n\n\t// reuse precompiled package-level regex\n\t// (declared in decoder_yaml.go)\n\n\tfor {\n\n\t\treadline, errReading := reader.ReadString('\\n')\n\t\tif errReading != nil && !errors.Is(errReading, io.EOF) {\n\t\t\treturn errReading\n\t\t}\n\t\tif strings.Contains(readline, \"$yqDocSeparator$\") {\n\t\t\t// Preserve the original line ending (CRLF or LF)\n\t\t\tlineEnding := \"\\n\"\n\t\t\tif strings.HasSuffix(readline, \"\\r\\n\") {\n\t\t\t\tlineEnding = \"\\r\\n\"\n\t\t\t}\n\t\t\tif PrintDocSeparators {\n\t\t\t\tif err := writeString(writer, \"---\"+lineEnding); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\n\t\t} else {\n\t\t\tif len(readline) > 0 && readline != \"\\n\" && readline[0] != '%' && !commentLineRe.MatchString(readline) {\n\t\t\t\treadline = \"# \" + readline\n\t\t\t}\n\t\t\tif ColorsEnabled && strings.TrimSpace(readline) != \"\" {\n\t\t\t\treadline = format(color.FgHiBlack) + readline + format(color.Reset)\n\t\t\t}\n\t\t\tif err := writeString(writer, readline); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\tif errors.Is(errReading, io.EOF) {\n\t\t\tif readline != \"\" {\n\t\t\t\t// the last comment we read didn't have a newline, put one in\n\t\t\t\tif err := writeString(writer, \"\\n\"); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t}\n\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/encoder_base64.go", "content": "//go:build !yq_nobase64\n\npackage yqlib\n\nimport (\n\t\"encoding/base64\"\n\t\"fmt\"\n\t\"io\"\n)\n\ntype base64Encoder struct {\n\tencoding base64.Encoding\n}\n\nfunc NewBase64Encoder() Encoder {\n\treturn &base64Encoder{encoding: *base64.StdEncoding}\n}\n\nfunc (e *base64Encoder) CanHandleAliases() bool {\n\treturn false\n}\n\nfunc (e *base64Encoder) PrintDocumentSeparator(_ io.Writer) error {\n\treturn nil\n}\n\nfunc (e *base64Encoder) PrintLeadingContent(_ io.Writer, _ string) error {\n\treturn nil\n}\n\nfunc (e *base64Encoder) Encode(writer io.Writer, node *CandidateNode) error {\n\tif node.guessTagFromCustomType() != \"!!str\" {\n\t\treturn fmt.Errorf(\"cannot encode %v as base64, can only operate on strings\", node.Tag)\n\t}\n\t_, err := writer.Write([]byte(e.encoding.EncodeToString([]byte(node.Value))))\n\treturn err\n}\n" }, { "path": "pkg/yqlib/encoder_csv.go", "content": "//go:build !yq_nocsv\n\npackage yqlib\n\nimport (\n\t\"encoding/csv\"\n\t\"fmt\"\n\t\"io\"\n)\n\ntype csvEncoder struct {\n\tseparator rune\n}\n\nfunc NewCsvEncoder(prefs CsvPreferences) Encoder {\n\treturn &csvEncoder{separator: prefs.Separator}\n}\n\nfunc (e *csvEncoder) CanHandleAliases() bool {\n\treturn false\n}\n\nfunc (e *csvEncoder) PrintDocumentSeparator(_ io.Writer) error {\n\treturn nil\n}\n\nfunc (e *csvEncoder) PrintLeadingContent(_ io.Writer, _ string) error {\n\treturn nil\n}\n\nfunc (e *csvEncoder) encodeRow(csvWriter *csv.Writer, contents []*CandidateNode) error {\n\tstringValues := make([]string, len(contents))\n\n\tfor i, child := range contents {\n\n\t\tif child.Kind != ScalarNode {\n\t\t\treturn fmt.Errorf(\"csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v\", i, child.Tag)\n\t\t}\n\t\tstringValues[i] = child.Value\n\t}\n\treturn csvWriter.Write(stringValues)\n}\n\nfunc (e *csvEncoder) encodeArrays(csvWriter *csv.Writer, content []*CandidateNode) error {\n\tfor i, child := range content {\n\n\t\tif child.Kind != SequenceNode {\n\t\t\treturn fmt.Errorf(\"csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v\", i, child.Tag)\n\t\t}\n\t\terr := e.encodeRow(csvWriter, child.Content)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (e *csvEncoder) extractHeader(child *CandidateNode) ([]*CandidateNode, error) {\n\tif child.Kind != MappingNode {\n\t\treturn nil, fmt.Errorf(\"csv object encoding only works for arrays of flat objects (string key => string/numbers/boolean value), child[0] is a %v\", child.Tag)\n\t}\n\tmapKeys := getMapKeys(child)\n\treturn mapKeys.Content, nil\n}\n\nfunc (e *csvEncoder) createChildRow(child *CandidateNode, headers []*CandidateNode) []*CandidateNode {\n\tchildRow := make([]*CandidateNode, 0)\n\tfor _, header := range headers {\n\t\tkeyIndex := findKeyInMap(child, header)\n\t\tvalue := createScalarNode(nil, \"\")\n\t\tif keyIndex != -1 {\n\t\t\tvalue = child.Content[keyIndex+1]\n\t\t}\n\t\tchildRow = append(childRow, value)\n\t}\n\treturn childRow\n\n}\n\nfunc (e *csvEncoder) encodeObjects(csvWriter *csv.Writer, content []*CandidateNode) error {\n\theaders, err := e.extractHeader(content[0])\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\terr = e.encodeRow(csvWriter, headers)\n\tif err != nil {\n\t\treturn nil\n\t}\n\n\tfor i, child := range content {\n\t\tif child.Kind != MappingNode {\n\t\t\treturn fmt.Errorf(\"csv object encoding only works for arrays of flat objects (string key => string/numbers/boolean value), child[%v] is a %v\", i, child.Tag)\n\t\t}\n\t\trow := e.createChildRow(child, headers)\n\t\terr = e.encodeRow(csvWriter, row)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t}\n\treturn nil\n}\n\nfunc (e *csvEncoder) Encode(writer io.Writer, node *CandidateNode) error {\n\tif node.Kind == ScalarNode {\n\t\treturn writeString(writer, node.Value+\"\\n\")\n\t}\n\n\tcsvWriter := csv.NewWriter(writer)\n\tcsvWriter.Comma = e.separator\n\n\t// node must be a sequence\n\tif node.Kind != SequenceNode {\n\t\treturn fmt.Errorf(\"csv encoding only works for arrays, got: %v\", node.Tag)\n\t} else if len(node.Content) == 0 {\n\t\treturn nil\n\t}\n\tif node.Content[0].Kind == ScalarNode {\n\t\treturn e.encodeRow(csvWriter, node.Content)\n\t}\n\n\tif node.Content[0].Kind == MappingNode {\n\t\treturn e.encodeObjects(csvWriter, node.Content)\n\t}\n\n\treturn e.encodeArrays(csvWriter, node.Content)\n\n}\n" }, { "path": "pkg/yqlib/encoder_hcl.go", "content": "//go:build !yq_nohcl\n\npackage yqlib\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"regexp\"\n\t\"strings\"\n\n\t\"github.com/fatih/color\"\n\t\"github.com/hashicorp/hcl/v2\"\n\t\"github.com/hashicorp/hcl/v2/hclsyntax\"\n\thclwrite \"github.com/hashicorp/hcl/v2/hclwrite\"\n\t\"github.com/zclconf/go-cty/cty\"\n)\n\ntype hclEncoder struct {\n\tprefs HclPreferences\n}\n\n// commentPathSep is used to join path segments when collecting comments.\n// It uses a rarely used ASCII control character to avoid collisions with\n// normal key names (including dots).\nconst commentPathSep = \"\\x1e\"\n\n// NewHclEncoder creates a new HCL encoder\nfunc NewHclEncoder(prefs HclPreferences) Encoder {\n\treturn &hclEncoder{prefs: prefs}\n}\n\nfunc (he *hclEncoder) CanHandleAliases() bool {\n\treturn false\n}\n\nfunc (he *hclEncoder) PrintDocumentSeparator(_ io.Writer) error {\n\treturn nil\n}\n\nfunc (he *hclEncoder) PrintLeadingContent(_ io.Writer, _ string) error {\n\treturn nil\n}\n\nfunc (he *hclEncoder) Encode(writer io.Writer, node *CandidateNode) error {\n\tlog.Debugf(\"I need to encode %v\", NodeToString(node))\n\tif node.Kind == ScalarNode {\n\t\treturn writeString(writer, node.Value+\"\\n\")\n\t}\n\n\tf := hclwrite.NewEmptyFile()\n\tbody := f.Body()\n\n\t// Collect comments as we encode\n\tcommentMap := make(map[string]string)\n\the.collectComments(node, \"\", commentMap)\n\n\tif err := he.encodeNode(body, node); err != nil {\n\t\treturn fmt.Errorf(\"failed to encode HCL: %w\", err)\n\t}\n\n\t// Get the formatted output and remove extra spacing before '='\n\toutput := f.Bytes()\n\tcompactOutput := he.compactSpacing(output)\n\n\t// Inject comments back into the output\n\tfinalOutput := he.injectComments(compactOutput, commentMap)\n\n\tif he.prefs.ColorsEnabled {\n\t\tcolourized := he.colorizeHcl(finalOutput)\n\t\t_, err := writer.Write(colourized)\n\t\treturn err\n\t}\n\n\t_, err := writer.Write(finalOutput)\n\treturn err\n}\n\n// compactSpacing removes extra whitespace before '=' in attribute assignments\nfunc (he *hclEncoder) compactSpacing(input []byte) []byte {\n\t// Use regex to replace multiple spaces before = with single space\n\tre := regexp.MustCompile(`(\\S)\\s{2,}=`)\n\treturn re.ReplaceAll(input, []byte(\"$1 =\"))\n}\n\n// collectComments recursively collects comments from nodes for later injection\nfunc (he *hclEncoder) collectComments(node *CandidateNode, prefix string, commentMap map[string]string) {\n\tif node == nil {\n\t\treturn\n\t}\n\n\t// For mapping nodes, collect comments from keys and values\n\tif node.Kind == MappingNode {\n\t\t// Collect root-level head comment if at root (prefix is empty)\n\t\tif prefix == \"\" && node.HeadComment != \"\" {\n\t\t\tcommentMap[joinCommentPath(\"__root__\", \"head\")] = node.HeadComment\n\t\t}\n\n\t\tfor i := 0; i < len(node.Content); i += 2 {\n\t\t\tkeyNode := node.Content[i]\n\t\t\tvalueNode := node.Content[i+1]\n\t\t\tkey := keyNode.Value\n\n\t\t\t// Create a path for this key\n\t\t\tpath := joinCommentPath(prefix, key)\n\n\t\t\t// Store comments from the key (head comments appear before the attribute)\n\t\t\tif keyNode.HeadComment != \"\" {\n\t\t\t\tcommentMap[joinCommentPath(path, \"head\")] = keyNode.HeadComment\n\t\t\t}\n\t\t\t// Store comments from the value (line comments appear after the value)\n\t\t\tif valueNode.LineComment != \"\" {\n\t\t\t\tcommentMap[joinCommentPath(path, \"line\")] = valueNode.LineComment\n\t\t\t}\n\t\t\tif valueNode.FootComment != \"\" {\n\t\t\t\tcommentMap[joinCommentPath(path, \"foot\")] = valueNode.FootComment\n\t\t\t}\n\n\t\t\t// Recurse into nested mappings\n\t\t\tif valueNode.Kind == MappingNode {\n\t\t\t\the.collectComments(valueNode, path, commentMap)\n\t\t\t}\n\t\t}\n\t}\n}\n\n// joinCommentPath concatenates path segments using commentPathSep, safely handling empty prefixes.\nfunc joinCommentPath(prefix, segment string) string {\n\tif prefix == \"\" {\n\t\treturn segment\n\t}\n\treturn prefix + commentPathSep + segment\n}\n\n// injectComments adds collected comments back into the HCL output\nfunc (he *hclEncoder) injectComments(output []byte, commentMap map[string]string) []byte {\n\t// Convert output to string for easier manipulation\n\tresult := string(output)\n\n\t// Root-level head comment (stored on the synthetic __root__/head path)\n\tfor path, comment := range commentMap {\n\t\tif path == joinCommentPath(\"__root__\", \"head\") {\n\t\t\ttrimmed := strings.TrimSpace(comment)\n\t\t\tif trimmed != \"\" && !strings.HasPrefix(result, trimmed) {\n\t\t\t\tresult = trimmed + \"\\n\" + result\n\t\t\t}\n\t\t}\n\t}\n\n\t// Attribute head comments: insert above matching assignment\n\tfor path, comment := range commentMap {\n\t\tparts := strings.Split(path, commentPathSep)\n\t\tif len(parts) < 2 {\n\t\t\tcontinue\n\t\t}\n\n\t\tcommentType := parts[len(parts)-1]\n\t\tkey := parts[len(parts)-2]\n\t\tif commentType != \"head\" || key == \"\" {\n\t\t\tcontinue\n\t\t}\n\n\t\ttrimmed := strings.TrimSpace(comment)\n\t\tif trimmed == \"\" {\n\t\t\tcontinue\n\t\t}\n\n\t\tre := regexp.MustCompile(`(?m)^(\\s*)` + regexp.QuoteMeta(key) + `\\s*=`)\n\t\tif re.MatchString(result) {\n\t\t\tresult = re.ReplaceAllString(result, \"$1\"+trimmed+\"\\n$0\")\n\t\t}\n\t}\n\n\treturn []byte(result)\n}\n\nfunc (he *hclEncoder) colorizeHcl(input []byte) []byte {\n\thcl := string(input)\n\tresult := strings.Builder{}\n\n\t// Create colour functions for different token types\n\tcommentColor := color.New(color.FgHiBlack).SprintFunc()\n\tstringColor := color.New(color.FgGreen).SprintFunc()\n\tnumberColor := color.New(color.FgHiMagenta).SprintFunc()\n\tkeyColor := color.New(color.FgCyan).SprintFunc()\n\tboolColor := color.New(color.FgHiMagenta).SprintFunc()\n\n\t// Simple tokenization for HCL colouring\n\ti := 0\n\tfor i < len(hcl) {\n\t\tch := hcl[i]\n\n\t\t// Comments - from # to end of line\n\t\tif ch == '#' {\n\t\t\tend := i\n\t\t\tfor end < len(hcl) && hcl[end] != '\\n' {\n\t\t\t\tend++\n\t\t\t}\n\t\t\tresult.WriteString(commentColor(hcl[i:end]))\n\t\t\ti = end\n\t\t\tcontinue\n\t\t}\n\n\t\t// Strings - quoted text\n\t\tif ch == '\"' || ch == '\\'' {\n\t\t\tquote := ch\n\t\t\tend := i + 1\n\t\t\tfor end < len(hcl) && hcl[end] != quote {\n\t\t\t\tif hcl[end] == '\\\\' {\n\t\t\t\t\tend++ // skip escaped char\n\t\t\t\t}\n\t\t\t\tend++\n\t\t\t}\n\t\t\tif end < len(hcl) {\n\t\t\t\tend++ // include closing quote\n\t\t\t}\n\t\t\tresult.WriteString(stringColor(hcl[i:end]))\n\t\t\ti = end\n\t\t\tcontinue\n\t\t}\n\n\t\t// Numbers - sequences of digits, possibly with decimal point or minus\n\t\tif (ch >= '0' && ch <= '9') || (ch == '-' && i+1 < len(hcl) && hcl[i+1] >= '0' && hcl[i+1] <= '9') {\n\t\t\tend := i\n\t\t\tif ch == '-' {\n\t\t\t\tend++\n\t\t\t}\n\t\t\tfor end < len(hcl) && ((hcl[end] >= '0' && hcl[end] <= '9') || hcl[end] == '.') {\n\t\t\t\tend++\n\t\t\t}\n\t\t\tresult.WriteString(numberColor(hcl[i:end]))\n\t\t\ti = end\n\t\t\tcontinue\n\t\t}\n\n\t\t// Identifiers/keys - alphanumeric + underscore\n\t\tif (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '_' {\n\t\t\tend := i\n\t\t\tfor end < len(hcl) && ((hcl[end] >= 'a' && hcl[end] <= 'z') ||\n\t\t\t\t(hcl[end] >= 'A' && hcl[end] <= 'Z') ||\n\t\t\t\t(hcl[end] >= '0' && hcl[end] <= '9') ||\n\t\t\t\thcl[end] == '_' || hcl[end] == '-') {\n\t\t\t\tend++\n\t\t\t}\n\t\t\tident := hcl[i:end]\n\n\t\t\t// Check if this is a keyword/reserved word\n\t\t\tswitch ident {\n\t\t\tcase \"true\", \"false\", \"null\":\n\t\t\t\tresult.WriteString(boolColor(ident))\n\t\t\tdefault:\n\t\t\t\t// Check if followed by = (it's a key)\n\t\t\t\tj := end\n\t\t\t\tfor j < len(hcl) && (hcl[j] == ' ' || hcl[j] == '\\t') {\n\t\t\t\t\tj++\n\t\t\t\t}\n\t\t\t\tif j < len(hcl) && hcl[j] == '=' {\n\t\t\t\t\tresult.WriteString(keyColor(ident))\n\t\t\t\t} else if j < len(hcl) && hcl[j] == '{' {\n\t\t\t\t\t// Block type\n\t\t\t\t\tresult.WriteString(keyColor(ident))\n\t\t\t\t} else {\n\t\t\t\t\tresult.WriteString(ident) // plain text for other identifiers\n\t\t\t\t}\n\t\t\t}\n\t\t\ti = end\n\t\t\tcontinue\n\t\t}\n\n\t\t// Everything else (whitespace, operators, brackets) - no color\n\t\tresult.WriteByte(ch)\n\t\ti++\n\t}\n\n\treturn []byte(result.String())\n}\n\n// Helper runes for unquoted identifiers\nfunc isHCLIdentifierStart(r rune) bool {\n\treturn (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || r == '_'\n}\n\nfunc isHCLIdentifierPart(r rune) bool {\n\treturn (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' || r == '-'\n}\n\nfunc isValidHCLIdentifier(s string) bool {\n\tif s == \"\" {\n\t\treturn false\n\t}\n\t// HCL identifiers must start with a letter or underscore\n\t// and contain only letters, digits, underscores, and hyphens\n\tfor i, r := range s {\n\t\tif i == 0 {\n\t\t\tif !isHCLIdentifierStart(r) {\n\t\t\t\treturn false\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tif !isHCLIdentifierPart(r) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// tokensForRawHCLExpr produces a minimal token stream for a simple HCL expression so we can\n// write it without introducing quotes (e.g. function calls like upper(message)).\nfunc tokensForRawHCLExpr(expr string) (hclwrite.Tokens, error) {\n\tvar tokens hclwrite.Tokens\n\tfor i := 0; i < len(expr); {\n\t\tch := expr[i]\n\t\tswitch {\n\t\tcase ch == ' ' || ch == '\\t':\n\t\t\ti++\n\t\t\tcontinue\n\t\tcase isHCLIdentifierStart(rune(ch)):\n\t\t\tstart := i\n\t\t\ti++\n\t\t\tfor i < len(expr) && isHCLIdentifierPart(rune(expr[i])) {\n\t\t\t\ti++\n\t\t\t}\n\t\t\ttokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenIdent, Bytes: []byte(expr[start:i])})\n\t\t\tcontinue\n\t\tcase ch >= '0' && ch <= '9':\n\t\t\tstart := i\n\t\t\ti++\n\t\t\tfor i < len(expr) && ((expr[i] >= '0' && expr[i] <= '9') || expr[i] == '.') {\n\t\t\t\ti++\n\t\t\t}\n\t\t\ttokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenNumberLit, Bytes: []byte(expr[start:i])})\n\t\t\tcontinue\n\t\tcase ch == '(':\n\t\t\ttokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenOParen, Bytes: []byte{'('}})\n\t\tcase ch == ')':\n\t\t\ttokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenCParen, Bytes: []byte{')'}})\n\t\tcase ch == ',':\n\t\t\ttokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenComma, Bytes: []byte{','}})\n\t\tcase ch == '.':\n\t\t\ttokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenDot, Bytes: []byte{'.'}})\n\t\tcase ch == '+':\n\t\t\ttokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenPlus, Bytes: []byte{'+'}})\n\t\tcase ch == '-':\n\t\t\ttokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenMinus, Bytes: []byte{'-'}})\n\t\tcase ch == '*':\n\t\t\ttokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenStar, Bytes: []byte{'*'}})\n\t\tcase ch == '/':\n\t\t\ttokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenSlash, Bytes: []byte{'/'}})\n\t\tdefault:\n\t\t\treturn nil, fmt.Errorf(\"unsupported character %q in raw HCL expression\", ch)\n\t\t}\n\t\ti++\n\t}\n\treturn tokens, nil\n}\n\n// encodeAttribute encodes a value as an HCL attribute\nfunc (he *hclEncoder) encodeAttribute(body *hclwrite.Body, key string, valueNode *CandidateNode) error {\n\tif valueNode.Kind == ScalarNode && valueNode.Tag == \"!!str\" {\n\t\t// Handle unquoted expressions (as-is, without quotes)\n\t\tif valueNode.Style == 0 {\n\t\t\ttokens, err := tokensForRawHCLExpr(valueNode.Value)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tbody.SetAttributeRaw(key, tokens)\n\t\t\treturn nil\n\t\t}\n\t\tif valueNode.Style&LiteralStyle != 0 {\n\t\t\ttokens, err := tokensForRawHCLExpr(valueNode.Value)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tbody.SetAttributeRaw(key, tokens)\n\t\t\treturn nil\n\t\t}\n\t\t// Check if template with interpolation\n\t\tif valueNode.Style&DoubleQuotedStyle != 0 && strings.Contains(valueNode.Value, \"${\") {\n\t\t\treturn he.encodeTemplateAttribute(body, key, valueNode.Value)\n\t\t}\n\t\t// Check if unquoted identifier\n\t\tif isValidHCLIdentifier(valueNode.Value) && valueNode.Style == 0 {\n\t\t\ttraversal := hcl.Traversal{\n\t\t\t\thcl.TraverseRoot{Name: valueNode.Value},\n\t\t\t}\n\t\t\tbody.SetAttributeTraversal(key, traversal)\n\t\t\treturn nil\n\t\t}\n\t}\n\t// Default: use cty.Value for quoted strings and all other types\n\tctyValue, err := nodeToCtyValue(valueNode)\n\tif err != nil {\n\t\treturn err\n\t}\n\tbody.SetAttributeValue(key, ctyValue)\n\treturn nil\n}\n\n// encodeTemplateAttribute encodes a template string with ${} interpolations\nfunc (he *hclEncoder) encodeTemplateAttribute(body *hclwrite.Body, key string, templateStr string) error {\n\ttokens := hclwrite.Tokens{\n\t\t{Type: hclsyntax.TokenOQuote, Bytes: []byte{'\"'}},\n\t}\n\n\tfor i := 0; i < len(templateStr); i++ {\n\t\tif i < len(templateStr)-1 && templateStr[i] == '$' && templateStr[i+1] == '{' {\n\t\t\t// Start of template interpolation\n\t\t\ttokens = append(tokens, &hclwrite.Token{\n\t\t\t\tType: hclsyntax.TokenTemplateInterp,\n\t\t\t\tBytes: []byte(\"${\"),\n\t\t\t})\n\t\t\ti++ // skip the '{'\n\t\t\t// Find the matching '}'\n\t\t\tstart := i + 1\n\t\t\tdepth := 1\n\t\t\tfor i++; i < len(templateStr) && depth > 0; i++ {\n\t\t\t\tswitch templateStr[i] {\n\t\t\t\tcase '{':\n\t\t\t\t\tdepth++\n\t\t\t\tcase '}':\n\t\t\t\t\tdepth--\n\t\t\t\t}\n\t\t\t}\n\t\t\ti-- // back up to the '}'\n\t\t\tinterpExpr := templateStr[start:i]\n\t\t\ttokens = append(tokens, &hclwrite.Token{\n\t\t\t\tType: hclsyntax.TokenIdent,\n\t\t\t\tBytes: []byte(interpExpr),\n\t\t\t})\n\t\t\ttokens = append(tokens, &hclwrite.Token{\n\t\t\t\tType: hclsyntax.TokenTemplateSeqEnd,\n\t\t\t\tBytes: []byte(\"}\"),\n\t\t\t})\n\t\t} else {\n\t\t\t// Regular character\n\t\t\ttokens = append(tokens, &hclwrite.Token{\n\t\t\t\tType: hclsyntax.TokenQuotedLit,\n\t\t\t\tBytes: []byte{templateStr[i]},\n\t\t\t})\n\t\t}\n\t}\n\ttokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenCQuote, Bytes: []byte{'\"'}})\n\tbody.SetAttributeRaw(key, tokens)\n\treturn nil\n}\n\n// encodeBlockIfMapping attempts to encode a value as a block. Returns true if it was encoded as a block.\nfunc (he *hclEncoder) encodeBlockIfMapping(body *hclwrite.Body, key string, valueNode *CandidateNode) bool {\n\tif valueNode.Kind != MappingNode || valueNode.Style == FlowStyle {\n\t\treturn false\n\t}\n\n\t// If EncodeSeparate is set, emit children as separate blocks regardless of label extraction\n\tif valueNode.EncodeSeparate {\n\t\tif handled, _ := he.encodeMappingChildrenAsBlocks(body, key, valueNode); handled {\n\t\t\treturn true\n\t\t}\n\t}\n\n\t// Try to extract block labels from a single-entry mapping chain\n\tif labels, bodyNode, ok := extractBlockLabels(valueNode); ok {\n\t\tif len(labels) > 1 && mappingChildrenAllMappings(bodyNode) {\n\t\t\tprimaryLabels := labels[:len(labels)-1]\n\t\t\tnestedType := labels[len(labels)-1]\n\t\t\tblock := body.AppendNewBlock(key, primaryLabels)\n\t\t\tif handled, err := he.encodeMappingChildrenAsBlocks(block.Body(), nestedType, bodyNode); err == nil && handled {\n\t\t\t\treturn true\n\t\t\t}\n\t\t\tif err := he.encodeNodeAttributes(block.Body(), bodyNode); err == nil {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t\tblock := body.AppendNewBlock(key, labels)\n\t\tif err := he.encodeNodeAttributes(block.Body(), bodyNode); err == nil {\n\t\t\treturn true\n\t\t}\n\t}\n\n\t// If all child values are mappings, treat each child key as a labelled instance of this block type\n\tif handled, _ := he.encodeMappingChildrenAsBlocks(body, key, valueNode); handled {\n\t\treturn true\n\t}\n\n\t// No labels detected, render as unlabelled block\n\tblock := body.AppendNewBlock(key, nil)\n\tif err := he.encodeNodeAttributes(block.Body(), valueNode); err == nil {\n\t\treturn true\n\t}\n\n\treturn false\n}\n\n// encodeNode encodes a CandidateNode directly to HCL, preserving style information\nfunc (he *hclEncoder) encodeNode(body *hclwrite.Body, node *CandidateNode) error {\n\tif node.Kind != MappingNode {\n\t\treturn fmt.Errorf(\"HCL encoder expects a mapping at the root level, got %v\", kindToString(node.Kind))\n\t}\n\n\tfor i := 0; i < len(node.Content); i += 2 {\n\t\tkeyNode := node.Content[i]\n\t\tvalueNode := node.Content[i+1]\n\t\tkey := keyNode.Value\n\n\t\t// Render as block or attribute depending on value type\n\t\tif he.encodeBlockIfMapping(body, key, valueNode) {\n\t\t\tcontinue\n\t\t}\n\n\t\t// Render as attribute: key = value\n\t\tif err := he.encodeAttribute(body, key, valueNode); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// mappingChildrenAllMappings reports whether all values in a mapping node are non-flow mappings.\nfunc mappingChildrenAllMappings(node *CandidateNode) bool {\n\tif node == nil || node.Kind != MappingNode || node.Style == FlowStyle {\n\t\treturn false\n\t}\n\tif len(node.Content) == 0 {\n\t\treturn false\n\t}\n\tfor i := 0; i < len(node.Content); i += 2 {\n\t\tchildVal := node.Content[i+1]\n\t\tif childVal.Kind != MappingNode || childVal.Style == FlowStyle {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// encodeMappingChildrenAsBlocks emits a block for each mapping child, treating the child key as a label.\n// Returns handled=true when it emitted blocks.\nfunc (he *hclEncoder) encodeMappingChildrenAsBlocks(body *hclwrite.Body, blockType string, valueNode *CandidateNode) (bool, error) {\n\tif !mappingChildrenAllMappings(valueNode) {\n\t\treturn false, nil\n\t}\n\n\t// Only emit as separate blocks if EncodeSeparate is true\n\t// This allows the encoder to respect the original block structure preserved by the decoder\n\tif !valueNode.EncodeSeparate {\n\t\treturn false, nil\n\t}\n\n\tfor i := 0; i < len(valueNode.Content); i += 2 {\n\t\tchildKey := valueNode.Content[i].Value\n\t\tchildVal := valueNode.Content[i+1]\n\n\t\t// Check if this child also represents multiple blocks (all children are mappings)\n\t\tif mappingChildrenAllMappings(childVal) {\n\t\t\t// Recursively emit each grandchild as a separate block with extended labels\n\t\t\tfor j := 0; j < len(childVal.Content); j += 2 {\n\t\t\t\tgrandchildKey := childVal.Content[j].Value\n\t\t\t\tgrandchildVal := childVal.Content[j+1]\n\t\t\t\tlabels := []string{childKey, grandchildKey}\n\n\t\t\t\t// Try to extract additional labels if this is a single-entry chain\n\t\t\t\tif extraLabels, bodyNode, ok := extractBlockLabels(grandchildVal); ok {\n\t\t\t\t\tlabels = append(labels, extraLabels...)\n\t\t\t\t\tgrandchildVal = bodyNode\n\t\t\t\t}\n\n\t\t\t\tblock := body.AppendNewBlock(blockType, labels)\n\t\t\t\tif err := he.encodeNodeAttributes(block.Body(), grandchildVal); err != nil {\n\t\t\t\t\treturn true, err\n\t\t\t\t}\n\t\t\t}\n\t\t} else {\n\t\t\t// Single block with this child as label(s)\n\t\t\tlabels := []string{childKey}\n\t\t\tif extraLabels, bodyNode, ok := extractBlockLabels(childVal); ok {\n\t\t\t\tlabels = append(labels, extraLabels...)\n\t\t\t\tchildVal = bodyNode\n\t\t\t}\n\t\t\tblock := body.AppendNewBlock(blockType, labels)\n\t\t\tif err := he.encodeNodeAttributes(block.Body(), childVal); err != nil {\n\t\t\t\treturn true, err\n\t\t\t}\n\t\t}\n\t}\n\n\treturn true, nil\n}\n\n// encodeNodeAttributes encodes the attributes of a mapping node (used for blocks)\nfunc (he *hclEncoder) encodeNodeAttributes(body *hclwrite.Body, node *CandidateNode) error {\n\tif node.Kind != MappingNode {\n\t\treturn fmt.Errorf(\"expected mapping node for block body\")\n\t}\n\n\tfor i := 0; i < len(node.Content); i += 2 {\n\t\tkeyNode := node.Content[i]\n\t\tvalueNode := node.Content[i+1]\n\t\tkey := keyNode.Value\n\n\t\t// Render as block or attribute depending on value type\n\t\tif he.encodeBlockIfMapping(body, key, valueNode) {\n\t\t\tcontinue\n\t\t}\n\n\t\t// Render attribute for non-block value\n\t\tif err := he.encodeAttribute(body, key, valueNode); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// extractBlockLabels detects a chain of single-entry mappings that encode block labels.\n// It returns the collected labels and the final mapping to be used as the block body.\n// Pattern: {label1: {label2: { ... {bodyMap} }}}\nfunc extractBlockLabels(node *CandidateNode) ([]string, *CandidateNode, bool) {\n\tvar labels []string\n\tcurrent := node\n\tfor current != nil && current.Kind == MappingNode && len(current.Content) == 2 {\n\t\tkeyNode := current.Content[0]\n\t\tvalNode := current.Content[1]\n\t\tif valNode.Kind != MappingNode {\n\t\t\tbreak\n\t\t}\n\t\tlabels = append(labels, keyNode.Value)\n\t\t// If the child is itself a single mapping entry with a mapping value, keep descending.\n\t\tif len(valNode.Content) == 2 && valNode.Content[1].Kind == MappingNode {\n\t\t\tcurrent = valNode\n\t\t\tcontinue\n\t\t}\n\t\t// Otherwise, we have reached the body mapping.\n\t\treturn labels, valNode, true\n\t}\n\treturn nil, nil, false\n}\n\n// nodeToCtyValue converts a CandidateNode directly to cty.Value, preserving order\nfunc nodeToCtyValue(node *CandidateNode) (cty.Value, error) {\n\tswitch node.Kind {\n\tcase ScalarNode:\n\t\t// Parse scalar value based on its tag\n\t\tswitch node.Tag {\n\t\tcase \"!!bool\":\n\t\t\treturn cty.BoolVal(node.Value == \"true\"), nil\n\t\tcase \"!!int\":\n\t\t\tvar i int64\n\t\t\t_, err := fmt.Sscanf(node.Value, \"%d\", &i)\n\t\t\tif err != nil {\n\t\t\t\treturn cty.NilVal, err\n\t\t\t}\n\t\t\treturn cty.NumberIntVal(i), nil\n\t\tcase \"!!float\":\n\t\t\tvar f float64\n\t\t\t_, err := fmt.Sscanf(node.Value, \"%f\", &f)\n\t\t\tif err != nil {\n\t\t\t\treturn cty.NilVal, err\n\t\t\t}\n\t\t\treturn cty.NumberFloatVal(f), nil\n\t\tcase \"!!null\":\n\t\t\treturn cty.NullVal(cty.DynamicPseudoType), nil\n\t\tdefault:\n\t\t\t// Default to string\n\t\t\treturn cty.StringVal(node.Value), nil\n\t\t}\n\tcase MappingNode:\n\t\t// Preserve order by iterating Content directly\n\t\tm := make(map[string]cty.Value)\n\t\tfor i := 0; i < len(node.Content); i += 2 {\n\t\t\tkeyNode := node.Content[i]\n\t\t\tvalueNode := node.Content[i+1]\n\t\t\tv, err := nodeToCtyValue(valueNode)\n\t\t\tif err != nil {\n\t\t\t\treturn cty.NilVal, err\n\t\t\t}\n\t\t\tm[keyNode.Value] = v\n\t\t}\n\t\treturn cty.ObjectVal(m), nil\n\tcase SequenceNode:\n\t\tvals := make([]cty.Value, len(node.Content))\n\t\tfor i, item := range node.Content {\n\t\t\tv, err := nodeToCtyValue(item)\n\t\t\tif err != nil {\n\t\t\t\treturn cty.NilVal, err\n\t\t\t}\n\t\t\tvals[i] = v\n\t\t}\n\t\treturn cty.TupleVal(vals), nil\n\tcase AliasNode:\n\t\treturn cty.NilVal, fmt.Errorf(\"HCL encoder does not support aliases\")\n\tdefault:\n\t\treturn cty.NilVal, fmt.Errorf(\"unsupported node kind: %v\", node.Kind)\n\t}\n}\n" }, { "path": "pkg/yqlib/encoder_ini.go", "content": "//go:build !yq_noini\n\npackage yqlib\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"io\"\n\n\t\"github.com/go-ini/ini\"\n)\n\ntype iniEncoder struct {\n\tindentString string\n}\n\n// NewINIEncoder creates a new INI encoder\nfunc NewINIEncoder() Encoder {\n\t// Hardcoded indent value of 0, meaning no additional spacing.\n\treturn &iniEncoder{\"\"}\n}\n\n// CanHandleAliases indicates whether the encoder supports aliases. INI does not support aliases.\nfunc (ie *iniEncoder) CanHandleAliases() bool {\n\treturn false\n}\n\n// PrintDocumentSeparator is a no-op since INI does not support multiple documents.\nfunc (ie *iniEncoder) PrintDocumentSeparator(_ io.Writer) error {\n\treturn nil\n}\n\n// PrintLeadingContent is a no-op since INI does not support leading content or comments at the encoder level.\nfunc (ie *iniEncoder) PrintLeadingContent(_ io.Writer, _ string) error {\n\treturn nil\n}\n\n// Encode converts a CandidateNode into INI format and writes it to the provided writer.\nfunc (ie *iniEncoder) Encode(writer io.Writer, node *CandidateNode) error {\n\tlog.Debugf(\"I need to encode %v\", NodeToString(node))\n\tlog.Debugf(\"kids %v\", len(node.Content))\n\n\tif node.Kind == ScalarNode {\n\t\treturn writeStringINI(writer, node.Value+\"\\n\")\n\t}\n\n\t// Create a new INI configuration.\n\tcfg := ini.Empty()\n\n\tif node.Kind == MappingNode {\n\t\t// Default section for key-value pairs at the root level.\n\t\tdefaultSection, err := cfg.NewSection(ini.DefaultSection)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\t// Process the node's content.\n\t\tfor i := 0; i < len(node.Content); i += 2 {\n\t\t\tkeyNode := node.Content[i]\n\t\t\tvalueNode := node.Content[i+1]\n\t\t\tkey := keyNode.Value\n\n\t\t\tswitch valueNode.Kind {\n\t\t\tcase ScalarNode:\n\t\t\t\t// Add key-value pair to the default section.\n\t\t\t\t_, err := defaultSection.NewKey(key, valueNode.Value)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\tcase MappingNode:\n\t\t\t\t// Create a new section for nested MappingNode.\n\t\t\t\tsection, err := cfg.NewSection(key)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\t// Process nested key-value pairs.\n\t\t\t\tfor j := 0; j < len(valueNode.Content); j += 2 {\n\t\t\t\t\tnestedKeyNode := valueNode.Content[j]\n\t\t\t\t\tnestedValueNode := valueNode.Content[j+1]\n\t\t\t\t\tif nestedValueNode.Kind == ScalarNode {\n\t\t\t\t\t\t_, err := section.NewKey(nestedKeyNode.Value, nestedValueNode.Value)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\tlog.Debugf(\"Skipping nested non-scalar value for key %s: %v\", nestedKeyNode.Value, nestedValueNode.Kind)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\tdefault:\n\t\t\t\tlog.Debugf(\"Skipping non-scalar value for key %s: %v\", key, valueNode.Kind)\n\t\t\t}\n\t\t}\n\t} else {\n\t\treturn fmt.Errorf(\"INI encoder supports only MappingNode at the root level, got %v\", node.Kind)\n\t}\n\n\t// Use a buffer to store the INI output as the library doesn't support direct io.Writer with indent.\n\tvar buffer bytes.Buffer\n\t_, err := cfg.WriteToIndent(&buffer, ie.indentString)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Write the buffer content to the provided writer.\n\t_, err = writer.Write(buffer.Bytes())\n\treturn err\n}\n\n// writeStringINI is a helper function to write a string to the provided writer for INI encoder.\nfunc writeStringINI(writer io.Writer, content string) error {\n\t_, err := writer.Write([]byte(content))\n\treturn err\n}\n" }, { "path": "pkg/yqlib/encoder_json.go", "content": "//go:build !yq_nojson\n\npackage yqlib\n\nimport (\n\t\"bytes\"\n\t\"io\"\n\n\t\"github.com/goccy/go-json\"\n)\n\ntype jsonEncoder struct {\n\tprefs JsonPreferences\n\tindentString string\n}\n\nfunc NewJSONEncoder(prefs JsonPreferences) Encoder {\n\tvar indentString = \"\"\n\n\tfor index := 0; index < prefs.Indent; index++ {\n\t\tindentString = indentString + \" \"\n\t}\n\n\treturn &jsonEncoder{prefs, indentString}\n}\n\nfunc (je *jsonEncoder) CanHandleAliases() bool {\n\treturn false\n}\n\nfunc (je *jsonEncoder) PrintDocumentSeparator(_ io.Writer) error {\n\treturn nil\n}\n\nfunc (je *jsonEncoder) PrintLeadingContent(_ io.Writer, _ string) error {\n\treturn nil\n}\n\nfunc (je *jsonEncoder) Encode(writer io.Writer, node *CandidateNode) error {\n\tlog.Debugf(\"I need to encode %v\", NodeToString(node))\n\tlog.Debugf(\"kids %v\", len(node.Content))\n\n\tif node.Kind == ScalarNode && je.prefs.UnwrapScalar {\n\t\treturn writeString(writer, node.Value+\"\\n\")\n\t}\n\n\tdestination := writer\n\ttempBuffer := bytes.NewBuffer(nil)\n\tif je.prefs.ColorsEnabled {\n\t\tdestination = tempBuffer\n\t}\n\n\tvar encoder = json.NewEncoder(destination)\n\tencoder.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >\n\tencoder.SetIndent(\"\", je.indentString)\n\n\terr := encoder.Encode(node)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif je.prefs.ColorsEnabled {\n\t\treturn colorizeAndPrint(tempBuffer.Bytes(), writer)\n\t}\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/encoder_kyaml.go", "content": "//go:build !yq_nokyaml\n\npackage yqlib\n\nimport (\n\t\"bytes\"\n\t\"io\"\n\t\"regexp\"\n\t\"strconv\"\n\t\"strings\"\n)\n\ntype kyamlEncoder struct {\n\tprefs KYamlPreferences\n}\n\nfunc NewKYamlEncoder(prefs KYamlPreferences) Encoder {\n\treturn &kyamlEncoder{prefs: prefs}\n}\n\nfunc (ke *kyamlEncoder) CanHandleAliases() bool {\n\t// KYAML is a restricted subset; avoid emitting anchors/aliases.\n\treturn false\n}\n\nfunc (ke *kyamlEncoder) PrintDocumentSeparator(writer io.Writer) error {\n\treturn PrintYAMLDocumentSeparator(writer, ke.prefs.PrintDocSeparators)\n}\n\nfunc (ke *kyamlEncoder) PrintLeadingContent(writer io.Writer, content string) error {\n\treturn PrintYAMLLeadingContent(writer, content, ke.prefs.PrintDocSeparators, ke.prefs.ColorsEnabled)\n}\n\nfunc (ke *kyamlEncoder) Encode(writer io.Writer, node *CandidateNode) error {\n\tlog.Debug(\"encoderKYaml - going to print %v\", NodeToString(node))\n\tif node.Kind == ScalarNode && ke.prefs.UnwrapScalar {\n\t\treturn writeString(writer, node.Value+\"\\n\")\n\t}\n\n\tdestination := writer\n\ttempBuffer := bytes.NewBuffer(nil)\n\tif ke.prefs.ColorsEnabled {\n\t\tdestination = tempBuffer\n\t}\n\n\t// Mirror the YAML encoder behaviour: trailing comments on the document root\n\t// are stored in FootComment and need to be printed after the document.\n\ttrailingContent := node.FootComment\n\n\tif err := ke.writeCommentBlock(destination, node.HeadComment, 0); err != nil {\n\t\treturn err\n\t}\n\tif err := ke.writeNode(destination, node, 0); err != nil {\n\t\treturn err\n\t}\n\tif err := ke.writeInlineComment(destination, node.LineComment); err != nil {\n\t\treturn err\n\t}\n\tif err := writeString(destination, \"\\n\"); err != nil {\n\t\treturn err\n\t}\n\tif err := ke.PrintLeadingContent(destination, trailingContent); err != nil {\n\t\treturn err\n\t}\n\n\tif ke.prefs.ColorsEnabled {\n\t\treturn colorizeAndPrint(tempBuffer.Bytes(), writer)\n\t}\n\treturn nil\n}\n\nfunc (ke *kyamlEncoder) writeNode(writer io.Writer, node *CandidateNode, indent int) error {\n\tswitch node.Kind {\n\tcase MappingNode:\n\t\treturn ke.writeMapping(writer, node, indent)\n\tcase SequenceNode:\n\t\treturn ke.writeSequence(writer, node, indent)\n\tcase ScalarNode:\n\t\treturn writeString(writer, ke.formatScalar(node))\n\tcase AliasNode:\n\t\t// Should have been exploded by the printer, but handle defensively.\n\t\tif node.Alias == nil {\n\t\t\treturn writeString(writer, \"null\")\n\t\t}\n\t\treturn ke.writeNode(writer, node.Alias, indent)\n\tdefault:\n\t\treturn writeString(writer, \"null\")\n\t}\n}\n\nfunc (ke *kyamlEncoder) writeMapping(writer io.Writer, node *CandidateNode, indent int) error {\n\tif len(node.Content) == 0 {\n\t\treturn writeString(writer, \"{}\")\n\t}\n\tif err := writeString(writer, \"{\\n\"); err != nil {\n\t\treturn err\n\t}\n\n\tfor i := 0; i+1 < len(node.Content); i += 2 {\n\t\tkeyNode := node.Content[i]\n\t\tvalueNode := node.Content[i+1]\n\n\t\tentryIndent := indent + ke.prefs.Indent\n\t\tif err := ke.writeCommentBlock(writer, keyNode.HeadComment, entryIndent); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif valueNode.HeadComment != \"\" && valueNode.HeadComment != keyNode.HeadComment {\n\t\t\tif err := ke.writeCommentBlock(writer, valueNode.HeadComment, entryIndent); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\tif err := ke.writeIndent(writer, entryIndent); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := writeString(writer, ke.formatKey(keyNode)); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := writeString(writer, \": \"); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := ke.writeNode(writer, valueNode, entryIndent); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\t// Always emit a trailing comma; KYAML encourages explicit separators,\n\t\t// and this ensures all quoted strings have a trailing `\",` as requested.\n\t\tif err := writeString(writer, \",\"); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tinline := valueNode.LineComment\n\t\tif inline == \"\" {\n\t\t\tinline = keyNode.LineComment\n\t\t}\n\t\tif err := ke.writeInlineComment(writer, inline); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := writeString(writer, \"\\n\"); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tfoot := valueNode.FootComment\n\t\tif foot == \"\" {\n\t\t\tfoot = keyNode.FootComment\n\t\t}\n\t\tif err := ke.writeCommentBlock(writer, foot, entryIndent); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif err := ke.writeIndent(writer, indent); err != nil {\n\t\treturn err\n\t}\n\treturn writeString(writer, \"}\")\n}\n\nfunc (ke *kyamlEncoder) writeSequence(writer io.Writer, node *CandidateNode, indent int) error {\n\tif len(node.Content) == 0 {\n\t\treturn writeString(writer, \"[]\")\n\t}\n\tif err := writeString(writer, \"[\\n\"); err != nil {\n\t\treturn err\n\t}\n\n\tfor _, child := range node.Content {\n\t\titemIndent := indent + ke.prefs.Indent\n\t\tif err := ke.writeCommentBlock(writer, child.HeadComment, itemIndent); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := ke.writeIndent(writer, itemIndent); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := ke.writeNode(writer, child, itemIndent); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := writeString(writer, \",\"); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := ke.writeInlineComment(writer, child.LineComment); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := writeString(writer, \"\\n\"); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := ke.writeCommentBlock(writer, child.FootComment, itemIndent); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif err := ke.writeIndent(writer, indent); err != nil {\n\t\treturn err\n\t}\n\treturn writeString(writer, \"]\")\n}\n\nfunc (ke *kyamlEncoder) writeIndent(writer io.Writer, indent int) error {\n\tif indent <= 0 {\n\t\treturn nil\n\t}\n\treturn writeString(writer, strings.Repeat(\" \", indent))\n}\n\nfunc (ke *kyamlEncoder) formatKey(keyNode *CandidateNode) string {\n\t// KYAML examples use bare keys. Quote keys only when needed.\n\tkey := keyNode.Value\n\tif isValidKYamlBareKey(key) {\n\t\treturn key\n\t}\n\treturn `\"` + escapeDoubleQuotedString(key) + `\"`\n}\n\nfunc (ke *kyamlEncoder) formatScalar(node *CandidateNode) string {\n\tswitch node.Tag {\n\tcase \"!!null\":\n\t\treturn \"null\"\n\tcase \"!!bool\":\n\t\treturn strings.ToLower(node.Value)\n\tcase \"!!int\", \"!!float\":\n\t\treturn node.Value\n\tcase \"!!str\":\n\t\treturn `\"` + escapeDoubleQuotedString(node.Value) + `\"`\n\tdefault:\n\t\t// Fall back to a string representation to avoid implicit typing surprises.\n\t\treturn `\"` + escapeDoubleQuotedString(node.Value) + `\"`\n\t}\n}\n\nvar kyamlBareKeyRe = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_-]*$`)\n\nfunc isValidKYamlBareKey(s string) bool {\n\t// Conservative: require an identifier-like key; otherwise quote.\n\tif s == \"\" {\n\t\treturn false\n\t}\n\treturn kyamlBareKeyRe.MatchString(s)\n}\n\nfunc escapeDoubleQuotedString(s string) string {\n\tvar b strings.Builder\n\tb.Grow(len(s) + 2)\n\n\tfor _, r := range s {\n\t\tswitch r {\n\t\tcase '\\\\':\n\t\t\tb.WriteString(`\\\\`)\n\t\tcase '\"':\n\t\t\tb.WriteString(`\\\"`)\n\t\tcase '\\n':\n\t\t\tb.WriteString(`\\n`)\n\t\tcase '\\r':\n\t\t\tb.WriteString(`\\r`)\n\t\tcase '\\t':\n\t\t\tb.WriteString(`\\t`)\n\t\tdefault:\n\t\t\tif r < 0x20 {\n\t\t\t\t// YAML double-quoted strings support \\uXXXX escapes.\n\t\t\t\tb.WriteString(`\\u`)\n\t\t\t\thex := \"0000\" + strings.ToUpper(strconv.FormatInt(int64(r), 16))\n\t\t\t\tb.WriteString(hex[len(hex)-4:])\n\t\t\t} else {\n\t\t\t\tb.WriteRune(r)\n\t\t\t}\n\t\t}\n\t}\n\treturn b.String()\n}\n\nfunc (ke *kyamlEncoder) writeCommentBlock(writer io.Writer, comment string, indent int) error {\n\tif strings.TrimSpace(comment) == \"\" {\n\t\treturn nil\n\t}\n\n\tlines := strings.Split(strings.ReplaceAll(comment, \"\\r\\n\", \"\\n\"), \"\\n\")\n\tfor _, line := range lines {\n\t\ttrimmed := strings.TrimSpace(line)\n\t\tif trimmed == \"\" {\n\t\t\tcontinue\n\t\t}\n\n\t\tif err := ke.writeIndent(writer, indent); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\ttoWrite := line\n\t\tif !commentLineRe.MatchString(toWrite) {\n\t\t\ttoWrite = \"# \" + toWrite\n\t\t}\n\t\tif err := writeString(writer, toWrite); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := writeString(writer, \"\\n\"); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (ke *kyamlEncoder) writeInlineComment(writer io.Writer, comment string) error {\n\tcomment = strings.TrimSpace(strings.ReplaceAll(comment, \"\\r\\n\", \"\\n\"))\n\tif comment == \"\" {\n\t\treturn nil\n\t}\n\n\tlines := strings.Split(comment, \"\\n\")\n\tfirst := strings.TrimSpace(lines[0])\n\tif first == \"\" {\n\t\treturn nil\n\t}\n\n\tif !strings.HasPrefix(first, \"#\") {\n\t\tfirst = \"# \" + first\n\t}\n\n\tif err := writeString(writer, \" \"); err != nil {\n\t\treturn err\n\t}\n\treturn writeString(writer, first)\n}\n" }, { "path": "pkg/yqlib/encoder_lua.go", "content": "//go:build !yq_nolua\n\npackage yqlib\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"strings\"\n)\n\ntype luaEncoder struct {\n\tdocPrefix string\n\tdocSuffix string\n\tindent int\n\tindentStr string\n\tunquoted bool\n\tglobals bool\n\tescape *strings.Replacer\n}\n\nfunc (le *luaEncoder) CanHandleAliases() bool {\n\treturn false\n}\n\nfunc NewLuaEncoder(prefs LuaPreferences) Encoder {\n\tescape := strings.NewReplacer(\n\t\t\"\\000\", \"\\\\000\",\n\t\t\"\\001\", \"\\\\001\",\n\t\t\"\\002\", \"\\\\002\",\n\t\t\"\\003\", \"\\\\003\",\n\t\t\"\\004\", \"\\\\004\",\n\t\t\"\\005\", \"\\\\005\",\n\t\t\"\\006\", \"\\\\006\",\n\t\t\"\\007\", \"\\\\a\",\n\t\t\"\\010\", \"\\\\b\",\n\t\t\"\\011\", \"\\\\t\",\n\t\t\"\\012\", \"\\\\n\",\n\t\t\"\\013\", \"\\\\v\",\n\t\t\"\\014\", \"\\\\f\",\n\t\t\"\\015\", \"\\\\r\",\n\t\t\"\\016\", \"\\\\014\",\n\t\t\"\\017\", \"\\\\015\",\n\t\t\"\\020\", \"\\\\016\",\n\t\t\"\\021\", \"\\\\017\",\n\t\t\"\\022\", \"\\\\018\",\n\t\t\"\\023\", \"\\\\019\",\n\t\t\"\\024\", \"\\\\020\",\n\t\t\"\\025\", \"\\\\021\",\n\t\t\"\\026\", \"\\\\022\",\n\t\t\"\\027\", \"\\\\023\",\n\t\t\"\\030\", \"\\\\024\",\n\t\t\"\\031\", \"\\\\025\",\n\t\t\"\\032\", \"\\\\026\",\n\t\t\"\\033\", \"\\\\027\",\n\t\t\"\\034\", \"\\\\028\",\n\t\t\"\\035\", \"\\\\029\",\n\t\t\"\\036\", \"\\\\030\",\n\t\t\"\\037\", \"\\\\031\",\n\t\t\"\\\"\", \"\\\\\\\"\",\n\t\t\"'\", \"\\\\'\",\n\t\t\"\\\\\", \"\\\\\\\\\",\n\t\t\"\\177\", \"\\\\127\",\n\t)\n\tunescape := strings.NewReplacer(\n\t\t\"\\\\'\", \"'\",\n\t\t\"\\\\\\\"\", \"\\\"\",\n\t\t\"\\\\n\", \"\\n\",\n\t\t\"\\\\r\", \"\\r\",\n\t\t\"\\\\t\", \"\\t\",\n\t\t\"\\\\\\\\\", \"\\\\\",\n\t)\n\treturn &luaEncoder{unescape.Replace(prefs.DocPrefix), unescape.Replace(prefs.DocSuffix), 0, \"\\t\", prefs.UnquotedKeys, prefs.Globals, escape}\n}\n\nfunc (le *luaEncoder) PrintDocumentSeparator(_ io.Writer) error {\n\treturn nil\n}\n\nfunc (le *luaEncoder) PrintLeadingContent(_ io.Writer, _ string) error {\n\treturn nil\n}\n\nfunc (le *luaEncoder) encodeString(writer io.Writer, node *CandidateNode) error {\n\tquote := \"\\\"\"\n\tswitch node.Style {\n\tcase LiteralStyle, FoldedStyle, FlowStyle:\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tif !strings.Contains(node.Value, \"]\"+strings.Repeat(\"=\", i)+\"]\") {\n\t\t\t\terr := writeString(writer, \"[\"+strings.Repeat(\"=\", i)+\"[\\n\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\terr = writeString(writer, node.Value)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\treturn writeString(writer, \"]\"+strings.Repeat(\"=\", i)+\"]\")\n\t\t\t}\n\t\t}\n\tcase SingleQuotedStyle:\n\t\tquote = \"'\"\n\n\t\t// fallthrough to regular ol' string\n\t}\n\treturn writeString(writer, quote+le.escape.Replace(node.Value)+quote)\n}\n\nfunc (le *luaEncoder) writeIndent(writer io.Writer) error {\n\tif le.indentStr == \"\" {\n\t\treturn nil\n\t}\n\terr := writeString(writer, \"\\n\")\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn writeString(writer, strings.Repeat(le.indentStr, le.indent))\n}\n\nfunc (le *luaEncoder) encodeArray(writer io.Writer, node *CandidateNode) error {\n\terr := writeString(writer, \"{\")\n\tif err != nil {\n\t\treturn err\n\t}\n\tle.indent++\n\tfor _, child := range node.Content {\n\t\terr = le.writeIndent(writer)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\terr := le.encodeAny(writer, child)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\terr = writeString(writer, \",\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif child.LineComment != \"\" {\n\t\t\tsansPrefix, _ := strings.CutPrefix(child.LineComment, \"#\")\n\t\t\terr = writeString(writer, \" --\"+sansPrefix)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\tle.indent--\n\tif len(node.Content) != 0 {\n\t\terr = le.writeIndent(writer)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn writeString(writer, \"}\")\n}\n\nfunc needsQuoting(s string) bool {\n\t// known keywords as of Lua 5.4\n\tswitch s {\n\tcase \"do\", \"and\", \"else\", \"break\",\n\t\t\"if\", \"end\", \"goto\", \"false\",\n\t\t\"in\", \"for\", \"then\", \"local\",\n\t\t\"or\", \"nil\", \"true\", \"until\",\n\t\t\"elseif\", \"function\", \"not\",\n\t\t\"repeat\", \"return\", \"while\":\n\t\treturn true\n\t}\n\t// [%a_][%w_]*\n\tfor i, c := range s {\n\t\tif i == 0 {\n\t\t\t// keeping for legacy reasons, upgraded linter\n\t\t\t//nolint:staticcheck\n\t\t\tif !((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_') {\n\t\t\t\treturn true\n\t\t\t}\n\t\t} else {\n\t\t\t// keeping for legacy reasons, upgraded linter\n\t\t\t//nolint:staticcheck\n\t\t\tif !((c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_') {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bool) error {\n\tif !global {\n\t\terr := writeString(writer, \"{\")\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tle.indent++\n\t}\n\tfor i, child := range node.Content {\n\t\tif (i % 2) == 1 {\n\t\t\t// value\n\t\t\terr := le.encodeAny(writer, child)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\terr = writeString(writer, \";\")\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t} else {\n\t\t\t// key\n\t\t\tif !global || i > 0 {\n\t\t\t\terr := le.writeIndent(writer)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t\tif (le.unquoted || global) && child.Tag == \"!!str\" && !needsQuoting(child.Value) {\n\t\t\t\terr := writeString(writer, child.Value+\" = \")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tif global {\n\t\t\t\t\t// This only works in Lua 5.2+\n\t\t\t\t\terr := writeString(writer, \"_ENV\")\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\terr := writeString(writer, \"[\")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\terr = le.encodeAny(writer, child)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\terr = writeString(writer, \"] = \")\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif child.LineComment != \"\" {\n\t\t\tsansPrefix, _ := strings.CutPrefix(child.LineComment, \"#\")\n\t\t\terr := writeString(writer, strings.Repeat(\" \", i%2)+\"--\"+sansPrefix)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif (i % 2) == 0 {\n\t\t\t\t// newline and indent after comments on keys\n\t\t\t\terr = le.writeIndent(writer)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tif global {\n\t\treturn writeString(writer, \"\\n\")\n\t}\n\tle.indent--\n\tif len(node.Content) != 0 {\n\t\terr := le.writeIndent(writer)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn writeString(writer, \"}\")\n}\n\nfunc (le *luaEncoder) encodeAny(writer io.Writer, node *CandidateNode) error {\n\tswitch node.Kind {\n\tcase SequenceNode:\n\t\treturn le.encodeArray(writer, node)\n\tcase MappingNode:\n\t\treturn le.encodeMap(writer, node, false)\n\tcase ScalarNode:\n\t\tswitch node.Tag {\n\t\tcase \"!!str\":\n\t\t\treturn le.encodeString(writer, node)\n\t\tcase \"!!null\":\n\t\t\t// TODO reject invalid use as a table key\n\t\t\treturn writeString(writer, \"nil\")\n\t\tcase \"!!bool\":\n\t\t\t// Yaml 1.2 has case variation e.g. True, FALSE etc but Lua only has\n\t\t\t// lower case\n\t\t\treturn writeString(writer, strings.ToLower(node.Value))\n\t\tcase \"!!int\":\n\t\t\tif strings.HasPrefix(node.Value, \"0o\") {\n\t\t\t\t_, octalValue, err := parseInt64(node.Value)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\treturn writeString(writer, fmt.Sprintf(\"%d\", octalValue))\n\t\t\t}\n\t\t\treturn writeString(writer, strings.ToLower(node.Value))\n\t\tcase \"!!float\":\n\t\t\tswitch strings.ToLower(node.Value) {\n\t\t\tcase \".inf\", \"+.inf\":\n\t\t\t\treturn writeString(writer, \"(1/0)\")\n\t\t\tcase \"-.inf\":\n\t\t\t\treturn writeString(writer, \"(-1/0)\")\n\t\t\tcase \".nan\":\n\t\t\t\treturn writeString(writer, \"(0/0)\")\n\t\t\tdefault:\n\t\t\t\treturn writeString(writer, node.Value)\n\t\t\t}\n\t\tdefault:\n\t\t\treturn fmt.Errorf(\"lua encoder NYI -- %s\", node.Tag)\n\t\t}\n\tdefault:\n\t\treturn fmt.Errorf(\"lua encoder NYI -- %s\", node.Tag)\n\t}\n}\n\nfunc (le *luaEncoder) encodeTopLevel(writer io.Writer, node *CandidateNode) error {\n\terr := writeString(writer, le.docPrefix)\n\tif err != nil {\n\t\treturn err\n\t}\n\terr = le.encodeAny(writer, node)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn writeString(writer, le.docSuffix)\n}\n\nfunc (le *luaEncoder) Encode(writer io.Writer, node *CandidateNode) error {\n\n\tif le.globals {\n\t\tif node.Kind != MappingNode {\n\t\t\treturn fmt.Errorf(\"--lua-global requires a top level MappingNode\")\n\t\t}\n\t\treturn le.encodeMap(writer, node, true)\n\t}\n\treturn le.encodeTopLevel(writer, node)\n}\n" }, { "path": "pkg/yqlib/encoder_properties.go", "content": "//go:build !yq_noprops\n\npackage yqlib\n\nimport (\n\t\"bufio\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"strings\"\n\n\t\"github.com/magiconair/properties\"\n)\n\ntype propertiesEncoder struct {\n\tprefs PropertiesPreferences\n}\n\nfunc NewPropertiesEncoder(prefs PropertiesPreferences) Encoder {\n\treturn &propertiesEncoder{\n\t\tprefs: prefs,\n\t}\n}\n\nfunc (pe *propertiesEncoder) CanHandleAliases() bool {\n\treturn false\n}\n\nfunc (pe *propertiesEncoder) PrintDocumentSeparator(_ io.Writer) error {\n\treturn nil\n}\n\nfunc (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content string) error {\n\treader := bufio.NewReader(strings.NewReader(content))\n\tfor {\n\n\t\treadline, errReading := reader.ReadString('\\n')\n\t\tif errReading != nil && !errors.Is(errReading, io.EOF) {\n\t\t\treturn errReading\n\t\t}\n\t\tif strings.Contains(readline, \"$yqDocSeparator$\") {\n\n\t\t\tif err := pe.PrintDocumentSeparator(writer); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t} else {\n\t\t\tif err := writeString(writer, readline); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\tif errors.Is(errReading, io.EOF) {\n\t\t\tif readline != \"\" {\n\t\t\t\t// the last comment we read didn't have a newline, put one in\n\t\t\t\tif err := writeString(writer, \"\\n\"); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (pe *propertiesEncoder) Encode(writer io.Writer, node *CandidateNode) error {\n\n\tif node.Kind == ScalarNode {\n\t\treturn writeString(writer, node.Value+\"\\n\")\n\t}\n\n\tmapKeysToStrings(node)\n\tp := properties.NewProperties()\n\tp.WriteSeparator = pe.prefs.KeyValueSeparator\n\terr := pe.doEncode(p, node, \"\", nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t_, err = p.WriteComment(writer, \"#\", properties.UTF8)\n\treturn err\n}\n\nfunc (pe *propertiesEncoder) doEncode(p *properties.Properties, node *CandidateNode, path string, keyNode *CandidateNode) error {\n\n\tcomments := \"\"\n\tif keyNode != nil {\n\t\t// include the key node comments if present\n\t\tcomments = headAndLineComment(keyNode)\n\t}\n\tcomments = comments + headAndLineComment(node)\n\tcommentsWithSpaces := strings.ReplaceAll(comments, \"\\n\", \"\\n \")\n\tp.SetComments(path, strings.Split(commentsWithSpaces, \"\\n\"))\n\n\tswitch node.Kind {\n\tcase ScalarNode:\n\t\tvar nodeValue string\n\t\tif pe.prefs.UnwrapScalar || !strings.Contains(node.Value, \" \") {\n\t\t\tnodeValue = node.Value\n\t\t} else {\n\t\t\tnodeValue = fmt.Sprintf(\"%q\", node.Value)\n\t\t}\n\t\t_, _, err := p.Set(path, nodeValue)\n\t\treturn err\n\tcase SequenceNode:\n\t\treturn pe.encodeArray(p, node.Content, path)\n\tcase MappingNode:\n\t\treturn pe.encodeMap(p, node.Content, path)\n\tcase AliasNode:\n\t\treturn pe.doEncode(p, node.Alias, path, nil)\n\tdefault:\n\t\treturn fmt.Errorf(\"unsupported node %v\", node.Tag)\n\t}\n}\n\nfunc (pe *propertiesEncoder) appendPath(path string, key interface{}) string {\n\tif path == \"\" {\n\t\treturn fmt.Sprintf(\"%v\", key)\n\t}\n\tswitch key.(type) {\n\tcase int:\n\t\tif pe.prefs.UseArrayBrackets {\n\t\t\treturn fmt.Sprintf(\"%v[%v]\", path, key)\n\t\t}\n\n\t}\n\treturn fmt.Sprintf(\"%v.%v\", path, key)\n}\n\nfunc (pe *propertiesEncoder) encodeArray(p *properties.Properties, kids []*CandidateNode, path string) error {\n\tfor index, child := range kids {\n\t\terr := pe.doEncode(p, child, pe.appendPath(path, index), nil)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (pe *propertiesEncoder) encodeMap(p *properties.Properties, kids []*CandidateNode, path string) error {\n\tfor index := 0; index < len(kids); index = index + 2 {\n\t\tkey := kids[index]\n\t\tvalue := kids[index+1]\n\t\terr := pe.doEncode(p, value, pe.appendPath(path, key.Value), key)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/encoder_properties_test.go", "content": "package yqlib\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\ntype keyValuePair struct {\n\tkey string\n\tvalue string\n\tcomment string\n}\n\nfunc (kv *keyValuePair) String(unwrap bool, sep string) string {\n\tbuilder := strings.Builder{}\n\n\tif kv.comment != \"\" {\n\t\tbuilder.WriteString(kv.comment)\n\t\tbuilder.WriteString(\"\\n\")\n\t}\n\n\tbuilder.WriteString(kv.key)\n\tbuilder.WriteString(sep)\n\n\tif unwrap {\n\t\tbuilder.WriteString(kv.value)\n\t} else {\n\t\tbuilder.WriteString(\"\\\"\")\n\t\tbuilder.WriteString(kv.value)\n\t\tbuilder.WriteString(\"\\\"\")\n\t}\n\n\treturn builder.String()\n}\n\ntype testProperties struct {\n\tpairs []keyValuePair\n}\n\nfunc (tp *testProperties) String(unwrap bool, sep string) string {\n\tkvs := []string{}\n\n\tfor _, kv := range tp.pairs {\n\t\tkvs = append(kvs, kv.String(unwrap, sep))\n\t}\n\n\treturn strings.Join(kvs, \"\\n\")\n}\n\nfunc yamlToProps(sampleYaml string, unwrapScalar bool, separator string) string {\n\tvar output bytes.Buffer\n\twriter := bufio.NewWriter(&output)\n\n\tvar propsEncoder = NewPropertiesEncoder(PropertiesPreferences{KeyValueSeparator: separator, UnwrapScalar: unwrapScalar})\n\tinputs, err := readDocuments(strings.NewReader(sampleYaml), \"sample.yml\", 0, NewYamlDecoder(ConfiguredYamlPreferences))\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tnode := inputs.Front().Value.(*CandidateNode)\n\terr = propsEncoder.Encode(writer, node)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\twriter.Flush()\n\n\treturn strings.TrimSuffix(output.String(), \"\\n\")\n}\n\nfunc doTest(t *testing.T, sampleYaml string, props testProperties, testUnwrapped, testWrapped bool) {\n\twraps := []bool{}\n\tif testUnwrapped {\n\t\twraps = append(wraps, true)\n\t}\n\tif testWrapped {\n\t\twraps = append(wraps, false)\n\t}\n\n\tfor _, unwrap := range wraps {\n\t\tfor _, sep := range []string{\" = \", \";\", \"=\", \" \"} {\n\t\t\tvar actualProps = yamlToProps(sampleYaml, unwrap, sep)\n\t\t\ttest.AssertResult(t, props.String(unwrap, sep), actualProps)\n\t\t}\n\t}\n}\n\nfunc TestPropertiesEncoderSimple(t *testing.T) {\n\tvar sampleYaml = `a: 'bob cool'`\n\n\tdoTest(\n\t\tt, sampleYaml,\n\t\ttestProperties{\n\t\t\tpairs: []keyValuePair{\n\t\t\t\t{\n\t\t\t\t\tkey: \"a\",\n\t\t\t\t\tvalue: \"bob cool\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\ttrue, true,\n\t)\n}\n\nfunc TestPropertiesEncoderSimpleWithComments(t *testing.T) {\n\tvar sampleYaml = `a: 'bob cool' # line`\n\n\tdoTest(\n\t\tt, sampleYaml,\n\t\ttestProperties{\n\t\t\tpairs: []keyValuePair{\n\t\t\t\t{\n\t\t\t\t\tkey: \"a\",\n\t\t\t\t\tvalue: \"bob cool\",\n\t\t\t\t\tcomment: \"# line\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\ttrue, true,\n\t)\n}\n\nfunc TestPropertiesEncoderDeep(t *testing.T) {\n\tvar sampleYaml = `a: \n b: \"bob cool\"\n`\n\n\tdoTest(\n\t\tt, sampleYaml,\n\t\ttestProperties{\n\t\t\tpairs: []keyValuePair{\n\t\t\t\t{\n\t\t\t\t\tkey: \"a.b\",\n\t\t\t\t\tvalue: \"bob cool\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\ttrue, true,\n\t)\n}\n\nfunc TestPropertiesEncoderDeepWithComments(t *testing.T) {\n\tvar sampleYaml = `a: # a thing\n b: \"bob cool\" # b thing\n`\n\n\tdoTest(\n\t\tt, sampleYaml,\n\t\ttestProperties{\n\t\t\tpairs: []keyValuePair{\n\t\t\t\t{\n\t\t\t\t\tkey: \"a.b\",\n\t\t\t\t\tvalue: \"bob cool\",\n\t\t\t\t\tcomment: \"# b thing\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\ttrue, true,\n\t)\n}\n\nfunc TestPropertiesEncoderArray_Unwrapped(t *testing.T) {\n\tvar sampleYaml = `a: \n b: [{c: dog}, {c: cat}]\n`\n\n\tdoTest(\n\t\tt, sampleYaml,\n\t\ttestProperties{\n\t\t\tpairs: []keyValuePair{\n\t\t\t\t{\n\t\t\t\t\tkey: \"a.b.0.c\",\n\t\t\t\t\tvalue: \"dog\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tkey: \"a.b.1.c\",\n\t\t\t\t\tvalue: \"cat\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\ttrue, false,\n\t)\n}\n\nfunc TestPropertiesEncoderArray_Wrapped(t *testing.T) {\n\tvar sampleYaml = `a: \n b: [{c: dog named jim}, {c: cat named jim}]\n`\n\n\tdoTest(\n\t\tt, sampleYaml,\n\t\ttestProperties{\n\t\t\tpairs: []keyValuePair{\n\t\t\t\t{\n\t\t\t\t\tkey: \"a.b.0.c\",\n\t\t\t\t\tvalue: \"dog named jim\",\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tkey: \"a.b.1.c\",\n\t\t\t\t\tvalue: \"cat named jim\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\tfalse, true,\n\t)\n}\n" }, { "path": "pkg/yqlib/encoder_sh.go", "content": "//go:build !yq_nosh\n\npackage yqlib\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"regexp\"\n\t\"strings\"\n)\n\nvar unsafeChars = regexp.MustCompile(`[^\\w@%+=:,./-]`)\n\ntype shEncoder struct {\n\tquoteAll bool\n}\n\nfunc NewShEncoder() Encoder {\n\treturn &shEncoder{false}\n}\n\nfunc (e *shEncoder) CanHandleAliases() bool {\n\treturn false\n}\n\nfunc (e *shEncoder) PrintDocumentSeparator(_ io.Writer) error {\n\treturn nil\n}\n\nfunc (e *shEncoder) PrintLeadingContent(_ io.Writer, _ string) error {\n\treturn nil\n}\n\nfunc (e *shEncoder) Encode(writer io.Writer, node *CandidateNode) error {\n\tif node.guessTagFromCustomType() != \"!!str\" {\n\t\treturn fmt.Errorf(\"cannot encode %v as URI, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string\", node.Tag)\n\t}\n\n\treturn writeString(writer, e.encode(node.Value))\n}\n\n// put any (shell-unsafe) characters into a single-quoted block, close the block lazily\nfunc (e *shEncoder) encode(input string) string {\n\tconst quote = '\\''\n\tvar inQuoteBlock = false\n\tvar encoded strings.Builder\n\tencoded.Grow(len(input))\n\n\tfor _, ir := range input {\n\t\t// open or close a single-quote block\n\t\tif ir == quote {\n\t\t\tif inQuoteBlock {\n\t\t\t\t// get out of a quote block for an input quote\n\t\t\t\tencoded.WriteRune(quote)\n\t\t\t\tinQuoteBlock = !inQuoteBlock\n\t\t\t}\n\t\t\t// escape the quote with a backslash\n\t\t\tencoded.WriteRune('\\\\')\n\t\t} else {\n\t\t\tif e.shouldQuote(ir) && !inQuoteBlock {\n\t\t\t\t// start a quote block for any (unsafe) characters\n\t\t\t\tencoded.WriteRune(quote)\n\t\t\t\tinQuoteBlock = !inQuoteBlock\n\t\t\t}\n\t\t}\n\t\t// pass on the input character\n\t\tencoded.WriteRune(ir)\n\t}\n\t// close any pending quote block\n\tif inQuoteBlock {\n\t\tencoded.WriteRune(quote)\n\t}\n\treturn encoded.String()\n}\n\nfunc (e *shEncoder) shouldQuote(ir rune) bool {\n\treturn e.quoteAll || unsafeChars.MatchString(string(ir))\n}\n" }, { "path": "pkg/yqlib/encoder_shellvariables.go", "content": "//go:build !yq_noshell\n\npackage yqlib\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"strings\"\n\t\"unicode/utf8\"\n\n\t\"golang.org/x/text/unicode/norm\"\n)\n\ntype shellVariablesEncoder struct {\n\tprefs ShellVariablesPreferences\n}\n\nfunc NewShellVariablesEncoder() Encoder {\n\treturn &shellVariablesEncoder{\n\t\tprefs: ConfiguredShellVariablesPreferences,\n\t}\n}\n\nfunc (pe *shellVariablesEncoder) CanHandleAliases() bool {\n\treturn false\n}\n\nfunc (pe *shellVariablesEncoder) PrintDocumentSeparator(_ io.Writer) error {\n\treturn nil\n}\n\nfunc (pe *shellVariablesEncoder) PrintLeadingContent(_ io.Writer, _ string) error {\n\treturn nil\n}\n\nfunc (pe *shellVariablesEncoder) Encode(writer io.Writer, node *CandidateNode) error {\n\n\tmapKeysToStrings(node)\n\terr := pe.doEncode(&writer, node, \"\")\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn err\n}\n\nfunc (pe *shellVariablesEncoder) doEncode(w *io.Writer, node *CandidateNode, path string) error {\n\n\t// Note this drops all comments.\n\n\tswitch node.Kind {\n\tcase ScalarNode:\n\t\tnonemptyPath := path\n\t\tif path == \"\" {\n\t\t\t// We can't assign an empty variable \"=somevalue\" because that would error out if sourced in a shell,\n\t\t\t// nor can we use \"_\" as a variable name ($_ is a special shell variable that can't be assigned)...\n\t\t\t// let's just pick a fallback key to use if we are encoding a single scalar\n\t\t\tnonemptyPath = \"value\"\n\t\t}\n\t\tvar valueString string\n\t\tif pe.prefs.UnwrapScalar {\n\t\t\tvalueString = node.Value\n\t\t} else {\n\t\t\tvalueString = quoteValue(node.Value)\n\t\t}\n\t\t_, err := io.WriteString(*w, nonemptyPath+\"=\"+valueString+\"\\n\")\n\t\treturn err\n\tcase SequenceNode:\n\t\tfor index, child := range node.Content {\n\t\t\terr := pe.doEncode(w, child, pe.appendPath(path, index))\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\treturn nil\n\tcase MappingNode:\n\t\tfor index := 0; index < len(node.Content); index = index + 2 {\n\t\t\tkey := node.Content[index]\n\t\t\tvalue := node.Content[index+1]\n\t\t\terr := pe.doEncode(w, value, pe.appendPath(path, key.Value))\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\treturn nil\n\tcase AliasNode:\n\t\treturn pe.doEncode(w, node.Alias, path)\n\tdefault:\n\t\treturn fmt.Errorf(\"unsupported node %v\", node.Tag)\n\t}\n}\n\nfunc (pe *shellVariablesEncoder) appendPath(cookedPath string, rawKey interface{}) string {\n\n\t// Shell variable names must match\n\t// [a-zA-Z_]+[a-zA-Z0-9_]*\n\t//\n\t// While this is not mandated by POSIX, which is quite lenient, it is\n\t// what shells (for example busybox ash *) allow in practice.\n\t//\n\t// Since yaml names can contain basically any character, we will process them according to these steps:\n\t//\n\t// 1. apply unicode compatibility decomposition NFKD (this will convert accented\n\t// letters to letters followed by accents, split ligatures, replace exponents\n\t// with the corresponding digit, etc.\n\t//\n\t// 2. discard non-ASCII characters as well as ASCII control characters (ie. anything\n\t// with code point < 32 or > 126), this will eg. discard accents but keep the base\n\t// unaccented letter because of NFKD above\n\t//\n\t// 3. replace all non-alphanumeric characters with _\n\t//\n\t// Moreover, for the root key only, we will prepend an underscore if what results from the steps above\n\t// does not start with [a-zA-Z_] (ie. if the root key starts with a digit).\n\t//\n\t// Note this is NOT a 1:1 mapping.\n\t//\n\t// (*) see endofname.c from https://git.busybox.net/busybox/tag/?h=1_36_0\n\n\t// XXX empty strings\n\n\tkey := strings.Map(func(r rune) rune {\n\t\tif isAlphaNumericOrUnderscore(r) {\n\t\t\treturn r\n\t\t} else if r < 32 || 126 < r {\n\t\t\treturn -1\n\t\t}\n\t\treturn '_'\n\t}, norm.NFKD.String(fmt.Sprintf(\"%v\", rawKey)))\n\n\tif cookedPath == \"\" {\n\t\tfirstRune, _ := utf8.DecodeRuneInString(key)\n\t\tif !isAlphaOrUnderscore(firstRune) {\n\t\t\treturn \"_\" + key\n\t\t}\n\t\treturn key\n\t}\n\treturn cookedPath + pe.prefs.KeySeparator + key\n}\n\nfunc quoteValue(value string) string {\n\tneedsQuoting := false\n\tfor _, r := range value {\n\t\tif !isAlphaNumericOrUnderscore(r) {\n\t\t\tneedsQuoting = true\n\t\t\tbreak\n\t\t}\n\t}\n\tif needsQuoting {\n\t\treturn \"'\" + strings.ReplaceAll(value, \"'\", \"'\\\"'\\\"'\") + \"'\"\n\t}\n\treturn value\n}\n\nfunc isAlphaOrUnderscore(r rune) bool {\n\treturn ('a' <= r && r <= 'z') || ('A' <= r && r <= 'Z') || r == '_'\n}\n\nfunc isAlphaNumericOrUnderscore(r rune) bool {\n\treturn isAlphaOrUnderscore(r) || ('0' <= r && r <= '9')\n}\n" }, { "path": "pkg/yqlib/encoder_shellvariables_test.go", "content": "package yqlib\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nfunc assertEncodesTo(t *testing.T, yaml string, shellvars string) {\n\tvar output bytes.Buffer\n\twriter := bufio.NewWriter(&output)\n\n\tvar encoder = NewShellVariablesEncoder()\n\tinputs, err := readDocuments(strings.NewReader(yaml), \"test.yml\", 0, NewYamlDecoder(ConfiguredYamlPreferences))\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tnode := inputs.Front().Value.(*CandidateNode)\n\terr = encoder.Encode(writer, node)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\twriter.Flush()\n\n\ttest.AssertResult(t, shellvars, strings.TrimSuffix(output.String(), \"\\n\"))\n}\n\nfunc TestShellVariablesEncoderNonquoting(t *testing.T) {\n\tassertEncodesTo(t, \"a: alice\", \"a=alice\")\n}\n\nfunc TestShellVariablesEncoderQuoting(t *testing.T) {\n\tassertEncodesTo(t, \"a: Lewis Carroll\", \"a='Lewis Carroll'\")\n}\n\nfunc TestShellVariablesEncoderQuotesQuoting(t *testing.T) {\n\tassertEncodesTo(t, \"a: Lewis Carroll's Alice\", \"a='Lewis Carroll'\\\"'\\\"'s Alice'\")\n}\n\nfunc TestShellVariablesEncoderStripComments(t *testing.T) {\n\tassertEncodesTo(t, \"a: Alice # comment\", \"a=Alice\")\n}\n\nfunc TestShellVariablesEncoderMap(t *testing.T) {\n\tassertEncodesTo(t, \"a:\\n b: Lewis\\n c: Carroll\", \"a_b=Lewis\\na_c=Carroll\")\n}\n\nfunc TestShellVariablesEncoderArray_Unwrapped(t *testing.T) {\n\tassertEncodesTo(t, \"a: [{n: Alice}, {n: Bob}]\", \"a_0_n=Alice\\na_1_n=Bob\")\n}\n\nfunc TestShellVariablesEncoderKeyNonPrintable(t *testing.T) {\n\tassertEncodesTo(t, `\"be\\all\": ring!`, \"bell='ring!'\")\n}\n\nfunc TestShellVariablesEncoderKeyPrintableNonAlphaNumeric(t *testing.T) {\n\tassertEncodesTo(t, `\"b-e l=l\": ring!`, \"b_e_l_l='ring!'\")\n}\n\nfunc TestShellVariablesEncoderKeyPrintableNonAscii(t *testing.T) {\n\tassertEncodesTo(t, `\"b\\u00e9ll\": ring!`, \"bell='ring!'\")\n}\n\nfunc TestShellVariablesEncoderRootKeyStartingWithDigit(t *testing.T) {\n\tassertEncodesTo(t, \"1a: onea\", \"_1a=onea\")\n}\n\nfunc TestShellVariablesEncoderRootKeyStartingWithUnderscore(t *testing.T) {\n\tassertEncodesTo(t, \"_key: value\", \"_key=value\")\n}\n\nfunc TestShellVariablesEncoderChildStartingWithUnderscore(t *testing.T) {\n\tassertEncodesTo(t, \"root:\\n _child: value\", \"root__child=value\")\n}\n\nfunc TestShellVariablesEncoderEmptyValue(t *testing.T) {\n\tassertEncodesTo(t, \"empty:\", \"empty=\")\n}\n\nfunc TestShellVariablesEncoderEmptyArray(t *testing.T) {\n\tassertEncodesTo(t, \"empty: []\", \"\")\n}\n\nfunc TestShellVariablesEncoderEmptyMap(t *testing.T) {\n\tassertEncodesTo(t, \"empty: {}\", \"\")\n}\n\nfunc TestShellVariablesEncoderScalarNode(t *testing.T) {\n\tassertEncodesTo(t, \"some string\", \"value='some string'\")\n}\n\nfunc assertEncodesToWithSeparator(t *testing.T, yaml string, shellvars string, separator string) {\n\tvar output bytes.Buffer\n\twriter := bufio.NewWriter(&output)\n\n\t// Save the original separator\n\toriginalSeparator := ConfiguredShellVariablesPreferences.KeySeparator\n\tdefer func() {\n\t\tConfiguredShellVariablesPreferences.KeySeparator = originalSeparator\n\t}()\n\n\t// Set the custom separator\n\tConfiguredShellVariablesPreferences.KeySeparator = separator\n\n\tvar encoder = NewShellVariablesEncoder()\n\tinputs, err := readDocuments(strings.NewReader(yaml), \"test.yml\", 0, NewYamlDecoder(ConfiguredYamlPreferences))\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tnode := inputs.Front().Value.(*CandidateNode)\n\terr = encoder.Encode(writer, node)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\twriter.Flush()\n\n\ttest.AssertResult(t, shellvars, strings.TrimSuffix(output.String(), \"\\n\"))\n}\n\nfunc TestShellVariablesEncoderCustomSeparator(t *testing.T) {\n\tassertEncodesToWithSeparator(t, \"a:\\n b: Lewis\\n c: Carroll\", \"a__b=Lewis\\na__c=Carroll\", \"__\")\n}\n\nfunc TestShellVariablesEncoderCustomSeparatorNested(t *testing.T) {\n\tassertEncodesToWithSeparator(t, \"my_app:\\n db_config:\\n host: localhost\", \"my_app__db_config__host=localhost\", \"__\")\n}\n\nfunc TestShellVariablesEncoderCustomSeparatorArray(t *testing.T) {\n\tassertEncodesToWithSeparator(t, \"a: [{n: Alice}, {n: Bob}]\", \"a__0__n=Alice\\na__1__n=Bob\", \"__\")\n}\n\nfunc TestShellVariablesEncoderCustomSeparatorSingleChar(t *testing.T) {\n\tassertEncodesToWithSeparator(t, \"a:\\n b: value\", \"aXb=value\", \"X\")\n}\n\nfunc assertEncodesToUnwrapped(t *testing.T, yaml string, shellvars string) {\n\tvar output bytes.Buffer\n\twriter := bufio.NewWriter(&output)\n\n\toriginalUnwrapScalar := ConfiguredShellVariablesPreferences.UnwrapScalar\n\tdefer func() {\n\t\tConfiguredShellVariablesPreferences.UnwrapScalar = originalUnwrapScalar\n\t}()\n\n\tConfiguredShellVariablesPreferences.UnwrapScalar = true\n\n\tvar encoder = NewShellVariablesEncoder()\n\tinputs, err := readDocuments(strings.NewReader(yaml), \"test.yml\", 0, NewYamlDecoder(ConfiguredYamlPreferences))\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tnode := inputs.Front().Value.(*CandidateNode)\n\terr = encoder.Encode(writer, node)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\twriter.Flush()\n\n\ttest.AssertResult(t, shellvars, strings.TrimSuffix(output.String(), \"\\n\"))\n}\n\nfunc TestShellVariablesEncoderUnwrapScalar(t *testing.T) {\n\tassertEncodesToUnwrapped(t, \"a: Lewis Carroll\", \"a=Lewis Carroll\")\n\tassertEncodesToUnwrapped(t, \"b: 123\", \"b=123\")\n\tassertEncodesToUnwrapped(t, \"c: true\", \"c=true\")\n\tassertEncodesToUnwrapped(t, \"d: value with spaces\", \"d=value with spaces\")\n}\n" }, { "path": "pkg/yqlib/encoder_test.go", "content": "//go:build !yq_nojson\n\npackage yqlib\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nfunc yamlToJSON(t *testing.T, sampleYaml string, indent int) string {\n\tt.Helper()\n\tvar output bytes.Buffer\n\twriter := bufio.NewWriter(&output)\n\n\tprefs := ConfiguredJSONPreferences.Copy()\n\tprefs.Indent = indent\n\tprefs.UnwrapScalar = false\n\tvar jsonEncoder = NewJSONEncoder(prefs)\n\tinputs, err := readDocuments(strings.NewReader(sampleYaml), \"sample.yml\", 0, NewYamlDecoder(ConfiguredYamlPreferences))\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tnode := inputs.Front().Value.(*CandidateNode)\n\tlog.Debugf(\"%v\", NodeToString(node))\n\t// log.Debugf(\"Content[0] %v\", NodeToString(node.Content[0]))\n\n\terr = jsonEncoder.Encode(writer, node)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\twriter.Flush()\n\n\treturn strings.TrimSuffix(output.String(), \"\\n\")\n}\n\nfunc TestJSONEncoderPreservesObjectOrder(t *testing.T) {\n\tvar sampleYaml = `zabbix: winner\napple: great\nbanana:\n- {cobra: kai, angus: bob}\n`\n\tvar expectedJSON = `{\n \"zabbix\": \"winner\",\n \"apple\": \"great\",\n \"banana\": [\n {\n \"cobra\": \"kai\",\n \"angus\": \"bob\"\n }\n ]\n}`\n\tvar actualJSON = yamlToJSON(t, sampleYaml, 2)\n\ttest.AssertResult(t, expectedJSON, actualJSON)\n}\n\nfunc TestJsonNullInArray(t *testing.T) {\n\tvar sampleYaml = `[null]`\n\tvar actualJSON = yamlToJSON(t, sampleYaml, 0)\n\ttest.AssertResult(t, sampleYaml, actualJSON)\n}\n\nfunc TestJsonNull(t *testing.T) {\n\tvar sampleYaml = `null`\n\tvar actualJSON = yamlToJSON(t, sampleYaml, 0)\n\ttest.AssertResult(t, sampleYaml, actualJSON)\n}\n\nfunc TestJsonNullInObject(t *testing.T) {\n\tvar sampleYaml = `{x: null}`\n\tvar actualJSON = yamlToJSON(t, sampleYaml, 0)\n\ttest.AssertResult(t, `{\"x\":null}`, actualJSON)\n}\n\nfunc TestJsonEncoderDoesNotEscapeHTMLChars(t *testing.T) {\n\tvar sampleYaml = `build: \"( ./lint && ./format && ./compile ) < src.code\"`\n\tvar expectedJSON = `{\"build\":\"( ./lint && ./format && ./compile ) < src.code\"}`\n\tvar actualJSON = yamlToJSON(t, sampleYaml, 0)\n\ttest.AssertResult(t, expectedJSON, actualJSON)\n}\n" }, { "path": "pkg/yqlib/encoder_toml.go", "content": "//go:build !yq_notoml\n\npackage yqlib\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"io\"\n\t\"strings\"\n\n\t\"github.com/fatih/color\"\n)\n\ntype tomlEncoder struct {\n\twroteRootAttr bool // Track if we wrote root-level attributes before tables\n\tprefs TomlPreferences\n}\n\nfunc NewTomlEncoder() Encoder {\n\treturn NewTomlEncoderWithPrefs(ConfiguredTomlPreferences)\n}\n\nfunc NewTomlEncoderWithPrefs(prefs TomlPreferences) Encoder {\n\treturn &tomlEncoder{prefs: prefs}\n}\n\nfunc (te *tomlEncoder) Encode(writer io.Writer, node *CandidateNode) error {\n\tif node.Kind != MappingNode {\n\t\t// For standalone selections, TOML tests expect raw value for scalars\n\t\tif node.Kind == ScalarNode {\n\t\t\treturn writeString(writer, node.Value+\"\\n\")\n\t\t}\n\t\treturn fmt.Errorf(\"TOML encoder expects a mapping at the root level\")\n\t}\n\n\t// Encode to a buffer first if colors are enabled\n\tvar buf bytes.Buffer\n\tvar targetWriter io.Writer\n\ttargetWriter = writer\n\tif te.prefs.ColorsEnabled {\n\t\ttargetWriter = &buf\n\t}\n\n\t// Encode a root mapping as a sequence of attributes, tables, and arrays of tables\n\tif err := te.encodeRootMapping(targetWriter, node); err != nil {\n\t\treturn err\n\t}\n\n\tif te.prefs.ColorsEnabled {\n\t\tcolourised := te.colorizeToml(buf.Bytes())\n\t\t_, err := writer.Write(colourised)\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc (te *tomlEncoder) PrintDocumentSeparator(_ io.Writer) error {\n\treturn nil\n}\n\nfunc (te *tomlEncoder) PrintLeadingContent(_ io.Writer, _ string) error {\n\treturn nil\n}\n\nfunc (te *tomlEncoder) CanHandleAliases() bool {\n\treturn false\n}\n\n// ---- helpers ----\n\nfunc (te *tomlEncoder) writeComment(w io.Writer, comment string) error {\n\tif comment == \"\" {\n\t\treturn nil\n\t}\n\tlines := strings.Split(comment, \"\\n\")\n\tfor _, line := range lines {\n\t\tline = strings.TrimSpace(line)\n\t\tif !strings.HasPrefix(line, \"#\") {\n\t\t\tline = \"# \" + line\n\t\t}\n\t\tif _, err := w.Write([]byte(line + \"\\n\")); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (te *tomlEncoder) formatScalar(node *CandidateNode) string {\n\tswitch node.Tag {\n\tcase \"!!str\":\n\t\t// Quote strings per TOML spec\n\t\treturn fmt.Sprintf(\"%q\", node.Value)\n\tcase \"!!bool\", \"!!int\", \"!!float\":\n\t\treturn node.Value\n\tcase \"!!null\":\n\t\t// TOML does not have null; encode as empty string\n\t\treturn `\"\"`\n\tdefault:\n\t\treturn node.Value\n\t}\n}\n\nfunc (te *tomlEncoder) encodeRootMapping(w io.Writer, node *CandidateNode) error {\n\tte.wroteRootAttr = false // Reset state\n\n\t// Write root head comment if present (at the very beginning, no leading blank line)\n\tif node.HeadComment != \"\" {\n\t\tif err := te.writeComment(w, node.HeadComment); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// Preserve existing order by iterating Content\n\tfor i := 0; i < len(node.Content); i += 2 {\n\t\tkeyNode := node.Content[i]\n\t\tvalNode := node.Content[i+1]\n\t\tif err := te.encodeTopLevelEntry(w, []string{keyNode.Value}, valNode); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// encodeTopLevelEntry encodes a key/value at the root, dispatching to attribute, table, or array-of-tables\nfunc (te *tomlEncoder) encodeTopLevelEntry(w io.Writer, path []string, node *CandidateNode) error {\n\tif len(path) == 0 {\n\t\treturn fmt.Errorf(\"cannot encode TOML entry with empty path\")\n\t}\n\n\tswitch node.Kind {\n\tcase ScalarNode:\n\t\t// key = value\n\t\treturn te.writeAttribute(w, path[len(path)-1], node)\n\tcase SequenceNode:\n\t\t// Empty arrays should be encoded as [] attributes\n\t\tif len(node.Content) == 0 {\n\t\t\treturn te.writeArrayAttribute(w, path[len(path)-1], node)\n\t\t}\n\n\t\t// If all items are mappings => array of tables; else => array attribute\n\t\tallMaps := true\n\t\tfor _, it := range node.Content {\n\t\t\tif it.Kind != MappingNode {\n\t\t\t\tallMaps = false\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif allMaps {\n\t\t\tkey := path[len(path)-1]\n\t\t\tfor _, it := range node.Content {\n\t\t\t\t// [[key]] then body\n\t\t\t\tif _, err := w.Write([]byte(\"[[\" + key + \"]]\\n\")); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif err := te.encodeMappingBodyWithPath(w, []string{key}, it); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\t\t// Regular array attribute\n\t\treturn te.writeArrayAttribute(w, path[len(path)-1], node)\n\tcase MappingNode:\n\t\t// Inline table if not EncodeSeparate, else emit separate tables/arrays of tables for children under this path\n\t\tif !node.EncodeSeparate {\n\t\t\t// If children contain mappings or arrays of mappings, prefer separate sections\n\t\t\tif te.hasEncodeSeparateChild(node) || te.hasStructuralChildren(node) {\n\t\t\t\treturn te.encodeSeparateMapping(w, path, node)\n\t\t\t}\n\t\t\treturn te.writeInlineTableAttribute(w, path[len(path)-1], node)\n\t\t}\n\t\treturn te.encodeSeparateMapping(w, path, node)\n\tdefault:\n\t\treturn fmt.Errorf(\"unsupported node kind for TOML: %v\", node.Kind)\n\t}\n}\n\nfunc (te *tomlEncoder) writeAttribute(w io.Writer, key string, value *CandidateNode) error {\n\tte.wroteRootAttr = true // Mark that we wrote a root attribute\n\n\t// Write head comment before the attribute\n\tif err := te.writeComment(w, value.HeadComment); err != nil {\n\t\treturn err\n\t}\n\n\t// Write the attribute\n\tline := key + \" = \" + te.formatScalar(value)\n\n\t// Add line comment if present\n\tif value.LineComment != \"\" {\n\t\tlineComment := strings.TrimSpace(value.LineComment)\n\t\tif !strings.HasPrefix(lineComment, \"#\") {\n\t\t\tlineComment = \"# \" + lineComment\n\t\t}\n\t\tline += \" \" + lineComment\n\t}\n\n\t_, err := w.Write([]byte(line + \"\\n\"))\n\treturn err\n}\n\nfunc (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *CandidateNode) error {\n\tte.wroteRootAttr = true // Mark that we wrote a root attribute\n\n\t// Write head comment before the array\n\tif err := te.writeComment(w, seq.HeadComment); err != nil {\n\t\treturn err\n\t}\n\n\t// Handle empty arrays\n\tif len(seq.Content) == 0 {\n\t\tline := key + \" = []\"\n\t\tif seq.LineComment != \"\" {\n\t\t\tlineComment := strings.TrimSpace(seq.LineComment)\n\t\t\tif !strings.HasPrefix(lineComment, \"#\") {\n\t\t\t\tlineComment = \"# \" + lineComment\n\t\t\t}\n\t\t\tline += \" \" + lineComment\n\t\t}\n\t\t_, err := w.Write([]byte(line + \"\\n\"))\n\t\treturn err\n\t}\n\n\t// Check if any array elements have head comments - if so, use multiline format\n\thasElementComments := false\n\tfor _, it := range seq.Content {\n\t\tif it.HeadComment != \"\" {\n\t\t\thasElementComments = true\n\t\t\tbreak\n\t\t}\n\t}\n\n\tif hasElementComments {\n\t\t// Write multiline array format with comments\n\t\tif _, err := w.Write([]byte(key + \" = [\\n\")); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tfor i, it := range seq.Content {\n\t\t\t// Write head comment for this element\n\t\t\tif it.HeadComment != \"\" {\n\t\t\t\tcommentLines := strings.Split(it.HeadComment, \"\\n\")\n\t\t\t\tfor _, commentLine := range commentLines {\n\t\t\t\t\tif strings.TrimSpace(commentLine) != \"\" {\n\t\t\t\t\t\tif !strings.HasPrefix(strings.TrimSpace(commentLine), \"#\") {\n\t\t\t\t\t\t\tcommentLine = \"# \" + commentLine\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif _, err := w.Write([]byte(\" \" + commentLine + \"\\n\")); err != nil {\n\t\t\t\t\t\t\treturn err\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\t// Write the element value\n\t\t\tvar itemStr string\n\t\t\tswitch it.Kind {\n\t\t\tcase ScalarNode:\n\t\t\t\titemStr = te.formatScalar(it)\n\t\t\tcase SequenceNode:\n\t\t\t\tnested, err := te.sequenceToInlineArray(it)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\titemStr = nested\n\t\t\tcase MappingNode:\n\t\t\t\tinline, err := te.mappingToInlineTable(it)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\titemStr = inline\n\t\t\tcase AliasNode:\n\t\t\t\treturn fmt.Errorf(\"aliases are not supported in TOML\")\n\t\t\tdefault:\n\t\t\t\treturn fmt.Errorf(\"unsupported array item kind: %v\", it.Kind)\n\t\t\t}\n\n\t\t\t// Always add trailing comma in multiline arrays\n\t\t\titemStr += \",\"\n\n\t\t\tif _, err := w.Write([]byte(\" \" + itemStr + \"\\n\")); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\t// Add blank line between elements (except after the last one)\n\t\t\tif i < len(seq.Content)-1 {\n\t\t\t\tif _, err := w.Write([]byte(\"\\n\")); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif _, err := w.Write([]byte(\"]\\n\")); err != nil {\n\t\t\treturn err\n\t\t}\n\t\treturn nil\n\t}\n\n\t// Join scalars or nested arrays recursively into TOML array syntax\n\titems := make([]string, 0, len(seq.Content))\n\tfor _, it := range seq.Content {\n\t\tswitch it.Kind {\n\t\tcase ScalarNode:\n\t\t\titems = append(items, te.formatScalar(it))\n\t\tcase SequenceNode:\n\t\t\t// Nested arrays: encode inline\n\t\t\tnested, err := te.sequenceToInlineArray(it)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\titems = append(items, nested)\n\t\tcase MappingNode:\n\t\t\t// Inline table inside array\n\t\t\tinline, err := te.mappingToInlineTable(it)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\titems = append(items, inline)\n\t\tcase AliasNode:\n\t\t\treturn fmt.Errorf(\"aliases are not supported in TOML\")\n\t\tdefault:\n\t\t\treturn fmt.Errorf(\"unsupported array item kind: %v\", it.Kind)\n\t\t}\n\t}\n\n\tline := key + \" = [\" + strings.Join(items, \", \") + \"]\"\n\n\t// Add line comment if present\n\tif seq.LineComment != \"\" {\n\t\tlineComment := strings.TrimSpace(seq.LineComment)\n\t\tif !strings.HasPrefix(lineComment, \"#\") {\n\t\t\tlineComment = \"# \" + lineComment\n\t\t}\n\t\tline += \" \" + lineComment\n\t}\n\n\t_, err := w.Write([]byte(line + \"\\n\"))\n\treturn err\n}\n\nfunc (te *tomlEncoder) sequenceToInlineArray(seq *CandidateNode) (string, error) {\n\titems := make([]string, 0, len(seq.Content))\n\tfor _, it := range seq.Content {\n\t\tswitch it.Kind {\n\t\tcase ScalarNode:\n\t\t\titems = append(items, te.formatScalar(it))\n\t\tcase SequenceNode:\n\t\t\tnested, err := te.sequenceToInlineArray(it)\n\t\t\tif err != nil {\n\t\t\t\treturn \"\", err\n\t\t\t}\n\t\t\titems = append(items, nested)\n\t\tcase MappingNode:\n\t\t\tinline, err := te.mappingToInlineTable(it)\n\t\t\tif err != nil {\n\t\t\t\treturn \"\", err\n\t\t\t}\n\t\t\titems = append(items, inline)\n\t\tdefault:\n\t\t\treturn \"\", fmt.Errorf(\"unsupported array item kind: %v\", it.Kind)\n\t\t}\n\t}\n\treturn \"[\" + strings.Join(items, \", \") + \"]\", nil\n}\n\nfunc (te *tomlEncoder) mappingToInlineTable(m *CandidateNode) (string, error) {\n\t// key = { a = 1, b = \"x\" }\n\tparts := make([]string, 0, len(m.Content)/2)\n\tfor i := 0; i < len(m.Content); i += 2 {\n\t\tk := m.Content[i].Value\n\t\tv := m.Content[i+1]\n\t\tswitch v.Kind {\n\t\tcase ScalarNode:\n\t\t\tparts = append(parts, fmt.Sprintf(\"%s = %s\", k, te.formatScalar(v)))\n\t\tcase SequenceNode:\n\t\t\t// inline array in inline table\n\t\t\tarr, err := te.sequenceToInlineArray(v)\n\t\t\tif err != nil {\n\t\t\t\treturn \"\", err\n\t\t\t}\n\t\t\tparts = append(parts, fmt.Sprintf(\"%s = %s\", k, arr))\n\t\tcase MappingNode:\n\t\t\t// nested inline table\n\t\t\tinline, err := te.mappingToInlineTable(v)\n\t\t\tif err != nil {\n\t\t\t\treturn \"\", err\n\t\t\t}\n\t\t\tparts = append(parts, fmt.Sprintf(\"%s = %s\", k, inline))\n\t\tdefault:\n\t\t\treturn \"\", fmt.Errorf(\"unsupported inline table value kind: %v\", v.Kind)\n\t\t}\n\t}\n\treturn \"{ \" + strings.Join(parts, \", \") + \" }\", nil\n}\n\nfunc (te *tomlEncoder) writeInlineTableAttribute(w io.Writer, key string, m *CandidateNode) error {\n\tinline, err := te.mappingToInlineTable(m)\n\tif err != nil {\n\t\treturn err\n\t}\n\t_, err = w.Write([]byte(key + \" = \" + inline + \"\\n\"))\n\treturn err\n}\n\nfunc (te *tomlEncoder) writeTableHeader(w io.Writer, path []string, m *CandidateNode) error {\n\t// Add blank line before table header (or before comment if present) if we wrote root attributes\n\tneedsBlankLine := te.wroteRootAttr\n\tif needsBlankLine {\n\t\tif _, err := w.Write([]byte(\"\\n\")); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tte.wroteRootAttr = false // Only add once\n\t}\n\n\t// Write head comment before the table header\n\tif m.HeadComment != \"\" {\n\t\tif err := te.writeComment(w, m.HeadComment); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\t// Write table header [a.b.c]\n\theader := \"[\" + strings.Join(path, \".\") + \"]\\n\"\n\t_, err := w.Write([]byte(header))\n\treturn err\n}\n\n// encodeSeparateMapping handles a mapping that should be encoded as table sections.\n// It emits the table header for this mapping if it has any content, then processes children.\nfunc (te *tomlEncoder) encodeSeparateMapping(w io.Writer, path []string, m *CandidateNode) error {\n\t// Check if this mapping has any non-mapping, non-array-of-tables children (i.e., attributes)\n\thasAttrs := false\n\tfor i := 0; i < len(m.Content); i += 2 {\n\t\tv := m.Content[i+1]\n\t\tif v.Kind == ScalarNode {\n\t\t\thasAttrs = true\n\t\t\tbreak\n\t\t}\n\t\tif v.Kind == SequenceNode {\n\t\t\t// Check if it's NOT an array of tables\n\t\t\tallMaps := true\n\t\t\tfor _, it := range v.Content {\n\t\t\t\tif it.Kind != MappingNode {\n\t\t\t\t\tallMaps = false\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !allMaps {\n\t\t\t\thasAttrs = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\n\t// If there are attributes or if the mapping is empty, emit the table header\n\tif hasAttrs || len(m.Content) == 0 {\n\t\tif err := te.writeTableHeader(w, path, m); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := te.encodeMappingBodyWithPath(w, path, m); err != nil {\n\t\t\treturn err\n\t\t}\n\t\treturn nil\n\t}\n\n\t// No attributes, just nested structures - process children\n\tfor i := 0; i < len(m.Content); i += 2 {\n\t\tk := m.Content[i].Value\n\t\tv := m.Content[i+1]\n\t\tswitch v.Kind {\n\t\tcase MappingNode:\n\t\t\t// Emit [path.k]\n\t\t\tnewPath := append(append([]string{}, path...), k)\n\t\t\tif err := te.writeTableHeader(w, newPath, v); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif err := te.encodeMappingBodyWithPath(w, newPath, v); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase SequenceNode:\n\t\t\t// If sequence of maps, emit [[path.k]] per element\n\t\t\tallMaps := true\n\t\t\tfor _, it := range v.Content {\n\t\t\t\tif it.Kind != MappingNode {\n\t\t\t\t\tallMaps = false\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tif allMaps {\n\t\t\t\tkey := strings.Join(append(append([]string{}, path...), k), \".\")\n\t\t\t\tfor _, it := range v.Content {\n\t\t\t\t\tif _, err := w.Write([]byte(\"[[\" + key + \"]]\\n\")); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tif err := te.encodeMappingBodyWithPath(w, append(append([]string{}, path...), k), it); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\t// Regular array attribute under the current table path\n\t\t\t\tif err := te.writeArrayAttribute(w, k, v); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\tcase ScalarNode:\n\t\t\t// Attributes directly under the current table path\n\t\t\tif err := te.writeAttribute(w, k, v); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (te *tomlEncoder) hasEncodeSeparateChild(m *CandidateNode) bool {\n\tfor i := 0; i < len(m.Content); i += 2 {\n\t\tv := m.Content[i+1]\n\t\tif v.Kind == MappingNode && v.EncodeSeparate {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (te *tomlEncoder) hasStructuralChildren(m *CandidateNode) bool {\n\tfor i := 0; i < len(m.Content); i += 2 {\n\t\tv := m.Content[i+1]\n\t\t// Only consider it structural if mapping has EncodeSeparate or is non-empty\n\t\tif v.Kind == MappingNode && v.EncodeSeparate {\n\t\t\treturn true\n\t\t}\n\t\tif v.Kind == SequenceNode {\n\t\t\tallMaps := true\n\t\t\tfor _, it := range v.Content {\n\t\t\t\tif it.Kind != MappingNode {\n\t\t\t\t\tallMaps = false\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tif allMaps {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n\n// encodeMappingBodyWithPath encodes attributes and nested arrays of tables using full dotted path context\nfunc (te *tomlEncoder) encodeMappingBodyWithPath(w io.Writer, path []string, m *CandidateNode) error {\n\t// First, attributes (scalars and non-map arrays)\n\tfor i := 0; i < len(m.Content); i += 2 {\n\t\tk := m.Content[i].Value\n\t\tv := m.Content[i+1]\n\t\tswitch v.Kind {\n\t\tcase ScalarNode:\n\t\t\tif err := te.writeAttribute(w, k, v); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\tcase SequenceNode:\n\t\t\tallMaps := true\n\t\t\tfor _, it := range v.Content {\n\t\t\t\tif it.Kind != MappingNode {\n\t\t\t\t\tallMaps = false\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !allMaps {\n\t\t\t\tif err := te.writeArrayAttribute(w, k, v); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// Then, nested arrays of tables with full path\n\tfor i := 0; i < len(m.Content); i += 2 {\n\t\tk := m.Content[i].Value\n\t\tv := m.Content[i+1]\n\t\tif v.Kind == SequenceNode {\n\t\t\tallMaps := true\n\t\t\tfor _, it := range v.Content {\n\t\t\t\tif it.Kind != MappingNode {\n\t\t\t\t\tallMaps = false\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tif allMaps {\n\t\t\t\tdotted := strings.Join(append(append([]string{}, path...), k), \".\")\n\t\t\t\tfor _, it := range v.Content {\n\t\t\t\t\tif _, err := w.Write([]byte(\"[[\" + dotted + \"]]\\n\")); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tif err := te.encodeMappingBodyWithPath(w, append(append([]string{}, path...), k), it); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\t// Finally, child mappings that are not marked EncodeSeparate get inlined as attributes\n\tfor i := 0; i < len(m.Content); i += 2 {\n\t\tk := m.Content[i].Value\n\t\tv := m.Content[i+1]\n\t\tif v.Kind == MappingNode && !v.EncodeSeparate {\n\t\t\tif err := te.writeInlineTableAttribute(w, k, v); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\n// colorizeToml applies syntax highlighting to TOML output using fatih/color\nfunc (te *tomlEncoder) colorizeToml(input []byte) []byte {\n\ttoml := string(input)\n\tresult := strings.Builder{}\n\n\t// Force color output (don't check for TTY)\n\tcolor.NoColor = false\n\n\t// Create color functions for different token types\n\t// Use EnableColor() to ensure colors work even when NO_COLOR env is set\n\tcommentColorObj := color.New(color.FgHiBlack)\n\tcommentColorObj.EnableColor()\n\tstringColorObj := color.New(color.FgGreen)\n\tstringColorObj.EnableColor()\n\tnumberColorObj := color.New(color.FgHiMagenta)\n\tnumberColorObj.EnableColor()\n\tkeyColorObj := color.New(color.FgCyan)\n\tkeyColorObj.EnableColor()\n\tboolColorObj := color.New(color.FgHiMagenta)\n\tboolColorObj.EnableColor()\n\tsectionColorObj := color.New(color.FgYellow, color.Bold)\n\tsectionColorObj.EnableColor()\n\n\tcommentColor := commentColorObj.SprintFunc()\n\tstringColor := stringColorObj.SprintFunc()\n\tnumberColor := numberColorObj.SprintFunc()\n\tkeyColor := keyColorObj.SprintFunc()\n\tboolColor := boolColorObj.SprintFunc()\n\tsectionColor := sectionColorObj.SprintFunc()\n\n\t// Simple tokenization for TOML colouring\n\ti := 0\n\tfor i < len(toml) {\n\t\tch := toml[i]\n\n\t\t// Comments - from # to end of line\n\t\tif ch == '#' {\n\t\t\tend := i\n\t\t\tfor end < len(toml) && toml[end] != '\\n' {\n\t\t\t\tend++\n\t\t\t}\n\t\t\tresult.WriteString(commentColor(toml[i:end]))\n\t\t\ti = end\n\t\t\tcontinue\n\t\t}\n\n\t\t// Table sections - [section] or [[array]]\n\t\t// Only treat '[' as a table section if it appears at the start of the line\n\t\t// (possibly after whitespace). This avoids mis-colouring inline arrays like\n\t\t// \"ports = [8000, 8001]\" as table sections.\n\t\tif ch == '[' {\n\t\t\tisSectionHeader := true\n\t\t\tif i > 0 {\n\t\t\t\tisSectionHeader = false\n\t\t\t\tj := i - 1\n\t\t\t\tfor j >= 0 && toml[j] != '\\n' {\n\t\t\t\t\tif toml[j] != ' ' && toml[j] != '\\t' && toml[j] != '\\r' {\n\t\t\t\t\t\t// Found a non-whitespace character before this '[' on the same line,\n\t\t\t\t\t\t// so this is not a table header.\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t\tj--\n\t\t\t\t}\n\t\t\t\tif j < 0 || toml[j] == '\\n' {\n\t\t\t\t\t// Reached the start of the string or a newline without encountering\n\t\t\t\t\t// any non-whitespace, so '[' is at the logical start of the line.\n\t\t\t\t\tisSectionHeader = true\n\t\t\t\t}\n\t\t\t}\n\t\t\tif isSectionHeader {\n\t\t\t\tend := i + 1\n\t\t\t\t// Check for [[\n\t\t\t\tif end < len(toml) && toml[end] == '[' {\n\t\t\t\t\tend++\n\t\t\t\t}\n\t\t\t\t// Find closing ]\n\t\t\t\tfor end < len(toml) && toml[end] != ']' {\n\t\t\t\t\tend++\n\t\t\t\t}\n\t\t\t\t// Include closing ]\n\t\t\t\tif end < len(toml) {\n\t\t\t\t\tend++\n\t\t\t\t\t// Check for ]]\n\t\t\t\t\tif end < len(toml) && toml[end] == ']' {\n\t\t\t\t\t\tend++\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tresult.WriteString(sectionColor(toml[i:end]))\n\t\t\t\ti = end\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\t// Strings - quoted text (double or single quotes)\n\t\tif ch == '\"' || ch == '\\'' {\n\t\t\tquote := ch\n\t\t\tend := i + 1\n\t\t\tfor end < len(toml) {\n\t\t\t\tif toml[end] == quote {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tif toml[end] == '\\\\' && end+1 < len(toml) {\n\t\t\t\t\t// Skip the backslash and the escaped character\n\t\t\t\t\tend += 2\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tend++\n\t\t\t}\n\t\t\tif end < len(toml) {\n\t\t\t\tend++ // include closing quote\n\t\t\t}\n\t\t\tresult.WriteString(stringColor(toml[i:end]))\n\t\t\ti = end\n\t\t\tcontinue\n\t\t}\n\n\t\t// Numbers - sequences of digits, possibly with decimal point or minus\n\t\tif (ch >= '0' && ch <= '9') || (ch == '-' && i+1 < len(toml) && toml[i+1] >= '0' && toml[i+1] <= '9') {\n\t\t\tend := i\n\t\t\tif ch == '-' {\n\t\t\t\tend++\n\t\t\t}\n\t\t\tfor end < len(toml) {\n\t\t\t\tc := toml[end]\n\t\t\t\tif (c >= '0' && c <= '9') || c == '.' || c == 'e' || c == 'E' {\n\t\t\t\t\tend++\n\t\t\t\t} else if (c == '+' || c == '-') && end > 0 && (toml[end-1] == 'e' || toml[end-1] == 'E') {\n\t\t\t\t\t// Only allow + or - immediately after 'e' or 'E' for scientific notation\n\t\t\t\t\tend++\n\t\t\t\t} else {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tresult.WriteString(numberColor(toml[i:end]))\n\t\t\ti = end\n\t\t\tcontinue\n\t\t}\n\n\t\t// Identifiers/keys - alphanumeric + underscore + dash\n\t\tif (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '_' {\n\t\t\tend := i\n\t\t\tfor end < len(toml) && ((toml[end] >= 'a' && toml[end] <= 'z') ||\n\t\t\t\t(toml[end] >= 'A' && toml[end] <= 'Z') ||\n\t\t\t\t(toml[end] >= '0' && toml[end] <= '9') ||\n\t\t\t\ttoml[end] == '_' || toml[end] == '-') {\n\t\t\t\tend++\n\t\t\t}\n\t\t\tident := toml[i:end]\n\n\t\t\t// Check if this is a boolean/null keyword\n\t\t\tswitch ident {\n\t\t\tcase \"true\", \"false\":\n\t\t\t\tresult.WriteString(boolColor(ident))\n\t\t\tdefault:\n\t\t\t\t// Check if followed by = or whitespace then = (it's a key)\n\t\t\t\tj := end\n\t\t\t\tfor j < len(toml) && (toml[j] == ' ' || toml[j] == '\\t') {\n\t\t\t\t\tj++\n\t\t\t\t}\n\t\t\t\tif j < len(toml) && toml[j] == '=' {\n\t\t\t\t\tresult.WriteString(keyColor(ident))\n\t\t\t\t} else {\n\t\t\t\t\tresult.WriteString(ident) // plain text for other identifiers\n\t\t\t\t}\n\t\t\t}\n\t\t\ti = end\n\t\t\tcontinue\n\t\t}\n\n\t\t// Everything else (whitespace, operators, brackets) - no color\n\t\tresult.WriteByte(ch)\n\t\ti++\n\t}\n\n\treturn []byte(result.String())\n}\n" }, { "path": "pkg/yqlib/encoder_uri.go", "content": "//go:build !yq_nouri\n\npackage yqlib\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"net/url\"\n)\n\ntype uriEncoder struct {\n}\n\nfunc NewUriEncoder() Encoder {\n\treturn &uriEncoder{}\n}\n\nfunc (e *uriEncoder) CanHandleAliases() bool {\n\treturn false\n}\n\nfunc (e *uriEncoder) PrintDocumentSeparator(_ io.Writer) error {\n\treturn nil\n}\n\nfunc (e *uriEncoder) PrintLeadingContent(_ io.Writer, _ string) error {\n\treturn nil\n}\n\nfunc (e *uriEncoder) Encode(writer io.Writer, node *CandidateNode) error {\n\tif node.guessTagFromCustomType() != \"!!str\" {\n\t\treturn fmt.Errorf(\"cannot encode %v as URI, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string\", node.Tag)\n\t}\n\t_, err := writer.Write([]byte(url.QueryEscape(node.Value)))\n\treturn err\n}\n" }, { "path": "pkg/yqlib/encoder_xml.go", "content": "//go:build !yq_noxml\n\npackage yqlib\n\nimport (\n\t\"encoding/xml\"\n\t\"fmt\"\n\t\"io\"\n\t\"regexp\"\n\t\"strings\"\n)\n\ntype xmlEncoder struct {\n\tindentString string\n\twriter io.Writer\n\tprefs XmlPreferences\n\tleadingContent string\n}\n\nfunc NewXMLEncoder(prefs XmlPreferences) Encoder {\n\tvar indentString = \"\"\n\n\tfor index := 0; index < prefs.Indent; index++ {\n\t\tindentString = indentString + \" \"\n\t}\n\treturn &xmlEncoder{indentString, nil, prefs, \"\"}\n}\n\nfunc (e *xmlEncoder) CanHandleAliases() bool {\n\treturn false\n}\n\nfunc (e *xmlEncoder) PrintDocumentSeparator(_ io.Writer) error {\n\treturn nil\n}\n\nfunc (e *xmlEncoder) PrintLeadingContent(_ io.Writer, content string) error {\n\te.leadingContent = content\n\treturn nil\n}\n\nfunc (e *xmlEncoder) Encode(writer io.Writer, node *CandidateNode) error {\n\tencoder := xml.NewEncoder(writer)\n\t// hack so we can manually add newlines to procInst and directives\n\te.writer = writer\n\tencoder.Indent(\"\", e.indentString)\n\tvar newLine xml.CharData = []byte(\"\\n\")\n\n\tif node.Tag == \"!!map\" {\n\t\t// make sure processing instructions are encoded first\n\t\tfor i := 0; i < len(node.Content); i += 2 {\n\t\t\tkey := node.Content[i]\n\t\t\tvalue := node.Content[i+1]\n\n\t\t\tif key.Value == (e.prefs.ProcInstPrefix + \"xml\") {\n\t\t\t\tname := strings.Replace(key.Value, e.prefs.ProcInstPrefix, \"\", 1)\n\t\t\t\tprocInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)}\n\t\t\t\tif err := encoder.EncodeToken(procInst); err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tif _, err := e.writer.Write([]byte(\"\\n\")); err != nil {\n\t\t\t\t\tlog.Warning(\"Unable to write newline, skipping: %w\", err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tif e.leadingContent != \"\" {\n\n\t\t// remove first and last newlines if present\n\t\terr := e.encodeComment(encoder, e.leadingContent)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\terr = encoder.EncodeToken(newLine)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tswitch node.Kind {\n\tcase MappingNode:\n\t\terr := e.encodeTopLevelMap(encoder, node)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\tcase ScalarNode:\n\t\tvar charData xml.CharData = []byte(node.Value)\n\t\terr := encoder.EncodeToken(charData)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\treturn encoder.Flush()\n\tdefault:\n\t\treturn fmt.Errorf(\"cannot encode %v to XML - only maps can be encoded\", node.Tag)\n\t}\n\n\treturn encoder.EncodeToken(newLine)\n\n}\n\nfunc (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *CandidateNode) error {\n\terr := e.encodeComment(encoder, headAndLineComment(node))\n\tif err != nil {\n\t\treturn err\n\t}\n\tfor i := 0; i < len(node.Content); i += 2 {\n\t\tkey := node.Content[i]\n\t\tvalue := node.Content[i+1]\n\n\t\tstart := xml.StartElement{Name: xml.Name{Local: key.Value}}\n\t\tlog.Debugf(\"comments of key %v\", key.Value)\n\t\terr := e.encodeComment(encoder, headAndLineComment(key))\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif headAndLineComment(key) != \"\" {\n\t\t\tvar newLine xml.CharData = []byte(\"\\n\")\n\t\t\terr = encoder.EncodeToken(newLine)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\tif key.Value == (e.prefs.ProcInstPrefix + \"xml\") { //nolint\n\t\t\t// dont double process these.\n\t\t} else if strings.HasPrefix(key.Value, e.prefs.ProcInstPrefix) {\n\t\t\tname := strings.Replace(key.Value, e.prefs.ProcInstPrefix, \"\", 1)\n\t\t\tprocInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)}\n\t\t\tif err := encoder.EncodeToken(procInst); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif _, err := e.writer.Write([]byte(\"\\n\")); err != nil {\n\t\t\t\tlog.Warning(\"Unable to write newline, skipping: %w\", err)\n\t\t\t}\n\t\t} else if key.Value == e.prefs.DirectiveName {\n\t\t\tvar directive xml.Directive = []byte(value.Value)\n\t\t\tif err := encoder.EncodeToken(directive); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif _, err := e.writer.Write([]byte(\"\\n\")); err != nil {\n\t\t\t\tlog.Warning(\"Unable to write newline, skipping: %w\", err)\n\t\t\t}\n\t\t} else {\n\n\t\t\tlog.Debugf(\"recursing\")\n\n\t\t\terr = e.doEncode(encoder, value, start)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\terr = e.encodeComment(encoder, footComment(key))\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn e.encodeComment(encoder, footComment(node))\n}\n\nfunc (e *xmlEncoder) encodeStart(encoder *xml.Encoder, node *CandidateNode, start xml.StartElement) error {\n\terr := encoder.EncodeToken(start)\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn e.encodeComment(encoder, headComment(node))\n}\n\nfunc (e *xmlEncoder) encodeEnd(encoder *xml.Encoder, node *CandidateNode, start xml.StartElement) error {\n\terr := encoder.EncodeToken(start.End())\n\tif err != nil {\n\t\treturn err\n\t}\n\treturn e.encodeComment(encoder, footComment(node))\n}\n\nfunc (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *CandidateNode, start xml.StartElement) error {\n\tswitch node.Kind {\n\tcase MappingNode:\n\t\treturn e.encodeMap(encoder, node, start)\n\tcase SequenceNode:\n\t\treturn e.encodeArray(encoder, node, start)\n\tcase ScalarNode:\n\t\terr := e.encodeStart(encoder, node, start)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tvar charData xml.CharData = []byte(node.Value)\n\t\terr = encoder.EncodeToken(charData)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tif err = e.encodeComment(encoder, lineComment(node)); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\treturn e.encodeEnd(encoder, node, start)\n\t}\n\treturn fmt.Errorf(\"unsupported type %v\", node.Tag)\n}\n\nvar xmlEncodeMultilineCommentRegex = regexp.MustCompile(`(^|\\n) *# ?(.*)`)\nvar xmlEncodeSingleLineCommentRegex = regexp.MustCompile(`^\\s*#(.*)\\n?`)\nvar chompRegexp = regexp.MustCompile(`\\n$`)\n\nfunc (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) error {\n\tif commentStr != \"\" {\n\t\tlog.Debugf(\"got comment [%v]\", commentStr)\n\t\t// multi line string\n\t\tif len(commentStr) > 2 && strings.Contains(commentStr[1:len(commentStr)-1], \"\\n\") {\n\t\t\tcommentStr = chompRegexp.ReplaceAllString(commentStr, \"\")\n\t\t\tlog.Debugf(\"chompRegexp [%v]\", commentStr)\n\t\t\tcommentStr = xmlEncodeMultilineCommentRegex.ReplaceAllString(commentStr, \"$1$2\")\n\t\t\tlog.Debugf(\"processed multiline [%v]\", commentStr)\n\t\t\t// if the first line is non blank, add a space\n\t\t\tif commentStr[0] != '\\n' && commentStr[0] != ' ' {\n\t\t\t\tcommentStr = \" \" + commentStr\n\t\t\t}\n\n\t\t} else {\n\t\t\tcommentStr = xmlEncodeSingleLineCommentRegex.ReplaceAllString(commentStr, \"$1\")\n\t\t}\n\n\t\tif !strings.HasSuffix(commentStr, \" \") && !strings.HasSuffix(commentStr, \"\\n\") {\n\t\t\tcommentStr = commentStr + \" \"\n\t\t\tlog.Debugf(\"added suffix [%v]\", commentStr)\n\t\t}\n\t\tlog.Debugf(\"encoding comment [%v]\", commentStr)\n\n\t\tvar comment xml.Comment = []byte(commentStr)\n\t\terr := encoder.EncodeToken(comment)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *CandidateNode, start xml.StartElement) error {\n\n\tif err := e.encodeComment(encoder, headAndLineComment(node)); err != nil {\n\t\treturn err\n\t}\n\n\tfor i := 0; i < len(node.Content); i++ {\n\t\tvalue := node.Content[i]\n\t\tif err := e.doEncode(encoder, value, start.Copy()); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn e.encodeComment(encoder, footComment(node))\n}\n\nfunc (e *xmlEncoder) isAttribute(name string) bool {\n\treturn strings.HasPrefix(name, e.prefs.AttributePrefix) &&\n\t\tname != e.prefs.ContentName &&\n\t\tname != e.prefs.DirectiveName &&\n\t\t!strings.HasPrefix(name, e.prefs.ProcInstPrefix)\n}\n\nfunc (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *CandidateNode, start xml.StartElement) error {\n\tlog.Debug(\"its a map\")\n\n\t//first find all the attributes and put them on the start token\n\tfor i := 0; i < len(node.Content); i += 2 {\n\t\tkey := node.Content[i]\n\t\tvalue := node.Content[i+1]\n\n\t\tif e.isAttribute(key.Value) {\n\t\t\tif value.Kind == ScalarNode {\n\t\t\t\tattributeName := strings.Replace(key.Value, e.prefs.AttributePrefix, \"\", 1)\n\t\t\t\tstart.Attr = append(start.Attr, xml.Attr{Name: xml.Name{Local: attributeName}, Value: value.Value})\n\t\t\t} else {\n\t\t\t\treturn fmt.Errorf(\"cannot use %v as attribute, only scalars are supported\", value.Tag)\n\t\t\t}\n\t\t}\n\t}\n\n\terr := e.encodeStart(encoder, node, start)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t//now we encode non attribute tokens\n\tfor i := 0; i < len(node.Content); i += 2 {\n\t\tkey := node.Content[i]\n\t\tvalue := node.Content[i+1]\n\n\t\terr := e.encodeComment(encoder, headAndLineComment(key))\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif strings.HasPrefix(key.Value, e.prefs.ProcInstPrefix) {\n\t\t\tname := strings.Replace(key.Value, e.prefs.ProcInstPrefix, \"\", 1)\n\t\t\tprocInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)}\n\t\t\tif err := encoder.EncodeToken(procInst); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t} else if key.Value == e.prefs.DirectiveName {\n\t\t\tvar directive xml.Directive = []byte(value.Value)\n\t\t\tif err := encoder.EncodeToken(directive); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t} else if key.Value == e.prefs.ContentName {\n\t\t\t// directly encode the contents\n\t\t\terr = e.encodeComment(encoder, headAndLineComment(value))\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tvar charData xml.CharData = []byte(value.Value)\n\t\t\terr = encoder.EncodeToken(charData)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\terr = e.encodeComment(encoder, footComment(value))\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t} else if !e.isAttribute(key.Value) {\n\t\t\tstart := xml.StartElement{Name: xml.Name{Local: key.Value}}\n\t\t\terr := e.doEncode(encoder, value, start)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\terr = e.encodeComment(encoder, footComment(key))\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn e.encodeEnd(encoder, node, start)\n}\n" }, { "path": "pkg/yqlib/encoder_yaml.go", "content": "package yqlib\n\nimport (\n\t\"bytes\"\n\t\"io\"\n\t\"strings\"\n\n\t\"go.yaml.in/yaml/v4\"\n)\n\ntype yamlEncoder struct {\n\tprefs YamlPreferences\n}\n\nfunc NewYamlEncoder(prefs YamlPreferences) Encoder {\n\treturn &yamlEncoder{prefs}\n}\n\nfunc (ye *yamlEncoder) CanHandleAliases() bool {\n\treturn true\n}\n\nfunc (ye *yamlEncoder) PrintDocumentSeparator(writer io.Writer) error {\n\treturn PrintYAMLDocumentSeparator(writer, ye.prefs.PrintDocSeparators)\n}\n\nfunc (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) error {\n\treturn PrintYAMLLeadingContent(writer, content, ye.prefs.PrintDocSeparators, ye.prefs.ColorsEnabled)\n}\n\nfunc (ye *yamlEncoder) Encode(writer io.Writer, node *CandidateNode) error {\n\tlog.Debug(\"encoderYaml - going to print %v\", NodeToString(node))\n\t// Detect line ending style from LeadingContent\n\tlineEnding := \"\\n\"\n\tif strings.Contains(node.LeadingContent, \"\\r\\n\") {\n\t\tlineEnding = \"\\r\\n\"\n\t}\n\tif node.Kind == ScalarNode && ye.prefs.UnwrapScalar {\n\t\tvalueToPrint := node.Value\n\t\tif node.LeadingContent == \"\" || valueToPrint != \"\" {\n\t\t\tvalueToPrint = valueToPrint + lineEnding\n\t\t}\n\t\treturn writeString(writer, valueToPrint)\n\t}\n\n\tdestination := writer\n\ttempBuffer := bytes.NewBuffer(nil)\n\tif ye.prefs.ColorsEnabled {\n\t\tdestination = tempBuffer\n\t}\n\n\tvar encoder = yaml.NewEncoder(destination)\n\n\tencoder.SetIndent(ye.prefs.Indent)\n\tif ye.prefs.CompactSequenceIndent {\n\t\tencoder.CompactSeqIndent()\n\t}\n\n\ttarget, err := node.MarshalYAML()\n\n\tif err != nil {\n\t\treturn err\n\t}\n\n\ttrailingContent := target.FootComment\n\ttarget.FootComment = \"\"\n\n\tif err := encoder.Encode(target); err != nil {\n\t\treturn err\n\t}\n\n\tif err := ye.PrintLeadingContent(destination, trailingContent); err != nil {\n\t\treturn err\n\t}\n\n\tif ye.prefs.ColorsEnabled {\n\t\treturn colorizeAndPrint(tempBuffer.Bytes(), writer)\n\t}\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/expression_parser.go", "content": "package yqlib\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n)\n\ntype ExpressionNode struct {\n\tOperation *Operation\n\tLHS *ExpressionNode\n\tRHS *ExpressionNode\n\tParent *ExpressionNode\n}\n\ntype ExpressionParserInterface interface {\n\tParseExpression(expression string) (*ExpressionNode, error)\n}\n\ntype expressionParserImpl struct {\n\tpathTokeniser expressionTokeniser\n\tpathPostFixer expressionPostFixer\n}\n\nfunc newExpressionParser() ExpressionParserInterface {\n\treturn &expressionParserImpl{newParticipleLexer(), newExpressionPostFixer()}\n}\n\nfunc (p *expressionParserImpl) ParseExpression(expression string) (*ExpressionNode, error) {\n\tlog.Debug(\"Parsing expression: [%v]\", expression)\n\ttokens, err := p.pathTokeniser.Tokenise(expression)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tvar Operations []*Operation\n\tOperations, err = p.pathPostFixer.ConvertToPostfix(tokens)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn p.createExpressionTree(Operations)\n}\n\nfunc (p *expressionParserImpl) createExpressionTree(postFixPath []*Operation) (*ExpressionNode, error) {\n\tvar stack = make([]*ExpressionNode, 0)\n\n\tif len(postFixPath) == 0 {\n\t\treturn nil, nil\n\t}\n\n\tfor _, Operation := range postFixPath {\n\t\tvar newNode = ExpressionNode{Operation: Operation}\n\t\tlog.Debugf(\"pathTree %v \", Operation.toString())\n\t\tif Operation.OperationType.NumArgs > 0 {\n\t\t\tnumArgs := Operation.OperationType.NumArgs\n\t\t\tswitch numArgs {\n\t\t\tcase 1:\n\t\t\t\tif len(stack) < 1 {\n\t\t\t\t\t// Allow certain unary ops to accept zero args by interpreting missing RHS as nil\n\t\t\t\t\t// TODO - make this more general on OperationType\n\t\t\t\t\tif Operation.OperationType == firstOpType {\n\t\t\t\t\t\t// no RHS provided; proceed without popping\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t\treturn nil, fmt.Errorf(\"'%v' expects 1 arg but received none\", strings.TrimSpace(Operation.StringValue))\n\t\t\t\t}\n\t\t\t\tremaining, rhs := stack[:len(stack)-1], stack[len(stack)-1]\n\t\t\t\tnewNode.RHS = rhs\n\t\t\t\trhs.Parent = &newNode\n\t\t\t\tstack = remaining\n\t\t\tcase 2:\n\t\t\t\tif len(stack) < 2 {\n\t\t\t\t\treturn nil, fmt.Errorf(\"'%v' expects 2 args but there is %v\", strings.TrimSpace(Operation.StringValue), len(stack))\n\t\t\t\t}\n\t\t\t\tremaining, lhs, rhs := stack[:len(stack)-2], stack[len(stack)-2], stack[len(stack)-1]\n\t\t\t\tnewNode.LHS = lhs\n\t\t\t\tlhs.Parent = &newNode\n\n\t\t\t\tnewNode.RHS = rhs\n\t\t\t\trhs.Parent = &newNode\n\n\t\t\t\tstack = remaining\n\t\t\t}\n\t\t}\n\t\tstack = append(stack, &newNode)\n\t}\n\tif len(stack) != 1 {\n\t\treturn nil, fmt.Errorf(\"bad expression, please check expression syntax\")\n\t}\n\treturn stack[0], nil\n}\n" }, { "path": "pkg/yqlib/expression_parser_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nfunc getExpressionParser() ExpressionParserInterface {\n\tInitExpressionParser()\n\treturn ExpressionParser\n}\n\nfunc TestParserCreateMapColonOnItsOwn(t *testing.T) {\n\t_, err := getExpressionParser().ParseExpression(\":\")\n\ttest.AssertResultComplex(t, \"':' expects 2 args but there is 0\", err.Error())\n}\n\nfunc TestParserNoMatchingCloseBracket(t *testing.T) {\n\t_, err := getExpressionParser().ParseExpression(\".cat | with(.;.bob\")\n\ttest.AssertResultComplex(t, \"bad expression - probably missing close bracket on WITH\", err.Error())\n}\n\nfunc TestParserNoMatchingCloseCollect(t *testing.T) {\n\t_, err := getExpressionParser().ParseExpression(\"[1,2\")\n\ttest.AssertResultComplex(t, \"bad expression, could not find matching `]`\", err.Error())\n}\nfunc TestParserNoMatchingCloseObjectInCollect(t *testing.T) {\n\t_, err := getExpressionParser().ParseExpression(`[{\"b\": \"c\"]`)\n\ttest.AssertResultComplex(t, \"bad expression, could not find matching `}`\", err.Error())\n}\n\nfunc TestParserNoMatchingCloseInCollect(t *testing.T) {\n\t_, err := getExpressionParser().ParseExpression(`[(.a]`)\n\ttest.AssertResultComplex(t, \"bad expression, could not find matching `)`\", err.Error())\n}\n\nfunc TestParserNoMatchingCloseCollectObject(t *testing.T) {\n\t_, err := getExpressionParser().ParseExpression(`{\"a\": \"b\"`)\n\ttest.AssertResultComplex(t, \"bad expression, could not find matching `}`\", err.Error())\n}\n\nfunc TestParserNoMatchingCloseCollectInCollectObject(t *testing.T) {\n\t_, err := getExpressionParser().ParseExpression(`{\"b\": [1}`)\n\ttest.AssertResultComplex(t, \"bad expression, could not find matching `]`\", err.Error())\n}\n\nfunc TestParserNoMatchingCloseBracketInCollectObject(t *testing.T) {\n\t_, err := getExpressionParser().ParseExpression(`{\"b\": (1}`)\n\ttest.AssertResultComplex(t, \"bad expression, could not find matching `)`\", err.Error())\n}\n\nfunc TestParserNoArgsForTwoArgOp(t *testing.T) {\n\t_, err := getExpressionParser().ParseExpression(\"=\")\n\ttest.AssertResultComplex(t, \"'=' expects 2 args but there is 0\", err.Error())\n}\n\nfunc TestParserOneLhsArgsForTwoArgOp(t *testing.T) {\n\t_, err := getExpressionParser().ParseExpression(\".a =\")\n\ttest.AssertResultComplex(t, \"'=' expects 2 args but there is 1\", err.Error())\n}\n\nfunc TestParserOneRhsArgsForTwoArgOp(t *testing.T) {\n\t_, err := getExpressionParser().ParseExpression(\"= .a\")\n\ttest.AssertResultComplex(t, \"'=' expects 2 args but there is 1\", err.Error())\n}\n\nfunc TestParserTwoArgsForTwoArgOp(t *testing.T) {\n\t_, err := getExpressionParser().ParseExpression(\".a = .b\")\n\ttest.AssertResultComplex(t, nil, err)\n}\n\nfunc TestParserNoArgsForOneArgOp(t *testing.T) {\n\t_, err := getExpressionParser().ParseExpression(\"explode\")\n\ttest.AssertResultComplex(t, \"'explode' expects 1 arg but received none\", err.Error())\n}\n\nfunc TestParserOneArgForOneArgOp(t *testing.T) {\n\t_, err := getExpressionParser().ParseExpression(\"explode(.)\")\n\ttest.AssertResultComplex(t, nil, err)\n}\n\nfunc TestParserExtraArgs(t *testing.T) {\n\t_, err := getExpressionParser().ParseExpression(\"sortKeys(.) explode(.)\")\n\ttest.AssertResultComplex(t, \"bad expression, please check expression syntax\", err.Error())\n}\n\nfunc TestParserEmptyExpression(t *testing.T) {\n\t_, err := getExpressionParser().ParseExpression(\"\")\n\ttest.AssertResultComplex(t, nil, err)\n}\n\nfunc TestParserSingleOperation(t *testing.T) {\n\tresult, err := getExpressionParser().ParseExpression(\".\")\n\ttest.AssertResultComplex(t, nil, err)\n\tif result == nil {\n\t\tt.Fatal(\"Expected non-nil result for single operation\")\n\t}\n\tif result.Operation == nil {\n\t\tt.Fatal(\"Expected operation to be set\")\n\t}\n}\n\nfunc TestParserFirstOpWithZeroArgs(t *testing.T) {\n\t// Test the special case where firstOpType can accept zero args\n\tresult, err := getExpressionParser().ParseExpression(\"first\")\n\ttest.AssertResultComplex(t, nil, err)\n\tif result == nil {\n\t\tt.Fatal(\"Expected non-nil result for first operation with zero args\")\n\t}\n}\n\nfunc TestParserInvalidExpressionTree(t *testing.T) {\n\t// This tests the createExpressionTree function with malformed postfix\n\tparser := getExpressionParser().(*expressionParserImpl)\n\n\t// Create invalid postfix operations that would leave more than one item on stack\n\tinvalidOps := []*Operation{\n\t\t{OperationType: &operationType{NumArgs: 0}},\n\t\t{OperationType: &operationType{NumArgs: 0}},\n\t}\n\n\t_, err := parser.createExpressionTree(invalidOps)\n\ttest.AssertResultComplex(t, \"bad expression, please check expression syntax\", err.Error())\n}\n" }, { "path": "pkg/yqlib/expression_postfix.go", "content": "package yqlib\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\n\tlogging \"gopkg.in/op/go-logging.v1\"\n)\n\ntype expressionPostFixer interface {\n\tConvertToPostfix([]*token) ([]*Operation, error)\n}\n\ntype expressionPostFixerImpl struct {\n}\n\nfunc newExpressionPostFixer() expressionPostFixer {\n\treturn &expressionPostFixerImpl{}\n}\n\nfunc popOpToResult(opStack []*token, result []*Operation) ([]*token, []*Operation) {\n\tvar newOp *token\n\topStack, newOp = opStack[0:len(opStack)-1], opStack[len(opStack)-1]\n\tlog.Debugf(\"popped %v from opstack to results\", newOp.toString(true))\n\treturn opStack, append(result, newOp.Operation)\n}\n\nfunc validateNoOpenTokens(token *token) error {\n\tswitch token.TokenType {\n\tcase openCollect:\n\t\treturn fmt.Errorf((\"bad expression, could not find matching `]`\"))\n\tcase openCollectObject:\n\t\treturn fmt.Errorf((\"bad expression, could not find matching `}`\"))\n\tcase openBracket:\n\t\treturn fmt.Errorf((\"bad expression, could not find matching `)`\"))\n\t}\n\treturn nil\n}\n\nfunc (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Operation, error) {\n\tvar result []*Operation\n\t// surround the whole thing with brackets\n\tvar opStack = []*token{{TokenType: openBracket}}\n\tvar tokens = append(infixTokens, &token{TokenType: closeBracket})\n\n\tfor _, currentToken := range tokens {\n\t\tlog.Debugf(\"postfix processing currentToken %v\", currentToken.toString(true))\n\t\tswitch currentToken.TokenType {\n\t\tcase openBracket, openCollect, openCollectObject:\n\t\t\topStack = append(opStack, currentToken)\n\t\t\tlog.Debugf(\"put %v onto the opstack\", currentToken.toString(true))\n\t\tcase closeCollect, closeCollectObject:\n\t\t\tvar opener tokenType = openCollect\n\t\t\tvar collectOperator = collectOpType\n\t\t\tif currentToken.TokenType == closeCollectObject {\n\t\t\t\topener = openCollectObject\n\t\t\t\tcollectOperator = collectObjectOpType\n\t\t\t}\n\n\t\t\tfor len(opStack) > 0 && opStack[len(opStack)-1].TokenType != opener {\n\t\t\t\tmissingClosingTokenErr := validateNoOpenTokens(opStack[len(opStack)-1])\n\t\t\t\tif missingClosingTokenErr != nil {\n\t\t\t\t\treturn nil, missingClosingTokenErr\n\t\t\t\t}\n\t\t\t\topStack, result = popOpToResult(opStack, result)\n\t\t\t}\n\t\t\tif len(opStack) == 0 {\n\t\t\t\treturn nil, errors.New(\"bad path expression, got close collect brackets without matching opening bracket\")\n\t\t\t}\n\t\t\t// now we should have [ as the last element on the opStack, get rid of it\n\t\t\topStack = opStack[0 : len(opStack)-1]\n\t\t\tlog.Debugf(\"deleting open bracket from opstack\")\n\n\t\t\t//and append a collect to the result\n\n\t\t\t// hack - see if there's the optional traverse flag\n\t\t\t// on the close op - move it to the traverse array op\n\t\t\t// allows for .[\"cat\"]?\n\t\t\tprefs := traversePreferences{}\n\t\t\tcloseTokenMatch := currentToken.Match\n\t\t\tif closeTokenMatch[len(closeTokenMatch)-1:] == \"?\" {\n\t\t\t\tprefs.OptionalTraverse = true\n\t\t\t}\n\t\t\tresult = append(result, &Operation{OperationType: collectOperator})\n\t\t\tlog.Debugf(\"put collect onto the result\")\n\t\t\tif opener != openCollect {\n\t\t\t\tresult = append(result, &Operation{OperationType: shortPipeOpType})\n\t\t\t\tlog.Debugf(\"put shortpipe onto the result\")\n\t\t\t}\n\n\t\t\t//traverseArrayCollect is a sneaky op that needs to be included too\n\t\t\t//when closing a ]\n\t\t\tif len(opStack) > 0 && opStack[len(opStack)-1].Operation != nil && opStack[len(opStack)-1].Operation.OperationType == traverseArrayOpType {\n\t\t\t\topStack[len(opStack)-1].Operation.Preferences = prefs\n\t\t\t\topStack, result = popOpToResult(opStack, result)\n\t\t\t}\n\n\t\tcase closeBracket:\n\t\t\tfor len(opStack) > 0 && opStack[len(opStack)-1].TokenType != openBracket {\n\t\t\t\tmissingClosingTokenErr := validateNoOpenTokens(opStack[len(opStack)-1])\n\t\t\t\tif missingClosingTokenErr != nil {\n\t\t\t\t\treturn nil, missingClosingTokenErr\n\t\t\t\t}\n\n\t\t\t\topStack, result = popOpToResult(opStack, result)\n\t\t\t}\n\t\t\tif len(opStack) == 0 {\n\t\t\t\treturn nil, errors.New(\"bad expression, got close brackets without matching opening bracket\")\n\t\t\t}\n\t\t\t// now we should have ( as the last element on the opStack, get rid of it\n\t\t\topStack = opStack[0 : len(opStack)-1]\n\n\t\tdefault:\n\t\t\tvar currentPrecedence = currentToken.Operation.OperationType.Precedence\n\t\t\t// pop off higher precedent operators onto the result\n\t\t\tfor len(opStack) > 0 &&\n\t\t\t\topStack[len(opStack)-1].TokenType == operationToken &&\n\t\t\t\topStack[len(opStack)-1].Operation.OperationType.Precedence > currentPrecedence {\n\t\t\t\topStack, result = popOpToResult(opStack, result)\n\t\t\t}\n\t\t\t// add this operator to the opStack\n\t\t\topStack = append(opStack, currentToken)\n\t\t\tlog.Debugf(\"put %v onto the opstack\", currentToken.toString(true))\n\t\t}\n\t}\n\n\tlog.Debugf(\"opstackLen: %v\", len(opStack))\n\tif len(opStack) > 0 {\n\t\tlog.Debugf(\"opstack:\")\n\t\tfor _, token := range opStack {\n\t\t\tlog.Debugf(\"- %v\", token.toString(true))\n\t\t}\n\n\t\treturn nil, fmt.Errorf(\"bad expression - probably missing close bracket on %v\", opStack[len(opStack)-1].toString(false))\n\t}\n\n\tif log.IsEnabledFor(logging.DEBUG) {\n\t\tlog.Debugf(\"PostFix Result:\")\n\t\tfor _, currentToken := range result {\n\t\t\tlog.Debugf(\"> %v\", currentToken.toString())\n\t\t}\n\t}\n\n\treturn result, nil\n}\n" }, { "path": "pkg/yqlib/expression_processing_test.go", "content": "package yqlib\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nvar variableWithNewLine = `\"cat\n\"`\n\nvar pathTests = []struct {\n\tpath string\n\texpectedTokens []interface{}\n\texpectedPostFix []interface{}\n}{\n\t{\n\t\t`envsubst(ne)`,\n\t\tappend(make([]interface{}, 0), \"ENVSUBST_NO_EMPTY\"),\n\t\tappend(make([]interface{}, 0), \"ENVSUBST_NO_EMPTY\"),\n\t},\n\t{\n\t\t`envsubst(nu)`,\n\t\tappend(make([]interface{}, 0), \"ENVSUBST_NO_UNSET\"),\n\t\tappend(make([]interface{}, 0), \"ENVSUBST_NO_UNSET\"),\n\t},\n\t{\n\t\t`envsubst(nu, ne)`,\n\t\tappend(make([]interface{}, 0), \"ENVSUBST_NO_EMPTY_NO_UNSET\"),\n\t\tappend(make([]interface{}, 0), \"ENVSUBST_NO_EMPTY_NO_UNSET\"),\n\t},\n\t{\n\t\t`envsubst(ne, nu)`,\n\t\tappend(make([]interface{}, 0), \"ENVSUBST_NO_EMPTY_NO_UNSET\"),\n\t\tappend(make([]interface{}, 0), \"ENVSUBST_NO_EMPTY_NO_UNSET\"),\n\t},\n\t{\n\t\t`[.a, .b]`,\n\t\tappend(make([]interface{}, 0), \"[\", \"a\", \"UNION\", \"b\", \"]\"),\n\t\tappend(make([]interface{}, 0), \"a\", \"b\", \"UNION\", \"COLLECT\"),\n\t},\n\t{\n\t\t`.[env(myenv)]`,\n\t\tappend(make([]interface{}, 0), \"SELF\", \"TRAVERSE_ARRAY\", \"[\", \"ENV\", \"]\"),\n\t\tappend(make([]interface{}, 0), \"SELF\", \"ENV\", \"COLLECT\", \"TRAVERSE_ARRAY\"),\n\t},\n\t{\n\t\t`.[\"cat\"].[\"dog\"]`,\n\t\tappend(make([]interface{}, 0), \"SELF\", \"TRAVERSE_ARRAY\", \"[\", \"cat (string)\", \"]\", \"SHORT_PIPE\", \"SELF\", \"TRAVERSE_ARRAY\", \"[\", \"dog (string)\", \"]\"),\n\t\tappend(make([]interface{}, 0), \"SELF\", \"cat (string)\", \"COLLECT\", \"TRAVERSE_ARRAY\", \"SELF\", \"dog (string)\", \"COLLECT\", \"TRAVERSE_ARRAY\", \"SHORT_PIPE\"),\n\t},\n\t{\n\t\t`.[\"cat\"]`,\n\t\tappend(make([]interface{}, 0), \"SELF\", \"TRAVERSE_ARRAY\", \"[\", \"cat (string)\", \"]\"),\n\t\tappend(make([]interface{}, 0), \"SELF\", \"cat (string)\", \"COLLECT\", \"TRAVERSE_ARRAY\"),\n\t},\n\t{\n\t\t\"with(.a;.=3)\",\n\t\tappend(make([]interface{}, 0), \"WITH\", \"(\", \"a\", \"BLOCK\", \"SELF\", \"ASSIGN\", \"3 (int64)\", \")\"),\n\t\tappend(make([]interface{}, 0), \"a\", \"SELF\", \"3 (int64)\", \"ASSIGN\", \"BLOCK\", \"WITH\"),\n\t},\n\t{\n\t\t\"0x12\",\n\t\tappend(make([]interface{}, 0), \"18 (int64)\"),\n\t\tappend(make([]interface{}, 0), \"18 (int64)\"),\n\t},\n\t{\n\t\t\"0X12\",\n\t\tappend(make([]interface{}, 0), \"18 (int64)\"),\n\t\tappend(make([]interface{}, 0), \"18 (int64)\"),\n\t},\n\t{\n\t\t\".a\\n\",\n\t\tappend(make([]interface{}, 0), \"a\"),\n\t\tappend(make([]interface{}, 0), \"a\"),\n\t},\n\t{\n\t\tvariableWithNewLine,\n\t\tappend(make([]interface{}, 0), \"cat\\n (string)\"),\n\t\tappend(make([]interface{}, 0), \"cat\\n (string)\"),\n\t},\n\t{\n\t\t`.[0]`,\n\t\tappend(make([]interface{}, 0), \"SELF\", \"TRAVERSE_ARRAY\", \"[\", \"0 (int64)\", \"]\"),\n\t\tappend(make([]interface{}, 0), \"SELF\", \"0 (int64)\", \"COLLECT\", \"TRAVERSE_ARRAY\"),\n\t},\n\t{\n\t\t`.[0][1]`,\n\t\tappend(make([]interface{}, 0), \"SELF\", \"TRAVERSE_ARRAY\", \"[\", \"0 (int64)\", \"]\", \"TRAVERSE_ARRAY\", \"[\", \"1 (int64)\", \"]\"),\n\t\tappend(make([]interface{}, 0), \"SELF\", \"0 (int64)\", \"COLLECT\", \"TRAVERSE_ARRAY\", \"1 (int64)\", \"COLLECT\", \"TRAVERSE_ARRAY\"),\n\t},\n\t{\n\t\t`\"\\\"\"`,\n\t\tappend(make([]interface{}, 0), \"\\\" (string)\"),\n\t\tappend(make([]interface{}, 0), \"\\\" (string)\"),\n\t},\n\t{\n\t\t`[]|join(\".\")`,\n\t\tappend(make([]interface{}, 0), \"[\", \"EMPTY\", \"]\", \"PIPE\", \"JOIN\", \"(\", \". (string)\", \")\"),\n\t\tappend(make([]interface{}, 0), \"EMPTY\", \"COLLECT\", \". (string)\", \"JOIN\", \"PIPE\"),\n\t},\n\t{\n\t\t`{\"cool\": .b or .c}`,\n\t\tappend(make([]interface{}, 0), \"{\", \"cool (string)\", \"CREATE_MAP\", \"b\", \"OR\", \"c\", \"}\"),\n\t\tappend(make([]interface{}, 0), \"cool (string)\", \"b\", \"c\", \"OR\", \"CREATE_MAP\", \"COLLECT_OBJECT\", \"SHORT_PIPE\"),\n\t},\n\t{\n\t\t`{\"cool\": []|join(\".\")}`,\n\t\tappend(make([]interface{}, 0), \"{\", \"cool (string)\", \"CREATE_MAP\", \"[\", \"EMPTY\", \"]\", \"PIPE\", \"JOIN\", \"(\", \". (string)\", \")\", \"}\"),\n\t\tappend(make([]interface{}, 0), \"cool (string)\", \"EMPTY\", \"COLLECT\", \". (string)\", \"JOIN\", \"PIPE\", \"CREATE_MAP\", \"COLLECT_OBJECT\", \"SHORT_PIPE\"),\n\t},\n\t{\n\t\t`.a as $item ireduce (0; . + $item)`, // note - add code to shuffle reduce to this position for postfix\n\t\tappend(make([]interface{}, 0), \"a\", \"ASSIGN_VARIABLE\", \"GET_VARIABLE\", \"REDUCE\", \"(\", \"0 (int64)\", \"BLOCK\", \"SELF\", \"ADD\", \"GET_VARIABLE\", \")\"),\n\t\tappend(make([]interface{}, 0), \"a\", \"GET_VARIABLE\", \"ASSIGN_VARIABLE\", \"0 (int64)\", \"SELF\", \"GET_VARIABLE\", \"ADD\", \"BLOCK\", \"REDUCE\"),\n\t},\n\t{\n\t\t`.a | .b | .c`,\n\t\tappend(make([]interface{}, 0), \"a\", \"PIPE\", \"b\", \"PIPE\", \"c\"),\n\t\tappend(make([]interface{}, 0), \"a\", \"b\", \"c\", \"PIPE\", \"PIPE\"),\n\t},\n\t{\n\t\t`[]`,\n\t\tappend(make([]interface{}, 0), \"[\", \"EMPTY\", \"]\"),\n\t\tappend(make([]interface{}, 0), \"EMPTY\", \"COLLECT\"),\n\t},\n\t{\n\t\t`{}`,\n\t\tappend(make([]interface{}, 0), \"{\", \"EMPTY\", \"}\"),\n\t\tappend(make([]interface{}, 0), \"EMPTY\", \"COLLECT_OBJECT\", \"SHORT_PIPE\"),\n\t},\n\t{\n\t\t`[{}]`,\n\t\tappend(make([]interface{}, 0), \"[\", \"{\", \"EMPTY\", \"}\", \"]\"),\n\t\tappend(make([]interface{}, 0), \"EMPTY\", \"COLLECT_OBJECT\", \"SHORT_PIPE\", \"COLLECT\"),\n\t},\n\t{\n\t\t`.realnames as $names | $names[\"anon\"]`,\n\t\tappend(make([]interface{}, 0), \"realnames\", \"ASSIGN_VARIABLE\", \"GET_VARIABLE\", \"PIPE\", \"GET_VARIABLE\", \"TRAVERSE_ARRAY\", \"[\", \"anon (string)\", \"]\"),\n\t\tappend(make([]interface{}, 0), \"realnames\", \"GET_VARIABLE\", \"ASSIGN_VARIABLE\", \"GET_VARIABLE\", \"anon (string)\", \"COLLECT\", \"TRAVERSE_ARRAY\", \"PIPE\"),\n\t},\n\t{\n\t\t`.b[.a]`,\n\t\tappend(make([]interface{}, 0), \"b\", \"TRAVERSE_ARRAY\", \"[\", \"a\", \"]\"),\n\t\tappend(make([]interface{}, 0), \"b\", \"a\", \"COLLECT\", \"TRAVERSE_ARRAY\"),\n\t},\n\t{\n\t\t`.b[.a]?`,\n\t\tappend(make([]interface{}, 0), \"b\", \"TRAVERSE_ARRAY\", \"[\", \"a\", \"]\"),\n\t\tappend(make([]interface{}, 0), \"b\", \"a\", \"COLLECT\", \"TRAVERSE_ARRAY\"),\n\t},\n\t{\n\t\t`.[]`,\n\t\tappend(make([]interface{}, 0), \"SELF\", \"TRAVERSE_ARRAY\", \"[\", \"EMPTY\", \"]\"),\n\t\tappend(make([]interface{}, 0), \"SELF\", \"EMPTY\", \"COLLECT\", \"TRAVERSE_ARRAY\"),\n\t},\n\t{\n\t\t`.a[]`,\n\t\tappend(make([]interface{}, 0), \"a\", \"TRAVERSE_ARRAY\", \"[\", \"EMPTY\", \"]\"),\n\t\tappend(make([]interface{}, 0), \"a\", \"EMPTY\", \"COLLECT\", \"TRAVERSE_ARRAY\"),\n\t},\n\t{\n\t\t`.a[]?`,\n\t\tappend(make([]interface{}, 0), \"a\", \"TRAVERSE_ARRAY\", \"[\", \"EMPTY\", \"]\"),\n\t\tappend(make([]interface{}, 0), \"a\", \"EMPTY\", \"COLLECT\", \"TRAVERSE_ARRAY\"),\n\t},\n\t{\n\t\t`.a.[]`,\n\t\tappend(make([]interface{}, 0), \"a\", \"SHORT_PIPE\", \"SELF\", \"TRAVERSE_ARRAY\", \"[\", \"EMPTY\", \"]\"),\n\t\tappend(make([]interface{}, 0), \"a\", \"SELF\", \"EMPTY\", \"COLLECT\", \"TRAVERSE_ARRAY\", \"SHORT_PIPE\"),\n\t},\n\t{\n\t\t`.a[0]`,\n\t\tappend(make([]interface{}, 0), \"a\", \"TRAVERSE_ARRAY\", \"[\", \"0 (int64)\", \"]\"),\n\t\tappend(make([]interface{}, 0), \"a\", \"0 (int64)\", \"COLLECT\", \"TRAVERSE_ARRAY\"),\n\t},\n\t{\n\t\t`.a[0]?`,\n\t\tappend(make([]interface{}, 0), \"a\", \"TRAVERSE_ARRAY\", \"[\", \"0 (int64)\", \"]\"),\n\t\tappend(make([]interface{}, 0), \"a\", \"0 (int64)\", \"COLLECT\", \"TRAVERSE_ARRAY\"),\n\t},\n\t{\n\t\t`.a.[0]`,\n\t\tappend(make([]interface{}, 0), \"a\", \"SHORT_PIPE\", \"SELF\", \"TRAVERSE_ARRAY\", \"[\", \"0 (int64)\", \"]\"),\n\t\tappend(make([]interface{}, 0), \"a\", \"SELF\", \"0 (int64)\", \"COLLECT\", \"TRAVERSE_ARRAY\", \"SHORT_PIPE\"),\n\t},\n\t{\n\t\t`.a[].c`,\n\t\tappend(make([]interface{}, 0), \"a\", \"TRAVERSE_ARRAY\", \"[\", \"EMPTY\", \"]\", \"SHORT_PIPE\", \"c\"),\n\t\tappend(make([]interface{}, 0), \"a\", \"EMPTY\", \"COLLECT\", \"TRAVERSE_ARRAY\", \"c\", \"SHORT_PIPE\"),\n\t},\n\t{\n\t\t`[3]`,\n\t\tappend(make([]interface{}, 0), \"[\", \"3 (int64)\", \"]\"),\n\t\tappend(make([]interface{}, 0), \"3 (int64)\", \"COLLECT\"),\n\t},\n\t{\n\t\t`.key.array + .key.array2`,\n\t\tappend(make([]interface{}, 0), \"key\", \"SHORT_PIPE\", \"array\", \"ADD\", \"key\", \"SHORT_PIPE\", \"array2\"),\n\t\tappend(make([]interface{}, 0), \"key\", \"array\", \"SHORT_PIPE\", \"key\", \"array2\", \"SHORT_PIPE\", \"ADD\"),\n\t},\n\t{\n\t\t`.key.array * .key.array2`,\n\t\tappend(make([]interface{}, 0), \"key\", \"SHORT_PIPE\", \"array\", \"MULTIPLY\", \"key\", \"SHORT_PIPE\", \"array2\"),\n\t\tappend(make([]interface{}, 0), \"key\", \"array\", \"SHORT_PIPE\", \"key\", \"array2\", \"SHORT_PIPE\", \"MULTIPLY\"),\n\t},\n\t{\n\t\t`.key.array // .key.array2`,\n\t\tappend(make([]interface{}, 0), \"key\", \"SHORT_PIPE\", \"array\", \"ALTERNATIVE\", \"key\", \"SHORT_PIPE\", \"array2\"),\n\t\tappend(make([]interface{}, 0), \"key\", \"array\", \"SHORT_PIPE\", \"key\", \"array2\", \"SHORT_PIPE\", \"ALTERNATIVE\"),\n\t},\n\t{\n\t\t`.a | .[].b == \"apple\"`,\n\t\tappend(make([]interface{}, 0), \"a\", \"PIPE\", \"SELF\", \"TRAVERSE_ARRAY\", \"[\", \"EMPTY\", \"]\", \"SHORT_PIPE\", \"b\", \"EQUALS\", \"apple (string)\"),\n\t\tappend(make([]interface{}, 0), \"a\", \"SELF\", \"EMPTY\", \"COLLECT\", \"TRAVERSE_ARRAY\", \"b\", \"SHORT_PIPE\", \"apple (string)\", \"EQUALS\", \"PIPE\"),\n\t},\n\t{\n\t\t`(.a | .[].b) == \"apple\"`,\n\t\tappend(make([]interface{}, 0), \"(\", \"a\", \"PIPE\", \"SELF\", \"TRAVERSE_ARRAY\", \"[\", \"EMPTY\", \"]\", \"SHORT_PIPE\", \"b\", \")\", \"EQUALS\", \"apple (string)\"),\n\t\tappend(make([]interface{}, 0), \"a\", \"SELF\", \"EMPTY\", \"COLLECT\", \"TRAVERSE_ARRAY\", \"b\", \"SHORT_PIPE\", \"PIPE\", \"apple (string)\", \"EQUALS\"),\n\t},\n\t{\n\t\t`.[] | select(. == \"*at\")`,\n\t\tappend(make([]interface{}, 0), \"SELF\", \"TRAVERSE_ARRAY\", \"[\", \"EMPTY\", \"]\", \"PIPE\", \"SELECT\", \"(\", \"SELF\", \"EQUALS\", \"*at (string)\", \")\"),\n\t\tappend(make([]interface{}, 0), \"SELF\", \"EMPTY\", \"COLLECT\", \"TRAVERSE_ARRAY\", \"SELF\", \"*at (string)\", \"EQUALS\", \"SELECT\", \"PIPE\"),\n\t},\n\t{\n\t\t`[true]`,\n\t\tappend(make([]interface{}, 0), \"[\", \"true (bool)\", \"]\"),\n\t\tappend(make([]interface{}, 0), \"true (bool)\", \"COLLECT\"),\n\t},\n\t{\n\t\t`[true, false]`,\n\t\tappend(make([]interface{}, 0), \"[\", \"true (bool)\", \"UNION\", \"false (bool)\", \"]\"),\n\t\tappend(make([]interface{}, 0), \"true (bool)\", \"false (bool)\", \"UNION\", \"COLLECT\"),\n\t},\n\t{\n\t\t`\"mike\": .a`,\n\t\tappend(make([]interface{}, 0), \"mike (string)\", \"CREATE_MAP\", \"a\"),\n\t\tappend(make([]interface{}, 0), \"mike (string)\", \"a\", \"CREATE_MAP\"),\n\t},\n\t{\n\t\t`.a: \"mike\"`,\n\t\tappend(make([]interface{}, 0), \"a\", \"CREATE_MAP\", \"mike (string)\"),\n\t\tappend(make([]interface{}, 0), \"a\", \"mike (string)\", \"CREATE_MAP\"),\n\t},\n\t{\n\t\t`{\"mike\": .a}`,\n\t\tappend(make([]interface{}, 0), \"{\", \"mike (string)\", \"CREATE_MAP\", \"a\", \"}\"),\n\t\tappend(make([]interface{}, 0), \"mike (string)\", \"a\", \"CREATE_MAP\", \"COLLECT_OBJECT\", \"SHORT_PIPE\"),\n\t},\n\t{\n\t\t`{.a: \"mike\"}`,\n\t\tappend(make([]interface{}, 0), \"{\", \"a\", \"CREATE_MAP\", \"mike (string)\", \"}\"),\n\t\tappend(make([]interface{}, 0), \"a\", \"mike (string)\", \"CREATE_MAP\", \"COLLECT_OBJECT\", \"SHORT_PIPE\"),\n\t},\n\t{\n\t\t`{.a: .c, .b.[]: .f.g[]}`,\n\t\tappend(make([]interface{}, 0), \"{\", \"a\", \"CREATE_MAP\", \"c\", \"UNION\", \"b\", \"SHORT_PIPE\", \"SELF\", \"TRAVERSE_ARRAY\", \"[\", \"EMPTY\", \"]\", \"CREATE_MAP\", \"f\", \"SHORT_PIPE\", \"g\", \"TRAVERSE_ARRAY\", \"[\", \"EMPTY\", \"]\", \"}\"),\n\t\tappend(make([]interface{}, 0), \"a\", \"c\", \"CREATE_MAP\", \"b\", \"SELF\", \"EMPTY\", \"COLLECT\", \"TRAVERSE_ARRAY\", \"SHORT_PIPE\", \"f\", \"g\", \"EMPTY\", \"COLLECT\", \"TRAVERSE_ARRAY\", \"SHORT_PIPE\", \"CREATE_MAP\", \"UNION\", \"COLLECT_OBJECT\", \"SHORT_PIPE\"),\n\t},\n\t{\n\t\t`explode(.a.b)`,\n\t\tappend(make([]interface{}, 0), \"EXPLODE\", \"(\", \"a\", \"SHORT_PIPE\", \"b\", \")\"),\n\t\tappend(make([]interface{}, 0), \"a\", \"b\", \"SHORT_PIPE\", \"EXPLODE\"),\n\t},\n\t{\n\t\t`.a.b style=\"folded\"`,\n\t\tappend(make([]interface{}, 0), \"a\", \"SHORT_PIPE\", \"b\", \"ASSIGN_STYLE\", \"folded (string)\"),\n\t\tappend(make([]interface{}, 0), \"a\", \"b\", \"SHORT_PIPE\", \"folded (string)\", \"ASSIGN_STYLE\"),\n\t},\n\t{\n\t\t`tag == \"str\"`,\n\t\tappend(make([]interface{}, 0), \"GET_TAG\", \"EQUALS\", \"str (string)\"),\n\t\tappend(make([]interface{}, 0), \"GET_TAG\", \"str (string)\", \"EQUALS\"),\n\t},\n\t{\n\t\t`. tag= \"str\"`,\n\t\tappend(make([]interface{}, 0), \"SELF\", \"ASSIGN_TAG\", \"str (string)\"),\n\t\tappend(make([]interface{}, 0), \"SELF\", \"str (string)\", \"ASSIGN_TAG\"),\n\t},\n\t{\n\t\t`lineComment == \"str\"`,\n\t\tappend(make([]interface{}, 0), \"GET_COMMENT\", \"EQUALS\", \"str (string)\"),\n\t\tappend(make([]interface{}, 0), \"GET_COMMENT\", \"str (string)\", \"EQUALS\"),\n\t},\n\t{\n\t\t`. lineComment= \"str\"`,\n\t\tappend(make([]interface{}, 0), \"SELF\", \"ASSIGN_COMMENT\", \"str (string)\"),\n\t\tappend(make([]interface{}, 0), \"SELF\", \"str (string)\", \"ASSIGN_COMMENT\"),\n\t},\n\t{\n\t\t`. lineComment |= \"str\"`,\n\t\tappend(make([]interface{}, 0), \"SELF\", \"ASSIGN_COMMENT\", \"str (string)\"),\n\t\tappend(make([]interface{}, 0), \"SELF\", \"str (string)\", \"ASSIGN_COMMENT\"),\n\t},\n\t{\n\t\t`.a.b tag=\"!!str\"`,\n\t\tappend(make([]interface{}, 0), \"a\", \"SHORT_PIPE\", \"b\", \"ASSIGN_TAG\", \"!!str (string)\"),\n\t\tappend(make([]interface{}, 0), \"a\", \"b\", \"SHORT_PIPE\", \"!!str (string)\", \"ASSIGN_TAG\"),\n\t},\n\t{\n\t\t`\"\"`,\n\t\tappend(make([]interface{}, 0), \" (string)\"),\n\t\tappend(make([]interface{}, 0), \" (string)\"),\n\t},\n\t{\n\t\t`.foo* | (. style=\"flow\")`,\n\t\tappend(make([]interface{}, 0), \"foo*\", \"PIPE\", \"(\", \"SELF\", \"ASSIGN_STYLE\", \"flow (string)\", \")\"),\n\t\tappend(make([]interface{}, 0), \"foo*\", \"SELF\", \"flow (string)\", \"ASSIGN_STYLE\", \"PIPE\"),\n\t},\n}\n\nvar tokeniser = newParticipleLexer()\nvar postFixer = newExpressionPostFixer()\n\nfunc TestPathParsing(t *testing.T) {\n\tfor _, tt := range pathTests {\n\t\ttokens, err := tokeniser.Tokenise(tt.path)\n\t\tif err != nil {\n\t\t\tt.Error(tt.path, err)\n\t\t}\n\t\tvar tokenValues []interface{}\n\t\tfor _, token := range tokens {\n\t\t\ttokenValues = append(tokenValues, token.toString(false))\n\t\t}\n\t\ttest.AssertResultComplexWithContext(t, tt.expectedTokens, tokenValues, fmt.Sprintf(\"tokenise: %v\", tt.path))\n\n\t\tresults, errorP := postFixer.ConvertToPostfix(tokens)\n\n\t\tvar readableResults []interface{}\n\t\tfor _, token := range results {\n\t\t\treadableResults = append(readableResults, token.toString())\n\t\t}\n\n\t\tif errorP != nil {\n\t\t\tt.Error(tt.path, err)\n\t\t}\n\n\t\ttest.AssertResultComplexWithContext(t, tt.expectedPostFix, readableResults, fmt.Sprintf(\"postfix: %v\", tt.path))\n\n\t}\n}\n" }, { "path": "pkg/yqlib/file_utils.go", "content": "package yqlib\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n)\n\nfunc tryRenameFile(from string, to string) error {\n\tif info, err := os.Lstat(to); err == nil && info.Mode()&os.ModeSymlink != 0 {\n\t\tlog.Debug(\"Target file is symlink, skipping rename and attempting to copy contents\")\n\n\t\tif copyError := copyFileContents(from, to); copyError != nil {\n\t\t\treturn fmt.Errorf(\"failed copying from %v to %v: %w\", from, to, copyError)\n\t\t}\n\t\ttryRemoveTempFile(from)\n\t\treturn nil\n\t} else if renameError := os.Rename(from, to); renameError != nil {\n\t\tlog.Debugf(\"Error renaming from %v to %v, attempting to copy contents\", from, to)\n\t\tlog.Debug(renameError.Error())\n\t\tlog.Debug(\"going to try copying instead\")\n\t\t// can't do this rename when running in docker to a file targeted in a mounted volume,\n\t\t// so gracefully degrade to copying the entire contents.\n\t\tif copyError := copyFileContents(from, to); copyError != nil {\n\t\t\treturn fmt.Errorf(\"failed copying from %v to %v: %w\", from, to, copyError)\n\t\t}\n\t\ttryRemoveTempFile(from)\n\t}\n\treturn nil\n}\n\nfunc tryRemoveTempFile(filename string) {\n\tlog.Debug(\"Removing temp file: %v\", filename)\n\tremoveErr := os.Remove(filename)\n\tif removeErr != nil {\n\t\tlog.Errorf(\"Failed to remove temp file: %v\", filename)\n\t}\n}\n\n// thanks https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang\nfunc copyFileContents(src, dst string) (err error) {\n\t// ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory,\n\t// and ensuring that it's not possible to give a path to a file outside thar directory.\n\n\tin, err := os.Open(src) // #nosec\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer safelyCloseFile(in)\n\tout, err := os.Create(dst) // #nosec\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer safelyCloseFile(out)\n\tif _, err = io.Copy(out, in); err != nil {\n\t\treturn err\n\t}\n\treturn out.Sync()\n}\n\nfunc SafelyCloseReader(reader io.Reader) {\n\tswitch reader := reader.(type) {\n\tcase *os.File:\n\t\tsafelyCloseFile(reader)\n\t}\n}\n\nfunc safelyCloseFile(file *os.File) {\n\terr := file.Close()\n\tif err != nil {\n\t\tlog.Error(\"Error closing file!\")\n\t\tlog.Error(err.Error())\n\t}\n}\n\nfunc createTempFile() (*os.File, error) {\n\t_, err := os.Stat(os.TempDir())\n\tif os.IsNotExist(err) {\n\t\terr = os.Mkdir(os.TempDir(), 0700)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else if err != nil {\n\t\treturn nil, err\n\t}\n\n\tfile, err := os.CreateTemp(\"\", \"temp\")\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn file, err\n}\n" }, { "path": "pkg/yqlib/format.go", "content": "package yqlib\n\nimport (\n\t\"fmt\"\n\t\"path/filepath\"\n\t\"slices\"\n\t\"strings\"\n)\n\ntype EncoderFactoryFunction func() Encoder\ntype DecoderFactoryFunction func() Decoder\n\ntype Format struct {\n\tFormalName string\n\tNames []string\n\tEncoderFactory EncoderFactoryFunction\n\tDecoderFactory DecoderFactoryFunction\n}\n\nvar YamlFormat = &Format{\"yaml\", []string{\"y\", \"yml\"},\n\tfunc() Encoder { return NewYamlEncoder(ConfiguredYamlPreferences) },\n\tfunc() Decoder { return NewYamlDecoder(ConfiguredYamlPreferences) },\n}\n\nvar KYamlFormat = &Format{\"kyaml\", []string{\"ky\"},\n\tfunc() Encoder { return NewKYamlEncoder(ConfiguredKYamlPreferences) },\n\t// KYaml is stricter YAML\n\tfunc() Decoder { return NewYamlDecoder(ConfiguredYamlPreferences) },\n}\n\nvar JSONFormat = &Format{\"json\", []string{\"j\"},\n\tfunc() Encoder { return NewJSONEncoder(ConfiguredJSONPreferences) },\n\tfunc() Decoder { return NewJSONDecoder() },\n}\n\nvar PropertiesFormat = &Format{\"props\", []string{\"p\", \"properties\"},\n\tfunc() Encoder { return NewPropertiesEncoder(ConfiguredPropertiesPreferences) },\n\tfunc() Decoder { return NewPropertiesDecoder() },\n}\n\nvar CSVFormat = &Format{\"csv\", []string{\"c\"},\n\tfunc() Encoder { return NewCsvEncoder(ConfiguredCsvPreferences) },\n\tfunc() Decoder { return NewCSVObjectDecoder(ConfiguredCsvPreferences) },\n}\n\nvar TSVFormat = &Format{\"tsv\", []string{\"t\"},\n\tfunc() Encoder { return NewCsvEncoder(ConfiguredTsvPreferences) },\n\tfunc() Decoder { return NewCSVObjectDecoder(ConfiguredTsvPreferences) },\n}\n\nvar XMLFormat = &Format{\"xml\", []string{\"x\"},\n\tfunc() Encoder { return NewXMLEncoder(ConfiguredXMLPreferences) },\n\tfunc() Decoder { return NewXMLDecoder(ConfiguredXMLPreferences) },\n}\n\nvar Base64Format = &Format{\"base64\", []string{},\n\tfunc() Encoder { return NewBase64Encoder() },\n\tfunc() Decoder { return NewBase64Decoder() },\n}\n\nvar UriFormat = &Format{\"uri\", []string{},\n\tfunc() Encoder { return NewUriEncoder() },\n\tfunc() Decoder { return NewUriDecoder() },\n}\n\nvar ShFormat = &Format{\"\", nil,\n\tfunc() Encoder { return NewShEncoder() },\n\tnil,\n}\n\nvar TomlFormat = &Format{\"toml\", []string{},\n\tfunc() Encoder { return NewTomlEncoderWithPrefs(ConfiguredTomlPreferences) },\n\tfunc() Decoder { return NewTomlDecoder() },\n}\n\nvar HclFormat = &Format{\"hcl\", []string{\"h\", \"tf\"},\n\tfunc() Encoder { return NewHclEncoder(ConfiguredHclPreferences) },\n\tfunc() Decoder { return NewHclDecoder() },\n}\n\nvar ShellVariablesFormat = &Format{\"shell\", []string{\"s\", \"sh\"},\n\tfunc() Encoder { return NewShellVariablesEncoder() },\n\tnil,\n}\n\nvar LuaFormat = &Format{\"lua\", []string{\"l\"},\n\tfunc() Encoder { return NewLuaEncoder(ConfiguredLuaPreferences) },\n\tfunc() Decoder { return NewLuaDecoder(ConfiguredLuaPreferences) },\n}\n\nvar INIFormat = &Format{\"ini\", []string{\"i\"},\n\tfunc() Encoder { return NewINIEncoder() },\n\tfunc() Decoder { return NewINIDecoder() },\n}\n\nvar Formats = []*Format{\n\tYamlFormat,\n\tKYamlFormat,\n\tJSONFormat,\n\tPropertiesFormat,\n\tCSVFormat,\n\tTSVFormat,\n\tXMLFormat,\n\tBase64Format,\n\tUriFormat,\n\tShFormat,\n\tTomlFormat,\n\tHclFormat,\n\tShellVariablesFormat,\n\tLuaFormat,\n\tINIFormat,\n}\n\nfunc (f *Format) MatchesName(name string) bool {\n\tif f.FormalName == name {\n\t\treturn true\n\t}\n\treturn slices.Contains(f.Names, name)\n}\n\nfunc (f *Format) GetConfiguredEncoder() Encoder {\n\treturn f.EncoderFactory()\n}\n\nfunc FormatStringFromFilename(filename string) string {\n\tif filename != \"\" {\n\t\tGetLogger().Debugf(\"checking filename '%s' for auto format detection\", filename)\n\t\text := filepath.Ext(filename)\n\t\tif len(ext) >= 2 && ext[0] == '.' {\n\t\t\tformat := strings.ToLower(ext[1:])\n\t\t\tGetLogger().Debugf(\"detected format '%s'\", format)\n\t\t\treturn format\n\t\t}\n\t}\n\n\tGetLogger().Debugf(\"using default inputFormat 'yaml'\")\n\treturn \"yaml\"\n}\n\nfunc FormatFromString(format string) (*Format, error) {\n\tif format != \"\" {\n\t\tfor _, printerFormat := range Formats {\n\t\t\tif printerFormat.MatchesName(format) {\n\t\t\t\treturn printerFormat, nil\n\t\t\t}\n\t\t}\n\t}\n\treturn nil, fmt.Errorf(\"unknown format '%v' please use [%v]\", format, GetAvailableOutputFormatString())\n}\n\nfunc GetAvailableOutputFormats() []*Format {\n\tvar formats = []*Format{}\n\tfor _, printerFormat := range Formats {\n\t\tif printerFormat.EncoderFactory != nil {\n\t\t\tformats = append(formats, printerFormat)\n\t\t}\n\t}\n\treturn formats\n}\n\nfunc GetAvailableOutputFormatString() string {\n\tvar formats = []string{}\n\tfor _, printerFormat := range GetAvailableOutputFormats() {\n\n\t\tif printerFormat.FormalName != \"\" {\n\t\t\tformats = append(formats, printerFormat.FormalName)\n\t\t}\n\t\tif len(printerFormat.Names) >= 1 {\n\t\t\tformats = append(formats, printerFormat.Names[0])\n\t\t}\n\t}\n\treturn strings.Join(formats, \"|\")\n}\n\nfunc GetAvailableInputFormats() []*Format {\n\tvar formats = []*Format{}\n\tfor _, printerFormat := range Formats {\n\t\tif printerFormat.DecoderFactory != nil {\n\t\t\tformats = append(formats, printerFormat)\n\t\t}\n\t}\n\treturn formats\n}\n\nfunc GetAvailableInputFormatString() string {\n\tvar formats = []string{}\n\tfor _, printerFormat := range GetAvailableInputFormats() {\n\n\t\tif printerFormat.FormalName != \"\" {\n\t\t\tformats = append(formats, printerFormat.FormalName)\n\t\t}\n\t\tif len(printerFormat.Names) >= 1 {\n\t\t\tformats = append(formats, printerFormat.Names[0])\n\t\t}\n\t}\n\treturn strings.Join(formats, \"|\")\n}\n" }, { "path": "pkg/yqlib/format_test.go", "content": "package yqlib\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\ntype formatStringScenario struct {\n\tdescription string\n\tinput string\n\texpectedFormat *Format\n\texpectedError string\n}\n\nvar formatStringScenarios = []formatStringScenario{\n\t{\n\t\tdescription: \"yaml\",\n\t\tinput: \"yaml\",\n\t\texpectedFormat: YamlFormat,\n\t},\n\t{\n\t\tdescription: \"Unknown format type\",\n\t\tinput: \"doc\",\n\t\texpectedError: \"unknown format 'doc' please use\",\n\t},\n\t{\n\t\tdescription: \"blank should error\",\n\t\tinput: \"\",\n\t\texpectedError: \"unknown format '' please use\",\n\t},\n}\n\nfunc TestFormatFromString(t *testing.T) {\n\tfor _, tt := range formatStringScenarios {\n\t\tactualFormat, actualError := FormatFromString(tt.input)\n\n\t\tif tt.expectedError != \"\" {\n\t\t\tif actualError == nil {\n\t\t\t\tt.Errorf(\"Expected [%v] error but found none\", tt.expectedError)\n\t\t\t} else {\n\t\t\t\ttest.AssertResultWithContext(t, true, strings.Contains(actualError.Error(), tt.expectedError),\n\t\t\t\t\tfmt.Sprintf(\"Expected [%v] to contain [%v]\", actualError.Error(), tt.expectedError),\n\t\t\t\t)\n\t\t\t}\n\t\t} else {\n\t\t\ttest.AssertResult(t, tt.expectedFormat, actualFormat)\n\t\t}\n\t}\n}\n\nfunc TestFormatStringFromFilename(t *testing.T) {\n\ttest.AssertResult(t, \"yaml\", FormatStringFromFilename(\"test.Yaml\"))\n\ttest.AssertResult(t, \"yaml\", FormatStringFromFilename(\"test.index.Yaml\"))\n\ttest.AssertResult(t, \"yaml\", FormatStringFromFilename(\"test\"))\n\ttest.AssertResult(t, \"json\", FormatStringFromFilename(\"test.json\"))\n\ttest.AssertResult(t, \"json\", FormatStringFromFilename(\"TEST.JSON\"))\n\ttest.AssertResult(t, \"yaml\", FormatStringFromFilename(\"test.json/foo\"))\n\ttest.AssertResult(t, \"yaml\", FormatStringFromFilename(\"\"))\n}\n" }, { "path": "pkg/yqlib/formatting_expressions_test.go", "content": "package yqlib\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nvar formattingExpressionScenarios = []formatScenario{\n\t{\n\t\tdescription: \"Using expression files and comments\",\n\t\tskipDoc: true,\n\t\tinput: \"a:\\n b: old\",\n\t\texpression: \"#! yq\\n\\n# This is a yq expression that updates the map\\n# for several great reasons outlined here.\\n\\n.a.b = \\\"new\\\" # line comment here\\n| .a.c = \\\"frog\\\"\\n\\n# Now good things will happen.\\n\",\n\t\texpected: \"a:\\n b: new\\n c: frog\\n\",\n\t},\n\t{\n\t\tdescription: \"Using expression files and comments\",\n\t\tsubdescription: \"Note that you can execute the file directly - but make sure you make the expression file executable.\",\n\t\tinput: \"a:\\n b: old\",\n\t\texpression: \"#! yq\\n\\n# This is a yq expression that updates the map\\n# for several great reasons outlined here.\\n\\n.a.b = \\\"new\\\" # line comment here\\n| .a.c = \\\"frog\\\"\\n\\n# Now good things will happen.\\n\",\n\t\texpected: \"a:\\n b: new\\n c: frog\\n\",\n\t\tscenarioType: \"shebang\",\n\t},\n\t{\n\t\tdescription: \"Flags in expression files\",\n\t\tsubdescription: \"You can specify flags on the shebang line, this only works when executing the file directly.\",\n\t\tinput: \"a:\\n b: old\",\n\t\texpression: \"#! yq -oj\\n\\n# This is a yq expression that updates the map\\n# for several great reasons outlined here.\\n\\n.a.b = \\\"new\\\" # line comment here\\n| .a.c = \\\"frog\\\"\\n\\n# Now good things will happen.\\n\",\n\t\texpected: \"a:\\n b: new\\n c: frog\\n\",\n\t\tscenarioType: \"shebang-json\",\n\t},\n\t{\n\t\tdescription: \"Commenting out yq expressions\",\n\t\tsubdescription: \"Note that `c` is no longer set to 'frog'. In this example we're calling yq directly and passing the expression file into `--from-file`, this is no different from executing the expression file directly.\",\n\t\tinput: \"a:\\n b: old\",\n\t\texpression: \"#! yq\\n# This is a yq expression that updates the map\\n# for several great reasons outlined here.\\n\\n.a.b = \\\"new\\\" # line comment here\\n# | .a.c = \\\"frog\\\"\\n\\n# Now good things will happen.\\n\",\n\t\texpected: \"a:\\n b: new\\n\",\n\t},\n}\n\nfunc documentExpressionScenario(_ *testing.T, w *bufio.Writer, i interface{}) {\n\ts := i.(formatScenario)\n\n\tif s.skipDoc {\n\t\treturn\n\t}\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.yaml file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"And an 'update.yq' expression file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```bash\\n%v```\\n\", s.expression))\n\n\twriteOrPanic(w, \"then\\n\")\n\tif strings.HasPrefix(s.scenarioType, \"shebang\") {\n\t\twriteOrPanic(w, \"```bash\\n./update.yq sample.yaml\\n```\\n\")\n\t} else {\n\t\twriteOrPanic(w, \"```bash\\nyq --from-file update.yq sample.yml\\n```\\n\")\n\t}\n\twriteOrPanic(w, \"will output\\n\")\n\tencoder := NewYamlEncoder(ConfiguredYamlPreferences)\n\n\tif s.scenarioType == \"shebang-json\" {\n\t\tencoder = NewJSONEncoder(ConfiguredJSONPreferences)\n\t}\n\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), encoder)))\n}\n\nfunc TestExpressionCommentScenarios(t *testing.T) {\n\tfor _, tt := range formattingExpressionScenarios {\n\t\ttest.AssertResultComplexWithContext(t, tt.expected,\n\t\t\tmustProcessFormatScenario(tt, NewYamlDecoder(ConfiguredYamlPreferences), NewYamlEncoder(ConfiguredYamlPreferences)),\n\t\t\ttt.description)\n\t}\n\tgenericScenarios := make([]interface{}, len(formattingExpressionScenarios))\n\tfor i, s := range formattingExpressionScenarios {\n\t\tgenericScenarios[i] = s\n\t}\n\tdocumentScenarios(t, \"usage\", \"formatting-expressions\", genericScenarios, documentExpressionScenario)\n}\n" }, { "path": "pkg/yqlib/front_matter.go", "content": "package yqlib\n\nimport (\n\t\"bufio\"\n\t\"errors\"\n\t\"io\"\n\t\"os\"\n)\n\ntype frontMatterHandler interface {\n\tSplit() error\n\tGetYamlFrontMatterFilename() string\n\tGetContentReader() io.Reader\n\tCleanUp()\n}\n\ntype frontMatterHandlerImpl struct {\n\toriginalFilename string\n\tyamlFrontMatterFilename string\n\tcontentReader io.Reader\n}\n\nfunc NewFrontMatterHandler(originalFilename string) frontMatterHandler {\n\treturn &frontMatterHandlerImpl{originalFilename, \"\", nil}\n}\n\nfunc (f *frontMatterHandlerImpl) GetYamlFrontMatterFilename() string {\n\treturn f.yamlFrontMatterFilename\n}\n\nfunc (f *frontMatterHandlerImpl) GetContentReader() io.Reader {\n\treturn f.contentReader\n}\n\nfunc (f *frontMatterHandlerImpl) CleanUp() {\n\ttryRemoveTempFile(f.yamlFrontMatterFilename)\n}\n\n// Splits the given file by yaml front matter\n// yaml content will be saved to first temporary file\n// remaining content will be saved to second temporary file\nfunc (f *frontMatterHandlerImpl) Split() error {\n\tvar reader *bufio.Reader\n\tvar err error\n\tif f.originalFilename == \"-\" {\n\t\treader = bufio.NewReader(os.Stdin)\n\t} else {\n\t\tfile, err := os.Open(f.originalFilename) // #nosec\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\treader = bufio.NewReader(file)\n\t}\n\tf.contentReader = reader\n\n\tyamlTempFile, err := createTempFile()\n\tif err != nil {\n\t\treturn err\n\t}\n\tf.yamlFrontMatterFilename = yamlTempFile.Name()\n\tlog.Debug(\"yamlTempFile: %v\", yamlTempFile.Name())\n\n\tlineCount := 0\n\n\tfor {\n\t\tpeekBytes, err := reader.Peek(3)\n\t\tif errors.Is(err, io.EOF) {\n\t\t\t// we've finished reading the yaml content..I guess\n\t\t\tbreak\n\t\t} else if err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif lineCount > 0 && string(peekBytes) == \"---\" {\n\t\t\t// we've finished reading the yaml content..\n\t\t\tbreak\n\t\t}\n\t\tline, errReading := reader.ReadString('\\n')\n\t\tlineCount = lineCount + 1\n\t\tif errReading != nil && !errors.Is(errReading, io.EOF) {\n\t\t\treturn errReading\n\t\t}\n\n\t\t_, errWriting := yamlTempFile.WriteString(line)\n\n\t\tif errWriting != nil {\n\t\t\treturn errWriting\n\t\t}\n\t}\n\n\tsafelyCloseFile(yamlTempFile)\n\n\treturn nil\n\n}\n" }, { "path": "pkg/yqlib/front_matter_test.go", "content": "package yqlib\n\nimport (\n\t\"io\"\n\t\"os\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nfunc createTestFile(content string) string {\n\ttempFile, err := createTempFile()\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\t_, err = tempFile.Write([]byte(content))\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tsafelyCloseFile(tempFile)\n\n\treturn tempFile.Name()\n}\n\nfunc readFile(filename string) string {\n\tbytes, err := os.ReadFile(filename)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn string(bytes)\n}\n\nfunc TestFrontMatterSplitWithLeadingSep(t *testing.T) {\n\tfile := createTestFile(`---\na: apple\nb: banana\n---\nnot a \nyaml: doc\n`)\n\n\texpectedYamlFm := `---\na: apple\nb: banana\n`\n\n\texpectedContent := `---\nnot a \nyaml: doc\n`\n\n\tfmHandler := NewFrontMatterHandler(file)\n\terr := fmHandler.Split()\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tyamlFm := readFile(fmHandler.GetYamlFrontMatterFilename())\n\n\ttest.AssertResult(t, expectedYamlFm, yamlFm)\n\n\tcontentBytes, err := io.ReadAll(fmHandler.GetContentReader())\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\ttest.AssertResult(t, expectedContent, string(contentBytes))\n\n\ttryRemoveTempFile(file)\n\tfmHandler.CleanUp()\n}\n\nfunc TestFrontMatterSplitWithNoLeadingSep(t *testing.T) {\n\tfile := createTestFile(`a: apple\nb: banana\n---\nnot a \nyaml: doc\n`)\n\n\texpectedYamlFm := `a: apple\nb: banana\n`\n\n\texpectedContent := `---\nnot a \nyaml: doc\n`\n\n\tfmHandler := NewFrontMatterHandler(file)\n\terr := fmHandler.Split()\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tyamlFm := readFile(fmHandler.GetYamlFrontMatterFilename())\n\n\ttest.AssertResult(t, expectedYamlFm, yamlFm)\n\n\tcontentBytes, err := io.ReadAll(fmHandler.GetContentReader())\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\ttest.AssertResult(t, expectedContent, string(contentBytes))\n\n\ttryRemoveTempFile(file)\n\tfmHandler.CleanUp()\n}\n\nfunc TestFrontMatterSplitWithArray(t *testing.T) {\n\tfile := createTestFile(`[1,2,3]\n---\nnot a \nyaml: doc\n`)\n\n\texpectedYamlFm := \"[1,2,3]\\n\"\n\n\texpectedContent := `---\nnot a \nyaml: doc\n`\n\n\tfmHandler := NewFrontMatterHandler(file)\n\terr := fmHandler.Split()\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tyamlFm := readFile(fmHandler.GetYamlFrontMatterFilename())\n\n\ttest.AssertResult(t, expectedYamlFm, yamlFm)\n\n\tcontentBytes, err := io.ReadAll(fmHandler.GetContentReader())\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\ttest.AssertResult(t, expectedContent, string(contentBytes))\n\n\ttryRemoveTempFile(file)\n\tfmHandler.CleanUp()\n}\n" }, { "path": "pkg/yqlib/goccy_yaml_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nvar goccyYamlFormatScenarios = []formatScenario{\n\t{\n\t\tdescription: \"basic - 3\",\n\t\tskipDoc: true,\n\t\tinput: \"3\",\n\t\texpected: \"3\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - 3.1\",\n\t\tskipDoc: true,\n\t\tinput: \"3.1\",\n\t\texpected: \"3.1\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - mike\",\n\t\tskipDoc: true,\n\t\tinput: \"mike: 3\",\n\t\texpected: \"mike: 3\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - map multiple entries\",\n\t\tskipDoc: true,\n\t\tinput: \"mike: 3\\nfred: 12\\n\",\n\t\texpected: \"mike: 3\\nfred: 12\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - 3.1\",\n\t\tskipDoc: true,\n\t\tinput: \"{\\n mike: 3\\n}\",\n\t\texpected: \"{mike: 3}\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - tag with number\",\n\t\tskipDoc: true,\n\t\tinput: \"mike: !!cat 3\",\n\t\texpected: \"mike: !!cat 3\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - array of numbers\",\n\t\tskipDoc: true,\n\t\tinput: \"- 3\",\n\t\texpected: \"- 3\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - single line array\",\n\t\tskipDoc: true,\n\t\tinput: \"[3]\",\n\t\texpected: \"[3]\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - plain string\",\n\t\tskipDoc: true,\n\t\tinput: `a: meow`,\n\t\texpected: \"a: meow\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - double quoted string\",\n\t\tskipDoc: true,\n\t\tinput: `a: \"meow\"`,\n\t\texpected: \"a: \\\"meow\\\"\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - single quoted string\",\n\t\tskipDoc: true,\n\t\tinput: `a: 'meow'`,\n\t\texpected: \"a: 'meow'\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - string block\",\n\t\tskipDoc: true,\n\t\tinput: \"a: |\\n meow\\n\",\n\t\texpected: \"a: |\\n meow\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - long string\",\n\t\tskipDoc: true,\n\t\tinput: \"a: the cute cat wrote a long sentence that wasn't wrapped at all.\\n\",\n\t\texpected: \"a: the cute cat wrote a long sentence that wasn't wrapped at all.\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - string block\",\n\t\tskipDoc: true,\n\t\tinput: \"a: |-\\n meow\\n\",\n\t\texpected: \"a: |-\\n meow\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - line comment\",\n\t\tskipDoc: true,\n\t\tinput: \"a: meow # line comment\\n\",\n\t\texpected: \"a: meow # line comment\\n\",\n\t},\n\t// {\n\t// \tdescription: \"basic - head comment\",\n\t// \tskipDoc: true,\n\t// \tinput: \"# head comment\\na: meow\\n\",\n\t// \texpected: \"# head comment\\na: meow\\n\", // go-yaml does this\n\t// },\n\t// {\n\t// \tdescription: \"basic - head and line comment\",\n\t// \tskipDoc: true,\n\t// \tinput: \"# head comment\\na: #line comment\\n meow\\n\",\n\t// \texpected: \"# head comment\\na: meow #line comment\\n\", // go-yaml does this\n\t// },\n\t{\n\t\tdescription: \"basic - foot comment\",\n\t\tskipDoc: true,\n\t\tinput: \"a: meow\\n# foot comment\\n\",\n\t\texpected: \"a: meow\\n# foot comment\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - foot comment\",\n\t\tskipDoc: true,\n\t\tinput: \"a: meow\\nb: woof\\n# foot comment\\n\",\n\t\texpected: \"a: meow\\nb: woof\\n# foot comment\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - boolean\",\n\t\tskipDoc: true,\n\t\tinput: \"true\\n\",\n\t\texpected: \"true\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - null\",\n\t\tskipDoc: true,\n\t\tinput: \"a: null\\n\",\n\t\texpected: \"a: null\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - ~\",\n\t\tskipDoc: true,\n\t\tinput: \"a: ~\\n\",\n\t\texpected: \"a: ~\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - ~\",\n\t\tskipDoc: true,\n\t\tinput: \"null\\n\",\n\t\texpected: \"null\\n\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"blank value round trip\",\n\t\tinput: \"test:\",\n\t\texpected: \"test:\\n\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"trailing comment\",\n\t\tinput: \"test: null\\n# this comment will be removed\",\n\t\texpected: \"test: null\\n# this comment will be removed\\n\",\n\t},\n\t// {\n\t// \tdescription: \"doc separator\",\n\t// \tskipDoc: true,\n\t// \tinput: \"# hi\\n---\\na: cat\\n---\",\n\t// \texpected: \"---\\na: cat\\n\",\n\t// },\n\t// {\n\t// \tdescription: \"scalar with doc separator\",\n\t// \tskipDoc: true,\n\t// \tinput: \"--- cat\",\n\t// \texpected: \"---\\ncat\\n\",\n\t// },\n\t{\n\t\tdescription: \"scalar with doc separator\",\n\t\tskipDoc: true,\n\t\tinput: \"---cat\",\n\t\texpected: \"---cat\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - null\",\n\t\tskipDoc: true,\n\t\tinput: \"null\",\n\t\texpected: \"null\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - ~\",\n\t\tskipDoc: true,\n\t\tinput: \"~\",\n\t\texpected: \"~\\n\",\n\t},\n\t{\n\t\tdescription: \"octal\",\n\t\tskipDoc: true,\n\t\tinput: \"0o30\",\n\t\texpression: \"tag\",\n\t\texpected: \"!!int\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - [null]\",\n\t\tskipDoc: true,\n\t\tinput: \"[null]\",\n\t\texpected: \"[null]\\n\",\n\t},\n\t{\n\t\tdescription: \"multi document\",\n\t\tskipDoc: true,\n\t\tinput: \"a: mike\\n---\\nb: remember\",\n\t\texpected: \"a: mike\\n---\\nb: remember\\n\",\n\t},\n\t{\n\t\tdescription: \"single doc anchor map\",\n\t\tskipDoc: true,\n\t\tinput: \"a: &remember mike\\nb: *remember\",\n\t\texpected: \"a: &remember mike\\nb: *remember\\n\",\n\t},\n\t{\n\t\tdescription: \"explode doc anchor map\",\n\t\tskipDoc: true,\n\t\tinput: \"a: &remember mike\\nb: *remember\",\n\t\texpression: \"explode(.)\",\n\t\texpected: \"a: mike\\nb: mike\\n\",\n\t},\n\t{\n\t\tdescription: \"multi document anchor map\",\n\t\tskipDoc: true,\n\t\tinput: \"a: &remember mike\\n---\\nb: *remember\",\n\t\texpression: \"explode(.)\",\n\t\texpected: \"a: mike\\n---\\nb: mike\\n\",\n\t},\n\t{\n\t\tdescription: \"merge anchor\",\n\t\tskipDoc: true,\n\t\tinput: \"a: &remember\\n c: mike\\nb:\\n <<: *remember\",\n\t\t// fine to have !!merge as that's what the current impl does\n\t\texpected: \"a: &remember\\n c: mike\\nb:\\n !!merge <<: *remember\\n\",\n\t},\n\t{\n\t\tdescription: \"custom tag\",\n\t\tskipDoc: true,\n\t\tinput: \"a: !cat mike\",\n\t\texpected: \"a: !cat mike\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - [~]\",\n\t\tskipDoc: true,\n\t\tinput: \"[~]\",\n\t\texpected: \"[~]\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - null map value\",\n\t\tskipDoc: true,\n\t\tinput: \"a: null\",\n\t\texpected: \"a: null\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - number\",\n\t\tskipDoc: true,\n\t\tinput: \"3\",\n\t\texpected: \"3\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - float\",\n\t\tskipDoc: true,\n\t\tinput: \"3.1\",\n\t\texpected: \"3.1\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - float\",\n\t\tskipDoc: true,\n\t\tinput: \"[1, 2]\",\n\t\texpected: \"[1, 2]\\n\",\n\t},\n}\n\nfunc testGoccyYamlScenario(t *testing.T, s formatScenario) {\n\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewGoccyYAMLDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)\n}\n\nfunc TestGoccyYmlFormatScenarios(t *testing.T) {\n\tfor _, tt := range goccyYamlFormatScenarios {\n\t\ttestGoccyYamlScenario(t, tt)\n\t}\n}\n" }, { "path": "pkg/yqlib/hcl.go", "content": "package yqlib\n\ntype HclPreferences struct {\n\tColorsEnabled bool\n}\n\nfunc NewDefaultHclPreferences() HclPreferences {\n\treturn HclPreferences{ColorsEnabled: false}\n}\n\nfunc (p *HclPreferences) Copy() HclPreferences {\n\treturn HclPreferences{ColorsEnabled: p.ColorsEnabled}\n}\n\nvar ConfiguredHclPreferences = NewDefaultHclPreferences()\n" }, { "path": "pkg/yqlib/hcl_test.go", "content": "//go:build !yq_nohcl\n\npackage yqlib\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nvar nestedExample = `service \"http\" \"web_proxy\" {\n listen_addr = \"127.0.0.1:8080\"\n}`\n\nvar nestedExampleYaml = \"service:\\n http:\\n web_proxy:\\n listen_addr: \\\"127.0.0.1:8080\\\"\\n\"\n\nvar multipleBlockLabelKeys = `service \"cat\" {\n process \"main\" {\n command = [\"/usr/local/bin/awesome-app\", \"server\"]\n }\n\n process \"management\" {\n command = [\"/usr/local/bin/awesome-app\", \"management\"]\n }\n}\n`\nvar multipleBlockLabelKeysExpected = `service \"cat\" {\n process \"main\" {\n command = [\"/usr/local/bin/awesome-app\", \"server\"]\n }\n process \"management\" {\n command = [\"/usr/local/bin/awesome-app\", \"management\"]\n }\n}\n`\n\nvar multipleBlockLabelKeysExpectedUpdate = `service \"cat\" {\n process \"main\" {\n command = [\"/usr/local/bin/awesome-app\", \"server\", \"meow\"]\n }\n process \"management\" {\n command = [\"/usr/local/bin/awesome-app\", \"management\"]\n }\n}\n`\n\nvar multipleBlockLabelKeysExpectedYaml = `service:\n cat:\n process:\n main:\n command:\n - \"/usr/local/bin/awesome-app\"\n - \"server\"\n management:\n command:\n - \"/usr/local/bin/awesome-app\"\n - \"management\"\n`\n\nvar simpleSample = `# Arithmetic with literals and application-provided variables\nsum = 1 + addend\n\n# String interpolation and templates\nmessage = \"Hello, ${name}!\"\n\n# Application-provided functions\nshouty_message = upper(message)`\n\nvar simpleSampleExpected = `# Arithmetic with literals and application-provided variables\nsum = 1 + addend\n# String interpolation and templates\nmessage = \"Hello, ${name}!\"\n# Application-provided functions\nshouty_message = upper(message)\n`\n\nvar simpleSampleExpectedYaml = `# Arithmetic with literals and application-provided variables\nsum: 1 + addend\n# String interpolation and templates\nmessage: \"Hello, ${name}!\"\n# Application-provided functions\nshouty_message: upper(message)\n`\n\nvar hclFormatScenarios = []formatScenario{\n\t{\n\t\tdescription: \"Parse HCL\",\n\t\tinput: `io_mode = \"async\"`,\n\t\texpected: \"io_mode: \\\"async\\\"\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Simple decode, no quotes\",\n\t\tskipDoc: true,\n\t\tinput: `io_mode = async`,\n\t\texpected: \"io_mode: async\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Simple roundtrip, no quotes\",\n\t\tskipDoc: true,\n\t\tinput: `io_mode = async`,\n\t\texpected: \"io_mode = async\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Nested decode\",\n\t\tskipDoc: true,\n\t\tinput: nestedExample,\n\t\texpected: nestedExampleYaml,\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Template decode\",\n\t\tskipDoc: true,\n\t\tinput: `message = \"Hello, ${name}!\"`,\n\t\texpected: \"message: \\\"Hello, ${name}!\\\"\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip: with template\",\n\t\tskipDoc: true,\n\t\tinput: `message = \"Hello, ${name}!\"`,\n\t\texpected: \"message = \\\"Hello, ${name}!\\\"\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip: with function\",\n\t\tskipDoc: true,\n\t\tinput: `shouty_message = upper(message)`,\n\t\texpected: \"shouty_message = upper(message)\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip: with arithmetic\",\n\t\tskipDoc: true,\n\t\tinput: `sum = 1 + addend`,\n\t\texpected: \"sum = 1 + addend\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Arithmetic decode\",\n\t\tskipDoc: true,\n\t\tinput: `sum = 1 + addend`,\n\t\texpected: \"sum: 1 + addend\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"number attribute\",\n\t\tskipDoc: true,\n\t\tinput: `port = 8080`,\n\t\texpected: \"port: 8080\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"float attribute\",\n\t\tskipDoc: true,\n\t\tinput: `pi = 3.14`,\n\t\texpected: \"pi: 3.14\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"boolean attribute\",\n\t\tskipDoc: true,\n\t\tinput: `enabled = true`,\n\t\texpected: \"enabled: true\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"object/map attribute\",\n\t\tskipDoc: true,\n\t\tinput: `obj = { a = 1, b = \"two\" }`,\n\t\texpected: \"obj: {a: 1, b: \\\"two\\\"}\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"nested block\",\n\t\tskipDoc: true,\n\t\tinput: `server { port = 8080 }`,\n\t\texpected: \"server:\\n port: 8080\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"multiple attributes\",\n\t\tskipDoc: true,\n\t\tinput: \"name = \\\"app\\\"\\nversion = 1\\nenabled = true\",\n\t\texpected: \"name: \\\"app\\\"\\nversion: 1\\nenabled: true\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"binary expression\",\n\t\tskipDoc: true,\n\t\tinput: `count = 0 - 42`,\n\t\texpected: \"count: -42\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"negative number\",\n\t\tskipDoc: true,\n\t\tinput: `count = -42`,\n\t\texpected: \"count: -42\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"scientific notation\",\n\t\tskipDoc: true,\n\t\tinput: `value = 1e-3`,\n\t\texpected: \"value: 0.001\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"nested object\",\n\t\tskipDoc: true,\n\t\tinput: `config = { db = { host = \"localhost\", port = 5432 } }`,\n\t\texpected: \"config: {db: {host: \\\"localhost\\\", port: 5432}}\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"mixed list\",\n\t\tskipDoc: true,\n\t\tinput: `values = [1, \"two\", true]`,\n\t\texpected: \"values:\\n - 1\\n - \\\"two\\\"\\n - true\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip: Sample Doc\",\n\t\tinput: multipleBlockLabelKeys,\n\t\texpected: multipleBlockLabelKeysExpected,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip: With an update\",\n\t\tinput: multipleBlockLabelKeys,\n\t\texpression: `.service.cat.process.main.command += \"meow\"`,\n\t\texpected: multipleBlockLabelKeysExpectedUpdate,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Parse HCL: Sample Doc\",\n\t\tinput: multipleBlockLabelKeys,\n\t\texpected: multipleBlockLabelKeysExpectedYaml,\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"block with labels\",\n\t\tskipDoc: true,\n\t\tinput: `resource \"aws_instance\" \"example\" { ami = \"ami-12345\" }`,\n\t\texpected: \"resource:\\n aws_instance:\\n example:\\n ami: \\\"ami-12345\\\"\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"block with labels roundtrip\",\n\t\tskipDoc: true,\n\t\tinput: `resource \"aws_instance\" \"example\" { ami = \"ami-12345\" }`,\n\t\texpected: \"resource \\\"aws_instance\\\" \\\"example\\\" {\\n ami = \\\"ami-12345\\\"\\n}\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"roundtrip simple attribute\",\n\t\tskipDoc: true,\n\t\tinput: `io_mode = \"async\"`,\n\t\texpected: `io_mode = \"async\"` + \"\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"roundtrip number attribute\",\n\t\tskipDoc: true,\n\t\tinput: `port = 8080`,\n\t\texpected: \"port = 8080\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"roundtrip float attribute\",\n\t\tskipDoc: true,\n\t\tinput: `pi = 3.14`,\n\t\texpected: \"pi = 3.14\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"roundtrip boolean attribute\",\n\t\tskipDoc: true,\n\t\tinput: `enabled = true`,\n\t\texpected: \"enabled = true\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"roundtrip list of strings\",\n\t\tskipDoc: true,\n\t\tinput: `tags = [\"a\", \"b\"]`,\n\t\texpected: \"tags = [\\\"a\\\", \\\"b\\\"]\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"roundtrip object/map attribute\",\n\t\tskipDoc: true,\n\t\tinput: `obj = { a = 1, b = \"two\" }`,\n\t\texpected: \"obj = {\\n a = 1\\n b = \\\"two\\\"\\n}\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"roundtrip nested block\",\n\t\tskipDoc: true,\n\t\tinput: `server { port = 8080 }`,\n\t\texpected: \"server {\\n port = 8080\\n}\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"roundtrip multiple attributes\",\n\t\tskipDoc: true,\n\t\tinput: \"name = \\\"app\\\"\\nversion = 1\\nenabled = true\",\n\t\texpected: \"name = \\\"app\\\"\\nversion = 1\\nenabled = true\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Parse HCL: with comments\",\n\t\tinput: \"# Configuration\\nport = 8080 # server port\",\n\t\texpected: \"# Configuration\\nport: 8080 # server port\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip: with comments\",\n\t\tinput: \"# Configuration\\nport = 8080\",\n\t\texpected: \"# Configuration\\nport = 8080\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip: extraction\",\n\t\tskipDoc: true,\n\t\tinput: simpleSample,\n\t\texpression: \".shouty_message\",\n\t\texpected: \"upper(message)\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip: With templates, functions and arithmetic\",\n\t\tinput: simpleSample,\n\t\texpected: simpleSampleExpected,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"roundtrip example\",\n\t\tskipDoc: true,\n\t\tinput: simpleSample,\n\t\texpected: simpleSampleExpectedYaml,\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Parse HCL: List of strings\",\n\t\tskipDoc: true,\n\t\tinput: `tags = [\"a\", \"b\"]`,\n\t\texpected: \"tags:\\n - \\\"a\\\"\\n - \\\"b\\\"\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"roundtrip list of objects\",\n\t\tskipDoc: true,\n\t\tinput: `items = [{ name = \"a\", value = 1 }, { name = \"b\", value = 2 }]`,\n\t\texpected: \"items = [{\\n name = \\\"a\\\"\\n value = 1\\n }, {\\n name = \\\"b\\\"\\n value = 2\\n}]\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"roundtrip nested blocks with same name\",\n\t\tskipDoc: true,\n\t\tinput: \"database \\\"primary\\\" {\\n host = \\\"localhost\\\"\\n port = 5432\\n}\\ndatabase \\\"replica\\\" {\\n host = \\\"replica.local\\\"\\n port = 5433\\n}\",\n\t\texpected: \"database \\\"primary\\\" {\\n host = \\\"localhost\\\"\\n port = 5432\\n}\\ndatabase \\\"replica\\\" {\\n host = \\\"replica.local\\\"\\n port = 5433\\n}\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"roundtrip mixed nested structure\",\n\t\tskipDoc: true,\n\t\tinput: \"servers \\\"web\\\" {\\n addresses = [\\\"10.0.1.1\\\", \\\"10.0.1.2\\\"]\\n port = 8080\\n}\",\n\t\texpected: \"servers \\\"web\\\" {\\n addresses = [\\\"10.0.1.1\\\", \\\"10.0.1.2\\\"]\\n port = 8080\\n}\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"roundtrip null value\",\n\t\tskipDoc: true,\n\t\tinput: `value = null`,\n\t\texpected: \"value = null\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"roundtrip empty list\",\n\t\tskipDoc: true,\n\t\tinput: `items = []`,\n\t\texpected: \"items = []\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"roundtrip empty object\",\n\t\tskipDoc: true,\n\t\tinput: `config = {}`,\n\t\texpected: \"config = {}\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip: Separate blocks with same name.\",\n\t\tinput: \"resource \\\"aws_instance\\\" \\\"web\\\" {\\n ami = \\\"ami-12345\\\"\\n}\\nresource \\\"aws_instance\\\" \\\"db\\\" {\\n ami = \\\"ami-67890\\\"\\n}\",\n\t\texpected: \"resource \\\"aws_instance\\\" \\\"web\\\" {\\n ami = \\\"ami-12345\\\"\\n}\\nresource \\\"aws_instance\\\" \\\"db\\\" {\\n ami = \\\"ami-67890\\\"\\n}\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"roundtrip deeply nested structure\",\n\t\tskipDoc: true,\n\t\tinput: \"app \\\"database\\\" \\\"primary\\\" \\\"connection\\\" {\\n host = \\\"db.local\\\"\\n port = 5432\\n}\",\n\t\texpected: \"app \\\"database\\\" \\\"primary\\\" \\\"connection\\\" {\\n host = \\\"db.local\\\"\\n port = 5432\\n}\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"roundtrip with leading comments\",\n\t\tskipDoc: true,\n\t\tinput: \"# Main config\\nenabled = true\\nport = 8080\",\n\t\texpected: \"# Main config\\nenabled = true\\nport = 8080\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Multiple attributes with comments (comment safety with safe path separator)\",\n\t\tskipDoc: true,\n\t\tinput: \"# Database config\\ndb_host = \\\"localhost\\\"\\n# Connection pool\\ndb_pool = 10\",\n\t\texpected: \"# Database config\\ndb_host = \\\"localhost\\\"\\n# Connection pool\\ndb_pool = 10\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Nested blocks with head comments\",\n\t\tskipDoc: true,\n\t\tinput: \"service \\\"api\\\" {\\n # Listen address\\n listen = \\\"0.0.0.0:8080\\\"\\n # TLS enabled\\n tls = true\\n}\",\n\t\texpected: \"service \\\"api\\\" {\\n # Listen address\\n listen = \\\"0.0.0.0:8080\\\"\\n # TLS enabled\\n tls = true\\n}\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Multiple blocks with EncodeSeparate preservation\",\n\t\tskipDoc: true,\n\t\tinput: \"resource \\\"aws_s3_bucket\\\" \\\"bucket1\\\" {\\n bucket = \\\"my-bucket-1\\\"\\n}\\nresource \\\"aws_s3_bucket\\\" \\\"bucket2\\\" {\\n bucket = \\\"my-bucket-2\\\"\\n}\",\n\t\texpected: \"resource \\\"aws_s3_bucket\\\" \\\"bucket1\\\" {\\n bucket = \\\"my-bucket-1\\\"\\n}\\nresource \\\"aws_s3_bucket\\\" \\\"bucket2\\\" {\\n bucket = \\\"my-bucket-2\\\"\\n}\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Blocks with same name handled separately\",\n\t\tskipDoc: true,\n\t\tinput: \"server \\\"primary\\\" { port = 8080 }\\nserver \\\"backup\\\" { port = 8081 }\",\n\t\texpected: \"server \\\"primary\\\" {\\n port = 8080\\n}\\nserver \\\"backup\\\" {\\n port = 8081\\n}\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Block label with dot roundtrip (commentPathSep)\",\n\t\tskipDoc: true,\n\t\tinput: \"service \\\"api.service\\\" {\\n port = 8080\\n}\",\n\t\texpected: \"service \\\"api.service\\\" {\\n port = 8080\\n}\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Nested template expression\",\n\t\tskipDoc: true,\n\t\tinput: `message = \"User: ${username}, Role: ${user_role}\"`,\n\t\texpected: \"message = \\\"User: ${username}, Role: ${user_role}\\\"\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Empty object roundtrip\",\n\t\tskipDoc: true,\n\t\tinput: `obj = {}`,\n\t\texpected: \"obj = {}\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Null value in block\",\n\t\tskipDoc: true,\n\t\tinput: `service { optional_field = null }`,\n\t\texpected: \"service {\\n optional_field = null\\n}\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n}\n\nfunc testHclScenario(t *testing.T, s formatScenario) {\n\tswitch s.scenarioType {\n\tcase \"decode\":\n\t\tresult := mustProcessFormatScenario(s, NewHclDecoder(), NewYamlEncoder(ConfiguredYamlPreferences))\n\t\ttest.AssertResultWithContext(t, s.expected, result, s.description)\n\tcase \"roundtrip\":\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewHclDecoder(), NewHclEncoder(ConfiguredHclPreferences)), s.description)\n\t}\n}\n\nfunc documentHclScenario(_ *testing.T, w *bufio.Writer, i interface{}) {\n\ts := i.(formatScenario)\n\n\tif s.skipDoc {\n\t\treturn\n\t}\n\tswitch s.scenarioType {\n\tcase \"\", \"decode\":\n\t\tdocumentHclDecodeScenario(w, s)\n\tcase \"roundtrip\":\n\t\tdocumentHclRoundTripScenario(w, s)\n\tdefault:\n\t\tpanic(fmt.Sprintf(\"unhandled scenario type %q\", s.scenarioType))\n\t}\n}\n\nfunc documentHclDecodeScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.hcl file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```hcl\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\texpression := s.expression\n\tif s.expression != \"\" {\n\t\texpression = fmt.Sprintf(\" '%v'\", s.expression)\n\t}\n\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -oy%v sample.hcl\\n```\\n\", expression))\n\twriteOrPanic(w, \"will output\\n\")\n\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewHclDecoder(), NewYamlEncoder(ConfiguredYamlPreferences))))\n}\n\nfunc documentHclRoundTripScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.hcl file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```hcl\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\texpression := s.expression\n\tif s.expression != \"\" {\n\t\texpression = fmt.Sprintf(\" '%v'\", s.expression)\n\t}\n\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq%v sample.hcl\\n```\\n\", expression))\n\twriteOrPanic(w, \"will output\\n\")\n\n\twriteOrPanic(w, fmt.Sprintf(\"```hcl\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewHclDecoder(), NewHclEncoder(ConfiguredHclPreferences))))\n}\n\nfunc TestHclEncoderPrintDocumentSeparator(t *testing.T) {\n\tencoder := NewHclEncoder(ConfiguredHclPreferences)\n\tvar buf bytes.Buffer\n\twriter := bufio.NewWriter(&buf)\n\n\terr := encoder.PrintDocumentSeparator(writer)\n\twriter.Flush()\n\n\ttest.AssertResult(t, nil, err)\n\ttest.AssertResult(t, \"\", buf.String())\n}\n\nfunc TestHclEncoderPrintLeadingContent(t *testing.T) {\n\tencoder := NewHclEncoder(ConfiguredHclPreferences)\n\tvar buf bytes.Buffer\n\twriter := bufio.NewWriter(&buf)\n\n\terr := encoder.PrintLeadingContent(writer, \"some content\")\n\twriter.Flush()\n\n\ttest.AssertResult(t, nil, err)\n\ttest.AssertResult(t, \"\", buf.String())\n}\n\nfunc TestHclEncoderCanHandleAliases(t *testing.T) {\n\tencoder := NewHclEncoder(ConfiguredHclPreferences)\n\ttest.AssertResult(t, false, encoder.CanHandleAliases())\n}\n\nfunc TestHclFormatScenarios(t *testing.T) {\n\tfor _, tt := range hclFormatScenarios {\n\t\ttestHclScenario(t, tt)\n\t}\n\tgenericScenarios := make([]interface{}, len(hclFormatScenarios))\n\tfor i, s := range hclFormatScenarios {\n\t\tgenericScenarios[i] = s\n\t}\n\tdocumentScenarios(t, \"usage\", \"hcl\", genericScenarios, documentHclScenario)\n}\n" }, { "path": "pkg/yqlib/ini.go", "content": "package yqlib\n\ntype INIPreferences struct {\n\tColorsEnabled bool\n}\n\nfunc NewDefaultINIPreferences() INIPreferences {\n\treturn INIPreferences{\n\t\tColorsEnabled: false,\n\t}\n}\n\nfunc (p *INIPreferences) Copy() INIPreferences {\n\treturn INIPreferences{\n\t\tColorsEnabled: p.ColorsEnabled,\n\t}\n}\n\nvar ConfiguredINIPreferences = NewDefaultINIPreferences()\n" }, { "path": "pkg/yqlib/ini_test.go", "content": "//go:build !yq_noini\n\npackage yqlib\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nconst simpleINIInput = `[section]\nkey = value\n`\n\nconst expectedSimpleINIOutput = `[section]\nkey = value\n`\n\nconst expectedSimpleINIYaml = `section:\n key: value\n`\n\nvar iniScenarios = []formatScenario{\n\t{\n\t\tdescription: \"Parse INI: simple\",\n\t\tinput: simpleINIInput,\n\t\tscenarioType: \"decode\",\n\t\texpected: expectedSimpleINIYaml,\n\t},\n\t{\n\t\tdescription: \"Encode INI: simple\",\n\t\tinput: `section: {key: value}`,\n\t\texpected: expectedSimpleINIOutput,\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip INI: simple\",\n\t\tinput: simpleINIInput,\n\t\texpected: expectedSimpleINIOutput,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"bad ini\",\n\t\tinput: `[section\\nkey = value`,\n\t\texpectedError: `bad file 'sample.yml': failed to parse INI content: unclosed section: [section\\nkey = value`,\n\t\tscenarioType: \"decode-error\",\n\t},\n}\n\nfunc documentRoundtripINIScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.ini file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```ini\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\n\texpression := s.expression\n\tif expression != \"\" {\n\t\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -p=ini -o=ini '%v' sample.ini\\n```\\n\", expression))\n\t} else {\n\t\twriteOrPanic(w, \"```bash\\nyq -p=ini -o=ini sample.ini\\n```\\n\")\n\t}\n\n\twriteOrPanic(w, \"will output\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```ini\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewINIDecoder(), NewINIEncoder())))\n}\n\nfunc documentDecodeINIScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.ini file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```ini\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\n\texpression := s.expression\n\tif expression != \"\" {\n\t\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -p=ini '%v' sample.ini\\n```\\n\", expression))\n\t} else {\n\t\twriteOrPanic(w, \"```bash\\nyq -p=ini sample.ini\\n```\\n\")\n\t}\n\n\twriteOrPanic(w, \"will output\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewINIDecoder(), NewYamlEncoder(ConfiguredYamlPreferences))))\n}\n\nfunc testINIScenario(t *testing.T, s formatScenario) {\n\tswitch s.scenarioType {\n\tcase \"encode\":\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewINIEncoder()), s.description)\n\tcase \"decode\":\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewINIDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)\n\tcase \"roundtrip\":\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewINIDecoder(), NewINIEncoder()), s.description)\n\tcase \"decode-error\":\n\t\tresult, err := processFormatScenario(s, NewINIDecoder(), NewINIEncoder())\n\t\tif err == nil {\n\t\t\tt.Errorf(\"Expected error '%v' but it worked: %v\", s.expectedError, result)\n\t\t} else {\n\t\t\ttest.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description)\n\t\t}\n\tdefault:\n\t\tpanic(fmt.Sprintf(\"unhandled scenario type %q\", s.scenarioType))\n\t}\n}\n\nfunc documentINIScenario(_ *testing.T, w *bufio.Writer, i interface{}) {\n\ts := i.(formatScenario)\n\tif s.skipDoc {\n\t\treturn\n\t}\n\tswitch s.scenarioType {\n\tcase \"encode\":\n\t\tdocumentINIEncodeScenario(w, s)\n\tcase \"decode\":\n\t\tdocumentDecodeINIScenario(w, s)\n\tcase \"roundtrip\":\n\t\tdocumentRoundtripINIScenario(w, s)\n\tcase \"decode-error\":\n\t\tdocumentDecodeErrorINIScenario(w, s)\n\tdefault:\n\t\tpanic(fmt.Sprintf(\"unhandled scenario type %q\", s.scenarioType))\n\t}\n}\n\nfunc documentINIEncodeScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.yml file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\n\texpression := s.expression\n\tif expression == \"\" {\n\t\texpression = \".\"\n\t}\n\n\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -o=ini '%v' sample.yml\\n```\\n\", expression))\n\n\twriteOrPanic(w, \"will output\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```ini\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewINIEncoder())))\n}\n\nfunc documentDecodeErrorINIScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.ini file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```ini\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then an error is expected:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```\\n%v\\n```\\n\\n\", s.expectedError))\n}\n\nfunc TestINIScenarios(t *testing.T) {\n\tfor _, tt := range iniScenarios {\n\t\ttestINIScenario(t, tt)\n\t}\n\tgenericScenarios := make([]interface{}, len(iniScenarios))\n\tfor i, s := range iniScenarios {\n\t\tgenericScenarios[i] = s\n\t}\n\tdocumentScenarios(t, \"usage\", \"convert\", genericScenarios, documentINIScenario)\n}\n" }, { "path": "pkg/yqlib/json.go", "content": "package yqlib\n\ntype JsonPreferences struct {\n\tIndent int\n\tColorsEnabled bool\n\tUnwrapScalar bool\n}\n\nfunc NewDefaultJsonPreferences() JsonPreferences {\n\treturn JsonPreferences{\n\t\tIndent: 2,\n\t\tColorsEnabled: true,\n\t\tUnwrapScalar: true,\n\t}\n}\n\nfunc (p *JsonPreferences) Copy() JsonPreferences {\n\treturn JsonPreferences{\n\t\tIndent: p.Indent,\n\t\tColorsEnabled: p.ColorsEnabled,\n\t\tUnwrapScalar: p.UnwrapScalar,\n\t}\n}\n\nvar ConfiguredJSONPreferences = NewDefaultJsonPreferences()\n" }, { "path": "pkg/yqlib/json_test.go", "content": "//go:build !yq_nojson\n\npackage yqlib\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nconst complexExpectYaml = `a: Easy! as one two three\nb:\n c: 2\n d:\n - 3\n - 4\n`\n\nconst sampleNdJson = `{\"this\": \"is a multidoc json file\"}\n{\"each\": [\"line is a valid json document\"]}\n{\"a number\": 4}\n`\n\nconst sampleNdJsonKey = `{\"a\": \"first\", \"b\": \"next\", \"ab\": \"last\"}`\n\nconst expectedJsonKeysInOrder = `a: first\nb: next\nab: last\n`\n\nconst expectedNdJsonYaml = `this: is a multidoc json file\n---\neach:\n - line is a valid json document\n---\na number: 4\n`\n\nconst expectedRoundTripSampleNdJson = `{\"this\":\"is a multidoc json file\"}\n{\"each\":[\"line is a valid json document\"]}\n{\"a number\":4}\n`\n\nconst expectedUpdatedMultilineJson = `{\"this\":\"is a multidoc json file\"}\n{\"each\":[\"line is a valid json document\",\"cool\"]}\n{\"a number\":4}\n`\n\nconst sampleMultiLineJson = `{\n\t\"this\": \"is a multidoc json file\"\n}\n{\n\t\"it\": [\n\t\t\"has\",\n\t\t\"consecutive\",\n\t\t\"json documents\"\n\t]\n}\n{\n\t\"a number\": 4\n}\n`\n\nconst roundTripMultiLineJson = `{\n \"this\": \"is a multidoc json file\"\n}\n{\n \"it\": [\n \"has\",\n \"consecutive\",\n \"json documents\"\n ]\n}\n{\n \"a number\": 4\n}\n`\n\nvar jsonScenarios = []formatScenario{\n\t{\n\t\tdescription: \"array empty\",\n\t\tskipDoc: true,\n\t\tinput: \"[]\",\n\t\tscenarioType: \"roundtrip-ndjson\",\n\t\tindent: 0,\n\t\texpected: \"[]\\n\",\n\t},\n\t{\n\t\tdescription: \"array has scalar\",\n\t\tskipDoc: true,\n\t\tinput: \"[3]\",\n\t\tscenarioType: \"roundtrip-ndjson\",\n\t\tindent: 0,\n\t\texpected: \"[3]\\n\",\n\t},\n\t{\n\t\tdescription: \"array has object\",\n\t\tskipDoc: true,\n\t\tinput: `[{\"x\": 3}]`,\n\t\tscenarioType: \"roundtrip-ndjson\",\n\t\tindent: 0,\n\t\texpected: \"[{\\\"x\\\":3}]\\n\",\n\t},\n\t{\n\t\tdescription: \"array null\",\n\t\tskipDoc: true,\n\t\tinput: \"[null]\",\n\t\tscenarioType: \"roundtrip-ndjson\",\n\t\tindent: 0,\n\t\texpected: \"[null]\\n\",\n\t},\n\t{\n\t\tdescription: \"set tags\",\n\t\tskipDoc: true,\n\t\tinput: \"[{}]\",\n\t\texpression: `[.. | type]`,\n\t\tscenarioType: \"roundtrip-ndjson\",\n\t\tindent: 0,\n\t\texpected: \"[\\\"!!seq\\\",\\\"!!map\\\"]\\n\",\n\t},\n\t{\n\t\tdescription: \"Parse json: simple\",\n\t\tsubdescription: \"JSON is a subset of yaml, so all you need to do is prettify the output\",\n\t\tinput: `{\"cat\": \"meow\"}`,\n\t\tscenarioType: \"decode-ndjson\",\n\t\texpected: \"cat: meow\\n\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Parse json: simple: key\",\n\t\tinput: `{\"cat\": \"meow\"}`,\n\t\texpression: \".cat | key\",\n\t\texpected: \"\\\"cat\\\"\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Parse json: simple: parent\",\n\t\tinput: `{\"cat\": \"meow\"}`,\n\t\texpression: \".cat | parent\",\n\t\texpected: \"{\\\"cat\\\":\\\"meow\\\"}\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Parse json: simple: path\",\n\t\tinput: `{\"cat\": \"meow\"}`,\n\t\texpression: \".cat | path\",\n\t\texpected: \"[\\\"cat\\\"]\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Parse json: deeper: path\",\n\t\tinput: `{\"cat\": {\"noises\": \"meow\"}}`,\n\t\texpression: \".cat.noises | path\",\n\t\texpected: \"[\\\"cat\\\",\\\"noises\\\"]\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Parse json: array path\",\n\t\tinput: `{\"cat\": {\"noises\": [\"meow\"]}}`,\n\t\texpression: \".cat.noises[0] | path\",\n\t\texpected: \"[\\\"cat\\\",\\\"noises\\\",0]\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"bad json\",\n\t\tskipDoc: true,\n\t\tinput: `{\"a\": 1 b\": 2}`,\n\t\texpectedError: `bad file 'sample.yml': json: string of object unexpected end of JSON input`,\n\t\tscenarioType: \"decode-error\",\n\t},\n\t{\n\t\tdescription: \"Parse json: complex\",\n\t\tsubdescription: \"JSON is a subset of yaml, so all you need to do is prettify the output\",\n\t\tinput: `{\"a\":\"Easy! as one two three\",\"b\":{\"c\":2,\"d\":[3,4]}}`,\n\t\texpected: complexExpectYaml,\n\t\tscenarioType: \"decode-ndjson\",\n\t},\n\t{\n\t\tdescription: \"Encode json: simple\",\n\t\tinput: `cat: meow`,\n\t\tindent: 2,\n\t\texpected: \"{\\n \\\"cat\\\": \\\"meow\\\"\\n}\\n\",\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tdescription: \"Encode json: simple - in one line\",\n\t\tinput: `cat: meow # this is a comment, and it will be dropped.`,\n\t\tindent: 0,\n\t\texpected: \"{\\\"cat\\\":\\\"meow\\\"}\\n\",\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tdescription: \"Encode json: comments\",\n\t\tinput: `cat: meow # this is a comment, and it will be dropped.`,\n\t\tindent: 2,\n\t\texpected: \"{\\n \\\"cat\\\": \\\"meow\\\"\\n}\\n\",\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tdescription: \"Encode json: anchors\",\n\t\tsubdescription: \"Anchors are dereferenced\",\n\t\tinput: \"cat: &ref meow\\nanotherCat: *ref\",\n\t\tindent: 2,\n\t\texpected: \"{\\n \\\"cat\\\": \\\"meow\\\",\\n \\\"anotherCat\\\": \\\"meow\\\"\\n}\\n\",\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tdescription: \"Encode json: multiple results\",\n\t\tsubdescription: \"Each matching node is converted into a json doc. This is best used with 0 indent (json document per line)\",\n\t\tinput: `things: [{stuff: cool}, {whatever: cat}]`,\n\t\texpression: `.things[]`,\n\t\tindent: 0,\n\t\texpected: \"{\\\"stuff\\\":\\\"cool\\\"}\\n{\\\"whatever\\\":\\\"cat\\\"}\\n\",\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip JSON Lines / NDJSON\",\n\t\tinput: sampleNdJson,\n\t\texpected: expectedRoundTripSampleNdJson,\n\t\tscenarioType: \"roundtrip-ndjson\",\n\t\tindent: 0,\n\t},\n\t{\n\t\tdescription: \"Roundtrip multi-document JSON\",\n\t\tsubdescription: \"The parser can also handle multiple multi-line json documents in a single file (despite this not being in the JSON Lines / NDJSON spec). Typically you would have one entire JSON document per line, but the parser also supports multiple multi-line json documents\",\n\t\tinput: sampleMultiLineJson,\n\t\texpected: roundTripMultiLineJson,\n\t\tscenarioType: \"roundtrip-multi\",\n\t},\n\t{\n\t\tdescription: \"Update a specific document in a multi-document json\",\n\t\tsubdescription: \"Documents are indexed by the `documentIndex` or `di` operator.\",\n\t\tinput: sampleNdJson,\n\t\texpected: expectedUpdatedMultilineJson,\n\t\texpression: `(select(di == 1) | .each ) += \"cool\"`,\n\t\tscenarioType: \"roundtrip-ndjson\",\n\t},\n\t{\n\t\tdescription: \"Find and update a specific document in a multi-document json\",\n\t\tsubdescription: \"Use expressions as you normally would.\",\n\t\tinput: sampleNdJson,\n\t\texpected: expectedUpdatedMultilineJson,\n\t\texpression: `(select(has(\"each\")) | .each ) += \"cool\"`,\n\t\tscenarioType: \"roundtrip-ndjson\",\n\t},\n\t{\n\t\tdescription: \"Decode JSON Lines / NDJSON\",\n\t\tinput: sampleNdJson,\n\t\texpected: expectedNdJsonYaml,\n\t\tscenarioType: \"decode-ndjson\",\n\t},\n\t{\n\t\tdescription: \"Decode JSON Lines / NDJSON, maintain key order\",\n\t\tskipDoc: true,\n\t\tinput: sampleNdJsonKey,\n\t\texpected: expectedJsonKeysInOrder,\n\t\tscenarioType: \"decode-ndjson\",\n\t},\n\t{\n\t\tdescription: \"numbers\",\n\t\tskipDoc: true,\n\t\tinput: \"[3, 3.0, 3.1, -1, 999999, 1000000, 1000001, 1.1]\",\n\t\texpected: \"- 3\\n- 3\\n- 3.1\\n- -1\\n- 999999\\n- 1000000\\n- 1000001\\n- 1.1\\n\",\n\t\tscenarioType: \"decode-ndjson\",\n\t},\n\t{\n\t\tdescription: \"number single\",\n\t\tskipDoc: true,\n\t\tinput: \"3\",\n\t\texpected: \"3\\n\",\n\t\tscenarioType: \"decode-ndjson\",\n\t},\n\t{\n\t\tdescription: \"empty string\",\n\t\tskipDoc: true,\n\t\tinput: `\"\"`,\n\t\texpected: \"\\n\",\n\t\tscenarioType: \"decode-ndjson\",\n\t},\n\t{\n\t\tdescription: \"strings\",\n\t\tskipDoc: true,\n\t\tinput: `[\"\", \"cat\"]`,\n\t\texpected: \"- \\\"\\\"\\n- cat\\n\",\n\t\tscenarioType: \"decode-ndjson\",\n\t},\n\t{\n\t\tdescription: \"null\",\n\t\tskipDoc: true,\n\t\tinput: `null`,\n\t\texpected: \"null\\n\",\n\t\tscenarioType: \"decode-ndjson\",\n\t},\n\t{\n\t\tdescription: \"booleans\",\n\t\tskipDoc: true,\n\t\tinput: `[true, false]`,\n\t\texpected: \"- true\\n- false\\n\",\n\t\tscenarioType: \"decode-ndjson\",\n\t},\n}\n\nfunc documentRoundtripNdJsonScenario(w *bufio.Writer, s formatScenario, indent int) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.json file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```json\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\n\texpression := s.expression\n\tif expression != \"\" {\n\t\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -p=json -o=json -I=%v '%v' sample.json\\n```\\n\", indent, expression))\n\t} else {\n\t\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -p=json -o=json -I=%v sample.json\\n```\\n\", indent))\n\t}\n\n\twriteOrPanic(w, \"will output\\n\")\n\tprefs := ConfiguredJSONPreferences.Copy()\n\tprefs.Indent = indent\n\tprefs.UnwrapScalar = false\n\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(prefs))))\n}\n\nfunc documentDecodeNdJsonScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.json file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```json\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\n\texpression := s.expression\n\tif expression != \"\" {\n\t\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -p=json '%v' sample.json\\n```\\n\", expression))\n\t} else {\n\t\twriteOrPanic(w, \"```bash\\nyq -p=json sample.json\\n```\\n\")\n\t}\n\n\twriteOrPanic(w, \"will output\\n\")\n\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(ConfiguredYamlPreferences))))\n}\n\nfunc decodeJSON(t *testing.T, jsonString string) *CandidateNode {\n\tdocs, err := readDocument(jsonString, \"sample.json\", 0)\n\n\tif err != nil {\n\t\tt.Error(err)\n\t\treturn nil\n\t}\n\n\texp, err := getExpressionParser().ParseExpression(PrettyPrintExp)\n\n\tif err != nil {\n\t\tt.Error(err)\n\t\treturn nil\n\t}\n\n\tcontext, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: docs}, exp)\n\n\tif err != nil {\n\t\tt.Error(err)\n\t\treturn nil\n\t}\n\n\treturn context.MatchingNodes.Front().Value.(*CandidateNode)\n}\n\nfunc testJSONScenario(t *testing.T, s formatScenario) {\n\tprefs := ConfiguredJSONPreferences.Copy()\n\tprefs.Indent = s.indent\n\tprefs.UnwrapScalar = false\n\tswitch s.scenarioType {\n\tcase \"encode\":\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(prefs)), s.description)\n\tcase \"decode\":\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(prefs)), s.description)\n\tcase \"decode-ndjson\":\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)\n\tcase \"roundtrip-ndjson\":\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(prefs)), s.description)\n\tcase \"roundtrip-multi\":\n\t\tprefs.Indent = 2\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(prefs)), s.description)\n\tcase \"decode-error\":\n\t\tresult, err := processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(prefs))\n\t\tif err == nil {\n\t\t\tt.Errorf(\"Expected error '%v' but it worked: %v\", s.expectedError, result)\n\t\t} else {\n\t\t\ttest.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description)\n\t\t}\n\tdefault:\n\t\tpanic(fmt.Sprintf(\"unhandled scenario type %q\", s.scenarioType))\n\t}\n}\n\nfunc documentJSONDecodeScenario(t *testing.T, w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.json file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```json\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\twriteOrPanic(w, \"```bash\\nyq -P '.' sample.json\\n```\\n\")\n\twriteOrPanic(w, \"will output\\n\")\n\n\tvar output bytes.Buffer\n\tprinter := NewSimpleYamlPrinter(bufio.NewWriter(&output), true, 2, true)\n\n\tnode := decodeJSON(t, s.input)\n\n\terr := printer.PrintResults(node.AsList())\n\tif err != nil {\n\t\tt.Error(err)\n\t\treturn\n\t}\n\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v```\\n\\n\", output.String()))\n}\n\nfunc documentJSONScenario(t *testing.T, w *bufio.Writer, i interface{}) {\n\ts := i.(formatScenario)\n\tif s.skipDoc {\n\t\treturn\n\t}\n\tswitch s.scenarioType {\n\tcase \"\":\n\t\tdocumentJSONDecodeScenario(t, w, s)\n\tcase \"encode\":\n\t\tdocumentJSONEncodeScenario(w, s)\n\tcase \"decode-ndjson\":\n\t\tdocumentDecodeNdJsonScenario(w, s)\n\tcase \"roundtrip-ndjson\":\n\t\tdocumentRoundtripNdJsonScenario(w, s, 0)\n\tcase \"roundtrip-multi\":\n\t\tdocumentRoundtripNdJsonScenario(w, s, 2)\n\n\tdefault:\n\t\tpanic(fmt.Sprintf(\"unhandled scenario type %q\", s.scenarioType))\n\t}\n}\n\nfunc documentJSONEncodeScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.yml file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\n\texpression := s.expression\n\tif expression == \"\" {\n\t\texpression = \".\"\n\t}\n\n\tif s.indent == 2 {\n\t\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -o=json '%v' sample.yml\\n```\\n\", expression))\n\t} else {\n\t\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -o=json -I=%v '%v' sample.yml\\n```\\n\", s.indent, expression))\n\t}\n\twriteOrPanic(w, \"will output\\n\")\n\tprefs := ConfiguredJSONPreferences.Copy()\n\tprefs.Indent = s.indent\n\tprefs.UnwrapScalar = false\n\n\twriteOrPanic(w, fmt.Sprintf(\"```json\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(prefs))))\n}\n\nfunc TestJSONScenarios(t *testing.T) {\n\tfor _, tt := range jsonScenarios {\n\t\ttestJSONScenario(t, tt)\n\t}\n\tgenericScenarios := make([]interface{}, len(jsonScenarios))\n\tfor i, s := range jsonScenarios {\n\t\tgenericScenarios[i] = s\n\t}\n\tdocumentScenarios(t, \"usage\", \"convert\", genericScenarios, documentJSONScenario)\n}\n" }, { "path": "pkg/yqlib/kyaml.go", "content": "//go:build !yq_nokyaml\n\npackage yqlib\n\ntype KYamlPreferences struct {\n\tIndent int\n\tColorsEnabled bool\n\tPrintDocSeparators bool\n\tUnwrapScalar bool\n}\n\nfunc NewDefaultKYamlPreferences() KYamlPreferences {\n\treturn KYamlPreferences{\n\t\tIndent: 2,\n\t\tColorsEnabled: false,\n\t\tPrintDocSeparators: true,\n\t\tUnwrapScalar: true,\n\t}\n}\n\nfunc (p *KYamlPreferences) Copy() KYamlPreferences {\n\treturn KYamlPreferences{\n\t\tIndent: p.Indent,\n\t\tColorsEnabled: p.ColorsEnabled,\n\t\tPrintDocSeparators: p.PrintDocSeparators,\n\t\tUnwrapScalar: p.UnwrapScalar,\n\t}\n}\n\nvar ConfiguredKYamlPreferences = NewDefaultKYamlPreferences()\n" }, { "path": "pkg/yqlib/kyaml_test.go", "content": "//go:build !yq_nokyaml\n\npackage yqlib\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"fmt\"\n\t\"regexp\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nvar ansiRe = regexp.MustCompile(`\\x1b\\[[0-9;]*m`)\n\nfunc stripANSI(s string) string {\n\treturn ansiRe.ReplaceAllString(s, \"\")\n}\n\nvar kyamlFormatScenarios = []formatScenario{\n\t{\n\t\tdescription: \"Encode kyaml: plain string scalar\",\n\t\tsubdescription: \"Strings are always double-quoted in KYaml output.\",\n\t\tscenarioType: \"encode\",\n\t\tindent: 2,\n\t\tinput: \"cat\\n\",\n\t\texpected: \"\\\"cat\\\"\\n\",\n\t},\n\t{\n\t\tdescription: \"encode plain int scalar\",\n\t\tscenarioType: \"encode\",\n\t\tindent: 2,\n\t\tinput: \"12\\n\",\n\t\texpected: \"12\\n\",\n\t\tskipDoc: true,\n\t},\n\t{\n\t\tdescription: \"encode plain bool scalar\",\n\t\tscenarioType: \"encode\",\n\t\tindent: 2,\n\t\tinput: \"true\\n\",\n\t\texpected: \"true\\n\",\n\t\tskipDoc: true,\n\t},\n\t{\n\t\tdescription: \"encode plain null scalar\",\n\t\tscenarioType: \"encode\",\n\t\tindent: 2,\n\t\tinput: \"null\\n\",\n\t\texpected: \"null\\n\",\n\t\tskipDoc: true,\n\t},\n\t{\n\t\tdescription: \"encode flow mapping and sequence\",\n\t\tscenarioType: \"encode\",\n\t\tindent: 2,\n\t\tinput: \"a: b\\nc:\\n - d\\n\",\n\t\texpected: \"{\\n\" +\n\t\t\t\" a: \\\"b\\\",\\n\" +\n\t\t\t\" c: [\\n\" +\n\t\t\t\" \\\"d\\\",\\n\" +\n\t\t\t\" ],\\n\" +\n\t\t\t\"}\\n\",\n\t},\n\t{\n\t\tdescription: \"encode non-string scalars\",\n\t\tscenarioType: \"encode\",\n\t\tindent: 2,\n\t\tinput: \"a: 12\\n\" +\n\t\t\t\"b: true\\n\" +\n\t\t\t\"c: null\\n\" +\n\t\t\t\"d: \\\"true\\\"\\n\",\n\t\texpected: \"{\\n\" +\n\t\t\t\" a: 12,\\n\" +\n\t\t\t\" b: true,\\n\" +\n\t\t\t\" c: null,\\n\" +\n\t\t\t\" d: \\\"true\\\",\\n\" +\n\t\t\t\"}\\n\",\n\t},\n\t{\n\t\tdescription: \"quote non-identifier keys\",\n\t\tscenarioType: \"encode\",\n\t\tindent: 2,\n\t\tinput: \"\\\"1a\\\": b\\n\\\"has space\\\": c\\n\",\n\t\texpected: \"{\\n\" +\n\t\t\t\" \\\"1a\\\": \\\"b\\\",\\n\" +\n\t\t\t\" \\\"has space\\\": \\\"c\\\",\\n\" +\n\t\t\t\"}\\n\",\n\t},\n\t{\n\t\tdescription: \"escape quoted strings\",\n\t\tscenarioType: \"encode\",\n\t\tindent: 2,\n\t\tinput: \"a: \\\"line1\\\\nline2\\\\t\\\\\\\"q\\\\\\\"\\\"\\n\",\n\t\texpected: \"{\\n\" +\n\t\t\t\" a: \\\"line1\\\\nline2\\\\t\\\\\\\"q\\\\\\\"\\\",\\n\" +\n\t\t\t\"}\\n\",\n\t},\n\t{\n\t\tdescription: \"preserve comments when encoding\",\n\t\tscenarioType: \"encode\",\n\t\tindent: 2,\n\t\tinput: \"# leading\\n\" +\n\t\t\t\"a: 1 # a line\\n\" +\n\t\t\t\"# head b\\n\" +\n\t\t\t\"b: 2\\n\" +\n\t\t\t\"c:\\n\" +\n\t\t\t\" # head d\\n\" +\n\t\t\t\" - d # d line\\n\" +\n\t\t\t\" - e\\n\" +\n\t\t\t\"# trailing\\n\",\n\t\texpected: \"# leading\\n\" +\n\t\t\t\"{\\n\" +\n\t\t\t\" a: 1, # a line\\n\" +\n\t\t\t\" # head b\\n\" +\n\t\t\t\" b: 2,\\n\" +\n\t\t\t\" c: [\\n\" +\n\t\t\t\" # head d\\n\" +\n\t\t\t\" \\\"d\\\", # d line\\n\" +\n\t\t\t\" \\\"e\\\",\\n\" +\n\t\t\t\" ],\\n\" +\n\t\t\t\" # trailing\\n\" +\n\t\t\t\"}\\n\",\n\t},\n\t{\n\t\tdescription: \"Encode kyaml: anchors and aliases\",\n\t\tsubdescription: \"KYaml output does not support anchors/aliases; they are expanded to concrete values.\",\n\t\tscenarioType: \"encode\",\n\t\tindent: 2,\n\t\tinput: \"base: &base\\n\" +\n\t\t\t\" a: b\\n\" +\n\t\t\t\"copy: *base\\n\",\n\t\texpected: \"{\\n\" +\n\t\t\t\" base: {\\n\" +\n\t\t\t\" a: \\\"b\\\",\\n\" +\n\t\t\t\" },\\n\" +\n\t\t\t\" copy: {\\n\" +\n\t\t\t\" a: \\\"b\\\",\\n\" +\n\t\t\t\" },\\n\" +\n\t\t\t\"}\\n\",\n\t},\n\t{\n\t\tdescription: \"Encode kyaml: yaml to kyaml shows formatting differences\",\n\t\tsubdescription: \"KYaml uses flow-style collections (braces/brackets) and explicit commas.\",\n\t\tscenarioType: \"encode\",\n\t\tindent: 2,\n\t\tinput: \"person:\\n\" +\n\t\t\t\" name: John\\n\" +\n\t\t\t\" pets:\\n\" +\n\t\t\t\" - cat\\n\" +\n\t\t\t\" - dog\\n\",\n\t\texpected: \"{\\n\" +\n\t\t\t\" person: {\\n\" +\n\t\t\t\" name: \\\"John\\\",\\n\" +\n\t\t\t\" pets: [\\n\" +\n\t\t\t\" \\\"cat\\\",\\n\" +\n\t\t\t\" \\\"dog\\\",\\n\" +\n\t\t\t\" ],\\n\" +\n\t\t\t\" },\\n\" +\n\t\t\t\"}\\n\",\n\t},\n\t{\n\t\tdescription: \"Encode kyaml: nested lists of objects\",\n\t\tsubdescription: \"Lists and objects can be nested arbitrarily; KYaml always uses flow-style collections.\",\n\t\tscenarioType: \"encode\",\n\t\tindent: 2,\n\t\tinput: \"- name: a\\n\" +\n\t\t\t\" items:\\n\" +\n\t\t\t\" - id: 1\\n\" +\n\t\t\t\" tags:\\n\" +\n\t\t\t\" - k: x\\n\" +\n\t\t\t\" v: y\\n\" +\n\t\t\t\" - k: x2\\n\" +\n\t\t\t\" v: y2\\n\" +\n\t\t\t\" - id: 2\\n\" +\n\t\t\t\" tags:\\n\" +\n\t\t\t\" - k: z\\n\" +\n\t\t\t\" v: w\\n\",\n\t\texpected: \"[\\n\" +\n\t\t\t\" {\\n\" +\n\t\t\t\" name: \\\"a\\\",\\n\" +\n\t\t\t\" items: [\\n\" +\n\t\t\t\" {\\n\" +\n\t\t\t\" id: 1,\\n\" +\n\t\t\t\" tags: [\\n\" +\n\t\t\t\" {\\n\" +\n\t\t\t\" k: \\\"x\\\",\\n\" +\n\t\t\t\" v: \\\"y\\\",\\n\" +\n\t\t\t\" },\\n\" +\n\t\t\t\" {\\n\" +\n\t\t\t\" k: \\\"x2\\\",\\n\" +\n\t\t\t\" v: \\\"y2\\\",\\n\" +\n\t\t\t\" },\\n\" +\n\t\t\t\" ],\\n\" +\n\t\t\t\" },\\n\" +\n\t\t\t\" {\\n\" +\n\t\t\t\" id: 2,\\n\" +\n\t\t\t\" tags: [\\n\" +\n\t\t\t\" {\\n\" +\n\t\t\t\" k: \\\"z\\\",\\n\" +\n\t\t\t\" v: \\\"w\\\",\\n\" +\n\t\t\t\" },\\n\" +\n\t\t\t\" ],\\n\" +\n\t\t\t\" },\\n\" +\n\t\t\t\" ],\\n\" +\n\t\t\t\" },\\n\" +\n\t\t\t\"]\\n\",\n\t},\n}\n\nfunc testKYamlScenario(t *testing.T, s formatScenario) {\n\tprefs := ConfiguredKYamlPreferences.Copy()\n\tprefs.Indent = s.indent\n\tprefs.UnwrapScalar = false\n\n\tswitch s.scenarioType {\n\tcase \"encode\":\n\t\ttest.AssertResultWithContext(\n\t\t\tt,\n\t\t\ts.expected,\n\t\t\tmustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewKYamlEncoder(prefs)),\n\t\t\ts.description,\n\t\t)\n\tdefault:\n\t\tpanic(fmt.Sprintf(\"unhandled scenario type %q\", s.scenarioType))\n\t}\n}\n\nfunc documentKYamlScenario(_ *testing.T, w *bufio.Writer, i interface{}) {\n\ts := i.(formatScenario)\n\tif s.skipDoc {\n\t\treturn\n\t}\n\n\tswitch s.scenarioType {\n\tcase \"encode\":\n\t\tdocumentKYamlEncodeScenario(w, s)\n\tdefault:\n\t\tpanic(fmt.Sprintf(\"unhandled scenario type %q\", s.scenarioType))\n\t}\n}\n\nfunc documentKYamlEncodeScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.yml file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\n\texpression := s.expression\n\tif expression == \"\" {\n\t\texpression = \".\"\n\t}\n\n\tif s.indent == 2 {\n\t\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -o=kyaml '%v' sample.yml\\n```\\n\", expression))\n\t} else {\n\t\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -o=kyaml -I=%v '%v' sample.yml\\n```\\n\", s.indent, expression))\n\t}\n\n\twriteOrPanic(w, \"will output\\n\")\n\n\tprefs := ConfiguredKYamlPreferences.Copy()\n\tprefs.Indent = s.indent\n\tprefs.UnwrapScalar = false\n\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewKYamlEncoder(prefs))))\n}\n\nfunc TestKYamlFormatScenarios(t *testing.T) {\n\tfor _, s := range kyamlFormatScenarios {\n\t\ttestKYamlScenario(t, s)\n\t}\n\n\tgenericScenarios := make([]interface{}, len(kyamlFormatScenarios))\n\tfor i, s := range kyamlFormatScenarios {\n\t\tgenericScenarios[i] = s\n\t}\n\tdocumentScenarios(t, \"usage\", \"kyaml\", genericScenarios, documentKYamlScenario)\n}\n\nfunc TestKYamlEncoderPrintDocumentSeparator(t *testing.T) {\n\tt.Run(\"enabled\", func(t *testing.T) {\n\t\tprefs := NewDefaultKYamlPreferences()\n\t\tprefs.PrintDocSeparators = true\n\n\t\tvar buf bytes.Buffer\n\t\terr := NewKYamlEncoder(prefs).PrintDocumentSeparator(&buf)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif buf.String() != \"---\\n\" {\n\t\t\tt.Fatalf(\"expected doc separator, got %q\", buf.String())\n\t\t}\n\t})\n\n\tt.Run(\"disabled\", func(t *testing.T) {\n\t\tprefs := NewDefaultKYamlPreferences()\n\t\tprefs.PrintDocSeparators = false\n\n\t\tvar buf bytes.Buffer\n\t\terr := NewKYamlEncoder(prefs).PrintDocumentSeparator(&buf)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif buf.String() != \"\" {\n\t\t\tt.Fatalf(\"expected no output, got %q\", buf.String())\n\t\t}\n\t})\n}\n\nfunc TestKYamlEncoderEncodeUnwrapScalar(t *testing.T) {\n\tprefs := NewDefaultKYamlPreferences()\n\tprefs.UnwrapScalar = true\n\n\tvar buf bytes.Buffer\n\terr := NewKYamlEncoder(prefs).Encode(&buf, &CandidateNode{\n\t\tKind: ScalarNode,\n\t\tTag: \"!!str\",\n\t\tValue: \"cat\",\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif buf.String() != \"cat\\n\" {\n\t\tt.Fatalf(\"expected unwrapped scalar, got %q\", buf.String())\n\t}\n}\n\nfunc TestKYamlEncoderEncodeColorsEnabled(t *testing.T) {\n\tprefs := NewDefaultKYamlPreferences()\n\tprefs.UnwrapScalar = false\n\tprefs.ColorsEnabled = true\n\n\tvar buf bytes.Buffer\n\terr := NewKYamlEncoder(prefs).Encode(&buf, &CandidateNode{\n\t\tKind: MappingNode,\n\t\tContent: []*CandidateNode{\n\t\t\t{Kind: ScalarNode, Tag: \"!!str\", Value: \"a\"},\n\t\t\t{Kind: ScalarNode, Tag: \"!!str\", Value: \"b\"},\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tout := stripANSI(buf.String())\n\tif !strings.Contains(out, \"a:\") || !strings.Contains(out, \"\\\"b\\\"\") {\n\t\tt.Fatalf(\"expected colourised output to contain rendered tokens, got %q\", out)\n\t}\n}\n\nfunc TestKYamlEncoderWriteNodeAliasAndUnknown(t *testing.T) {\n\tke := NewKYamlEncoder(NewDefaultKYamlPreferences()).(*kyamlEncoder)\n\n\tt.Run(\"alias_nil\", func(t *testing.T) {\n\t\tvar buf bytes.Buffer\n\t\terr := ke.writeNode(&buf, &CandidateNode{Kind: AliasNode}, 0)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif buf.String() != \"null\" {\n\t\t\tt.Fatalf(\"expected null for nil alias, got %q\", buf.String())\n\t\t}\n\t})\n\n\tt.Run(\"alias_value\", func(t *testing.T) {\n\t\tvar buf bytes.Buffer\n\t\terr := ke.writeNode(&buf, &CandidateNode{\n\t\t\tKind: AliasNode,\n\t\t\tAlias: &CandidateNode{Kind: ScalarNode, Tag: \"!!int\", Value: \"12\"},\n\t\t}, 0)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif buf.String() != \"12\" {\n\t\t\tt.Fatalf(\"expected dereferenced alias value, got %q\", buf.String())\n\t\t}\n\t})\n\n\tt.Run(\"unknown_kind\", func(t *testing.T) {\n\t\tvar buf bytes.Buffer\n\t\terr := ke.writeNode(&buf, &CandidateNode{Kind: Kind(12345)}, 0)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif buf.String() != \"null\" {\n\t\t\tt.Fatalf(\"expected null for unknown kind, got %q\", buf.String())\n\t\t}\n\t})\n}\n\nfunc TestKYamlEncoderEmptyCollections(t *testing.T) {\n\tke := NewKYamlEncoder(NewDefaultKYamlPreferences()).(*kyamlEncoder)\n\n\tt.Run(\"empty_mapping\", func(t *testing.T) {\n\t\tvar buf bytes.Buffer\n\t\terr := ke.writeNode(&buf, &CandidateNode{Kind: MappingNode}, 0)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif buf.String() != \"{}\" {\n\t\t\tt.Fatalf(\"expected empty mapping, got %q\", buf.String())\n\t\t}\n\t})\n\n\tt.Run(\"empty_sequence\", func(t *testing.T) {\n\t\tvar buf bytes.Buffer\n\t\terr := ke.writeNode(&buf, &CandidateNode{Kind: SequenceNode}, 0)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif buf.String() != \"[]\" {\n\t\t\tt.Fatalf(\"expected empty sequence, got %q\", buf.String())\n\t\t}\n\t})\n}\n\nfunc TestKYamlEncoderScalarFallbackAndEscaping(t *testing.T) {\n\tke := NewKYamlEncoder(NewDefaultKYamlPreferences()).(*kyamlEncoder)\n\n\tt.Run(\"unknown_tag_falls_back_to_string\", func(t *testing.T) {\n\t\tvar buf bytes.Buffer\n\t\terr := ke.writeNode(&buf, &CandidateNode{Kind: ScalarNode, Tag: \"!!timestamp\", Value: \"2020-01-01T00:00:00Z\"}, 0)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif buf.String() != \"\\\"2020-01-01T00:00:00Z\\\"\" {\n\t\t\tt.Fatalf(\"expected quoted fallback, got %q\", buf.String())\n\t\t}\n\t})\n\n\tt.Run(\"escape_double_quoted\", func(t *testing.T) {\n\t\tgot := escapeDoubleQuotedString(\"a\\\\b\\\"c\\n\\r\\t\" + string(rune(0x01)))\n\t\twant := \"a\\\\\\\\b\\\\\\\"c\\\\n\\\\r\\\\t\\\\u0001\"\n\t\tif got != want {\n\t\t\tt.Fatalf(\"expected %q, got %q\", want, got)\n\t\t}\n\t})\n\n\tt.Run(\"valid_bare_key\", func(t *testing.T) {\n\t\tif isValidKYamlBareKey(\"\") {\n\t\t\tt.Fatalf(\"expected empty string to be invalid\")\n\t\t}\n\t\tif isValidKYamlBareKey(\"1a\") {\n\t\t\tt.Fatalf(\"expected leading digit to be invalid\")\n\t\t}\n\t\tif !isValidKYamlBareKey(\"a_b-2\") {\n\t\t\tt.Fatalf(\"expected identifier-like key to be valid\")\n\t\t}\n\t})\n}\n\nfunc TestKYamlEncoderCommentsInMapping(t *testing.T) {\n\tprefs := NewDefaultKYamlPreferences()\n\tprefs.UnwrapScalar = false\n\tke := NewKYamlEncoder(prefs).(*kyamlEncoder)\n\n\tvar buf bytes.Buffer\n\terr := ke.writeNode(&buf, &CandidateNode{\n\t\tKind: MappingNode,\n\t\tContent: []*CandidateNode{\n\t\t\t{\n\t\t\t\tKind: ScalarNode,\n\t\t\t\tTag: \"!!str\",\n\t\t\t\tValue: \"a\",\n\t\t\t\tHeadComment: \"key head\",\n\t\t\t\tLineComment: \"key line\",\n\t\t\t\tFootComment: \"key foot\",\n\t\t\t},\n\t\t\t{\n\t\t\t\tKind: ScalarNode,\n\t\t\t\tTag: \"!!str\",\n\t\t\t\tValue: \"b\",\n\t\t\t\tHeadComment: \"value head\",\n\t\t\t},\n\t\t},\n\t}, 0)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tout := buf.String()\n\tif !strings.Contains(out, \"# key head\\n\") {\n\t\tt.Fatalf(\"expected key head comment, got %q\", out)\n\t}\n\tif !strings.Contains(out, \"# value head\\n\") {\n\t\tt.Fatalf(\"expected value head comment, got %q\", out)\n\t}\n\tif !strings.Contains(out, \", # key line\\n\") {\n\t\tt.Fatalf(\"expected inline key comment fallback, got %q\", out)\n\t}\n\tif !strings.Contains(out, \"# key foot\\n\") {\n\t\tt.Fatalf(\"expected foot comment fallback, got %q\", out)\n\t}\n}\n\nfunc TestKYamlEncoderCommentBlockAndInlineComment(t *testing.T) {\n\tke := NewKYamlEncoder(NewDefaultKYamlPreferences()).(*kyamlEncoder)\n\n\tt.Run(\"comment_block_prefixing_and_crlf\", func(t *testing.T) {\n\t\tvar buf bytes.Buffer\n\t\terr := ke.writeCommentBlock(&buf, \"line1\\r\\n\\r\\n# already\\r\\nline2\", 2)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\twant := \" # line1\\n # already\\n # line2\\n\"\n\t\tif buf.String() != want {\n\t\t\tt.Fatalf(\"expected %q, got %q\", want, buf.String())\n\t\t}\n\t})\n\n\tt.Run(\"inline_comment_prefix_and_first_line_only\", func(t *testing.T) {\n\t\tvar buf bytes.Buffer\n\t\terr := ke.writeInlineComment(&buf, \"hello\\r\\nsecond line\")\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif buf.String() != \" # hello\" {\n\t\t\tt.Fatalf(\"expected %q, got %q\", \" # hello\", buf.String())\n\t\t}\n\t})\n\n\tt.Run(\"inline_comment_already_prefixed\", func(t *testing.T) {\n\t\tvar buf bytes.Buffer\n\t\terr := ke.writeInlineComment(&buf, \"# hello\")\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif buf.String() != \" # hello\" {\n\t\t\tt.Fatalf(\"expected %q, got %q\", \" # hello\", buf.String())\n\t\t}\n\t})\n}\n" }, { "path": "pkg/yqlib/lexer.go", "content": "package yqlib\n\nimport (\n\t\"fmt\"\n\t\"regexp\"\n)\n\ntype expressionTokeniser interface {\n\tTokenise(expression string) ([]*token, error)\n}\n\ntype tokenType uint32\n\nconst (\n\toperationToken = 1 << iota\n\topenBracket\n\tcloseBracket\n\topenCollect\n\tcloseCollect\n\topenCollectObject\n\tcloseCollectObject\n\ttraverseArrayCollect\n)\n\ntype token struct {\n\tTokenType tokenType\n\tOperation *Operation\n\tAssignOperation *Operation // e.g. tag (GetTag) op becomes AssignTag if '=' follows it\n\tCheckForPostTraverse bool // e.g. [1]cat should really be [1].cat\n\tMatch string\n}\n\nfunc (t *token) toString(detail bool) string {\n\tswitch t.TokenType {\n\tcase operationToken:\n\t\tif detail {\n\t\t\treturn fmt.Sprintf(\"%v (%v)\", t.Operation.toString(), t.Operation.OperationType.Precedence)\n\t\t}\n\t\treturn t.Operation.toString()\n\tcase openBracket:\n\t\treturn \"(\"\n\tcase closeBracket:\n\t\treturn \")\"\n\tcase openCollect:\n\t\treturn \"[\"\n\tcase closeCollect:\n\t\treturn \"]\"\n\tcase openCollectObject:\n\t\treturn \"{\"\n\tcase closeCollectObject:\n\t\treturn \"}\"\n\tcase traverseArrayCollect:\n\t\treturn \".[\"\n\n\t}\n\treturn \"NFI\"\n}\n\nfunc unwrap(value string) string {\n\treturn value[1 : len(value)-1]\n}\n\nfunc extractNumberParameter(value string) (int, error) {\n\tparameterParser := regexp.MustCompile(`.*\\((-?[0-9]+)\\)`)\n\tmatches := parameterParser.FindStringSubmatch(value)\n\tvar indent, errParsingInt = parseInt(matches[1])\n\tif errParsingInt != nil {\n\t\treturn 0, errParsingInt\n\t}\n\treturn indent, nil\n}\n\nfunc hasOptionParameter(value string, option string) bool {\n\tparameterParser := regexp.MustCompile(`.*\\([^\\)]*\\)`)\n\tmatches := parameterParser.FindStringSubmatch(value)\n\tif len(matches) == 0 {\n\t\treturn false\n\t}\n\tparameterString := matches[0]\n\toptionParser := regexp.MustCompile(fmt.Sprintf(\"\\\\b%v\\\\b\", option))\n\treturn len(optionParser.FindStringSubmatch(parameterString)) > 0\n}\n\nfunc postProcessTokens(tokens []*token) []*token {\n\tvar postProcessedTokens = make([]*token, 0)\n\n\tskipNextToken := false\n\n\tfor index := range tokens {\n\t\tif skipNextToken {\n\t\t\tskipNextToken = false\n\t\t} else {\n\t\t\tpostProcessedTokens, skipNextToken = handleToken(tokens, index, postProcessedTokens)\n\t\t}\n\t}\n\n\treturn postProcessedTokens\n}\n\nfunc tokenIsOpType(token *token, opType *operationType) bool {\n\treturn token.TokenType == operationToken && token.Operation.OperationType == opType\n}\n\nfunc handleToken(tokens []*token, index int, postProcessedTokens []*token) (tokensAccum []*token, skipNextToken bool) {\n\tskipNextToken = false\n\tcurrentToken := tokens[index]\n\n\tlog.Debug(\"processing %v\", currentToken.toString(true))\n\n\tif currentToken.TokenType == traverseArrayCollect {\n\t\t// `.[exp]`` works by creating a traversal array of [self, exp] and piping that into the traverse array operator\n\t\t//need to put a traverse array then a collect currentToken\n\t\t// do this by adding traverse then converting currentToken to collect\n\n\t\tlog.Debug(\"adding self\")\n\t\top := &Operation{OperationType: selfReferenceOpType, StringValue: \"SELF\"}\n\t\tpostProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})\n\n\t\tlog.Debug(\"adding traverse array\")\n\t\top = &Operation{OperationType: traverseArrayOpType, StringValue: \"TRAVERSE_ARRAY\"}\n\t\tpostProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})\n\n\t\tcurrentToken = &token{TokenType: openCollect}\n\n\t}\n\n\tif tokenIsOpType(currentToken, createMapOpType) {\n\t\tlog.Debugf(\"tokenIsOpType: createMapOpType\")\n\t\t// check the previous token is '[', means we are slice, but dont have a first number\n\t\tif index > 0 && tokens[index-1].TokenType == traverseArrayCollect {\n\t\t\tlog.Debugf(\"previous token is : traverseArrayOpType\")\n\t\t\t// need to put the number 0 before this token, as that is implied\n\t\t\tpostProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: createValueOperation(0, \"0\")})\n\t\t}\n\t}\n\n\tif index != len(tokens)-1 && currentToken.AssignOperation != nil &&\n\t\ttokenIsOpType(tokens[index+1], assignOpType) {\n\t\tlog.Debug(\"its an update assign\")\n\t\tcurrentToken.Operation = currentToken.AssignOperation\n\t\tcurrentToken.Operation.UpdateAssign = tokens[index+1].Operation.UpdateAssign\n\t\tskipNextToken = true\n\t}\n\n\tlog.Debug(\"adding token to the fixed list\")\n\tpostProcessedTokens = append(postProcessedTokens, currentToken)\n\n\tif tokenIsOpType(currentToken, createMapOpType) {\n\t\tlog.Debugf(\"tokenIsOpType: createMapOpType\")\n\t\t// check the next token is ']', means we are slice, but dont have a second number\n\t\tif index != len(tokens)-1 && tokens[index+1].TokenType == closeCollect {\n\t\t\tlog.Debugf(\"next token is : closeCollect\")\n\t\t\t// need to put the number 0 before this token, as that is implied\n\t\t\tlengthOp := &Operation{OperationType: lengthOpType}\n\t\t\tpostProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: lengthOp})\n\t\t}\n\t}\n\n\tif index != len(tokens)-1 &&\n\t\t((currentToken.TokenType == openCollect && tokens[index+1].TokenType == closeCollect) ||\n\t\t\t(currentToken.TokenType == openCollectObject && tokens[index+1].TokenType == closeCollectObject)) {\n\t\tlog.Debug(\"adding empty\")\n\t\top := &Operation{OperationType: emptyOpType, StringValue: \"EMPTY\"}\n\t\tpostProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})\n\t}\n\n\tif index != len(tokens)-1 && currentToken.CheckForPostTraverse &&\n\n\t\t(tokenIsOpType(tokens[index+1], traversePathOpType) ||\n\t\t\t(tokens[index+1].TokenType == traverseArrayCollect)) {\n\t\tlog.Debug(\"adding pipe because the next thing is traverse\")\n\t\top := &Operation{OperationType: shortPipeOpType, Value: \"PIPE\", StringValue: \".\"}\n\t\tpostProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})\n\t}\n\tif index != len(tokens)-1 && currentToken.CheckForPostTraverse &&\n\t\ttokens[index+1].TokenType == openCollect {\n\n\t\tlog.Debug(\"adding traverseArray because next is opencollect\")\n\t\top := &Operation{OperationType: traverseArrayOpType}\n\t\tpostProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})\n\t}\n\treturn postProcessedTokens, skipNextToken\n}\n" }, { "path": "pkg/yqlib/lexer_participle.go", "content": "package yqlib\n\nimport (\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/alecthomas/participle/v2/lexer\"\n)\n\nvar participleYqRules = []*participleYqRule{\n\t{\"LINE_COMMENT\", `line_?comment|lineComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true}), 0},\n\t{\"HEAD_COMMENT\", `head_?comment|headComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{HeadComment: true}), 0},\n\t{\"FOOT_COMMENT\", `foot_?comment|footComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{FootComment: true}), 0},\n\n\t{\"OpenBracket\", `\\(`, literalToken(openBracket, false), 0},\n\t{\"CloseBracket\", `\\)`, literalToken(closeBracket, true), 0},\n\t{\"OpenTraverseArrayCollect\", `\\.\\[`, literalToken(traverseArrayCollect, false), 0},\n\n\t{\"OpenCollect\", `\\[`, literalToken(openCollect, false), 0},\n\t{\"CloseCollect\", `\\]\\??`, literalToken(closeCollect, true), 0},\n\n\t{\"OpenCollectObject\", `\\{`, literalToken(openCollectObject, false), 0},\n\t{\"CloseCollectObject\", `\\}`, literalToken(closeCollectObject, true), 0},\n\n\t{\"RecursiveDecentIncludingKeys\", `\\.\\.\\.`, recursiveDecentOpToken(true), 0},\n\t{\"RecursiveDecent\", `\\.\\.`, recursiveDecentOpToken(false), 0},\n\n\t{\"GetVariable\", `\\$[a-zA-Z_\\-0-9]+`, getVariableOpToken(), 0},\n\t{\"AssignAsVariable\", `as`, opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{}), 0},\n\t{\"AssignRefVariable\", `ref`, opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{IsReference: true}), 0},\n\n\t{\"CreateMap\", `:\\s*`, opToken(createMapOpType), 0},\n\tsimpleOp(\"length\", lengthOpType),\n\tsimpleOp(\"line\", lineOpType),\n\tsimpleOp(\"column\", columnOpType),\n\tsimpleOp(\"eval\", evalOpType),\n\tsimpleOp(\"to_?number\", toNumberOpType),\n\n\t{\"MapValues\", `map_?values`, opToken(mapValuesOpType), 0},\n\tsimpleOp(\"map\", mapOpType),\n\tsimpleOp(\"filter\", filterOpType),\n\tsimpleOp(\"pick\", pickOpType),\n\tsimpleOp(\"omit\", omitOpType),\n\n\t{\"FlattenWithDepth\", `flatten\\([0-9]+\\)`, flattenWithDepth(), 0},\n\t{\"Flatten\", `flatten`, opTokenWithPrefs(flattenOpType, nil, flattenPreferences{depth: -1}), 0},\n\n\tsimpleOp(\"format_datetime\", formatDateTimeOpType),\n\tsimpleOp(\"now\", nowOpType),\n\tsimpleOp(\"tz\", tzOpType),\n\tsimpleOp(\"from_?unix\", fromUnixOpType),\n\tsimpleOp(\"to_?unix\", toUnixOpType),\n\tsimpleOp(\"with_dtf\", withDtFormatOpType),\n\tsimpleOp(\"error\", errorOpType),\n\tsimpleOp(\"shuffle\", shuffleOpType),\n\tsimpleOp(\"sortKeys\", sortKeysOpType),\n\tsimpleOp(\"sort_?keys\", sortKeysOpType),\n\n\t{\"ArrayToMap\", \"array_?to_?map\", expressionOpToken(`(.[] | select(. != null) ) as $i ireduce({}; .[$i | key] = $i)`), 0},\n\t{\"Root\", \"root\", expressionOpToken(`parent(-1)`), 0},\n\t{\"YamlEncodeWithIndent\", `to_?yaml\\([0-9]+\\)`, encodeParseIndent(YamlFormat), 0},\n\t{\"XMLEncodeWithIndent\", `to_?xml\\([0-9]+\\)`, encodeParseIndent(XMLFormat), 0},\n\t{\"JSONEncodeWithIndent\", `to_?json\\([0-9]+\\)`, encodeParseIndent(JSONFormat), 0},\n\n\t{\"YamlDecode\", `from_?yaml|@yamld|from_?json|@jsond`, decodeOp(YamlFormat), 0},\n\t{\"YamlEncode\", `to_?yaml|@yaml`, encodeWithIndent(YamlFormat, 2), 0},\n\n\t{\"JSONEncode\", `to_?json`, encodeWithIndent(JSONFormat, 2), 0},\n\t{\"JSONEncodeNoIndent\", `@json`, encodeWithIndent(JSONFormat, 0), 0},\n\n\t{\"PropertiesDecode\", `from_?props|@propsd`, decodeOp(PropertiesFormat), 0},\n\t{\"PropsEncode\", `to_?props|@props`, encodeWithIndent(PropertiesFormat, 2), 0},\n\n\t{\"XmlDecode\", `from_?xml|@xmld`, decodeOp(XMLFormat), 0},\n\t{\"XMLEncode\", `to_?xml`, encodeWithIndent(XMLFormat, 2), 0},\n\t{\"XMLEncodeNoIndent\", `@xml`, encodeWithIndent(XMLFormat, 0), 0},\n\n\t{\"CSVDecode\", `from_?csv|@csvd`, decodeOp(CSVFormat), 0},\n\t{\"CSVEncode\", `to_?csv|@csv`, encodeWithIndent(CSVFormat, 0), 0},\n\n\t{\"TSVDecode\", `from_?tsv|@tsvd`, decodeOp(TSVFormat), 0},\n\t{\"TSVEncode\", `to_?tsv|@tsv`, encodeWithIndent(TSVFormat, 0), 0},\n\n\t{\"Base64d\", `@base64d`, decodeOp(Base64Format), 0},\n\t{\"Base64\", `@base64`, encodeWithIndent(Base64Format, 0), 0},\n\n\t{\"Urid\", `@urid`, decodeOp(UriFormat), 0},\n\t{\"Uri\", `@uri`, encodeWithIndent(UriFormat, 0), 0},\n\t{\"SH\", `@sh`, encodeWithIndent(ShFormat, 0), 0},\n\n\t{\"LoadXML\", `load_?xml|xml_?load`, loadOp(NewXMLDecoder(ConfiguredXMLPreferences)), 0},\n\n\t{\"LoadBase64\", `load_?base64`, loadOp(NewBase64Decoder()), 0},\n\n\t{\"LoadProperties\", `load_?props`, loadOp(NewPropertiesDecoder()), 0},\n\tsimpleOp(\"load_?str|str_?load\", loadStringOpType),\n\t{\"LoadYaml\", `load`, loadOp(NewYamlDecoder(LoadYamlPreferences)), 0},\n\n\t{\"SplitDocument\", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0},\n\n\tsimpleOp(\"select\", selectOpType),\n\tsimpleOp(\"has\", hasOpType),\n\tsimpleOp(\"unique_?by\", uniqueByOpType),\n\tsimpleOp(\"unique\", uniqueOpType),\n\n\tsimpleOp(\"group_?by\", groupByOpType),\n\tsimpleOp(\"explode\", explodeOpType),\n\tsimpleOp(\"or\", orOpType),\n\tsimpleOp(\"and\", andOpType),\n\tsimpleOp(\"not\", notOpType),\n\tsimpleOp(\"ireduce\", reduceOpType),\n\n\tsimpleOp(\"join\", joinStringOpType),\n\tsimpleOp(\"sub\", subStringOpType),\n\tsimpleOp(\"match\", matchOpType),\n\tsimpleOp(\"capture\", captureOpType),\n\tsimpleOp(\"test\", testOpType),\n\n\tsimpleOp(\"sort_?by\", sortByOpType),\n\tsimpleOp(\"sort\", sortOpType),\n\tsimpleOp(\"first\", firstOpType),\n\n\tsimpleOp(\"reverse\", reverseOpType),\n\n\tsimpleOp(\"any_c\", anyConditionOpType),\n\tsimpleOp(\"any\", anyOpType),\n\n\tsimpleOp(\"all_c\", allConditionOpType),\n\tsimpleOp(\"all\", allOpType),\n\n\tsimpleOp(\"contains\", containsOpType),\n\tsimpleOp(\"split\", splitStringOpType),\n\n\tsimpleOp(\"parents\", getParentsOpType),\n\t{\"ParentWithLevel\", `parent\\(-?[0-9]+\\)`, parentWithLevel(), 0},\n\t{\"ParentWithDefaultLevel\", `parent`, parentWithDefaultLevel(), 0},\n\n\tsimpleOp(\"keys\", keysOpType),\n\tsimpleOp(\"key\", getKeyOpType),\n\tsimpleOp(\"is_?key\", isKeyOpType),\n\n\tsimpleOp(\"file_?name|fileName\", getFilenameOpType),\n\tsimpleOp(\"file_?index|fileIndex|fi\", getFileIndexOpType),\n\tsimpleOp(\"path\", getPathOpType),\n\tsimpleOp(\"set_?path\", setPathOpType),\n\tsimpleOp(\"del_?paths\", delPathsOpType),\n\n\tsimpleOp(\"to_?entries|toEntries\", toEntriesOpType),\n\tsimpleOp(\"from_?entries|fromEntries\", fromEntriesOpType),\n\tsimpleOp(\"with_?entries|withEntries\", withEntriesOpType),\n\n\tsimpleOp(\"with\", withOpType),\n\n\tsimpleOp(\"collect\", collectOpType),\n\tsimpleOp(\"del\", deleteChildOpType),\n\n\tassignableOp(\"style\", getStyleOpType, assignStyleOpType),\n\tassignableOp(\"tag|type\", getTagOpType, assignTagOpType),\n\tsimpleOp(\"kind\", getKindOpType),\n\tassignableOp(\"anchor\", getAnchorOpType, assignAnchorOpType),\n\tassignableOp(\"alias\", getAliasOpType, assignAliasOpType),\n\n\t{\"ALL_COMMENTS\", `comments\\s*=`, assignAllCommentsOp(false), 0},\n\t{\"ALL_COMMENTS_ASSIGN_RELATIVE\", `comments\\s*\\|=`, assignAllCommentsOp(true), 0},\n\n\t{\"Block\", `;`, opToken(blockOpType), 0},\n\t{\"Alternative\", `\\/\\/`, opToken(alternativeOpType), 0},\n\n\t{\"DocumentIndex\", `documentIndex|document_?index|di`, opToken(getDocumentIndexOpType), 0},\n\n\t{\"Uppercase\", `upcase|ascii_?upcase`, opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: true}), 0},\n\t{\"Downcase\", `downcase|ascii_?downcase`, opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: false}), 0},\n\tsimpleOp(\"trim\", trimOpType),\n\tsimpleOp(\"to_?string\", toStringOpType),\n\n\t{\"HexValue\", `0[xX][0-9A-Fa-f]+`, hexValue(), 0},\n\t{\"FloatValueScientific\", `-?[1-9](\\.\\d+)?[Ee][-+]?\\d+`, floatValue(), 0},\n\t{\"FloatValue\", `-?\\d+(\\.\\d+)`, floatValue(), 0},\n\n\t{\"NumberValue\", `-?\\d+`, numberValue(), 0},\n\n\t{\"TrueBooleanValue\", `[Tt][Rr][Uu][Ee]`, booleanValue(true), 0},\n\t{\"FalseBooleanValue\", `[Ff][Aa][Ll][Ss][Ee]`, booleanValue(false), 0},\n\n\t{\"NullValue\", `[Nn][Uu][Ll][Ll]|~`, nullValue(), 0},\n\n\t{\"QuotedStringValue\", `\"([^\"\\\\]*(\\\\.[^\"\\\\]*)*)\"`, stringValue(), 0},\n\n\t{\"StrEnvOp\", `strenv\\([^\\)]+\\)`, envOp(true), 0},\n\t{\"EnvOp\", `env\\([^\\)]+\\)`, envOp(false), 0},\n\n\t{\"EnvSubstWithOptions\", `envsubst\\((ne|nu|ff| |,)+\\)`, envSubstWithOptions(), 0},\n\tsimpleOp(\"envsubst\", envsubstOpType),\n\n\t{\"Equals\", `\\s*==\\s*`, opToken(equalsOpType), 0},\n\t{\"NotEquals\", `\\s*!=\\s*`, opToken(notEqualsOpType), 0},\n\n\t{\"GreaterThanEquals\", `\\s*>=\\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: true, Greater: true}), 0},\n\t{\"LessThanEquals\", `\\s*<=\\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: true, Greater: false}), 0},\n\n\t{\"GreaterThan\", `\\s*>\\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: false, Greater: true}), 0},\n\t{\"LessThan\", `\\s*<\\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: false, Greater: false}), 0},\n\n\tsimpleOp(\"min\", minOpType),\n\tsimpleOp(\"max\", maxOpType),\n\n\t{\"AssignRelative\", `\\|=[c]*`, assignOpToken(true), 0},\n\t{\"Assign\", `=[c]*`, assignOpToken(false), 0},\n\n\t{`whitespace`, `[ \\t\\n]+`, nil, 0},\n\n\t{\"WrappedPathElement\", `\\.\"[^ \"]+\"\\??`, pathToken(true), 0},\n\t{\"PathElement\", `\\.[^ ;\\}\\{\\:\\[\\],\\|\\.\\[\\(\\)=\\n!]+\\??`, pathToken(false), 0},\n\t{\"Pipe\", `\\|`, opToken(pipeOpType), 0},\n\t{\"Self\", `\\.`, opToken(selfReferenceOpType), 0},\n\n\t{\"Union\", `,`, opToken(unionOpType), 0},\n\n\t{\"MultiplyAssign\", `\\*=[\\+|\\?cdn]*`, multiplyWithPrefs(multiplyAssignOpType), 0},\n\t{\"Multiply\", `\\*[\\+|\\?cdn]*`, multiplyWithPrefs(multiplyOpType), 0},\n\n\t{\"Divide\", `\\/`, opToken(divideOpType), 0},\n\n\t{\"Modulo\", `%`, opToken(moduloOpType), 0},\n\n\t{\"AddAssign\", `\\+=`, opToken(addAssignOpType), 0},\n\t{\"Add\", `\\+`, opToken(addOpType), 0},\n\n\t{\"SubtractAssign\", `\\-=`, opToken(subtractAssignOpType), 0},\n\t{\"Subtract\", `\\-`, opToken(subtractOpType), 0},\n\t{\"Comment\", `#.*`, nil, 0},\n\n\tsimpleOp(\"pivot\", pivotOpType),\n}\n\ntype yqAction func(lexer.Token) (*token, error)\n\ntype participleYqRule struct {\n\tName string\n\tPattern string\n\tCreateYqToken yqAction\n\tParticipleTokenType lexer.TokenType\n}\n\ntype participleLexer struct {\n\tlexerDefinition lexer.StringDefinition\n}\n\nfunc simpleOp(name string, opType *operationType) *participleYqRule {\n\treturn &participleYqRule{strings.ToUpper(string(name[1])) + name[1:], name, opToken(opType), 0}\n}\n\nfunc assignableOp(name string, opType *operationType, assignOpType *operationType) *participleYqRule {\n\treturn &participleYqRule{strings.ToUpper(string(name[1])) + name[1:], name, opTokenWithPrefs(opType, assignOpType, nil), 0}\n}\n\nfunc newParticipleLexer() expressionTokeniser {\n\tsimpleRules := make([]lexer.SimpleRule, len(participleYqRules))\n\tfor i, yqRule := range participleYqRules {\n\t\tsimpleRules[i] = lexer.SimpleRule{Name: yqRule.Name, Pattern: yqRule.Pattern}\n\t}\n\tlexerDefinition := lexer.MustSimple(simpleRules)\n\tsymbols := lexerDefinition.Symbols()\n\n\tfor _, yqRule := range participleYqRules {\n\t\tyqRule.ParticipleTokenType = symbols[yqRule.Name]\n\t}\n\n\treturn &participleLexer{lexerDefinition}\n}\n\nfunc pathToken(wrapped bool) yqAction {\n\treturn func(rawToken lexer.Token) (*token, error) {\n\t\tvalue := rawToken.Value\n\t\tprefs := traversePreferences{}\n\n\t\tif value[len(value)-1:] == \"?\" {\n\t\t\tprefs.OptionalTraverse = true\n\t\t\tvalue = value[:len(value)-1]\n\t\t}\n\n\t\tvalue = value[1:]\n\t\tif wrapped {\n\t\t\tvalue = unwrap(value)\n\t\t}\n\t\tlog.Debug(\"PathToken %v\", value)\n\t\top := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: prefs}\n\t\treturn &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil\n\t}\n}\n\nfunc recursiveDecentOpToken(includeMapKeys bool) yqAction {\n\tprefs := recursiveDescentPreferences{\n\t\tRecurseArray: true,\n\t\tTraversePreferences: traversePreferences{\n\t\t\tDontFollowAlias: true,\n\t\t\tIncludeMapKeys: includeMapKeys,\n\t\t},\n\t}\n\treturn opTokenWithPrefs(recursiveDescentOpType, nil, prefs)\n}\n\nfunc opTokenWithPrefs(opType *operationType, assignOpType *operationType, preferences interface{}) yqAction {\n\treturn func(rawToken lexer.Token) (*token, error) {\n\t\tvalue := rawToken.Value\n\t\top := &Operation{OperationType: opType, Value: opType.Type, StringValue: value, Preferences: preferences}\n\t\tvar assign *Operation\n\t\tif assignOpType != nil {\n\t\t\tassign = &Operation{OperationType: assignOpType, Value: assignOpType.Type, StringValue: value, Preferences: preferences}\n\t\t}\n\t\treturn &token{TokenType: operationToken, Operation: op, AssignOperation: assign, CheckForPostTraverse: op.OperationType.CheckForPostTraverse}, nil\n\t}\n}\n\nfunc expressionOpToken(expression string) yqAction {\n\treturn func(_ lexer.Token) (*token, error) {\n\t\tprefs := expressionOpPreferences{expression: expression}\n\t\texpressionOp := &Operation{OperationType: expressionOpType, Preferences: prefs}\n\t\treturn &token{TokenType: operationToken, Operation: expressionOp}, nil\n\t}\n}\n\nfunc flattenWithDepth() yqAction {\n\treturn func(rawToken lexer.Token) (*token, error) {\n\t\tvalue := rawToken.Value\n\t\tvar depth, errParsingInt = extractNumberParameter(value)\n\t\tif errParsingInt != nil {\n\t\t\treturn nil, errParsingInt\n\t\t}\n\n\t\tprefs := flattenPreferences{depth: depth}\n\t\top := &Operation{OperationType: flattenOpType, Value: flattenOpType.Type, StringValue: value, Preferences: prefs}\n\t\treturn &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: flattenOpType.CheckForPostTraverse}, nil\n\t}\n}\n\nfunc assignAllCommentsOp(updateAssign bool) yqAction {\n\treturn func(rawToken lexer.Token) (*token, error) {\n\t\tlog.Debug(\"assignAllCommentsOp %v\", rawToken.Value)\n\t\tvalue := rawToken.Value\n\t\top := &Operation{\n\t\t\tOperationType: assignCommentOpType,\n\t\t\tValue: assignCommentOpType.Type,\n\t\t\tStringValue: value,\n\t\t\tUpdateAssign: updateAssign,\n\t\t\tPreferences: commentOpPreferences{LineComment: true, HeadComment: true, FootComment: true},\n\t\t}\n\t\treturn &token{TokenType: operationToken, Operation: op}, nil\n\t}\n}\n\nfunc assignOpToken(updateAssign bool) yqAction {\n\treturn func(rawToken lexer.Token) (*token, error) {\n\t\tlog.Debug(\"assignOpToken %v\", rawToken.Value)\n\t\tvalue := rawToken.Value\n\t\tprefs := assignPreferences{DontOverWriteAnchor: true}\n\t\tif strings.Contains(value, \"c\") {\n\t\t\tprefs.ClobberCustomTags = true\n\t\t}\n\t\top := &Operation{OperationType: assignOpType, Value: assignOpType.Type, StringValue: value, UpdateAssign: updateAssign, Preferences: prefs}\n\t\treturn &token{TokenType: operationToken, Operation: op}, nil\n\t}\n}\n\nfunc booleanValue(val bool) yqAction {\n\treturn func(rawToken lexer.Token) (*token, error) {\n\t\treturn &token{TokenType: operationToken, Operation: createValueOperation(val, rawToken.Value)}, nil\n\t}\n}\n\nfunc nullValue() yqAction {\n\treturn func(rawToken lexer.Token) (*token, error) {\n\t\treturn &token{TokenType: operationToken, Operation: createValueOperation(nil, rawToken.Value)}, nil\n\t}\n}\n\nfunc stringValue() yqAction {\n\treturn func(rawToken lexer.Token) (*token, error) {\n\t\tlog.Debug(\"rawTokenvalue: %v\", rawToken.Value)\n\t\tvalue := unwrap(rawToken.Value)\n\t\tlog.Debug(\"unwrapped: %v\", value)\n\t\tvalue = processEscapeCharacters(value)\n\t\treturn &token{TokenType: operationToken, Operation: &Operation{\n\t\t\tOperationType: stringInterpolationOpType,\n\t\t\tStringValue: value,\n\t\t\tValue: value,\n\t\t}}, nil\n\t}\n}\n\nfunc envOp(strenv bool) yqAction {\n\treturn func(rawToken lexer.Token) (*token, error) {\n\t\tvalue := rawToken.Value\n\t\tpreferences := envOpPreferences{}\n\n\t\tif strenv {\n\t\t\t// strenv( )\n\t\t\tvalue = value[7 : len(value)-1]\n\t\t\tpreferences.StringValue = true\n\t\t} else {\n\t\t\t//env( )\n\t\t\tvalue = value[4 : len(value)-1]\n\t\t}\n\n\t\tenvOperation := createValueOperation(value, value)\n\t\tenvOperation.OperationType = envOpType\n\t\tenvOperation.Preferences = preferences\n\n\t\treturn &token{TokenType: operationToken, Operation: envOperation, CheckForPostTraverse: envOpType.CheckForPostTraverse}, nil\n\t}\n}\n\nfunc envSubstWithOptions() yqAction {\n\treturn func(rawToken lexer.Token) (*token, error) {\n\t\tvalue := rawToken.Value\n\t\tnoEmpty := hasOptionParameter(value, \"ne\")\n\t\tnoUnset := hasOptionParameter(value, \"nu\")\n\t\tfailFast := hasOptionParameter(value, \"ff\")\n\t\tenvsubstOpType.Type = \"ENVSUBST\"\n\t\tprefs := envOpPreferences{NoUnset: noUnset, NoEmpty: noEmpty, FailFast: failFast}\n\t\tif noEmpty {\n\t\t\tenvsubstOpType.Type = envsubstOpType.Type + \"_NO_EMPTY\"\n\t\t}\n\t\tif noUnset {\n\t\t\tenvsubstOpType.Type = envsubstOpType.Type + \"_NO_UNSET\"\n\t\t}\n\n\t\top := &Operation{OperationType: envsubstOpType, Value: envsubstOpType.Type, StringValue: value, Preferences: prefs}\n\t\treturn &token{TokenType: operationToken, Operation: op}, nil\n\t}\n}\n\nfunc multiplyWithPrefs(op *operationType) yqAction {\n\treturn func(rawToken lexer.Token) (*token, error) {\n\t\tprefs := multiplyPreferences{}\n\t\tprefs.AssignPrefs = assignPreferences{}\n\t\toptions := rawToken.Value\n\t\tif strings.Contains(options, \"+\") {\n\t\t\tprefs.AppendArrays = true\n\t\t}\n\t\tif strings.Contains(options, \"?\") {\n\t\t\tprefs.TraversePrefs = traversePreferences{DontAutoCreate: true}\n\t\t}\n\t\tif strings.Contains(options, \"n\") {\n\t\t\tprefs.AssignPrefs.OnlyWriteNull = true\n\t\t}\n\t\tif strings.Contains(options, \"d\") {\n\t\t\tprefs.DeepMergeArrays = true\n\t\t}\n\t\tif strings.Contains(options, \"c\") {\n\t\t\tprefs.AssignPrefs.ClobberCustomTags = true\n\t\t}\n\t\tprefs.TraversePrefs.DontFollowAlias = true\n\t\tprefs.TraversePrefs.ExactKeyMatch = true\n\t\top := &Operation{OperationType: op, Value: multiplyOpType.Type, StringValue: options, Preferences: prefs}\n\t\treturn &token{TokenType: operationToken, Operation: op}, nil\n\t}\n\n}\n\nfunc getVariableOpToken() yqAction {\n\treturn func(rawToken lexer.Token) (*token, error) {\n\t\tvalue := rawToken.Value\n\n\t\tvalue = value[1:]\n\n\t\tgetVarOperation := createValueOperation(value, value)\n\t\tgetVarOperation.OperationType = getVariableOpType\n\n\t\treturn &token{TokenType: operationToken, Operation: getVarOperation, CheckForPostTraverse: true}, nil\n\t}\n}\n\nfunc hexValue() yqAction {\n\treturn func(rawToken lexer.Token) (*token, error) {\n\t\tvar originalString = rawToken.Value\n\t\tvar numberString = originalString[2:]\n\t\tlog.Debugf(\"numberString: %v\", numberString)\n\t\tvar number, errParsingInt = strconv.ParseInt(numberString, 16, 64)\n\t\tif errParsingInt != nil {\n\t\t\treturn nil, errParsingInt\n\t\t}\n\n\t\treturn &token{TokenType: operationToken, Operation: createValueOperation(number, originalString)}, nil\n\t}\n}\n\nfunc floatValue() yqAction {\n\treturn func(rawToken lexer.Token) (*token, error) {\n\t\tvar numberString = rawToken.Value\n\t\tvar number, errParsingInt = strconv.ParseFloat(numberString, 64)\n\t\tif errParsingInt != nil {\n\t\t\treturn nil, errParsingInt\n\t\t}\n\t\treturn &token{TokenType: operationToken, Operation: createValueOperation(number, numberString)}, nil\n\t}\n}\n\nfunc numberValue() yqAction {\n\treturn func(rawToken lexer.Token) (*token, error) {\n\t\tvar numberString = rawToken.Value\n\t\tvar number, errParsingInt = strconv.ParseInt(numberString, 10, 64)\n\t\tif errParsingInt != nil {\n\t\t\treturn nil, errParsingInt\n\t\t}\n\n\t\treturn &token{TokenType: operationToken, Operation: createValueOperation(number, numberString)}, nil\n\t}\n}\n\nfunc parentWithLevel() yqAction {\n\treturn func(rawToken lexer.Token) (*token, error) {\n\t\tvalue := rawToken.Value\n\t\tvar level, errParsingInt = extractNumberParameter(value)\n\t\tif errParsingInt != nil {\n\t\t\treturn nil, errParsingInt\n\t\t}\n\n\t\tprefs := parentOpPreferences{Level: level}\n\t\top := &Operation{OperationType: getParentOpType, Value: getParentOpType.Type, StringValue: value, Preferences: prefs}\n\t\treturn &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil\n\t}\n}\n\nfunc parentWithDefaultLevel() yqAction {\n\treturn func(_ lexer.Token) (*token, error) {\n\t\tprefs := parentOpPreferences{Level: 1}\n\t\top := &Operation{OperationType: getParentOpType, Value: getParentOpType.Type, StringValue: getParentOpType.Type, Preferences: prefs}\n\t\treturn &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil\n\t}\n}\n\nfunc encodeParseIndent(outputFormat *Format) yqAction {\n\treturn func(rawToken lexer.Token) (*token, error) {\n\t\tvalue := rawToken.Value\n\t\tvar indent, errParsingInt = extractNumberParameter(value)\n\t\tif errParsingInt != nil {\n\t\t\treturn nil, errParsingInt\n\t\t}\n\n\t\tprefs := encoderPreferences{format: outputFormat, indent: indent}\n\t\top := &Operation{OperationType: encodeOpType, Value: encodeOpType.Type, StringValue: value, Preferences: prefs}\n\t\treturn &token{TokenType: operationToken, Operation: op}, nil\n\t}\n}\n\nfunc encodeWithIndent(outputFormat *Format, indent int) yqAction {\n\tprefs := encoderPreferences{format: outputFormat, indent: indent}\n\treturn opTokenWithPrefs(encodeOpType, nil, prefs)\n}\n\nfunc decodeOp(format *Format) yqAction {\n\tprefs := decoderPreferences{format: format}\n\treturn opTokenWithPrefs(decodeOpType, nil, prefs)\n}\n\nfunc loadOp(decoder Decoder) yqAction {\n\tprefs := loadPrefs{decoder}\n\treturn opTokenWithPrefs(loadOpType, nil, prefs)\n}\n\nfunc opToken(op *operationType) yqAction {\n\treturn opTokenWithPrefs(op, nil, nil)\n}\n\nfunc literalToken(tt tokenType, checkForPost bool) yqAction {\n\treturn func(rawToken lexer.Token) (*token, error) {\n\t\treturn &token{TokenType: tt, CheckForPostTraverse: checkForPost, Match: rawToken.Value}, nil\n\t}\n}\n\nfunc (p *participleLexer) getYqDefinition(rawToken lexer.Token) *participleYqRule {\n\tfor _, yqRule := range participleYqRules {\n\t\tif yqRule.ParticipleTokenType == rawToken.Type {\n\t\t\treturn yqRule\n\t\t}\n\t}\n\treturn &participleYqRule{}\n}\n\nfunc (p *participleLexer) Tokenise(expression string) ([]*token, error) {\n\tmyLexer, err := p.lexerDefinition.LexString(\"\", expression)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttokens := make([]*token, 0)\n\n\tfor {\n\t\trawToken, e := myLexer.Next()\n\t\tif e != nil {\n\t\t\treturn nil, e\n\t\t} else if rawToken.Type == lexer.EOF {\n\t\t\treturn postProcessTokens(tokens), nil\n\t\t}\n\n\t\tdefinition := p.getYqDefinition(rawToken)\n\t\tif definition.CreateYqToken != nil {\n\t\t\ttoken, e := definition.CreateYqToken(rawToken)\n\t\t\tif e != nil {\n\t\t\t\treturn nil, e\n\t\t\t}\n\t\t\ttokens = append(tokens, token)\n\t\t}\n\n\t}\n\n}\n" }, { "path": "pkg/yqlib/lexer_participle_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n\n\t\"github.com/alecthomas/repr\"\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\ntype participleLexerScenario struct {\n\texpression string\n\ttokens []*token\n}\n\nvar participleLexerScenarios = []participleLexerScenario{\n\t{\n\t\texpression: \"to_entries[]\",\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: toEntriesOpType,\n\t\t\t\t\tValue: \"TO_ENTRIES\",\n\t\t\t\t\tStringValue: \"to_entries\",\n\t\t\t\t},\n\t\t\t\tCheckForPostTraverse: true,\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: traverseArrayOpType,\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: openCollect,\n\t\t\t\tMatch: \"[\",\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: emptyOpType,\n\t\t\t\t\tStringValue: \"EMPTY\",\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: closeCollect,\n\t\t\t\tCheckForPostTraverse: true,\n\t\t\t\tMatch: \"]\",\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: \".a!=\",\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: traversePathOpType,\n\t\t\t\t\tValue: \"a\",\n\t\t\t\t\tStringValue: \"a\",\n\t\t\t\t\tPreferences: traversePreferences{},\n\t\t\t\t},\n\t\t\t\tCheckForPostTraverse: true,\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: notEqualsOpType,\n\t\t\t\t\tValue: \"NOT_EQUALS\",\n\t\t\t\t\tStringValue: \"!=\",\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: \".[:3]\",\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: selfReferenceOpType,\n\t\t\t\t\tStringValue: \"SELF\",\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: traverseArrayOpType,\n\t\t\t\t\tStringValue: \"TRAVERSE_ARRAY\",\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: openCollect,\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: valueOpType,\n\t\t\t\t\tValue: 0,\n\t\t\t\t\tStringValue: \"0\",\n\t\t\t\t\tCandidateNode: &CandidateNode{\n\t\t\t\t\t\tKind: ScalarNode,\n\t\t\t\t\t\tTag: \"!!int\",\n\t\t\t\t\t\tValue: \"0\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: createMapOpType,\n\t\t\t\t\tValue: \"CREATE_MAP\",\n\t\t\t\t\tStringValue: \":\",\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: valueOpType,\n\t\t\t\t\tValue: int64(3),\n\t\t\t\t\tStringValue: \"3\",\n\t\t\t\t\tCandidateNode: &CandidateNode{\n\t\t\t\t\t\tKind: ScalarNode,\n\t\t\t\t\t\tTag: \"!!int\",\n\t\t\t\t\t\tValue: \"3\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: closeCollect,\n\t\t\t\tCheckForPostTraverse: true,\n\t\t\t\tMatch: \"]\",\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: \".[-2:]\",\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: selfReferenceOpType,\n\t\t\t\t\tStringValue: \"SELF\",\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: traverseArrayOpType,\n\t\t\t\t\tStringValue: \"TRAVERSE_ARRAY\",\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: openCollect,\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: valueOpType,\n\t\t\t\t\tValue: int64(-2),\n\t\t\t\t\tStringValue: \"-2\",\n\t\t\t\t\tCandidateNode: &CandidateNode{\n\t\t\t\t\t\tKind: ScalarNode,\n\t\t\t\t\t\tTag: \"!!int\",\n\t\t\t\t\t\tValue: \"-2\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: createMapOpType,\n\t\t\t\t\tValue: \"CREATE_MAP\",\n\t\t\t\t\tStringValue: \":\",\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: lengthOpType,\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: closeCollect,\n\t\t\t\tCheckForPostTraverse: true,\n\t\t\t\tMatch: \"]\",\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: \".a\",\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: traversePathOpType,\n\t\t\t\t\tValue: \"a\",\n\t\t\t\t\tStringValue: \"a\",\n\t\t\t\t\tPreferences: traversePreferences{},\n\t\t\t\t},\n\t\t\t\tCheckForPostTraverse: true,\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: \".a.b\",\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: traversePathOpType,\n\t\t\t\t\tValue: \"a\",\n\t\t\t\t\tStringValue: \"a\",\n\t\t\t\t\tPreferences: traversePreferences{},\n\t\t\t\t},\n\t\t\t\tCheckForPostTraverse: true,\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: shortPipeOpType,\n\t\t\t\t\tValue: \"PIPE\",\n\t\t\t\t\tStringValue: \".\",\n\t\t\t\t\tPreferences: nil,\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: traversePathOpType,\n\t\t\t\t\tValue: \"b\",\n\t\t\t\t\tStringValue: \"b\",\n\t\t\t\t\tPreferences: traversePreferences{},\n\t\t\t\t},\n\t\t\t\tCheckForPostTraverse: true,\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: \".a.b?\",\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: traversePathOpType,\n\t\t\t\t\tValue: \"a\",\n\t\t\t\t\tStringValue: \"a\",\n\t\t\t\t\tPreferences: traversePreferences{},\n\t\t\t\t},\n\t\t\t\tCheckForPostTraverse: true,\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: shortPipeOpType,\n\t\t\t\t\tValue: \"PIPE\",\n\t\t\t\t\tStringValue: \".\",\n\t\t\t\t\tPreferences: nil,\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: traversePathOpType,\n\t\t\t\t\tValue: \"b\",\n\t\t\t\t\tStringValue: \"b\",\n\t\t\t\t\tPreferences: traversePreferences{\n\t\t\t\t\t\tOptionalTraverse: true,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tCheckForPostTraverse: true,\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: `.a.\"b?\"`,\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: traversePathOpType,\n\t\t\t\t\tValue: \"a\",\n\t\t\t\t\tStringValue: \"a\",\n\t\t\t\t\tPreferences: traversePreferences{},\n\t\t\t\t},\n\t\t\t\tCheckForPostTraverse: true,\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: shortPipeOpType,\n\t\t\t\t\tValue: \"PIPE\",\n\t\t\t\t\tStringValue: \".\",\n\t\t\t\t\tPreferences: nil,\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: traversePathOpType,\n\t\t\t\t\tValue: \"b?\",\n\t\t\t\t\tStringValue: \"b?\",\n\t\t\t\t\tPreferences: traversePreferences{},\n\t\t\t\t},\n\t\t\t\tCheckForPostTraverse: true,\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: ` .a .\"b?\"`,\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: traversePathOpType,\n\t\t\t\t\tValue: \"a\",\n\t\t\t\t\tStringValue: \"a\",\n\t\t\t\t\tPreferences: traversePreferences{},\n\t\t\t\t},\n\t\t\t\tCheckForPostTraverse: true,\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: shortPipeOpType,\n\t\t\t\t\tValue: \"PIPE\",\n\t\t\t\t\tStringValue: \".\",\n\t\t\t\t\tPreferences: nil,\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: traversePathOpType,\n\t\t\t\t\tValue: \"b?\",\n\t\t\t\t\tStringValue: \"b?\",\n\t\t\t\t\tPreferences: traversePreferences{},\n\t\t\t\t},\n\t\t\t\tCheckForPostTraverse: true,\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: `.a | .b`,\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: traversePathOpType,\n\t\t\t\t\tValue: \"a\",\n\t\t\t\t\tStringValue: \"a\",\n\t\t\t\t\tPreferences: traversePreferences{},\n\t\t\t\t},\n\t\t\t\tCheckForPostTraverse: true,\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: pipeOpType,\n\t\t\t\t\tValue: \"PIPE\",\n\t\t\t\t\tStringValue: \"|\",\n\t\t\t\t\tPreferences: nil,\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: traversePathOpType,\n\t\t\t\t\tValue: \"b\",\n\t\t\t\t\tStringValue: \"b\",\n\t\t\t\t\tPreferences: traversePreferences{},\n\t\t\t\t},\n\t\t\t\tCheckForPostTraverse: true,\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: \"(.a)\",\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: openBracket,\n\t\t\t\tMatch: \"(\",\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: traversePathOpType,\n\t\t\t\t\tValue: \"a\",\n\t\t\t\t\tStringValue: \"a\",\n\t\t\t\t\tPreferences: traversePreferences{},\n\t\t\t\t},\n\t\t\t\tCheckForPostTraverse: true,\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: closeBracket,\n\t\t\t\tMatch: \")\",\n\t\t\t\tCheckForPostTraverse: true,\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: \"..\",\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: recursiveDescentOpType,\n\t\t\t\t\tValue: \"RECURSIVE_DESCENT\",\n\t\t\t\t\tStringValue: \"..\",\n\t\t\t\t\tPreferences: recursiveDescentPreferences{\n\t\t\t\t\t\tRecurseArray: true,\n\t\t\t\t\t\tTraversePreferences: traversePreferences{\n\t\t\t\t\t\t\tDontFollowAlias: true,\n\t\t\t\t\t\t\tIncludeMapKeys: false,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: \"...\",\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: recursiveDescentOpType,\n\t\t\t\t\tValue: \"RECURSIVE_DESCENT\",\n\t\t\t\t\tStringValue: \"...\",\n\t\t\t\t\tPreferences: recursiveDescentPreferences{\n\t\t\t\t\t\tRecurseArray: true,\n\t\t\t\t\t\tTraversePreferences: traversePreferences{\n\t\t\t\t\t\t\tDontFollowAlias: true,\n\t\t\t\t\t\t\tIncludeMapKeys: true,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: \".a,.b\",\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: traversePathOpType,\n\t\t\t\t\tValue: \"a\",\n\t\t\t\t\tStringValue: \"a\",\n\t\t\t\t\tPreferences: traversePreferences{},\n\t\t\t\t},\n\t\t\t\tCheckForPostTraverse: true,\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: unionOpType,\n\t\t\t\t\tValue: \"UNION\",\n\t\t\t\t\tStringValue: \",\",\n\t\t\t\t\tPreferences: nil,\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: traversePathOpType,\n\t\t\t\t\tValue: \"b\",\n\t\t\t\t\tStringValue: \"b\",\n\t\t\t\t\tPreferences: traversePreferences{},\n\t\t\t\t},\n\t\t\t\tCheckForPostTraverse: true,\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: \"map_values\",\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: mapValuesOpType,\n\t\t\t\t\tValue: \"MAP_VALUES\",\n\t\t\t\t\tStringValue: \"map_values\",\n\t\t\t\t\tPreferences: nil,\n\t\t\t\t},\n\t\t\t\tCheckForPostTraverse: mapValuesOpType.CheckForPostTraverse,\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: \"mapvalues\",\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: mapValuesOpType,\n\t\t\t\t\tValue: \"MAP_VALUES\",\n\t\t\t\t\tStringValue: \"mapvalues\",\n\t\t\t\t\tPreferences: nil,\n\t\t\t\t},\n\t\t\t\tCheckForPostTraverse: mapValuesOpType.CheckForPostTraverse,\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: \"flatten(3)\",\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: flattenOpType,\n\t\t\t\t\tValue: \"FLATTEN_BY\",\n\t\t\t\t\tStringValue: \"flatten(3)\",\n\t\t\t\t\tPreferences: flattenPreferences{depth: 3},\n\t\t\t\t},\n\t\t\t\tCheckForPostTraverse: flattenOpType.CheckForPostTraverse,\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: \"flatten\",\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: flattenOpType,\n\t\t\t\t\tValue: \"FLATTEN_BY\",\n\t\t\t\t\tStringValue: \"flatten\",\n\t\t\t\t\tPreferences: flattenPreferences{depth: -1},\n\t\t\t\t},\n\t\t\t\tCheckForPostTraverse: flattenOpType.CheckForPostTraverse,\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: \"length\",\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: lengthOpType,\n\t\t\t\t\tValue: \"LENGTH\",\n\t\t\t\t\tStringValue: \"length\",\n\t\t\t\t\tPreferences: nil,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: \"format_datetime\",\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: formatDateTimeOpType,\n\t\t\t\t\tValue: \"FORMAT_DATE_TIME\",\n\t\t\t\t\tStringValue: \"format_datetime\",\n\t\t\t\t\tPreferences: nil,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: \"to_yaml(3)\",\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: encodeOpType,\n\t\t\t\t\tValue: \"ENCODE\",\n\t\t\t\t\tStringValue: \"to_yaml(3)\",\n\t\t\t\t\tPreferences: encoderPreferences{\n\t\t\t\t\t\tformat: YamlFormat,\n\t\t\t\t\t\tindent: 3,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: \"tojson(2)\",\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: encodeOpType,\n\t\t\t\t\tValue: \"ENCODE\",\n\t\t\t\t\tStringValue: \"tojson(2)\",\n\t\t\t\t\tPreferences: encoderPreferences{\n\t\t\t\t\t\tformat: JSONFormat,\n\t\t\t\t\t\tindent: 2,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: \"@yaml\",\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: encodeOpType,\n\t\t\t\t\tValue: \"ENCODE\",\n\t\t\t\t\tStringValue: \"@yaml\",\n\t\t\t\t\tPreferences: encoderPreferences{\n\t\t\t\t\t\tformat: YamlFormat,\n\t\t\t\t\t\tindent: 2,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: \"to_props\",\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: encodeOpType,\n\t\t\t\t\tValue: \"ENCODE\",\n\t\t\t\t\tStringValue: \"to_props\",\n\t\t\t\t\tPreferences: encoderPreferences{\n\t\t\t\t\t\tformat: PropertiesFormat,\n\t\t\t\t\t\tindent: 2,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: \"@base64d\",\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: decodeOpType,\n\t\t\t\t\tValue: \"DECODE\",\n\t\t\t\t\tStringValue: \"@base64d\",\n\t\t\t\t\tPreferences: decoderPreferences{\n\t\t\t\t\t\tformat: Base64Format,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: \"@base64\",\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: encodeOpType,\n\t\t\t\t\tValue: \"ENCODE\",\n\t\t\t\t\tStringValue: \"@base64\",\n\t\t\t\t\tPreferences: encoderPreferences{\n\t\t\t\t\t\tformat: Base64Format,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: \"@yamld\",\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: decodeOpType,\n\t\t\t\t\tValue: \"DECODE\",\n\t\t\t\t\tStringValue: \"@yamld\",\n\t\t\t\t\tPreferences: decoderPreferences{\n\t\t\t\t\t\tformat: YamlFormat,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: `\"string with a\\n\"`,\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: stringInterpolationOpType,\n\t\t\t\t\tValue: \"string with a\\n\",\n\t\t\t\t\tStringValue: \"string with a\\n\",\n\t\t\t\t\tPreferences: nil,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: `\"string with a \\\"\"`,\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: stringInterpolationOpType,\n\t\t\t\t\tValue: `string with a \"`,\n\t\t\t\t\tStringValue: `string with a \"`,\n\t\t\t\t\tPreferences: nil,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: `\"string with a\\r\"`,\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: stringInterpolationOpType,\n\t\t\t\t\tValue: \"string with a\\r\",\n\t\t\t\t\tStringValue: \"string with a\\r\",\n\t\t\t\t\tPreferences: nil,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: `\"string with a\\t\"`,\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: stringInterpolationOpType,\n\t\t\t\t\tValue: \"string with a\\t\",\n\t\t\t\t\tStringValue: \"string with a\\t\",\n\t\t\t\t\tPreferences: nil,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: `\"string with a\\f\"`,\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: stringInterpolationOpType,\n\t\t\t\t\tValue: \"string with a\\f\",\n\t\t\t\t\tStringValue: \"string with a\\f\",\n\t\t\t\t\tPreferences: nil,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: `\"string with a\\v\"`,\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: stringInterpolationOpType,\n\t\t\t\t\tValue: \"string with a\\v\",\n\t\t\t\t\tStringValue: \"string with a\\v\",\n\t\t\t\t\tPreferences: nil,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: `\"string with a\\b\"`,\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: stringInterpolationOpType,\n\t\t\t\t\tValue: \"string with a\\b\",\n\t\t\t\t\tStringValue: \"string with a\\b\",\n\t\t\t\t\tPreferences: nil,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\texpression: `\"string with a\\a\"`,\n\t\ttokens: []*token{\n\t\t\t{\n\t\t\t\tTokenType: operationToken,\n\t\t\t\tOperation: &Operation{\n\t\t\t\t\tOperationType: stringInterpolationOpType,\n\t\t\t\t\tValue: \"string with a\\a\",\n\t\t\t\t\tStringValue: \"string with a\\a\",\n\t\t\t\t\tPreferences: nil,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t},\n}\n\nfunc TestParticipleLexer(t *testing.T) {\n\tlexer := newParticipleLexer()\n\n\tfor _, scenario := range participleLexerScenarios {\n\t\tactual, err := lexer.Tokenise(scenario.expression)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t} else {\n\t\t\ttest.AssertResultWithContext(t, repr.String(scenario.tokens, repr.Indent(\" \")), repr.String(actual, repr.Indent(\" \")), scenario.expression)\n\t\t}\n\n\t}\n}\n" }, { "path": "pkg/yqlib/lib.go", "content": "// Use the top level Evaluator or StreamEvaluator to evaluate expressions and return matches.\npackage yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n\t\"math\"\n\t\"strconv\"\n\t\"strings\"\n\n\tlogging \"gopkg.in/op/go-logging.v1\"\n)\n\nvar ExpressionParser ExpressionParserInterface\n\nfunc InitExpressionParser() {\n\tif ExpressionParser == nil {\n\t\tExpressionParser = newExpressionParser()\n\t}\n}\n\nvar log = logging.MustGetLogger(\"yq-lib\")\n\nvar PrettyPrintExp = `(... | (select(tag != \"!!str\"), select(tag == \"!!str\") | select(test(\"(?i)^(y|yes|n|no|on|off)$\") | not)) ) style=\"\"`\n\n// GetLogger returns the yq logger instance.\nfunc GetLogger() *logging.Logger {\n\treturn log\n}\n\nfunc getContentValueByKey(content []*CandidateNode, key string) *CandidateNode {\n\tfor index := 0; index < len(content); index = index + 2 {\n\t\tkeyNode := content[index]\n\t\tvalueNode := content[index+1]\n\t\tif keyNode.Value == key {\n\t\t\treturn valueNode\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc recurseNodeArrayEqual(lhs *CandidateNode, rhs *CandidateNode) bool {\n\tif len(lhs.Content) != len(rhs.Content) {\n\t\treturn false\n\t}\n\n\tfor index := 0; index < len(lhs.Content); index = index + 1 {\n\t\tif !recursiveNodeEqual(lhs.Content[index], rhs.Content[index]) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc findInArray(array *CandidateNode, item *CandidateNode) int {\n\n\tfor index := 0; index < len(array.Content); index = index + 1 {\n\t\tif recursiveNodeEqual(array.Content[index], item) {\n\t\t\treturn index\n\t\t}\n\t}\n\treturn -1\n}\n\nfunc findKeyInMap(dataMap *CandidateNode, item *CandidateNode) int {\n\n\tfor index := 0; index < len(dataMap.Content); index = index + 2 {\n\t\tif recursiveNodeEqual(dataMap.Content[index], item) {\n\t\t\treturn index\n\t\t}\n\t}\n\treturn -1\n}\n\nfunc recurseNodeObjectEqual(lhs *CandidateNode, rhs *CandidateNode) bool {\n\tif len(lhs.Content) != len(rhs.Content) {\n\t\treturn false\n\t}\n\n\tfor index := 0; index < len(lhs.Content); index = index + 2 {\n\t\tkey := lhs.Content[index]\n\t\tvalue := lhs.Content[index+1]\n\n\t\tindexInRHS := findInArray(rhs, key)\n\n\t\tif indexInRHS == -1 || !recursiveNodeEqual(value, rhs.Content[indexInRHS+1]) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc parseSnippet(value string) (*CandidateNode, error) {\n\tif value == \"\" {\n\t\treturn &CandidateNode{\n\t\t\tKind: ScalarNode,\n\t\t\tTag: \"!!null\",\n\t\t}, nil\n\t}\n\tdecoder := NewYamlDecoder(ConfiguredYamlPreferences)\n\terr := decoder.Init(strings.NewReader(value))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tresult, err := decoder.Decode()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif result.Kind == ScalarNode {\n\t\tresult.LineComment = result.LeadingContent\n\t} else {\n\t\tresult.HeadComment = result.LeadingContent\n\t}\n\tresult.LeadingContent = \"\"\n\n\tif result.Tag == \"!!str\" {\n\t\t// use the original string value, as\n\t\t// decoding drops new lines\n\t\tnewNode := createScalarNode(value, value)\n\t\tnewNode.LineComment = result.LineComment\n\t\treturn newNode, nil\n\t}\n\tresult.Line = 0\n\tresult.Column = 0\n\treturn result, err\n}\n\nfunc recursiveNodeEqual(lhs *CandidateNode, rhs *CandidateNode) bool {\n\tif lhs.Kind != rhs.Kind {\n\t\treturn false\n\t}\n\n\tif lhs.Kind == ScalarNode {\n\t\t//process custom tags of scalar nodes.\n\t\t//dont worry about matching tags of maps or arrays.\n\n\t\tlhsTag := lhs.guessTagFromCustomType()\n\t\trhsTag := rhs.guessTagFromCustomType()\n\n\t\tif lhsTag != rhsTag {\n\t\t\treturn false\n\t\t}\n\t}\n\n\tif lhs.Tag == \"!!null\" {\n\t\treturn true\n\n\t} else if lhs.Kind == ScalarNode {\n\t\treturn lhs.Value == rhs.Value\n\t} else if lhs.Kind == SequenceNode {\n\t\treturn recurseNodeArrayEqual(lhs, rhs)\n\t} else if lhs.Kind == MappingNode {\n\t\treturn recurseNodeObjectEqual(lhs, rhs)\n\t}\n\treturn false\n}\n\n// yaml numbers can have underscores, be hex and octal encoded...\nfunc parseInt64(numberString string) (string, int64, error) {\n\tif strings.Contains(numberString, \"_\") {\n\t\tnumberString = strings.ReplaceAll(numberString, \"_\", \"\")\n\t}\n\n\tif strings.HasPrefix(numberString, \"0x\") ||\n\t\tstrings.HasPrefix(numberString, \"0X\") {\n\t\tnum, err := strconv.ParseInt(numberString[2:], 16, 64)\n\t\treturn \"0x%X\", num, err\n\t} else if strings.HasPrefix(numberString, \"0o\") {\n\t\tnum, err := strconv.ParseInt(numberString[2:], 8, 64)\n\t\treturn \"0o%o\", num, err\n\t}\n\tnum, err := strconv.ParseInt(numberString, 10, 64)\n\treturn \"%v\", num, err\n}\n\nfunc parseInt(numberString string) (int, error) {\n\t_, parsed, err := parseInt64(numberString)\n\n\tif err != nil {\n\t\treturn 0, err\n\t} else if parsed > math.MaxInt || parsed < math.MinInt {\n\t\treturn 0, fmt.Errorf(\"%v is not within [%v, %v]\", parsed, math.MinInt, math.MaxInt)\n\t}\n\n\treturn int(parsed), err\n}\n\nfunc processEscapeCharacters(original string) string {\n\tif original == \"\" {\n\t\treturn original\n\t}\n\n\tvar result strings.Builder\n\trunes := []rune(original)\n\n\tfor i := 0; i < len(runes); i++ {\n\t\tif runes[i] == '\\\\' && i < len(runes)-1 {\n\t\t\tnext := runes[i+1]\n\t\t\tswitch next {\n\t\t\tcase '\\\\':\n\t\t\t\t// Check if followed by opening bracket - if so, preserve both backslashes\n\t\t\t\t// this is required for string interpolation to work correctly.\n\t\t\t\tif i+2 < len(runes) && runes[i+2] == '(' {\n\t\t\t\t\t// Preserve \\\\ when followed by (\n\t\t\t\t\tresult.WriteRune('\\\\')\n\t\t\t\t\tresult.WriteRune('\\\\')\n\t\t\t\t\ti++ // Skip the next backslash (we'll process the ( normally on next iteration)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\t// Escaped backslash: \\\\ -> \\\n\t\t\t\tresult.WriteRune('\\\\')\n\t\t\t\ti++ // Skip the next backslash\n\t\t\t\tcontinue\n\t\t\tcase '\"':\n\t\t\t\tresult.WriteRune('\"')\n\t\t\t\ti++ // Skip the quote\n\t\t\t\tcontinue\n\t\t\tcase 'n':\n\t\t\t\tresult.WriteRune('\\n')\n\t\t\t\ti++ // Skip the 'n'\n\t\t\t\tcontinue\n\t\t\tcase 't':\n\t\t\t\tresult.WriteRune('\\t')\n\t\t\t\ti++ // Skip the 't'\n\t\t\t\tcontinue\n\t\t\tcase 'r':\n\t\t\t\tresult.WriteRune('\\r')\n\t\t\t\ti++ // Skip the 'r'\n\t\t\t\tcontinue\n\t\t\tcase 'f':\n\t\t\t\tresult.WriteRune('\\f')\n\t\t\t\ti++ // Skip the 'f'\n\t\t\t\tcontinue\n\t\t\tcase 'v':\n\t\t\t\tresult.WriteRune('\\v')\n\t\t\t\ti++ // Skip the 'v'\n\t\t\t\tcontinue\n\t\t\tcase 'b':\n\t\t\t\tresult.WriteRune('\\b')\n\t\t\t\ti++ // Skip the 'b'\n\t\t\t\tcontinue\n\t\t\tcase 'a':\n\t\t\t\tresult.WriteRune('\\a')\n\t\t\t\ti++ // Skip the 'a'\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\tresult.WriteRune(runes[i])\n\t}\n\n\tvalue := result.String()\n\tif value != original {\n\t\tlog.Debug(\"processEscapeCharacters from [%v] to [%v]\", original, value)\n\t}\n\treturn value\n}\n\nfunc headAndLineComment(node *CandidateNode) string {\n\treturn headComment(node) + lineComment(node)\n}\n\nfunc headComment(node *CandidateNode) string {\n\treturn strings.Replace(node.HeadComment, \"#\", \"\", 1)\n}\n\nfunc lineComment(node *CandidateNode) string {\n\treturn strings.Replace(node.LineComment, \"#\", \"\", 1)\n}\n\nfunc footComment(node *CandidateNode) string {\n\treturn strings.Replace(node.FootComment, \"#\", \"\", 1)\n}\n\n// use for debugging only\nfunc NodesToString(collection *list.List) string {\n\tif !log.IsEnabledFor(logging.DEBUG) {\n\t\treturn \"\"\n\t}\n\n\tresult := fmt.Sprintf(\"%v results\\n\", collection.Len())\n\tfor el := collection.Front(); el != nil; el = el.Next() {\n\t\tresult = result + \"\\n\" + NodeToString(el.Value.(*CandidateNode))\n\t}\n\treturn result\n}\n\nfunc NodeToString(node *CandidateNode) string {\n\tif !log.IsEnabledFor(logging.DEBUG) {\n\t\treturn \"\"\n\t}\n\tif node == nil {\n\t\treturn \"-- nil --\"\n\t}\n\ttag := node.Tag\n\tif node.Kind == AliasNode {\n\t\ttag = \"alias\"\n\t}\n\tvalueToUse := node.Value\n\tif valueToUse == \"\" {\n\t\tvalueToUse = fmt.Sprintf(\"%v kids\", len(node.Content))\n\t}\n\treturn fmt.Sprintf(`D%v, P%v, %v (%v)::%v`, node.GetDocument(), node.GetNicePath(), KindString(node.Kind), tag, valueToUse)\n}\n\nfunc NodeContentToString(node *CandidateNode, depth int) string {\n\tif !log.IsEnabledFor(logging.DEBUG) {\n\t\treturn \"\"\n\t}\n\tvar sb strings.Builder\n\tfor _, child := range node.Content {\n\t\tfor i := 0; i < depth; i++ {\n\t\t\tsb.WriteString(\" \")\n\t\t}\n\t\tsb.WriteString(\"- \")\n\t\tsb.WriteString(NodeToString(child))\n\t\tsb.WriteString(\"\\n\")\n\t\tsb.WriteString(NodeContentToString(child, depth+1))\n\t}\n\treturn sb.String()\n}\n\nfunc KindString(kind Kind) string {\n\tswitch kind {\n\tcase ScalarNode:\n\t\treturn \"ScalarNode\"\n\tcase SequenceNode:\n\t\treturn \"SequenceNode\"\n\tcase MappingNode:\n\t\treturn \"MappingNode\"\n\tcase AliasNode:\n\t\treturn \"AliasNode\"\n\tdefault:\n\t\treturn \"unknown!\"\n\t}\n}\n" }, { "path": "pkg/yqlib/lib_test.go", "content": "package yqlib\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nfunc TestGetLogger(t *testing.T) {\n\tl := GetLogger()\n\tif l != log {\n\t\tt.Fatal(\"GetLogger should return the yq logger instance, not a copy\")\n\t}\n}\n\ntype parseSnippetScenario struct {\n\tsnippet string\n\texpected *CandidateNode\n\texpectedError string\n}\n\nvar parseSnippetScenarios = []parseSnippetScenario{\n\t{\n\t\tsnippet: \":\",\n\t\texpectedError: \"yaml: did not find expected key\",\n\t},\n\t{\n\t\tsnippet: \"\",\n\t\texpected: &CandidateNode{\n\t\t\tKind: ScalarNode,\n\t\t\tTag: \"!!null\",\n\t\t},\n\t},\n\t{\n\t\tsnippet: \"null\",\n\t\texpected: &CandidateNode{\n\t\t\tKind: ScalarNode,\n\t\t\tTag: \"!!null\",\n\t\t\tValue: \"null\",\n\t\t\tLine: 0,\n\t\t\tColumn: 0,\n\t\t},\n\t},\n\t{\n\t\tsnippet: \"3\",\n\t\texpected: &CandidateNode{\n\t\t\tKind: ScalarNode,\n\t\t\tTag: \"!!int\",\n\t\t\tValue: \"3\",\n\t\t\tLine: 0,\n\t\t\tColumn: 0,\n\t\t},\n\t},\n\t{\n\t\tsnippet: \"cat\",\n\t\texpected: &CandidateNode{\n\t\t\tKind: ScalarNode,\n\t\t\tTag: \"!!str\",\n\t\t\tValue: \"cat\",\n\t\t\tLine: 0,\n\t\t\tColumn: 0,\n\t\t},\n\t},\n\t{\n\t\tsnippet: \"# things\",\n\t\texpected: &CandidateNode{\n\t\t\tKind: ScalarNode,\n\t\t\tTag: \"!!null\",\n\t\t\tLineComment: \"# things\",\n\t\t\tLine: 0,\n\t\t\tColumn: 0,\n\t\t},\n\t},\n\t{\n\t\tsnippet: \"3.1\",\n\t\texpected: &CandidateNode{\n\t\t\tKind: ScalarNode,\n\t\t\tTag: \"!!float\",\n\t\t\tValue: \"3.1\",\n\t\t\tLine: 0,\n\t\t\tColumn: 0,\n\t\t},\n\t},\n\t{\n\t\tsnippet: \"true\",\n\t\texpected: &CandidateNode{\n\t\t\tKind: ScalarNode,\n\t\t\tTag: \"!!bool\",\n\t\t\tValue: \"true\",\n\t\t\tLine: 0,\n\t\t\tColumn: 0,\n\t\t},\n\t},\n}\n\nfunc TestParseSnippet(t *testing.T) {\n\tfor _, tt := range parseSnippetScenarios {\n\t\tactual, err := parseSnippet(tt.snippet)\n\t\tif tt.expectedError != \"\" {\n\t\t\tif err == nil {\n\t\t\t\tt.Errorf(\"Expected error '%v' but it worked!\", tt.expectedError)\n\t\t\t} else {\n\t\t\t\ttest.AssertResultComplexWithContext(t, tt.expectedError, err.Error(), tt.snippet)\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Error(tt.snippet)\n\t\t\tt.Error(err)\n\t\t}\n\t\ttest.AssertResultComplexWithContext(t, tt.expected, actual, tt.snippet)\n\t}\n}\n\ntype parseInt64Scenario struct {\n\tnumberString string\n\texpectedParsedNumber int64\n\texpectedFormatString string\n}\n\nvar parseInt64Scenarios = []parseInt64Scenario{\n\t{\n\t\tnumberString: \"34\",\n\t\texpectedParsedNumber: 34,\n\t},\n\t{\n\t\tnumberString: \"10_000\",\n\t\texpectedParsedNumber: 10000,\n\t\texpectedFormatString: \"10000\",\n\t},\n\t{\n\t\tnumberString: \"0x10\",\n\t\texpectedParsedNumber: 16,\n\t},\n\t{\n\t\tnumberString: \"0x10_000\",\n\t\texpectedParsedNumber: 65536,\n\t\texpectedFormatString: \"0x10000\",\n\t},\n\t{\n\t\tnumberString: \"0o10\",\n\t\texpectedParsedNumber: 8,\n\t},\n}\n\nfunc TestParseInt64(t *testing.T) {\n\tfor _, tt := range parseInt64Scenarios {\n\t\tformat, actualNumber, err := parseInt64(tt.numberString)\n\n\t\tif err != nil {\n\t\t\tt.Error(tt.numberString)\n\t\t\tt.Error(err)\n\t\t}\n\t\ttest.AssertResultComplexWithContext(t, tt.expectedParsedNumber, actualNumber, tt.numberString)\n\t\tif tt.expectedFormatString == \"\" {\n\t\t\ttt.expectedFormatString = tt.numberString\n\t\t}\n\n\t\ttest.AssertResultComplexWithContext(t, tt.expectedFormatString, fmt.Sprintf(format, actualNumber), fmt.Sprintf(\"Formatting of: %v\", tt.numberString))\n\t}\n}\n\nfunc TestGetContentValueByKey(t *testing.T) {\n\t// Create content with key-value pairs\n\tkey1 := createStringScalarNode(\"key1\")\n\tvalue1 := createStringScalarNode(\"value1\")\n\tkey2 := createStringScalarNode(\"key2\")\n\tvalue2 := createStringScalarNode(\"value2\")\n\n\tcontent := []*CandidateNode{key1, value1, key2, value2}\n\n\t// Test finding existing key\n\tresult := getContentValueByKey(content, \"key1\")\n\ttest.AssertResult(t, value1, result)\n\n\t// Test finding another existing key\n\tresult = getContentValueByKey(content, \"key2\")\n\ttest.AssertResult(t, value2, result)\n\n\t// Test finding non-existing key\n\tresult = getContentValueByKey(content, \"nonexistent\")\n\ttest.AssertResult(t, (*CandidateNode)(nil), result)\n\n\t// Test with empty content\n\tresult = getContentValueByKey([]*CandidateNode{}, \"key1\")\n\ttest.AssertResult(t, (*CandidateNode)(nil), result)\n}\n\nfunc TestRecurseNodeArrayEqual(t *testing.T) {\n\t// Create two arrays with same content\n\tarray1 := &CandidateNode{\n\t\tKind: SequenceNode,\n\t\tContent: []*CandidateNode{\n\t\t\tcreateStringScalarNode(\"item1\"),\n\t\t\tcreateStringScalarNode(\"item2\"),\n\t\t},\n\t}\n\n\tarray2 := &CandidateNode{\n\t\tKind: SequenceNode,\n\t\tContent: []*CandidateNode{\n\t\t\tcreateStringScalarNode(\"item1\"),\n\t\t\tcreateStringScalarNode(\"item2\"),\n\t\t},\n\t}\n\n\tarray3 := &CandidateNode{\n\t\tKind: SequenceNode,\n\t\tContent: []*CandidateNode{\n\t\t\tcreateStringScalarNode(\"item1\"),\n\t\t\tcreateStringScalarNode(\"different\"),\n\t\t},\n\t}\n\n\tarray4 := &CandidateNode{\n\t\tKind: SequenceNode,\n\t\tContent: []*CandidateNode{\n\t\t\tcreateStringScalarNode(\"item1\"),\n\t\t},\n\t}\n\n\ttest.AssertResult(t, true, recurseNodeArrayEqual(array1, array2))\n\ttest.AssertResult(t, false, recurseNodeArrayEqual(array1, array3))\n\ttest.AssertResult(t, false, recurseNodeArrayEqual(array1, array4))\n}\n\nfunc TestFindInArray(t *testing.T) {\n\titem1 := createStringScalarNode(\"item1\")\n\titem2 := createStringScalarNode(\"item2\")\n\titem3 := createStringScalarNode(\"item3\")\n\n\tarray := &CandidateNode{\n\t\tKind: SequenceNode,\n\t\tContent: []*CandidateNode{item1, item2, item3},\n\t}\n\n\t// Test finding existing items\n\ttest.AssertResult(t, 0, findInArray(array, item1))\n\ttest.AssertResult(t, 1, findInArray(array, item2))\n\ttest.AssertResult(t, 2, findInArray(array, item3))\n\n\t// Test finding non-existing item\n\tnonExistent := createStringScalarNode(\"nonexistent\")\n\ttest.AssertResult(t, -1, findInArray(array, nonExistent))\n}\n\nfunc TestFindKeyInMap(t *testing.T) {\n\tkey1 := createStringScalarNode(\"key1\")\n\tvalue1 := createStringScalarNode(\"value1\")\n\tkey2 := createStringScalarNode(\"key2\")\n\tvalue2 := createStringScalarNode(\"value2\")\n\n\tmapNode := &CandidateNode{\n\t\tKind: MappingNode,\n\t\tContent: []*CandidateNode{key1, value1, key2, value2},\n\t}\n\n\t// Test finding existing keys\n\ttest.AssertResult(t, 0, findKeyInMap(mapNode, key1))\n\ttest.AssertResult(t, 2, findKeyInMap(mapNode, key2))\n\n\t// Test finding non-existing key\n\tnonExistent := createStringScalarNode(\"nonexistent\")\n\ttest.AssertResult(t, -1, findKeyInMap(mapNode, nonExistent))\n}\n\nfunc TestRecurseNodeObjectEqual(t *testing.T) {\n\t// Create two objects with same content\n\tkey1 := createStringScalarNode(\"key1\")\n\tvalue1 := createStringScalarNode(\"value1\")\n\tkey2 := createStringScalarNode(\"key2\")\n\tvalue2 := createStringScalarNode(\"value2\")\n\n\tobj1 := &CandidateNode{\n\t\tKind: MappingNode,\n\t\tContent: []*CandidateNode{key1, value1, key2, value2},\n\t}\n\n\tobj2 := &CandidateNode{\n\t\tKind: MappingNode,\n\t\tContent: []*CandidateNode{key1, value1, key2, value2},\n\t}\n\n\t// Create object with different values\n\tvalue3 := createStringScalarNode(\"value3\")\n\tobj3 := &CandidateNode{\n\t\tKind: MappingNode,\n\t\tContent: []*CandidateNode{key1, value3, key2, value2},\n\t}\n\n\t// Create object with different keys\n\tkey3 := createStringScalarNode(\"key3\")\n\tobj4 := &CandidateNode{\n\t\tKind: MappingNode,\n\t\tContent: []*CandidateNode{key1, value1, key3, value2},\n\t}\n\n\ttest.AssertResult(t, true, recurseNodeObjectEqual(obj1, obj2))\n\ttest.AssertResult(t, false, recurseNodeObjectEqual(obj1, obj3))\n\ttest.AssertResult(t, false, recurseNodeObjectEqual(obj1, obj4))\n}\n\nfunc TestParseInt(t *testing.T) {\n\ttype parseIntScenario struct {\n\t\tnumberString string\n\t\texpectedParsedNumber int\n\t\texpectedError string\n\t}\n\n\tscenarios := []parseIntScenario{\n\t\t{\n\t\t\tnumberString: \"34\",\n\t\t\texpectedParsedNumber: 34,\n\t\t},\n\t\t{\n\t\t\tnumberString: \"10_000\",\n\t\t\texpectedParsedNumber: 10000,\n\t\t},\n\t\t{\n\t\t\tnumberString: \"0x10\",\n\t\t\texpectedParsedNumber: 16,\n\t\t},\n\t\t{\n\t\t\tnumberString: \"0o10\",\n\t\t\texpectedParsedNumber: 8,\n\t\t},\n\t\t{\n\t\t\tnumberString: \"invalid\",\n\t\t\texpectedError: \"strconv.ParseInt\",\n\t\t},\n\t}\n\n\tfor _, tt := range scenarios {\n\t\tactualNumber, err := parseInt(tt.numberString)\n\t\tif tt.expectedError != \"\" {\n\t\t\tif err == nil {\n\t\t\t\tt.Errorf(\"Expected error for '%s' but got none\", tt.numberString)\n\t\t\t} else if !strings.Contains(err.Error(), tt.expectedError) {\n\t\t\t\tt.Errorf(\"Expected error containing '%s' for '%s', got '%s'\", tt.expectedError, tt.numberString, err.Error())\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Errorf(\"Unexpected error for '%s': %v\", tt.numberString, err)\n\t\t}\n\t\ttest.AssertResultComplexWithContext(t, tt.expectedParsedNumber, actualNumber, tt.numberString)\n\t}\n}\n\nfunc TestHeadAndLineComment(t *testing.T) {\n\tnode := &CandidateNode{\n\t\tHeadComment: \"# head comment\",\n\t\tLineComment: \"# line comment\",\n\t}\n\n\tresult := headAndLineComment(node)\n\ttest.AssertResult(t, \" head comment line comment\", result)\n}\n\nfunc TestHeadComment(t *testing.T) {\n\tnode := &CandidateNode{\n\t\tHeadComment: \"# head comment\",\n\t}\n\n\tresult := headComment(node)\n\ttest.AssertResult(t, \" head comment\", result)\n\n\t// Test without #\n\tnode.HeadComment = \"no hash comment\"\n\tresult = headComment(node)\n\ttest.AssertResult(t, \"no hash comment\", result)\n}\n\nfunc TestLineComment(t *testing.T) {\n\tnode := &CandidateNode{\n\t\tLineComment: \"# line comment\",\n\t}\n\n\tresult := lineComment(node)\n\ttest.AssertResult(t, \" line comment\", result)\n\n\t// Test without #\n\tnode.LineComment = \"no hash comment\"\n\tresult = lineComment(node)\n\ttest.AssertResult(t, \"no hash comment\", result)\n}\n\nfunc TestFootComment(t *testing.T) {\n\tnode := &CandidateNode{\n\t\tFootComment: \"# foot comment\",\n\t}\n\n\tresult := footComment(node)\n\ttest.AssertResult(t, \" foot comment\", result)\n\n\t// Test without #\n\tnode.FootComment = \"no hash comment\"\n\tresult = footComment(node)\n\ttest.AssertResult(t, \"no hash comment\", result)\n}\n\nfunc TestKindString(t *testing.T) {\n\ttest.AssertResult(t, \"ScalarNode\", KindString(ScalarNode))\n\ttest.AssertResult(t, \"SequenceNode\", KindString(SequenceNode))\n\ttest.AssertResult(t, \"MappingNode\", KindString(MappingNode))\n\ttest.AssertResult(t, \"AliasNode\", KindString(AliasNode))\n\ttest.AssertResult(t, \"unknown!\", KindString(Kind(999))) // Invalid kind\n}\n\ntype processEscapeCharactersScenario struct {\n\tinput string\n\texpected string\n}\n\nvar processEscapeCharactersScenarios = []processEscapeCharactersScenario{\n\t{\n\t\tinput: \"\",\n\t\texpected: \"\",\n\t},\n\t{\n\t\tinput: \"hello\",\n\t\texpected: \"hello\",\n\t},\n\t{\n\t\tinput: \"\\\\\\\"\",\n\t\texpected: \"\\\"\",\n\t},\n\t{\n\t\tinput: \"hello\\\\\\\"world\",\n\t\texpected: \"hello\\\"world\",\n\t},\n\t{\n\t\tinput: \"\\\\n\",\n\t\texpected: \"\\n\",\n\t},\n\t{\n\t\tinput: \"line1\\\\nline2\",\n\t\texpected: \"line1\\nline2\",\n\t},\n\t{\n\t\tinput: \"\\\\t\",\n\t\texpected: \"\\t\",\n\t},\n\t{\n\t\tinput: \"hello\\\\tworld\",\n\t\texpected: \"hello\\tworld\",\n\t},\n\t{\n\t\tinput: \"\\\\r\",\n\t\texpected: \"\\r\",\n\t},\n\t{\n\t\tinput: \"hello\\\\rworld\",\n\t\texpected: \"hello\\rworld\",\n\t},\n\t{\n\t\tinput: \"\\\\f\",\n\t\texpected: \"\\f\",\n\t},\n\t{\n\t\tinput: \"hello\\\\fworld\",\n\t\texpected: \"hello\\fworld\",\n\t},\n\t{\n\t\tinput: \"\\\\v\",\n\t\texpected: \"\\v\",\n\t},\n\t{\n\t\tinput: \"hello\\\\vworld\",\n\t\texpected: \"hello\\vworld\",\n\t},\n\t{\n\t\tinput: \"\\\\b\",\n\t\texpected: \"\\b\",\n\t},\n\t{\n\t\tinput: \"hello\\\\bworld\",\n\t\texpected: \"hello\\bworld\",\n\t},\n\t{\n\t\tinput: \"\\\\a\",\n\t\texpected: \"\\a\",\n\t},\n\t{\n\t\tinput: \"hello\\\\aworld\",\n\t\texpected: \"hello\\aworld\",\n\t},\n\t{\n\t\tinput: \"\\\\\\\"\\\\n\\\\t\\\\r\\\\f\\\\v\\\\b\\\\a\",\n\t\texpected: \"\\\"\\n\\t\\r\\f\\v\\b\\a\",\n\t},\n\t{\n\t\tinput: \"multiple\\\\nlines\\\\twith\\\\ttabs\",\n\t\texpected: \"multiple\\nlines\\twith\\ttabs\",\n\t},\n\t{\n\t\tinput: \"quote\\\\\\\"here\",\n\t\texpected: \"quote\\\"here\",\n\t},\n\t{\n\t\tinput: \"\\\\\\\\\",\n\t\texpected: \"\\\\\", // Backslash is processed: \"\\\\\\\\\" becomes \"\\\\\"\n\t},\n\t{\n\t\tinput: \"\\\\\\\"test\\\\\\\"\",\n\t\texpected: \"\\\"test\\\"\",\n\t},\n\t{\n\t\tinput: \"a\\\\\\\\b\",\n\t\texpected: \"a\\\\b\", // Tests roundtrip: \"a\\\\\\\\b\" should become \"a\\\\b\"\n\t},\n\t{\n\t\tinput: \"Hi \\\\\\\\(.value)\",\n\t\texpected: \"Hi \\\\\\\\(.value)\",\n\t},\n\t{\n\t\tinput: `a\\\\b`,\n\t\texpected: \"a\\\\b\",\n\t},\n}\n\nfunc TestProcessEscapeCharacters(t *testing.T) {\n\tfor _, tt := range processEscapeCharactersScenarios {\n\t\tactual := processEscapeCharacters(tt.input)\n\t\ttest.AssertResultComplexWithContext(t, tt.expected, actual, fmt.Sprintf(\"Input: %q\", tt.input))\n\t}\n}\n" }, { "path": "pkg/yqlib/lua.go", "content": "package yqlib\n\ntype LuaPreferences struct {\n\tDocPrefix string\n\tDocSuffix string\n\tUnquotedKeys bool\n\tGlobals bool\n}\n\nfunc NewDefaultLuaPreferences() LuaPreferences {\n\treturn LuaPreferences{\n\t\tDocPrefix: \"return \",\n\t\tDocSuffix: \";\\n\",\n\t\tUnquotedKeys: false,\n\t\tGlobals: false,\n\t}\n}\n\nvar ConfiguredLuaPreferences = NewDefaultLuaPreferences()\n" }, { "path": "pkg/yqlib/lua_test.go", "content": "//go:build !yq_nolua\n\npackage yqlib\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nvar luaScenarios = []formatScenario{\n\t{\n\t\tdescription: \"Basic input example\",\n\t\tinput: `return {\n\t[\"country\"] = \"Australia\"; -- this place\n\t[\"cities\"] = {\n\t\t\"Sydney\",\n\t\t\"Melbourne\",\n\t\t\"Brisbane\",\n\t\t\"Perth\",\n\t};\n};\n`,\n\t\texpected: `country: Australia\ncities:\n - Sydney\n - Melbourne\n - Brisbane\n - Perth\n`,\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"path\",\n\t\texpression: \".cities[2] | path\",\n\t\tinput: `return {\n\t[\"country\"] = \"Australia\"; -- this place\n\t[\"cities\"] = {\n\t\t\"Sydney\",\n\t\t\"Melbourne\",\n\t\t\"Brisbane\",\n\t\t\"Perth\",\n\t};\n};\n`,\n\t\texpected: \"- cities\\n- 2\\n\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"path\",\n\t\texpression: \".cities[2] | key\",\n\t\tinput: `return {\n\t[\"country\"] = \"Australia\"; -- this place\n\t[\"cities\"] = {\n\t\t\"Sydney\",\n\t\t\"Melbourne\",\n\t\t\"Brisbane\",\n\t\t\"Perth\",\n\t};\n};\n`,\n\t\texpected: \"2\\n\",\n\t},\n\t{\n\t\tdescription: \"Basic output example\",\n\t\tscenarioType: \"encode\",\n\t\tinput: `---\ncountry: Australia # this place\ncities:\n- Sydney\n- Melbourne\n- Brisbane\n- Perth`,\n\t\texpected: `return {\n\t[\"country\"] = \"Australia\"; -- this place\n\t[\"cities\"] = {\n\t\t\"Sydney\",\n\t\t\"Melbourne\",\n\t\t\"Brisbane\",\n\t\t\"Perth\",\n\t};\n};\n`,\n\t},\n\t{\n\t\tdescription: \"Basic roundtrip\",\n\t\tskipDoc: true,\n\t\tscenarioType: \"roundtrip\",\n\t\texpression: `.cities[0] = \"Adelaide\"`,\n\t\tinput: `return {\n\t[\"country\"] = \"Australia\"; -- this place\n\t[\"cities\"] = {\n\t\t\"Sydney\",\n\t\t\"Melbourne\",\n\t\t\"Brisbane\",\n\t\t\"Perth\",\n\t};\n};\n`,\n\t\texpected: `return {\n\t[\"country\"] = \"Australia\";\n\t[\"cities\"] = {\n\t\t\"Adelaide\",\n\t\t\"Melbourne\",\n\t\t\"Brisbane\",\n\t\t\"Perth\",\n\t};\n};\n`,\n\t},\n\t{\n\t\tdescription: \"Unquoted keys\",\n\t\tsubdescription: \"Uses the `--lua-unquoted` option to produce a nicer-looking output.\",\n\t\tscenarioType: \"unquoted-encode\",\n\t\tinput: `---\ncountry: Australia # this place\ncities:\n- Sydney\n- Melbourne\n- Brisbane\n- Perth`,\n\t\texpected: `return {\n\tcountry = \"Australia\"; -- this place\n\tcities = {\n\t\t\"Sydney\",\n\t\t\"Melbourne\",\n\t\t\"Brisbane\",\n\t\t\"Perth\",\n\t};\n};\n`,\n\t},\n\t{\n\t\tdescription: \"Globals\",\n\t\tsubdescription: \"Uses the `--lua-globals` option to export the values into the global scope.\",\n\t\tscenarioType: \"globals-encode\",\n\t\tinput: `---\ncountry: Australia # this place\ncities:\n- Sydney\n- Melbourne\n- Brisbane\n- Perth`,\n\t\texpected: `country = \"Australia\"; -- this place\ncities = {\n\t\"Sydney\",\n\t\"Melbourne\",\n\t\"Brisbane\",\n\t\"Perth\",\n};\n`,\n\t},\n\t{\n\t\tdescription: \"Elaborate example\",\n\t\tinput: `---\nhello: world\ntables:\n like: this\n keys: values\n ? look: non-string keys\n : True\nnumbers:\n - decimal: 12345\n - hex: 0x7fabc123\n - octal: 0o30\n - float: 123.45\n - infinity: .inf\n plus_infinity: +.inf\n minus_infinity: -.inf\n - not: .nan\n`,\n\t\texpected: `return {\n\t[\"hello\"] = \"world\";\n\t[\"tables\"] = {\n\t\t[\"like\"] = \"this\";\n\t\t[\"keys\"] = \"values\";\n\t\t[{\n\t\t\t[\"look\"] = \"non-string keys\";\n\t\t}] = true;\n\t};\n\t[\"numbers\"] = {\n\t\t{\n\t\t\t[\"decimal\"] = 12345;\n\t\t},\n\t\t{\n\t\t\t[\"hex\"] = 0x7fabc123;\n\t\t},\n\t\t{\n\t\t\t[\"octal\"] = 24;\n\t\t},\n\t\t{\n\t\t\t[\"float\"] = 123.45;\n\t\t},\n\t\t{\n\t\t\t[\"infinity\"] = (1/0);\n\t\t\t[\"plus_infinity\"] = (1/0);\n\t\t\t[\"minus_infinity\"] = (-1/0);\n\t\t},\n\t\t{\n\t\t\t[\"not\"] = (0/0);\n\t\t},\n\t};\n};\n`,\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Sequence\",\n\t\tinput: \"- a\\n- b\\n- c\\n\",\n\t\texpected: \"return {\\n\\t\\\"a\\\",\\n\\t\\\"b\\\",\\n\\t\\\"c\\\",\\n};\\n\",\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Mapping\",\n\t\tinput: \"a: b\\nc:\\n d: e\\nf: 0\\n\",\n\t\texpected: \"return {\\n\\t[\\\"a\\\"] = \\\"b\\\";\\n\\t[\\\"c\\\"] = {\\n\\t\\t[\\\"d\\\"] = \\\"e\\\";\\n\\t};\\n\\t[\\\"f\\\"] = 0;\\n};\\n\",\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Scalar str\",\n\t\tinput: \"str: |\\n foo\\n bar\\nanother: 'single'\\nand: \\\"double\\\"\",\n\t\texpected: \"return {\\n\\t[\\\"str\\\"] = [[\\nfoo\\nbar\\n]];\\n\\t[\\\"another\\\"] = 'single';\\n\\t[\\\"and\\\"] = \\\"double\\\";\\n};\\n\",\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Scalar null\",\n\t\tinput: \"x: null\\n\",\n\t\texpected: \"return {\\n\\t[\\\"x\\\"] = nil;\\n};\\n\",\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Scalar int\",\n\t\tinput: \"- 1\\n- 2\\n- 0x10\\n- 0o30\\n- -999\\n\",\n\t\texpected: \"return {\\n\\t1,\\n\\t2,\\n\\t0x10,\\n\\t24,\\n\\t-999,\\n};\\n\",\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Scalar float\",\n\t\tinput: \"- 1.0\\n- 3.14\\n- 1e100\\n- .Inf\\n- .NAN\\n\",\n\t\texpected: \"return {\\n\\t1.0,\\n\\t3.14,\\n\\t1e100,\\n\\t(1/0),\\n\\t(0/0),\\n};\\n\",\n\t\tscenarioType: \"encode\",\n\t},\n}\n\nfunc testLuaScenario(t *testing.T, s formatScenario) {\n\tswitch s.scenarioType {\n\tcase \"\", \"decode\":\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)\n\tcase \"encode\":\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(ConfiguredLuaPreferences)), s.description)\n\tcase \"roundtrip\":\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewLuaEncoder(ConfiguredLuaPreferences)), s.description)\n\tcase \"unquoted-encode\":\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(LuaPreferences{\n\t\t\tDocPrefix: \"return \",\n\t\t\tDocSuffix: \";\\n\",\n\t\t\tUnquotedKeys: true,\n\t\t\tGlobals: false,\n\t\t})), s.description)\n\tcase \"globals-encode\":\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(LuaPreferences{\n\t\t\tDocPrefix: \"return \",\n\t\t\tDocSuffix: \";\\n\",\n\t\t\tUnquotedKeys: false,\n\t\t\tGlobals: true,\n\t\t})), s.description)\n\tdefault:\n\t\tpanic(fmt.Sprintf(\"unhandled scenario type %q\", s.scenarioType))\n\t}\n}\n\nfunc documentLuaScenario(_ *testing.T, w *bufio.Writer, i interface{}) {\n\ts := i.(formatScenario)\n\n\tif s.skipDoc {\n\t\treturn\n\t}\n\tswitch s.scenarioType {\n\tcase \"\", \"decode\":\n\t\tdocumentLuaDecodeScenario(w, s)\n\tcase \"encode\", \"unquoted-encode\", \"globals-encode\":\n\t\tdocumentLuaEncodeScenario(w, s)\n\tcase \"roundtrip\":\n\t\tdocumentLuaRoundTripScenario(w, s)\n\tdefault:\n\t\tpanic(fmt.Sprintf(\"unhandled scenario type %q\", s.scenarioType))\n\t}\n}\n\nfunc documentLuaDecodeScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.lua file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```lua\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\texpression := s.expression\n\tif expression == \"\" {\n\t\texpression = \".\"\n\t}\n\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -oy '%v' sample.lua\\n```\\n\", expression))\n\twriteOrPanic(w, \"will output\\n\")\n\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewYamlEncoder(ConfiguredYamlPreferences))))\n}\n\nfunc documentLuaEncodeScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\tprefs := ConfiguredLuaPreferences\n\tswitch s.scenarioType {\n\tcase \"unquoted-encode\":\n\t\tprefs = LuaPreferences{\n\t\t\tDocPrefix: \"return \",\n\t\t\tDocSuffix: \";\\n\",\n\t\t\tUnquotedKeys: true,\n\t\t\tGlobals: false,\n\t\t}\n\tcase \"globals-encode\":\n\t\tprefs = LuaPreferences{\n\t\t\tDocPrefix: \"return \",\n\t\t\tDocSuffix: \";\\n\",\n\t\t\tUnquotedKeys: false,\n\t\t\tGlobals: true,\n\t\t}\n\t}\n\twriteOrPanic(w, \"Given a sample.yml file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\tswitch s.scenarioType {\n\tcase \"unquoted-encode\":\n\t\twriteOrPanic(w, \"```bash\\nyq -o=lua --lua-unquoted '.' sample.yml\\n```\\n\")\n\tcase \"globals-encode\":\n\t\twriteOrPanic(w, \"```bash\\nyq -o=lua --lua-globals '.' sample.yml\\n```\\n\")\n\tdefault:\n\t\twriteOrPanic(w, \"```bash\\nyq -o=lua '.' sample.yml\\n```\\n\")\n\t}\n\twriteOrPanic(w, \"will output\\n\")\n\n\twriteOrPanic(w, fmt.Sprintf(\"```lua\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(prefs))))\n}\n\nfunc documentLuaRoundTripScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.lua file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```lua\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\twriteOrPanic(w, \"```bash\\nyq '.' sample.lua\\n```\\n\")\n\twriteOrPanic(w, \"will output\\n\")\n\n\twriteOrPanic(w, fmt.Sprintf(\"```lua\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewLuaEncoder(ConfiguredLuaPreferences))))\n}\n\nfunc TestLuaScenarios(t *testing.T) {\n\tfor _, tt := range luaScenarios {\n\t\ttestLuaScenario(t, tt)\n\t}\n\tgenericScenarios := make([]interface{}, len(luaScenarios))\n\tfor i, s := range luaScenarios {\n\t\tgenericScenarios[i] = s\n\t}\n\tdocumentScenarios(t, \"usage\", \"lua\", genericScenarios, documentLuaScenario)\n}\n" }, { "path": "pkg/yqlib/matchKeyString.go", "content": "package yqlib\n\nfunc matchKey(name string, pattern string) (matched bool) {\n\tif pattern == \"\" {\n\t\treturn name == pattern\n\t}\n\tif pattern == \"*\" {\n\t\tlog.Debug(\"wild!\")\n\t\treturn true\n\t}\n\treturn deepMatch(name, pattern)\n}\n\n// deepMatch reports whether the name matches the pattern in linear time.\n// Source https://research.swtch.com/glob\nfunc deepMatch(name, pattern string) bool {\n\tpx := 0\n\tnx := 0\n\tnextPx := 0\n\tnextNx := 0\n\tfor px < len(pattern) || nx < len(name) {\n\t\tif px < len(pattern) {\n\t\t\tc := pattern[px]\n\t\t\tswitch c {\n\t\t\tdefault: // ordinary character\n\t\t\t\tif nx < len(name) && name[nx] == c {\n\t\t\t\t\tpx++\n\t\t\t\t\tnx++\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\tcase '?': // single-character wildcard\n\t\t\t\tif nx < len(name) {\n\t\t\t\t\tpx++\n\t\t\t\t\tnx++\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\tcase '*': // zero-or-more-character wildcard\n\t\t\t\t// Try to match at nx.\n\t\t\t\t// If that doesn't work out,\n\t\t\t\t// restart at nx+1 next.\n\t\t\t\tnextPx = px\n\t\t\t\tnextNx = nx + 1\n\t\t\t\tpx++\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\t// Mismatch. Maybe restart.\n\t\tif 0 < nextNx && nextNx <= len(name) {\n\t\t\tpx = nextPx\n\t\t\tnx = nextNx\n\t\t\tcontinue\n\t\t}\n\t\treturn false\n\t}\n\t// Matched all of pattern to all of name. Success.\n\treturn true\n}\n" }, { "path": "pkg/yqlib/matchKeyString_test.go", "content": "package yqlib\n\nimport (\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc TestDeepMatch(t *testing.T) {\n\tvar tests = []struct {\n\t\tname string\n\t\tpattern string\n\t\tok bool\n\t}{\n\t\t{\"\", \"\", true},\n\t\t{\"<<\", \"<<\", true},\n\t\t{\"\", \"x\", false},\n\t\t{\"x\", \"\", false},\n\t\t{\"abc\", \"abc\", true},\n\t\t{\"abc\", \"*\", true},\n\t\t{\"abc\", \"*c\", true},\n\t\t{\"abc\", \"*b\", false},\n\t\t{\"abc\", \"a*\", true},\n\t\t{\"abc\", \"b*\", false},\n\t\t{\"a\", \"a*\", true},\n\t\t{\"a\", \"*a\", true},\n\t\t{\"axbxcxdxe\", \"a*b*c*d*e*\", true},\n\t\t{\"axbxcxdxexxx\", \"a*b*c*d*e*\", true},\n\t\t{\"abxbbxdbxebxczzx\", \"a*b?c*x\", true},\n\t\t{\"abxbbxdbxebxczzy\", \"a*b?c*x\", false},\n\t\t{strings.Repeat(\"a\", 100), \"a*a*a*a*b\", false},\n\t\t{\"xxx\", \"*x\", true},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name+\" \"+tt.pattern, func(t *testing.T) {\n\t\t\tif want, got := tt.ok, deepMatch(tt.name, tt.pattern); want != got {\n\t\t\t\tt.Errorf(\"Expected %v got %v\", want, got)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "pkg/yqlib/no_base64.go", "content": "//go:build yq_nobase64\n\npackage yqlib\n\nfunc NewBase64Decoder() Decoder {\n\treturn nil\n}\n\nfunc NewBase64Encoder() Encoder {\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/no_csv.go", "content": "//go:build yq_nocsv\n\npackage yqlib\n\nfunc NewCSVObjectDecoder(prefs CsvPreferences) Decoder {\n\treturn nil\n}\n\nfunc NewCsvEncoder(prefs CsvPreferences) Encoder {\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/no_hcl.go", "content": "//go:build yq_nohcl\n\npackage yqlib\n\nfunc NewHclDecoder() Decoder {\n\treturn nil\n}\n\nfunc NewHclEncoder(_ HclPreferences) Encoder {\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/no_ini.go", "content": "//go:build yq_noini\n\npackage yqlib\n\nfunc NewINIDecoder() Decoder {\n\treturn nil\n}\n\nfunc NewINIEncoder() Encoder {\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/no_json.go", "content": "//go:build yq_nojson\n\npackage yqlib\n\nfunc NewJSONDecoder() Decoder {\n\treturn nil\n}\n\nfunc NewJSONEncoder(prefs JsonPreferences) Encoder {\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/no_kyaml.go", "content": "//go:build yq_nokyaml\n\npackage yqlib\n\nfunc NewKYamlEncoder(_ KYamlPreferences) Encoder {\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/no_lua.go", "content": "//go:build yq_nolua\n\npackage yqlib\n\nfunc NewLuaEncoder(prefs LuaPreferences) Encoder {\n\treturn nil\n}\n\nfunc NewLuaDecoder(prefs LuaPreferences) Decoder {\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/no_props.go", "content": "//go:build yq_noprops\n\npackage yqlib\n\nfunc NewPropertiesDecoder() Decoder {\n\treturn nil\n}\n\nfunc NewPropertiesEncoder(prefs PropertiesPreferences) Encoder {\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/no_sh.go", "content": "//go:build yq_nosh\n\npackage yqlib\n\nfunc NewShEncoder() Encoder {\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/no_shellvariables.go", "content": "//go:build yq_noshell\n\npackage yqlib\n\nfunc NewShellVariablesEncoder() Encoder {\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/no_toml.go", "content": "//go:build yq_notoml\n\npackage yqlib\n\nfunc NewTomlDecoder() Decoder {\n\treturn nil\n}\n\nfunc NewTomlEncoder() Encoder {\n\treturn nil\n}\n\nfunc NewTomlEncoderWithPrefs(prefs TomlPreferences) Encoder {\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/no_uri.go", "content": "//go:build yq_nouri\n\npackage yqlib\n\nfunc NewUriDecoder() Decoder {\n\treturn nil\n}\n\nfunc NewUriEncoder() Encoder {\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/no_xml.go", "content": "//go:build yq_noxml\n\npackage yqlib\n\nfunc NewXMLDecoder(prefs XmlPreferences) Decoder {\n\treturn nil\n}\n\nfunc NewXMLEncoder(prefs XmlPreferences) Encoder {\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/operation.go", "content": "package yqlib\n\nimport \"fmt\"\n\ntype Operation struct {\n\tOperationType *operationType\n\tValue interface{}\n\tStringValue string\n\tCandidateNode *CandidateNode // used for Value Path elements\n\tPreferences interface{}\n\tUpdateAssign bool // used for assign ops, when true it means we evaluate the rhs given the lhs\n}\n\ntype operationType struct {\n\tType string\n\tNumArgs uint // number of arguments to the op\n\tPrecedence uint\n\tHandler operatorHandler\n\tCheckForPostTraverse bool\n\tToString func(o *Operation) string\n}\n\nvar valueToStringFunc = func(p *Operation) string {\n\treturn fmt.Sprintf(\"%v (%T)\", p.Value, p.Value)\n}\n\nfunc createValueOperation(value interface{}, stringValue string) *Operation {\n\tlog.Debug(\"creating value op for string %v\", stringValue)\n\tvar node = createScalarNode(value, stringValue)\n\n\treturn &Operation{\n\t\tOperationType: valueOpType,\n\t\tValue: value,\n\t\tStringValue: stringValue,\n\t\tCandidateNode: node,\n\t}\n}\n\nvar orOpType = &operationType{Type: \"OR\", NumArgs: 2, Precedence: 20, Handler: orOperator}\nvar andOpType = &operationType{Type: \"AND\", NumArgs: 2, Precedence: 20, Handler: andOperator}\nvar reduceOpType = &operationType{Type: \"REDUCE\", NumArgs: 2, Precedence: 35, Handler: reduceOperator}\n\nvar blockOpType = &operationType{Type: \"BLOCK\", Precedence: 10, NumArgs: 2, Handler: emptyOperator}\n\nvar unionOpType = &operationType{Type: \"UNION\", NumArgs: 2, Precedence: 10, Handler: unionOperator}\n\nvar pipeOpType = &operationType{Type: \"PIPE\", NumArgs: 2, Precedence: 30, Handler: pipeOperator}\n\nvar assignOpType = &operationType{Type: \"ASSIGN\", NumArgs: 2, Precedence: 40, Handler: assignUpdateOperator}\nvar addAssignOpType = &operationType{Type: \"ADD_ASSIGN\", NumArgs: 2, Precedence: 40, Handler: addAssignOperator}\nvar subtractAssignOpType = &operationType{Type: \"SUBTRACT_ASSIGN\", NumArgs: 2, Precedence: 40, Handler: subtractAssignOperator}\n\nvar assignAttributesOpType = &operationType{Type: \"ASSIGN_ATTRIBUTES\", NumArgs: 2, Precedence: 40, Handler: assignAttributesOperator}\nvar assignStyleOpType = &operationType{Type: \"ASSIGN_STYLE\", NumArgs: 2, Precedence: 40, Handler: assignStyleOperator}\nvar assignVariableOpType = &operationType{Type: \"ASSIGN_VARIABLE\", NumArgs: 2, Precedence: 40, Handler: useWithPipe}\nvar assignTagOpType = &operationType{Type: \"ASSIGN_TAG\", NumArgs: 2, Precedence: 40, Handler: assignTagOperator}\nvar assignCommentOpType = &operationType{Type: \"ASSIGN_COMMENT\", NumArgs: 2, Precedence: 40, Handler: assignCommentsOperator}\nvar assignAnchorOpType = &operationType{Type: \"ASSIGN_ANCHOR\", NumArgs: 2, Precedence: 40, Handler: assignAnchorOperator}\nvar assignAliasOpType = &operationType{Type: \"ASSIGN_ALIAS\", NumArgs: 2, Precedence: 40, Handler: assignAliasOperator}\n\nvar multiplyOpType = &operationType{Type: \"MULTIPLY\", NumArgs: 2, Precedence: 42, Handler: multiplyOperator}\nvar multiplyAssignOpType = &operationType{Type: \"MULTIPLY_ASSIGN\", NumArgs: 2, Precedence: 42, Handler: multiplyAssignOperator}\n\nvar divideOpType = &operationType{Type: \"DIVIDE\", NumArgs: 2, Precedence: 42, Handler: divideOperator}\n\nvar moduloOpType = &operationType{Type: \"MODULO\", NumArgs: 2, Precedence: 42, Handler: moduloOperator}\n\nvar addOpType = &operationType{Type: \"ADD\", NumArgs: 2, Precedence: 42, Handler: addOperator}\nvar subtractOpType = &operationType{Type: \"SUBTRACT\", NumArgs: 2, Precedence: 42, Handler: subtractOperator}\nvar alternativeOpType = &operationType{Type: \"ALTERNATIVE\", NumArgs: 2, Precedence: 42, Handler: alternativeOperator}\n\nvar equalsOpType = &operationType{Type: \"EQUALS\", NumArgs: 2, Precedence: 40, Handler: equalsOperator}\nvar notEqualsOpType = &operationType{Type: \"NOT_EQUALS\", NumArgs: 2, Precedence: 40, Handler: notEqualsOperator}\n\nvar compareOpType = &operationType{Type: \"COMPARE\", NumArgs: 2, Precedence: 40, Handler: compareOperator}\nvar minOpType = &operationType{Type: \"MIN\", NumArgs: 0, Precedence: 40, Handler: minOperator}\nvar maxOpType = &operationType{Type: \"MAX\", NumArgs: 0, Precedence: 40, Handler: maxOperator}\n\n// createmap needs to be above union, as we use union to build the components of the objects\nvar createMapOpType = &operationType{Type: \"CREATE_MAP\", NumArgs: 2, Precedence: 15, Handler: createMapOperator}\n\nvar shortPipeOpType = &operationType{Type: \"SHORT_PIPE\", NumArgs: 2, Precedence: 45, Handler: pipeOperator}\n\nvar lengthOpType = &operationType{Type: \"LENGTH\", NumArgs: 0, Precedence: 50, Handler: lengthOperator}\nvar lineOpType = &operationType{Type: \"LINE\", NumArgs: 0, Precedence: 50, Handler: lineOperator}\nvar columnOpType = &operationType{Type: \"LINE\", NumArgs: 0, Precedence: 50, Handler: columnOperator}\n\n// Use this expression to create alias/syntactic sugar expressions (in lexer_participle).\nvar expressionOpType = &operationType{Type: \"EXP\", NumArgs: 0, Precedence: 50, Handler: expressionOperator}\n\nvar collectOpType = &operationType{Type: \"COLLECT\", NumArgs: 1, Precedence: 50, Handler: collectOperator}\nvar mapOpType = &operationType{Type: \"MAP\", NumArgs: 1, Precedence: 52, Handler: mapOperator, CheckForPostTraverse: true}\nvar filterOpType = &operationType{Type: \"FILTER\", NumArgs: 1, Precedence: 52, Handler: filterOperator, CheckForPostTraverse: true}\nvar errorOpType = &operationType{Type: \"ERROR\", NumArgs: 1, Precedence: 50, Handler: errorOperator}\nvar pickOpType = &operationType{Type: \"PICK\", NumArgs: 1, Precedence: 52, Handler: pickOperator, CheckForPostTraverse: true}\nvar omitOpType = &operationType{Type: \"OMIT\", NumArgs: 1, Precedence: 52, Handler: omitOperator, CheckForPostTraverse: true}\nvar evalOpType = &operationType{Type: \"EVAL\", NumArgs: 1, Precedence: 52, Handler: evalOperator, CheckForPostTraverse: true}\nvar mapValuesOpType = &operationType{Type: \"MAP_VALUES\", NumArgs: 1, Precedence: 52, Handler: mapValuesOperator, CheckForPostTraverse: true}\n\nvar formatDateTimeOpType = &operationType{Type: \"FORMAT_DATE_TIME\", NumArgs: 1, Precedence: 50, Handler: formatDateTime}\nvar withDtFormatOpType = &operationType{Type: \"WITH_DATE_TIME_FORMAT\", NumArgs: 1, Precedence: 50, Handler: withDateTimeFormat}\nvar nowOpType = &operationType{Type: \"NOW\", NumArgs: 0, Precedence: 50, Handler: nowOp}\nvar tzOpType = &operationType{Type: \"TIMEZONE\", NumArgs: 1, Precedence: 50, Handler: tzOp}\nvar fromUnixOpType = &operationType{Type: \"FROM_UNIX\", NumArgs: 0, Precedence: 50, Handler: fromUnixOp}\nvar toUnixOpType = &operationType{Type: \"TO_UNIX\", NumArgs: 0, Precedence: 50, Handler: toUnixOp}\n\nvar encodeOpType = &operationType{Type: \"ENCODE\", NumArgs: 0, Precedence: 50, Handler: encodeOperator}\nvar decodeOpType = &operationType{Type: \"DECODE\", NumArgs: 0, Precedence: 50, Handler: decodeOperator}\n\nvar anyOpType = &operationType{Type: \"ANY\", NumArgs: 0, Precedence: 50, Handler: anyOperator}\nvar allOpType = &operationType{Type: \"ALL\", NumArgs: 0, Precedence: 50, Handler: allOperator}\nvar containsOpType = &operationType{Type: \"CONTAINS\", NumArgs: 1, Precedence: 50, Handler: containsOperator}\nvar anyConditionOpType = &operationType{Type: \"ANY_CONDITION\", NumArgs: 1, Precedence: 50, Handler: anyOperator}\nvar allConditionOpType = &operationType{Type: \"ALL_CONDITION\", NumArgs: 1, Precedence: 50, Handler: allOperator}\n\nvar toEntriesOpType = &operationType{Type: \"TO_ENTRIES\", NumArgs: 0, Precedence: 52, Handler: toEntriesOperator, CheckForPostTraverse: true}\nvar fromEntriesOpType = &operationType{Type: \"FROM_ENTRIES\", NumArgs: 0, Precedence: 50, Handler: fromEntriesOperator}\nvar withEntriesOpType = &operationType{Type: \"WITH_ENTRIES\", NumArgs: 1, Precedence: 50, Handler: withEntriesOperator}\n\nvar withOpType = &operationType{Type: \"WITH\", NumArgs: 1, Precedence: 52, Handler: withOperator, CheckForPostTraverse: true}\n\nvar splitDocumentOpType = &operationType{Type: \"SPLIT_DOC\", NumArgs: 0, Precedence: 52, Handler: splitDocumentOperator, CheckForPostTraverse: true}\nvar getVariableOpType = &operationType{Type: \"GET_VARIABLE\", NumArgs: 0, Precedence: 55, Handler: getVariableOperator}\nvar getStyleOpType = &operationType{Type: \"GET_STYLE\", NumArgs: 0, Precedence: 50, Handler: getStyleOperator}\nvar getTagOpType = &operationType{Type: \"GET_TAG\", NumArgs: 0, Precedence: 50, Handler: getTagOperator}\nvar getKindOpType = &operationType{Type: \"GET_KIND\", NumArgs: 0, Precedence: 50, Handler: getKindOperator}\n\nvar getKeyOpType = &operationType{Type: \"GET_KEY\", NumArgs: 0, Precedence: 50, Handler: getKeyOperator}\nvar isKeyOpType = &operationType{Type: \"IS_KEY\", NumArgs: 0, Precedence: 50, Handler: isKeyOperator}\nvar getParentOpType = &operationType{Type: \"GET_PARENT\", NumArgs: 0, Precedence: 50, Handler: getParentOperator}\nvar getParentsOpType = &operationType{Type: \"GET_PARENTS\", NumArgs: 0, Precedence: 50, Handler: getParentsOperator}\n\nvar getCommentOpType = &operationType{Type: \"GET_COMMENT\", NumArgs: 0, Precedence: 50, Handler: getCommentsOperator}\nvar getAnchorOpType = &operationType{Type: \"GET_ANCHOR\", NumArgs: 0, Precedence: 50, Handler: getAnchorOperator}\nvar getAliasOpType = &operationType{Type: \"GET_ALIAS\", NumArgs: 0, Precedence: 50, Handler: getAliasOperator}\nvar getDocumentIndexOpType = &operationType{Type: \"GET_DOCUMENT_INDEX\", NumArgs: 0, Precedence: 50, Handler: getDocumentIndexOperator}\nvar getFilenameOpType = &operationType{Type: \"GET_FILENAME\", NumArgs: 0, Precedence: 50, Handler: getFilenameOperator}\nvar getFileIndexOpType = &operationType{Type: \"GET_FILE_INDEX\", NumArgs: 0, Precedence: 50, Handler: getFileIndexOperator}\n\nvar getPathOpType = &operationType{Type: \"GET_PATH\", NumArgs: 0, Precedence: 52, Handler: getPathOperator, CheckForPostTraverse: true}\nvar setPathOpType = &operationType{Type: \"SET_PATH\", NumArgs: 1, Precedence: 50, Handler: setPathOperator}\nvar delPathsOpType = &operationType{Type: \"DEL_PATHS\", NumArgs: 1, Precedence: 52, Handler: delPathsOperator, CheckForPostTraverse: true}\n\nvar explodeOpType = &operationType{Type: \"EXPLODE\", NumArgs: 1, Precedence: 52, Handler: explodeOperator, CheckForPostTraverse: true}\nvar sortByOpType = &operationType{Type: \"SORT_BY\", NumArgs: 1, Precedence: 52, Handler: sortByOperator, CheckForPostTraverse: true}\nvar firstOpType = &operationType{Type: \"FIRST\", NumArgs: 1, Precedence: 52, Handler: firstOperator, CheckForPostTraverse: true}\nvar reverseOpType = &operationType{Type: \"REVERSE\", NumArgs: 0, Precedence: 52, Handler: reverseOperator, CheckForPostTraverse: true}\nvar sortOpType = &operationType{Type: \"SORT\", NumArgs: 0, Precedence: 52, Handler: sortOperator, CheckForPostTraverse: true}\nvar shuffleOpType = &operationType{Type: \"SHUFFLE\", NumArgs: 0, Precedence: 52, Handler: shuffleOperator, CheckForPostTraverse: true}\n\nvar sortKeysOpType = &operationType{Type: \"SORT_KEYS\", NumArgs: 1, Precedence: 52, Handler: sortKeysOperator, CheckForPostTraverse: true}\n\nvar joinStringOpType = &operationType{Type: \"JOIN\", NumArgs: 1, Precedence: 50, Handler: joinStringOperator}\nvar subStringOpType = &operationType{Type: \"SUBSTR\", NumArgs: 1, Precedence: 50, Handler: substituteStringOperator}\nvar matchOpType = &operationType{Type: \"MATCH\", NumArgs: 1, Precedence: 50, Handler: matchOperator}\nvar captureOpType = &operationType{Type: \"CAPTURE\", NumArgs: 1, Precedence: 50, Handler: captureOperator}\nvar testOpType = &operationType{Type: \"TEST\", NumArgs: 1, Precedence: 50, Handler: testOperator}\nvar splitStringOpType = &operationType{Type: \"SPLIT\", NumArgs: 1, Precedence: 52, Handler: splitStringOperator, CheckForPostTraverse: true}\nvar changeCaseOpType = &operationType{Type: \"CHANGE_CASE\", NumArgs: 0, Precedence: 50, Handler: changeCaseOperator}\nvar trimOpType = &operationType{Type: \"TRIM\", NumArgs: 0, Precedence: 50, Handler: trimSpaceOperator}\nvar toStringOpType = &operationType{Type: \"TO_STRING\", NumArgs: 0, Precedence: 50, Handler: toStringOperator}\nvar stringInterpolationOpType = &operationType{Type: \"STRING_INT\", NumArgs: 0, Precedence: 50, Handler: stringInterpolationOperator, ToString: valueToStringFunc}\n\nvar loadOpType = &operationType{Type: \"LOAD\", NumArgs: 1, Precedence: 52, Handler: loadOperator, CheckForPostTraverse: true}\nvar loadStringOpType = &operationType{Type: \"LOAD_STRING\", NumArgs: 1, Precedence: 52, Handler: loadStringOperator}\n\nvar keysOpType = &operationType{Type: \"KEYS\", NumArgs: 0, Precedence: 52, Handler: keysOperator, CheckForPostTraverse: true}\n\nvar collectObjectOpType = &operationType{Type: \"COLLECT_OBJECT\", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator}\n\nvar traversePathOpType = &operationType{Type: \"TRAVERSE_PATH\", NumArgs: 0, Precedence: 55, Handler: traversePathOperator,\n\tToString: func(p *Operation) string {\n\t\treturn fmt.Sprintf(\"%v\", p.Value)\n\t}}\n\nvar traverseArrayOpType = &operationType{Type: \"TRAVERSE_ARRAY\", NumArgs: 2, Precedence: 50, Handler: traverseArrayOperator}\n\nvar selfReferenceOpType = &operationType{Type: \"SELF\", NumArgs: 0, Precedence: 55, Handler: selfOperator}\nvar valueOpType = &operationType{Type: \"VALUE\", NumArgs: 0, Precedence: 50, Handler: valueOperator, ToString: valueToStringFunc}\nvar referenceOpType = &operationType{Type: \"REF\", NumArgs: 0, Precedence: 50, Handler: referenceOperator}\nvar envOpType = &operationType{Type: \"ENV\", NumArgs: 0, Precedence: 52, Handler: envOperator, CheckForPostTraverse: true}\nvar notOpType = &operationType{Type: \"NOT\", NumArgs: 0, Precedence: 50, Handler: notOperator}\nvar toNumberOpType = &operationType{Type: \"TO_NUMBER\", NumArgs: 0, Precedence: 50, Handler: toNumberOperator}\nvar emptyOpType = &operationType{Type: \"EMPTY\", Precedence: 50, Handler: emptyOperator}\n\nvar envsubstOpType = &operationType{Type: \"ENVSUBST\", NumArgs: 0, Precedence: 50, Handler: envsubstOperator}\n\nvar recursiveDescentOpType = &operationType{Type: \"RECURSIVE_DESCENT\", NumArgs: 0, Precedence: 50, Handler: recursiveDescentOperator}\n\nvar selectOpType = &operationType{Type: \"SELECT\", NumArgs: 1, Precedence: 52, Handler: selectOperator, CheckForPostTraverse: true}\nvar hasOpType = &operationType{Type: \"HAS\", NumArgs: 1, Precedence: 50, Handler: hasOperator}\nvar uniqueOpType = &operationType{Type: \"UNIQUE\", NumArgs: 0, Precedence: 52, Handler: unique, CheckForPostTraverse: true}\nvar uniqueByOpType = &operationType{Type: \"UNIQUE_BY\", NumArgs: 1, Precedence: 52, Handler: uniqueBy, CheckForPostTraverse: true}\nvar groupByOpType = &operationType{Type: \"GROUP_BY\", NumArgs: 1, Precedence: 52, Handler: groupBy, CheckForPostTraverse: true}\nvar flattenOpType = &operationType{Type: \"FLATTEN_BY\", NumArgs: 0, Precedence: 52, Handler: flattenOp, CheckForPostTraverse: true}\nvar deleteChildOpType = &operationType{Type: \"DELETE\", NumArgs: 1, Precedence: 40, Handler: deleteChildOperator}\n\nvar pivotOpType = &operationType{Type: \"PIVOT\", NumArgs: 0, Precedence: 52, Handler: pivotOperator, CheckForPostTraverse: true}\n\n// debugging purposes only\nfunc (p *Operation) toString() string {\n\tif p == nil {\n\t\treturn \"OP IS NIL\"\n\t}\n\tif p.OperationType.ToString != nil {\n\t\treturn p.OperationType.ToString(p)\n\t}\n\treturn fmt.Sprintf(\"%v\", p.OperationType.Type)\n}\n" }, { "path": "pkg/yqlib/operator_add.go", "content": "package yqlib\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n)\n\nfunc createAddOp(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode {\n\treturn &ExpressionNode{Operation: &Operation{OperationType: addOpType},\n\t\tLHS: lhs,\n\t\tRHS: rhs}\n}\n\nfunc addAssignOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\treturn compoundAssignFunction(d, context, expressionNode, createAddOp)\n}\n\nfunc toNodes(candidate *CandidateNode, lhs *CandidateNode) []*CandidateNode {\n\tif candidate.Tag == \"!!null\" {\n\t\treturn []*CandidateNode{}\n\t}\n\n\tclone := candidate.Copy()\n\n\tswitch candidate.Kind {\n\tcase SequenceNode:\n\t\treturn clone.Content\n\tdefault:\n\t\tif len(lhs.Content) > 0 {\n\t\t\tclone.Style = lhs.Content[0].Style\n\t\t}\n\t\treturn []*CandidateNode{clone}\n\t}\n\n}\n\nfunc addOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"Add operator\")\n\t// only calculate when empty IF we are the root expression; OR\n\t// calcWhenEmpty := expressionNode.Parent == nil || expressionNode.Parent.LHS == expressionNode\n\tcalcWhenEmpty := context.MatchingNodes.Len() > 0\n\n\treturn crossFunction(d, context.ReadOnlyClone(), expressionNode, add, calcWhenEmpty)\n}\n\nfunc add(_ *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {\n\tlhsNode := lhs\n\n\tif lhs == nil && rhs == nil {\n\t\treturn nil, nil\n\t} else if lhs == nil {\n\t\treturn rhs.Copy(), nil\n\t} else if rhs == nil {\n\t\treturn lhs.Copy(), nil\n\t} else if lhsNode.Tag == \"!!null\" {\n\t\treturn lhs.CopyAsReplacement(rhs), nil\n\t}\n\n\ttarget := lhs.CopyWithoutContent()\n\n\tswitch lhsNode.Kind {\n\tcase MappingNode:\n\t\tif rhs.Kind != MappingNode {\n\t\t\treturn nil, fmt.Errorf(\"%v (%v) cannot be added to a %v (%v)\", rhs.Tag, rhs.GetNicePath(), lhsNode.Tag, lhs.GetNicePath())\n\t\t}\n\t\taddMaps(target, lhs, rhs)\n\tcase SequenceNode:\n\t\taddSequences(target, lhs, rhs)\n\tcase ScalarNode:\n\t\tif rhs.Kind != ScalarNode {\n\t\t\treturn nil, fmt.Errorf(\"%v (%v) cannot be added to a %v (%v)\", rhs.Tag, rhs.GetNicePath(), lhsNode.Tag, lhs.GetNicePath())\n\t\t}\n\t\ttarget.Kind = ScalarNode\n\t\ttarget.Style = lhsNode.Style\n\t\tif err := addScalars(context, target, lhsNode, rhs); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\treturn target, nil\n}\n\nfunc addScalars(context Context, target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error {\n\tlhsTag := lhs.Tag\n\trhsTag := rhs.guessTagFromCustomType()\n\tlhsIsCustom := false\n\tif !strings.HasPrefix(lhsTag, \"!!\") {\n\t\t// custom tag - we have to have a guess\n\t\tlhsTag = lhs.guessTagFromCustomType()\n\t\tlhsIsCustom = true\n\t}\n\n\tisDateTime := lhs.Tag == \"!!timestamp\"\n\n\t// if the lhs is a string, it might be a timestamp in a custom format.\n\tif lhsTag == \"!!str\" && context.GetDateTimeLayout() != time.RFC3339 {\n\t\t_, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value)\n\t\tisDateTime = err == nil\n\t}\n\n\tif isDateTime {\n\t\treturn addDateTimes(context.GetDateTimeLayout(), target, lhs, rhs)\n\n\t} else if lhsTag == \"!!str\" {\n\t\ttarget.Tag = lhs.Tag\n\t\tif rhsTag == \"!!null\" {\n\t\t\ttarget.Value = lhs.Value\n\t\t} else {\n\t\t\ttarget.Value = lhs.Value + rhs.Value\n\t\t}\n\n\t} else if rhsTag == \"!!str\" {\n\t\ttarget.Tag = rhs.Tag\n\t\ttarget.Value = lhs.Value + rhs.Value\n\t} else if lhsTag == \"!!int\" && rhsTag == \"!!int\" {\n\t\tformat, lhsNum, err := parseInt64(lhs.Value)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t_, rhsNum, err := parseInt64(rhs.Value)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tsum := lhsNum + rhsNum\n\t\ttarget.Tag = lhs.Tag\n\t\ttarget.Value = fmt.Sprintf(format, sum)\n\t} else if (lhsTag == \"!!int\" || lhsTag == \"!!float\") && (rhsTag == \"!!int\" || rhsTag == \"!!float\") {\n\t\tlhsNum, err := strconv.ParseFloat(lhs.Value, 64)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\trhsNum, err := strconv.ParseFloat(rhs.Value, 64)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tsum := lhsNum + rhsNum\n\t\tif lhsIsCustom {\n\t\t\ttarget.Tag = lhs.Tag\n\t\t} else {\n\t\t\ttarget.Tag = \"!!float\"\n\t\t}\n\t\ttarget.Value = fmt.Sprintf(\"%v\", sum)\n\t} else {\n\t\treturn fmt.Errorf(\"%v cannot be added to %v\", lhsTag, rhsTag)\n\t}\n\treturn nil\n}\n\nfunc addDateTimes(layout string, target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error {\n\n\tduration, err := time.ParseDuration(rhs.Value)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"unable to parse duration [%v]: %w\", rhs.Value, err)\n\t}\n\n\tcurrentTime, err := parseDateTime(layout, lhs.Value)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tnewTime := currentTime.Add(duration)\n\ttarget.Value = newTime.Format(layout)\n\treturn nil\n\n}\n\nfunc addSequences(target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) {\n\tlog.Debugf(\"adding sequences! target: %v; lhs %v; rhs: %v\", NodeToString(target), NodeToString(lhs), NodeToString(rhs))\n\ttarget.Kind = SequenceNode\n\tif len(lhs.Content) == 0 {\n\t\tlog.Debugf(\"dont copy lhs style\")\n\t\ttarget.Style = 0\n\t}\n\ttarget.Tag = lhs.Tag\n\n\textraNodes := toNodes(rhs, lhs)\n\n\ttarget.AddChildren(lhs.Content)\n\ttarget.AddChildren(extraNodes)\n}\n\nfunc addMaps(target *CandidateNode, lhsC *CandidateNode, rhsC *CandidateNode) {\n\tlhs := lhsC\n\trhs := rhsC\n\n\tif len(lhs.Content) == 0 {\n\t\tlog.Debugf(\"dont copy lhs style\")\n\t\ttarget.Style = 0\n\t}\n\n\ttarget.Content = make([]*CandidateNode, 0)\n\ttarget.AddChildren(lhs.Content)\n\n\tfor index := 0; index < len(rhs.Content); index = index + 2 {\n\t\tkey := rhs.Content[index]\n\t\tvalue := rhs.Content[index+1]\n\t\tlog.Debug(\"finding %v\", key.Value)\n\t\tindexInLHS := findKeyInMap(target, key)\n\t\tlog.Debug(\"indexInLhs %v\", indexInLHS)\n\t\tif indexInLHS < 0 {\n\t\t\t// not in there, append it\n\t\t\ttarget.AddKeyValueChild(key, value)\n\t\t} else {\n\t\t\t// it's there, replace it\n\t\t\toldValue := target.Content[indexInLHS+1]\n\t\t\tnewValueCopy := oldValue.CopyAsReplacement(value)\n\t\t\ttarget.Content[indexInLHS+1] = newValueCopy\n\t\t}\n\t}\n\ttarget.Kind = MappingNode\n\tif len(lhs.Content) > 0 {\n\t\ttarget.Style = lhs.Style\n\t}\n\ttarget.Tag = lhs.Tag\n}\n" }, { "path": "pkg/yqlib/operator_add_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar addOperatorScenarios = []expressionScenario{\n\t{\n\t\tskipDoc: true,\n\t\texpression: `\"foo\" + \"bar\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::foobar\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `[] | .[] | \"foo\" + .`,\n\t\texpected: []string{},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `[] | .[] | . + \"foo\"`,\n\t\texpected: []string{},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `select(.) | \"foo\" + \"bar\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::foobar\\n\", // jq does not do this :/ - but yq has for quite some time.\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"apples: 3\",\n\t\texpression: `.apples + 3`,\n\t\texpected: []string{\n\t\t\t\"D0, P[apples], (!!int)::6\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"apples: 3\",\n\t\texpression: `.bobo + 3`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!int)::3\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `select(.) | \"cat\" + .`,\n\t\texpected: []string{},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[]`,\n\t\texpression: `.[] | (.a + \"|\" + .b)`,\n\t\texpected: []string{},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[]`,\n\t\texpression: `.[] | (.a + \"|\")`,\n\t\texpected: []string{},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[]`,\n\t\texpression: `.[] | (\"|\" + .a)`,\n\t\texpected: []string{},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `resources: [foo, bar, baz]`,\n\t\texpression: `.missing + .resources | .[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[resources 0], (!!str)::foo\\n\",\n\t\t\t\"D0, P[resources 1], (!!str)::bar\\n\",\n\t\t\t\"D0, P[resources 2], (!!str)::baz\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `resources: [foo, bar, baz]`,\n\t\texpression: `. | .missing + .resources | .[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[resources 0], (!!str)::foo\\n\",\n\t\t\t\"D0, P[resources 1], (!!str)::bar\\n\",\n\t\t\t\"D0, P[resources 2], (!!str)::baz\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `resources: [foo, bar, baz]`,\n\t\texpression: `. | .missing + .resources`,\n\t\texpected: []string{\n\t\t\t\"D0, P[resources], (!!seq)::[foo, bar, baz]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `resources: [foo, bar, baz]`,\n\t\texpression: `. | .missing + .resources | .[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[resources 0], (!!str)::foo\\n\",\n\t\t\t\"D0, P[resources 1], (!!str)::bar\\n\",\n\t\t\t\"D0, P[resources 2], (!!str)::baz\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[{a: foo, b: bar}, {a: 1, b: 2}]`,\n\t\texpression: \".[] | .a + .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[0 a], (!!str)::foobar\\n\",\n\t\t\t\"D0, P[1 a], (!!int)::3\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"add sequence creates a new sequence\",\n\t\texpression: `[\"a\"] as $f | {0:$f + [\"b\"], 1:$f}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::0:\\n - a\\n - b\\n1:\\n - a\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a: key`,\n\t\texpression: `. += {\"key\": \"b\"}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: key\\nkey: b\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[[c], [b]]`,\n\t\texpression: `.[] | . += \"a\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!seq)::[c, a]\\n\",\n\t\t\t\"D0, P[1], (!!seq)::[b, a]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{}`,\n\t\texpression: \"(.a + .b) as $x | .\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a: 0`,\n\t\texpression: \".a += .b.c\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: 0\\n\",\n\t\t},\n\t},\n\n\t{\n\t\tdescription: \"Concatenate arrays\",\n\t\tdocument: `{a: [1,2], b: [3,4]}`,\n\t\texpression: `.a + .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!seq)::[1, 2, 3, 4]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Concatenate to existing array\",\n\t\tsubdescription: \"Note that the styling of `a` is kept.\",\n\t\tdocument: \"a: [1,2]\\nb:\\n - 3\\n - 4\",\n\t\tdontFormatInputForDoc: true,\n\t\texpression: `.a += .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: [1, 2, 3, 4]\\nb:\\n - 3\\n - 4\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `[1] + ([2], [3])`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- 1\\n- 2\\n\",\n\t\t\t\"D0, P[], (!!seq)::- 1\\n- 3\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Concatenate null to array\",\n\t\tdocument: `{a: [1,2]}`,\n\t\texpression: `.a + null`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!seq)::[1, 2]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Concatenate to empty array\",\n\t\tdocument: `{a: []}`,\n\t\texpression: `.a + \"cat\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!seq)::- cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Append to existing array\",\n\t\tsubdescription: \"Note that the styling is copied from existing array elements\",\n\t\tdontFormatInputForDoc: true,\n\t\tdocument: `a: ['dog']`,\n\t\texpression: `.a += \"cat\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: ['dog', 'cat']\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Prepend to existing array\",\n\t\tdocument: `a: [dog]`,\n\t\texpression: `.a = [\"cat\"] + .a`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: [cat, dog]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Concatenate to existing array\",\n\t\tsubdescription: \"does not modify original\",\n\t\tdocument: `{a: ['dog'], b: cat}`,\n\t\texpression: `.a = .a + .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: ['dog', 'cat'], b: cat}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Concatenate to empty array\",\n\t\tdocument: `a: []`,\n\t\texpression: `.a += \"cat\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a:\\n - cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Concatenate to existing array\",\n\t\tdocument: `a: [dog]`,\n\t\texpression: `.a += \"cat\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: [dog, cat]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Concatenate to empty object\",\n\t\tdocument: `{a: {}}`,\n\t\texpression: `.a + {\"b\": \"cat\"}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!map)::b: cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Concatenate to existing object\",\n\t\tdocument: `{a: {c: dog}}`,\n\t\texpression: `.a + {\"b\": \"cat\"}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!map)::{c: dog, b: cat}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Concatenate to existing object\",\n\t\tsubdescription: \"matches stylig\",\n\t\tdocument: \"a:\\n c: dog\",\n\t\texpression: `.a + {\"b\": \"cat\"}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!map)::c: dog\\nb: cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Concatenate to empty object in place\",\n\t\tdocument: `a: {}`,\n\t\texpression: `.a += {\"b\": \"cat\"}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a:\\n b: cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Concatenate to existing object in place\",\n\t\tdocument: `a: {c: dog}`,\n\t\texpression: `.a += {\"b\": \"cat\"}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: {c: dog, b: cat}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Add new object to array\",\n\t\tdocument: `a: [{dog: woof}]`,\n\t\texpression: `.a + {\"cat\": \"meow\"}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!seq)::[{dog: woof}, {cat: meow}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Relative append\",\n\t\tdocument: `a: { a1: {b: [cat]}, a2: {b: [dog]}, a3: {} }`,\n\t\texpression: `.a[].b += [\"mouse\"]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: {a1: {b: [cat, mouse]}, a2: {b: [dog, mouse]}, a3: {b: [mouse]}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"String concatenation\",\n\t\tdocument: `{a: cat, b: meow}`,\n\t\texpression: `.a += .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: catmeow, b: meow}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"String concatenation - str + int\",\n\t\tskipDoc: true,\n\t\tdocument: `{a: !cool cat, b: meow}`,\n\t\texpression: `.a + 3`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!cool)::cat3\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"String concatenation - int + str\",\n\t\tskipDoc: true,\n\t\tdocument: `{a: !cool cat, b: meow}`,\n\t\texpression: `3 + .a`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!cool)::3cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `null + \"cat\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `\"cat\" + null`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Number addition - float\",\n\t\tsubdescription: \"If the lhs or rhs are floats then the expression will be calculated with floats.\",\n\t\tdocument: `{a: 3, b: 4.9}`,\n\t\texpression: `.a = .a + .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: 7.9, b: 4.9}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Number addition - int\",\n\t\tsubdescription: \"If both the lhs and rhs are ints then the expression will be calculated with ints.\",\n\t\tdocument: `{a: 3, b: 4}`,\n\t\texpression: `.a = .a + .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: 7, b: 4}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Increment numbers\",\n\t\tdocument: `{a: 3, b: 5}`,\n\t\texpression: `.[] += 1`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: 4, b: 6}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Date addition\",\n\t\tsubdescription: \"You can add durations to dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.\",\n\t\tdocument: `a: 2021-01-01T00:00:00Z`,\n\t\texpression: `.a += \"3h10m\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: 2021-01-01T03:10:00Z\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Date addition -date only\",\n\t\tskipDoc: true,\n\t\tdocument: `a: 2021-01-01`,\n\t\texpression: `.a += \"24h\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: 2021-01-02T00:00:00Z\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Date addition - custom format\",\n\t\tsubdescription: \"You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.\",\n\t\tdocument: `a: Saturday, 15-Dec-01 at 2:59AM GMT`,\n\t\texpression: `with_dtf(\"Monday, 02-Jan-06 at 3:04PM MST\", .a += \"3h1m\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: Saturday, 15-Dec-01 at 6:00AM GMT\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Date addition - custom format\",\n\t\tsubdescription: \"You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.\",\n\t\tdocument: `a: !cat Saturday, 15-Dec-01 at 2:59AM GMT`,\n\t\texpression: `with_dtf(\"Monday, 02-Jan-06 at 3:04PM MST\", .a += \"3h1m\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !cat Saturday, 15-Dec-01 at 6:00AM GMT\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"empty add shouldn't add\",\n\t\tdocument: `[]`,\n\t\texpression: `.[] | (.a + \"cat\")`,\n\t\texpected: []string{},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"empty add shouldn't add\",\n\t\tdocument: `[]`,\n\t\texpression: `.[] | (.a + \"cat\" + .b)`,\n\t\texpected: []string{},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Add to empty\",\n\t\tsubdescription: \"should behave like null\",\n\t\texpression: `.nada + \"cat\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Add to null\",\n\t\tsubdescription: \"Adding to null simply returns the rhs\",\n\t\texpression: `null + \"cat\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Add maps to shallow merge\",\n\t\tsubdescription: \"Adding objects together shallow merges them. Use `*` to deeply merge.\",\n\t\tdocument: \"a: {thing: {name: Astuff, value: x}, a1: cool}\\nb: {thing: {name: Bstuff, legs: 3}, b1: neat}\",\n\t\texpression: `.a += .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: {thing: {name: Bstuff, legs: 3}, a1: cool, b1: neat}\\nb: {thing: {name: Bstuff, legs: 3}, b1: neat}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Custom types: that are really strings\",\n\t\tsubdescription: \"When custom tags are encountered, yq will try to decode the underlying type.\",\n\t\tdocument: \"a: !horse cat\\nb: !goat _meow\",\n\t\texpression: `.a += .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !horse cat_meow\\nb: !goat _meow\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Custom types: that are really numbers\",\n\t\tsubdescription: \"When custom tags are encountered, yq will try to decode the underlying type.\",\n\t\tdocument: \"a: !horse 1.2\\nb: !goat 2.3\",\n\t\texpression: `.a += .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !horse 3.5\\nb: !goat 2.3\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: !horse 2\\nb: !goat 2.3\",\n\t\texpression: `.a += .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !horse 4.3\\nb: !goat 2.3\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: 2\\nb: !goat 2.3\",\n\t\texpression: `.a += .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: 4.3\\nb: !goat 2.3\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Custom types: that are really ints\",\n\t\tdocument: \"a: !horse 2\\nb: !goat 3\",\n\t\texpression: `.a += .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !horse 5\\nb: !goat 3\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Custom types: that are really arrays\",\n\t\tskipDoc: true,\n\t\tsubdescription: \"when custom tags are encountered, yq will try to decode the underlying type.\",\n\t\tdocument: \"a: !horse [a]\\nb: !goat [b]\",\n\t\texpression: `.a += .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !horse [a, b]\\nb: !goat [b]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Keep anchors\",\n\t\tdocument: \"a: &horse [1]\",\n\t\texpression: `.a += 2`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: &horse [1, 2]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Add sequence to map\",\n\t\tdocument: \"a: {x: cool}\",\n\t\texpression: `.a += [2]`,\n\t\texpectedError: \"!!seq () cannot be added to a !!map (a)\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Add sequence to scalar\",\n\t\tdocument: \"a: cool\",\n\t\texpression: `.a += [2]`,\n\t\texpectedError: \"!!seq () cannot be added to a !!str (a)\",\n\t},\n}\n\nfunc TestAddOperatorScenarios(t *testing.T) {\n\tfor _, tt := range addOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"add\", addOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_alternative.go", "content": "package yqlib\n\nfunc alternativeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"alternative\")\n\tprefs := crossFunctionPreferences{\n\t\tCalcWhenEmpty: true,\n\t\tCalculation: alternativeFunc,\n\t\tLhsResultValue: func(lhs *CandidateNode) (*CandidateNode, error) {\n\t\t\tif lhs == nil {\n\t\t\t\treturn nil, nil\n\t\t\t}\n\t\t\ttruthy := isTruthyNode(lhs)\n\t\t\tif truthy {\n\t\t\t\treturn lhs, nil\n\t\t\t}\n\t\t\treturn nil, nil\n\t\t},\n\t}\n\treturn crossFunctionWithPrefs(d, context, expressionNode, prefs)\n}\n\nfunc alternativeFunc(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {\n\tif lhs == nil {\n\t\treturn rhs, nil\n\t}\n\tif rhs == nil {\n\t\treturn lhs, nil\n\t}\n\n\tisTrue := isTruthyNode(lhs)\n\tif isTrue {\n\t\treturn lhs, nil\n\t}\n\treturn rhs, nil\n}\n" }, { "path": "pkg/yqlib/operator_alternative_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar alternativeOperatorScenarios = []expressionScenario{\n\t{\n\t\t// to match jq - we do not use a readonly clone context on the LHS.\n\t\tskipDoc: true,\n\t\texpression: `.b // .c`,\n\t\tdocument: `a: bridge`,\n\t\texpected: []string{\n\t\t\t\"D0, P[c], (!!null)::null\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `(.b // \"hello\") as $x | .`,\n\t\tdocument: `a: bridge`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: bridge\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `.a // .b`,\n\t\tdocument: `a: 2`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!int)::2\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"LHS is defined\",\n\t\texpression: `.a // \"hello\"`,\n\t\tdocument: `{a: bridge}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!str)::bridge\\n\",\n\t\t},\n\t},\n\t{\n\t\texpression: `select(tag == \"seq\") // \"cat\"`,\n\t\tskipDoc: true,\n\t\tdocument: `a: frog`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"LHS is not defined\",\n\t\texpression: `.a // \"hello\"`,\n\t\tdocument: `{}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::hello\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"LHS is null\",\n\t\texpression: `.a // \"hello\"`,\n\t\tdocument: `{a: ~}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::hello\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"LHS is false\",\n\t\texpression: `.a // \"hello\"`,\n\t\tdocument: `{a: false}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::hello\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"RHS is an expression\",\n\t\texpression: `.a // .b`,\n\t\tdocument: `{a: false, b: cat}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[b], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `false // true`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Update or create - entity exists\",\n\t\tsubdescription: \"This initialises `a` if it's not present\",\n\t\texpression: \"(.a // (.a = 0)) += 1\",\n\t\tdocument: `a: 1`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: 2\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Update or create - entity does not exist\",\n\t\tsubdescription: \"This initialises `a` if it's not present\",\n\t\texpression: \"(.a // (.a = 0)) += 1\",\n\t\tdocument: `b: camel`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::b: camel\\na: 1\\n\",\n\t\t},\n\t},\n}\n\nfunc TestAlternativeOperatorScenarios(t *testing.T) {\n\tfor _, tt := range alternativeOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"alternative-default-value\", alternativeOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_anchors_aliases.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n)\n\nvar showMergeAnchorToSpecWarning = true\n\nfunc assignAliasOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\n\tlog.Debugf(\"AssignAlias operator!\")\n\n\taliasName := \"\"\n\tif !expressionNode.Operation.UpdateAssign {\n\t\trhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\tif rhs.MatchingNodes.Front() != nil {\n\t\t\taliasName = rhs.MatchingNodes.Front().Value.(*CandidateNode).Value\n\t\t}\n\t}\n\n\tlhs, err := d.GetMatchingNodes(context, expressionNode.LHS)\n\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\n\tfor el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tlog.Debugf(\"Setting aliasName : %v\", candidate.GetKey())\n\n\t\tif expressionNode.Operation.UpdateAssign {\n\t\t\trhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS)\n\t\t\tif err != nil {\n\t\t\t\treturn Context{}, err\n\t\t\t}\n\t\t\tif rhs.MatchingNodes.Front() != nil {\n\t\t\t\taliasName = rhs.MatchingNodes.Front().Value.(*CandidateNode).Value\n\t\t\t}\n\t\t}\n\n\t\tif aliasName != \"\" {\n\t\t\tcandidate.Kind = AliasNode\n\t\t\tcandidate.Value = aliasName\n\t\t}\n\t}\n\treturn context, nil\n}\n\nfunc getAliasOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"GetAlias operator!\")\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tresult := candidate.CreateReplacement(ScalarNode, \"!!str\", candidate.Value)\n\t\tresults.PushBack(result)\n\t}\n\treturn context.ChildContext(results), nil\n}\n\nfunc assignAnchorOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\n\tlog.Debugf(\"AssignAnchor operator!\")\n\n\tanchorName := \"\"\n\tif !expressionNode.Operation.UpdateAssign {\n\t\trhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t\tif rhs.MatchingNodes.Front() != nil {\n\t\t\tanchorName = rhs.MatchingNodes.Front().Value.(*CandidateNode).Value\n\t\t}\n\t}\n\n\tlhs, err := d.GetMatchingNodes(context, expressionNode.LHS)\n\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\n\tfor el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tlog.Debugf(\"Setting anchorName of : %v\", candidate.GetKey())\n\n\t\tif expressionNode.Operation.UpdateAssign {\n\t\t\trhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS)\n\t\t\tif err != nil {\n\t\t\t\treturn Context{}, err\n\t\t\t}\n\n\t\t\tif rhs.MatchingNodes.Front() != nil {\n\t\t\t\tanchorName = rhs.MatchingNodes.Front().Value.(*CandidateNode).Value\n\t\t\t}\n\t\t}\n\n\t\tcandidate.Anchor = anchorName\n\t}\n\treturn context, nil\n}\n\nfunc getAnchorOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"GetAnchor operator!\")\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tanchor := candidate.Anchor\n\t\tresult := candidate.CreateReplacement(ScalarNode, \"!!str\", anchor)\n\t\tresults.PushBack(result)\n\t}\n\treturn context.ChildContext(results), nil\n}\n\nfunc explodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"ExplodeOperation\")\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\trhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.RHS)\n\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\tfor childEl := rhs.MatchingNodes.Front(); childEl != nil; childEl = childEl.Next() {\n\t\t\terr = explodeNode(childEl.Value.(*CandidateNode), context)\n\t\t\tif err != nil {\n\t\t\t\treturn Context{}, err\n\t\t\t}\n\t\t}\n\n\t}\n\n\treturn context, nil\n}\n\nfunc fixedReconstructAliasedMap(node *CandidateNode) error {\n\tvar newContent = []*CandidateNode{}\n\n\tfor index := 0; index < len(node.Content); index = index + 2 {\n\t\tkeyNode := node.Content[index]\n\t\tvalueNode := node.Content[index+1]\n\t\tif keyNode.Tag != \"!!merge\" {\n\t\t\t// always add in plain nodes\n\t\t\t// explode both the key and value nodes\n\t\t\tif err := explodeNode(keyNode, Context{}); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif err := explodeNode(valueNode, Context{}); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tnewContent = append(newContent, keyNode, valueNode)\n\t\t} else {\n\t\t\tsequence := valueNode\n\t\t\tif sequence.Kind == AliasNode {\n\t\t\t\tsequence = sequence.Alias\n\t\t\t}\n\t\t\tif sequence.Kind != SequenceNode {\n\t\t\t\tsequence = &CandidateNode{Content: []*CandidateNode{sequence}}\n\t\t\t}\n\t\t\tfor index := 0; index < len(sequence.Content); index = index + 1 {\n\t\t\t\t// for merge anchors, we only set them if the key is not already in node or the newContent\n\t\t\t\tmergeNodeSeq := sequence.Content[index]\n\t\t\t\tif mergeNodeSeq.Kind == AliasNode {\n\t\t\t\t\tmergeNodeSeq = mergeNodeSeq.Alias\n\t\t\t\t}\n\t\t\t\tif mergeNodeSeq.Kind != MappingNode {\n\t\t\t\t\treturn fmt.Errorf(\"can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got sequence containing %v\", mergeNodeSeq.Tag)\n\t\t\t\t}\n\t\t\t\titemsToAdd := mergeNodeSeq.FilterMapContentByKey(func(keyNode *CandidateNode) bool {\n\t\t\t\t\treturn getContentValueByKey(node.Content, keyNode.Value) == nil &&\n\t\t\t\t\t\tgetContentValueByKey(newContent, keyNode.Value) == nil\n\t\t\t\t})\n\n\t\t\t\tfor _, item := range itemsToAdd {\n\t\t\t\t\t// copy to ensure exploding doesn't modify the original node\n\t\t\t\t\titemCopy := item.Copy()\n\t\t\t\t\tif err := explodeNode(itemCopy, Context{}); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t\tnewContent = append(newContent, itemCopy)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tnode.Content = newContent\n\treturn nil\n}\n\nfunc reconstructAliasedMap(node *CandidateNode, context Context) error {\n\tvar newContent = list.New()\n\t// can I short cut here by prechecking if there's an anchor in the map?\n\t// no it needs to recurse in overrideEntry.\n\n\tfor index := 0; index < len(node.Content); index = index + 2 {\n\t\tkeyNode := node.Content[index]\n\t\tvalueNode := node.Content[index+1]\n\t\tlog.Debugf(\"traversing %v\", keyNode.Value)\n\t\tif keyNode.Value != \"<<\" {\n\t\t\terr := overrideEntry(node, keyNode, valueNode, index, context.ChildContext(newContent))\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t} else {\n\t\t\tif valueNode.Kind == SequenceNode {\n\t\t\t\tlog.Debugf(\"an alias merge list!\")\n\t\t\t\tfor index := len(valueNode.Content) - 1; index >= 0; index = index - 1 {\n\t\t\t\t\taliasNode := valueNode.Content[index]\n\t\t\t\t\terr := applyAlias(node, aliasNode.Alias, index, context.ChildContext(newContent))\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tlog.Debugf(\"an alias merge!\")\n\t\t\t\terr := applyAlias(node, valueNode.Alias, index, context.ChildContext(newContent))\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tnode.Content = make([]*CandidateNode, 0)\n\tfor newEl := newContent.Front(); newEl != nil; newEl = newEl.Next() {\n\t\tnode.AddChild(newEl.Value.(*CandidateNode))\n\t}\n\treturn nil\n}\n\nfunc explodeNode(node *CandidateNode, context Context) error {\n\tlog.Debugf(\"explodeNode - %v\", NodeToString(node))\n\tnode.Anchor = \"\"\n\tswitch node.Kind {\n\tcase SequenceNode:\n\t\tfor index, contentNode := range node.Content {\n\t\t\tlog.Debugf(\"explodeNode - index %v\", index)\n\t\t\terrorInContent := explodeNode(contentNode, context)\n\t\t\tif errorInContent != nil {\n\t\t\t\treturn errorInContent\n\t\t\t}\n\t\t}\n\t\treturn nil\n\tcase AliasNode:\n\t\tlog.Debugf(\"explodeNode - an alias to %v\", NodeToString(node.Alias))\n\t\tif node.Alias != nil {\n\t\t\tnode.Kind = node.Alias.Kind\n\t\t\tnode.Style = node.Alias.Style\n\t\t\tnode.Tag = node.Alias.Tag\n\t\t\tnode.AddChildren(node.Alias.Content)\n\t\t\tnode.Value = node.Alias.Value\n\t\t\tnode.Alias = nil\n\t\t}\n\t\tlog.Debug(\"now I'm %v\", NodeToString(node))\n\t\treturn nil\n\tcase MappingNode:\n\t\t// //check the map has an alias in it\n\t\thasAlias := false\n\t\tfor index := 0; index < len(node.Content); index = index + 2 {\n\t\t\tkeyNode := node.Content[index]\n\t\t\tif keyNode.Value == \"<<\" {\n\t\t\t\thasAlias = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\tif hasAlias {\n\t\t\tif ConfiguredYamlPreferences.FixMergeAnchorToSpec {\n\t\t\t\treturn fixedReconstructAliasedMap(node)\n\t\t\t}\n\t\t\tif showMergeAnchorToSpecWarning {\n\t\t\t\tlog.Warning(\"--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025.\")\n\t\t\t\tshowMergeAnchorToSpecWarning = false\n\t\t\t}\n\t\t\t// this is a slow op, which is why we want to check before running it.\n\t\t\treturn reconstructAliasedMap(node, context)\n\t\t}\n\t\t// this map has no aliases, but it's kids might\n\t\tfor index := 0; index < len(node.Content); index = index + 2 {\n\t\t\tkeyNode := node.Content[index]\n\t\t\tvalueNode := node.Content[index+1]\n\t\t\terr := explodeNode(keyNode, context)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\terr = explodeNode(valueNode, context)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\treturn nil\n\tdefault:\n\t\treturn nil\n\t}\n}\n\nfunc applyAlias(node *CandidateNode, alias *CandidateNode, aliasIndex int, newContent Context) error {\n\tlog.Debug(\"alias is nil ?\")\n\tif alias == nil {\n\t\treturn nil\n\t}\n\tlog.Debug(\"alias: %v\", NodeToString(alias))\n\tif alias.Kind != MappingNode {\n\t\treturn fmt.Errorf(\"can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got sequence containing %v\", alias.Tag)\n\t}\n\tfor index := 0; index < len(alias.Content); index = index + 2 {\n\t\tkeyNode := alias.Content[index]\n\t\tlog.Debugf(\"applying alias key %v\", keyNode.Value)\n\t\tvalueNode := alias.Content[index+1]\n\t\terr := overrideEntry(node, keyNode, valueNode, aliasIndex, newContent)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc overrideEntry(node *CandidateNode, key *CandidateNode, value *CandidateNode, startIndex int, newContent Context) error {\n\n\terr := explodeNode(value, newContent)\n\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfor newEl := newContent.MatchingNodes.Front(); newEl != nil; newEl = newEl.Next() {\n\t\tvalueEl := newEl.Next() // move forward twice\n\t\tkeyNode := newEl.Value.(*CandidateNode)\n\t\tlog.Debugf(\"checking new content %v:%v\", keyNode.Value, valueEl.Value.(*CandidateNode).Value)\n\t\tif keyNode.Value == key.Value && keyNode.Alias == nil && key.Alias == nil {\n\t\t\tlog.Debugf(\"overridign new content\")\n\t\t\tvalueEl.Value = value\n\t\t\treturn nil\n\t\t}\n\t\tnewEl = valueEl // move forward twice\n\t}\n\n\tfor index := startIndex + 2; index < len(node.Content); index = index + 2 {\n\t\tkeyNode := node.Content[index]\n\n\t\tif keyNode.Value == key.Value && keyNode.Alias == nil {\n\t\t\tlog.Debugf(\"content will be overridden at index %v\", index)\n\t\t\treturn nil\n\t\t}\n\t}\n\n\terr = explodeNode(key, newContent)\n\tif err != nil {\n\t\treturn err\n\t}\n\tlog.Debugf(\"adding %v:%v\", key.Value, value.Value)\n\tnewContent.MatchingNodes.PushBack(key)\n\tnewContent.MatchingNodes.PushBack(value)\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/operator_anchors_aliases_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar specDocument = `- &CENTRE { x: 1, y: 2 }\n- &LEFT { x: 0, y: 2 }\n- &BIG { r: 10 }\n- &SMALL { r: 1 }\n`\n\nvar expectedSpecResult = \"D0, P[4], (!!map)::x: 1\\ny: 2\\nr: 10\\n\"\n\nvar simpleArrayRef = `item_value: &item_value\n value: true\n\nthingOne:\n name: item_1\n <<: *item_value\n\nthingTwo:\n name: item_2\n <<: *item_value\n`\n\nvar expectedUpdatedArrayRef = `D0, P[], (!!map)::item_value: &item_value\n value: true\nthingOne:\n name: item_1\n value: false\nthingTwo:\n name: item_2\n !!merge <<: *item_value\n`\n\nvar explodeMergeAnchorsFixedExpected = `D0, P[], (!!map)::foo:\n a: foo_a\n thing: foo_thing\n c: foo_c\nbar:\n b: bar_b\n thing: bar_thing\n c: bar_c\nfoobarList:\n b: foobarList_b\n a: foo_a\n thing: foo_thing\n c: foobarList_c\nfoobar:\n c: foobar_c\n a: foo_a\n thing: foobar_thing\n`\n\nvar explodeMergeAnchorsExpected = `D0, P[], (!!map)::foo:\n a: foo_a\n thing: foo_thing\n c: foo_c\nbar:\n b: bar_b\n thing: bar_thing\n c: bar_c\nfoobarList:\n b: bar_b\n thing: foo_thing\n c: foobarList_c\n a: foo_a\nfoobar:\n c: foo_c\n a: foo_a\n thing: foobar_thing\n`\n\nvar explodeWhenKeysExistDocument = `objects:\n - &circle\n name: circle\n shape: round\n - name: ellipse\n !!merge <<: *circle\n - !!merge <<: *circle\n name: egg\n`\n\nvar explodeWhenKeysExistLegacy = `D0, P[], (!!map)::objects:\n - name: circle\n shape: round\n - name: circle\n shape: round\n - shape: round\n name: egg\n`\n\nvar explodeWhenKeysExistExpected = `D0, P[], (!!map)::objects:\n - name: circle\n shape: round\n - name: ellipse\n shape: round\n - shape: round\n name: egg\n`\n\nvar fixedAnchorOperatorScenarios = []expressionScenario{\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"merge anchor after existing keys\",\n\t\tsubdescription: \"Does not override existing keys - note the name field in the second element is still ellipse.\",\n\t\tdocument: explodeWhenKeysExistDocument,\n\t\texpression: \"explode(.)\",\n\t\texpected: []string{explodeWhenKeysExistExpected},\n\t},\n\t{\n\t\tdescription: \"FIXED: Explode with merge anchors\",\n\t\tsubdescription: \"Observe that foobarList.b property is still foobarList_b.\",\n\t\tdocument: mergeDocSample,\n\t\texpression: `explode(.)`,\n\t\texpected: []string{explodeMergeAnchorsFixedExpected},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: mergeDocSample,\n\t\texpression: `.foo* | explode(.) | (. style=\"flow\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\\n\",\n\t\t\t\"D0, P[foobarList], (!!map)::{b: foobarList_b, a: foo_a, thing: foo_thing, c: foobarList_c}\\n\",\n\t\t\t\"D0, P[foobar], (!!map)::{c: foobar_c, a: foo_a, thing: foobar_thing}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: mergeDocSample,\n\t\texpression: `.foo* | explode(explode(.)) | (. style=\"flow\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\\n\",\n\t\t\t\"D0, P[foobarList], (!!map)::{b: foobarList_b, a: foo_a, thing: foo_thing, c: foobarList_c}\\n\",\n\t\t\t\"D0, P[foobar], (!!map)::{c: foobar_c, a: foo_a, thing: foobar_thing}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"FIXED: Merge multiple maps\",\n\t\tsubdescription: \"Taken from https://yaml.org/type/merge.html. Same values as legacy, but with the correct key order.\",\n\t\tdocument: specDocument + \"- << : [ *CENTRE, *BIG ]\\n\",\n\t\texpression: \".[4] | explode(.)\",\n\t\texpected: []string{\"D0, P[4], (!!map)::x: 1\\ny: 2\\nr: 10\\n\"},\n\t},\n\t{\n\t\tdescription: \"FIXED: Override\",\n\t\tsubdescription: \"Taken from https://yaml.org/type/merge.html. Same values as legacy, but with the correct key order.\",\n\t\tdocument: specDocument + \"- << : [ *BIG, *LEFT, *SMALL ]\\n x: 1\\n\",\n\t\texpression: \".[4] | explode(.)\",\n\t\texpected: []string{\"D0, P[4], (!!map)::r: 10\\ny: 2\\nx: 1\\n\"},\n\t},\n\t{\n\t\tdescription: \"Exploding inline merge anchor\",\n\t\t// subdescription: \"`<<` map must be exploded, otherwise `c: *b` will become invalid\",\n\t\tdocument: `{a: {b: &b 42}, <<: {c: *b}}`,\n\t\texpression: `explode(.) | sort_keys(.)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {b: 42}, c: 42}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Exploding inline merge anchor in sequence\",\n\t\tsubdescription: \"`<<` map must be exploded, otherwise `c: *b` will become invalid\",\n\t\tdocument: `{a: {b: &b 42}, <<: [{c: *b}]}`,\n\t\texpression: `explode(.) | sort_keys(.)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {b: 42}, c: 42}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Exploding merge anchor should not explode neighbours\",\n\t\tsubdescription: \"b must not be exploded, as `r: *a` will become invalid\",\n\t\tdocument: `{b: &b {a: &a 42}, r: *a, c: {<<: *b}}`,\n\t\texpression: `explode(.c)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{b: &b {a: &a 42}, r: *a, c: {a: 42}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Exploding sequence merge anchor should not explode neighbours\",\n\t\tsubdescription: \"b must not be exploded, as `r: *a` will become invalid\",\n\t\tdocument: `{b: &b {a: &a 42}, r: *a, c: {<<: [*b]}}`,\n\t\texpression: `explode(.c)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{b: &b {a: &a 42}, r: *a, c: {a: 42}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Merge anchor with inline map\",\n\t\tdocument: `{<<: {a: 42}}`,\n\t\texpression: `explode(.)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: 42}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Merge anchor with sequence with inline map\",\n\t\tdocument: `{<<: [{a: 42}]}`,\n\t\texpression: `explode(.)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: 42}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Merge anchor with aliased sequence with inline map\",\n\t\tdocument: `{s: &s [{a: 42}], m: {<<: *s}}`,\n\t\texpression: `.m | explode(.)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[m], (!!map)::{a: 42}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"deleting after explode\",\n\t\tdocument: \"x: 37\\na: &a\\n b: 42\\n<<: *a\",\n\t\texpression: `explode(.) | del(.x)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a:\\n b: 42\\nb: 42\\n\",\n\t\t},\n\t},\n}\n\nvar badAnchorOperatorScenarios = []expressionScenario{\n\t{\n\t\tskipDoc: true, // incorrect overrides\n\t\tdescription: \"LEGACY: merge anchor after existing keys\",\n\t\tdocument: explodeWhenKeysExistDocument,\n\t\texpression: \"explode(.)\",\n\t\texpected: []string{explodeWhenKeysExistLegacy},\n\t},\n\t{\n\t\tdescription: \"LEGACY: Explode with merge anchors\", // incorrect overrides\n\t\tsubdescription: \"Caution: this is for when --yaml-fix-merge-anchor-to-spec=false; it's not to YAML spec because the merge anchors incorrectly override the object values (foobarList.b is set to bar_b when it should still be foobarList_b). Flag will default to true in late 2025\",\n\t\tdocument: mergeDocSample,\n\t\texpression: `explode(.)`,\n\t\texpected: []string{explodeMergeAnchorsExpected},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: mergeDocSample, // incorrect overrides\n\t\texpression: `.foo* | explode(.) | (. style=\"flow\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\\n\",\n\t\t\t\"D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\\n\",\n\t\t\t\"D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: mergeDocSample,\n\t\texpression: `.foo* | explode(explode(.)) | (. style=\"flow\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\\n\",\n\t\t\t\"D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\\n\",\n\t\t\t\"D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"LEGACY: Merge multiple maps\",\n\t\tsubdescription: \"see https://yaml.org/type/merge.html. This has the correct data, but the wrong key order; set --yaml-fix-merge-anchor-to-spec=true to fix the key order.\",\n\t\tdocument: specDocument + \"- << : [ *CENTRE, *BIG ]\\n\",\n\t\texpression: \".[4] | explode(.)\",\n\t\texpected: []string{\"D0, P[4], (!!map)::r: 10\\nx: 1\\ny: 2\\n\"},\n\t},\n\t{\n\t\tdescription: \"LEGACY: Override\",\n\t\tsubdescription: \"see https://yaml.org/type/merge.html. This has the correct data, but the wrong key order; set --yaml-fix-merge-anchor-to-spec=true to fix the key order.\",\n\n\t\tdocument: specDocument + \"- << : [ *BIG, *LEFT, *SMALL ]\\n x: 1\\n\",\n\t\texpression: \".[4] | explode(.)\",\n\t\texpected: []string{\"D0, P[4], (!!map)::r: 10\\nx: 1\\ny: 2\\n\"},\n\t},\n}\n\nvar anchorOperatorScenarios = []expressionScenario{\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"merge anchor to alias alias\",\n\t\tdocument: \"b: &b 10\\na: &a { k: *b }\\nc:\\n <<: [*a]\",\n\t\texpression: \"explode(.)\",\n\t\texpected: []string{\"D0, P[], (!!map)::b: 10\\na: {k: 10}\\nc:\\n k: 10\\n\"},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"merge anchor not map\",\n\t\tdocument: \"a: &a\\n - 0\\nc:\\n <<: [*a]\\n\",\n\t\texpectedError: \"can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got sequence containing !!seq\",\n\t\texpression: \"explode(.)\",\n\t},\n\t{\n\t\tdescription: \"Merge one map\",\n\t\tsubdescription: \"see https://yaml.org/type/merge.html\",\n\t\tdocument: specDocument + \"- << : *CENTRE\\n r: 10\\n\",\n\t\texpression: \".[4] | explode(.)\",\n\t\texpected: []string{expectedSpecResult},\n\t},\n\n\t{\n\t\tdescription: \"Get anchor\",\n\t\tdocument: `a: &billyBob cat`,\n\t\texpression: `.a | anchor`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!str)::billyBob\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Set anchor\",\n\t\tdocument: `a: cat`,\n\t\texpression: `.a anchor = \"foobar\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: &foobar cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Set anchor relatively using assign-update\",\n\t\tdocument: `a: {b: cat}`,\n\t\texpression: `.a anchor |= .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: &cat {b: cat}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a: {c: cat}`,\n\t\texpression: `.a anchor |= .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: {c: cat}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a: {c: cat}`,\n\t\texpression: `.a anchor = .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: {c: cat}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Get alias\",\n\t\tdocument: `{b: &billyBob meow, a: *billyBob}`,\n\t\texpression: `.a | alias`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!str)::billyBob\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Set alias\",\n\t\tdocument: `{b: &meow purr, a: cat}`,\n\t\texpression: `.a alias = \"meow\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{b: &meow purr, a: *meow}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Set alias to blank does nothing\",\n\t\tdocument: `{b: &meow purr, a: cat}`,\n\t\texpression: `.a alias = \"\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{b: &meow purr, a: cat}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{b: &meow purr, a: cat}`,\n\t\texpression: `.a alias = .c`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{b: &meow purr, a: cat}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{b: &meow purr, a: cat}`,\n\t\texpression: `.a alias |= .c`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{b: &meow purr, a: cat}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Set alias relatively using assign-update\",\n\t\tdocument: `{b: &meow purr, a: {f: meow}}`,\n\t\texpression: `.a alias |= .f`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{b: &meow purr, a: *meow}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Dont explode alias and anchor - check alias parent\",\n\t\tskipDoc: true,\n\t\tdocument: `{a: &a [1], b: *a}`,\n\t\texpression: `.b[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a 0], (!!int)::1\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Explode alias and anchor - check alias parent\",\n\t\tskipDoc: true,\n\t\tdocument: `{a: &a cat, b: *a}`,\n\t\texpression: `explode(.) | .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[b], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Explode splat\",\n\t\tskipDoc: true,\n\t\tdocument: `{a: &a cat, b: *a}`,\n\t\texpression: `explode(.)[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!str)::cat\\n\",\n\t\t\t\"D0, P[b], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Explode alias and anchor - check original parent\",\n\t\tskipDoc: true,\n\t\tdocument: `{a: &a cat, b: *a}`,\n\t\texpression: `explode(.) | .a`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Explode alias and anchor\",\n\t\tdocument: `{f : {a: &a cat, b: *a}}`,\n\t\texpression: `explode(.f)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{f: {a: cat, b: cat}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Explode with no aliases or anchors\",\n\t\tdocument: `a: mike`,\n\t\texpression: `explode(.a)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: mike\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Explode with alias keys\",\n\t\tsubdescription: \"No space between alias\",\n\t\tskipDoc: true,\n\t\tdocument: `{f : {a: &a cat, *a: b}}`,\n\t\texpression: `explode(.f)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{f: {a: cat, cat: b}}\\n\",\n\t\t},\n\t\tskipForGoccy: true, // can't handle no space between alias\n\t},\n\t{\n\t\tdescription: \"Explode with alias keys\",\n\t\tdocument: `{f : {a: &a cat, *a : b}}`,\n\t\texpression: `explode(.f)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{f: {a: cat, cat: b}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{f : {a: &a cat, b: &b {foo: *a}, *a: *b}}`,\n\t\texpression: `explode(.f)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{f: {a: cat, b: {foo: cat}, cat: {foo: cat}}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Dereference and update a field\",\n\t\tsubdescription: \"Use explode with multiply to dereference an object\",\n\t\tdocument: simpleArrayRef,\n\t\texpression: `.thingOne |= (explode(.) | sort_keys(.)) * {\"value\": false}`,\n\t\texpected: []string{expectedUpdatedArrayRef},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Duplicate keys\",\n\t\tsubdescription: \"outside merge anchor\",\n\t\tdocument: `{a: 1, a: 2}`,\n\t\texpression: `explode(.)`,\n\t\texpected: []string{\n\t\t\t// {a: 2} would also be fine\n\t\t\t\"D0, P[], (!!map)::{a: 1, a: 2}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"!!str << should not be treated as merge anchor\",\n\t\tdocument: `{!!str <<: {a: 37}}`,\n\t\texpression: `explode(.).a`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!null)::null\\n\",\n\t\t},\n\t},\n}\n\nfunc TestAnchorAliasOperatorScenarios(t *testing.T) {\n\tfor _, tt := range append(anchorOperatorScenarios, badAnchorOperatorScenarios...) {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"anchor-and-alias-operators\", append(anchorOperatorScenarios, badAnchorOperatorScenarios...))\n}\n\nfunc TestAnchorAliasOperatorAlignedToSpecScenarios(t *testing.T) {\n\tConfiguredYamlPreferences.FixMergeAnchorToSpec = true\n\tfor _, tt := range append(fixedAnchorOperatorScenarios, anchorOperatorScenarios...) {\n\t\ttestScenario(t, &tt)\n\t}\n\n\tfor i, tt := range fixedAnchorOperatorScenarios {\n\t\tfixedAnchorOperatorScenarios[i].subdescription = \"Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour (flag will default to true in late 2025).\\n\" + tt.subdescription\n\t}\n\tappendOperatorDocumentScenario(t, \"anchor-and-alias-operators\", fixedAnchorOperatorScenarios)\n\tConfiguredYamlPreferences.FixMergeAnchorToSpec = false\n}\n" }, { "path": "pkg/yqlib/operator_array_to_map_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar arrayToMapScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Simple example\",\n\t\tdocument: `cool: [null, null, hello]`,\n\t\texpression: `.cool |= array_to_map`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::cool:\\n 2: hello\\n\",\n\t\t},\n\t},\n}\n\nfunc TestArrayToMapScenarios(t *testing.T) {\n\tfor _, tt := range arrayToMapScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"array-to-map\", arrayToMapScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_assign.go", "content": "package yqlib\n\ntype assignPreferences struct {\n\tDontOverWriteAnchor bool\n\tOnlyWriteNull bool\n\tClobberCustomTags bool\n}\n\nfunc assignUpdateFunc(prefs assignPreferences) crossFunctionCalculation {\n\treturn func(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {\n\t\tif !prefs.OnlyWriteNull || lhs.Tag == \"!!null\" {\n\t\t\tlhs.UpdateFrom(rhs, prefs)\n\t\t}\n\t\treturn lhs, nil\n\t}\n}\n\n// they way *= (multipleAssign) is handled, we set the multiplePrefs\n// on the node, not assignPrefs. Long story.\nfunc getAssignPreferences(preferences interface{}) assignPreferences {\n\tprefs := assignPreferences{}\n\n\tswitch typedPref := preferences.(type) {\n\tcase assignPreferences:\n\t\tprefs = typedPref\n\tcase multiplyPreferences:\n\t\tprefs = typedPref.AssignPrefs\n\t}\n\treturn prefs\n}\n\nfunc assignUpdateOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlhs, err := d.GetMatchingNodes(context, expressionNode.LHS)\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\n\tprefs := getAssignPreferences(expressionNode.Operation.Preferences)\n\n\tlog.Debug(\"assignUpdateOperator prefs: %v\", prefs)\n\n\tif !expressionNode.Operation.UpdateAssign {\n\t\t// this works because we already ran against LHS with an editable context.\n\t\t_, err := crossFunction(d, context.ReadOnlyClone(), expressionNode, assignUpdateFunc(prefs), false)\n\t\treturn context, err\n\t}\n\n\t//traverse backwards through the context -\n\t// like delete, we need to run against the children first.\n\t// (e.g. consider when running with expression '.. |= [.]' - we need\n\t// to wrap the children first\n\tfor el := lhs.MatchingNodes.Back(); el != nil; el = el.Prev() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\trhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.RHS)\n\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t\t// grab the first value\n\t\tfirst := rhs.MatchingNodes.Front()\n\n\t\tif first != nil {\n\t\t\trhsCandidate := first.Value.(*CandidateNode)\n\t\t\tcandidate.UpdateFrom(rhsCandidate, prefs)\n\t\t}\n\t}\n\n\treturn context, nil\n}\n\n// does not update content or values\nfunc assignAttributesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debug(\"getting lhs matching nodes for update\")\n\tlhs, err := d.GetMatchingNodes(context, expressionNode.LHS)\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\tfor el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\trhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS)\n\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t\t// grab the first value\n\t\tfirst := rhs.MatchingNodes.Front()\n\n\t\tif first != nil {\n\t\t\tprefs := assignPreferences{}\n\t\t\tif expressionNode.Operation.Preferences != nil {\n\t\t\t\tprefs = expressionNode.Operation.Preferences.(assignPreferences)\n\t\t\t}\n\t\t\tif !prefs.OnlyWriteNull || candidate.Tag == \"!!null\" {\n\t\t\t\tcandidate.UpdateAttributesFrom(first.Value.(*CandidateNode), prefs)\n\t\t\t}\n\t\t}\n\t}\n\treturn context, nil\n}\n" }, { "path": "pkg/yqlib/operator_assign_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar mergeAnchorAssign = `a: &a\n x: OriginalValue\nb:\n <<: *a`\n\nvar assignOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Create yaml file\",\n\t\texpression: `.a.b = \"cat\" | .x = \"frog\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], ()::a:\\n b: cat\\nx: frog\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Create yaml file\",\n\t\tdocument: \"a: {b: 3}\",\n\t\texpression: `.a |= .`,\n\t\tskipDoc: true,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: {b: 3}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"{}\",\n\t\texpression: `.a |= .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: null\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: mergeAnchorAssign,\n\t\texpression: `.c = .b | .a.x = \"ModifiedValue\" | explode(.)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a:\\n x: ModifiedValue\\nb:\\n x: ModifiedValue\\nc:\\n x: ModifiedValue\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"{}\",\n\t\texpression: `.a = .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: null\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"self reference\",\n\t\tdocument: \"a: cat\",\n\t\texpression: `.a = [.a]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a:\\n - cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"change to number when old value is valid number\",\n\t\tdocument: `a: \"3\"`,\n\t\texpression: `.a = 3`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: 3\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"change to bool when old value is valid bool\",\n\t\tdocument: `a: \"true\"`,\n\t\texpression: `.a = true`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: true\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"update custom tag string, dont clobber style\",\n\t\tdocument: `a: !cat \"meow\"`,\n\t\texpression: `.a = \"woof\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !cat \\\"woof\\\"\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Update node to be the child value\",\n\t\tdocument: `{a: {b: {g: foof}}}`,\n\t\texpression: `.a |= .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {g: foof}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Double elements in an array\",\n\t\tdocument: `[1,2,3]`,\n\t\texpression: `.[] |= . * 2`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[2, 4, 6]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Update node from another file\",\n\t\tsubdescription: \"Note this will also work when the second file is a scalar (string/number)\",\n\t\tdocument: `{a: apples}`,\n\t\tdocument2: \"{b: bob}\",\n\t\texpression: `select(fileIndex==0).a = select(fileIndex==1) | select(fileIndex==0)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {b: bob}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Update node to be the sibling value\",\n\t\tdocument: `{a: {b: child}, b: sibling}`,\n\t\texpression: `.a = .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: sibling, b: sibling}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Updated multiple paths\",\n\t\tdocument: `{a: fieldA, b: fieldB, c: fieldC}`,\n\t\texpression: `(.a, .c) = \"potato\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: potato, b: fieldB, c: potato}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Update string value\",\n\t\tdocument: `{a: {b: apple}}`,\n\t\texpression: `.a.b = \"frog\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {b: frog}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Update string value via |=\",\n\t\tsubdescription: \"Note there is no difference between `=` and `|=` when the RHS is a scalar\",\n\t\tdocument: `{a: {b: apple}}`,\n\t\texpression: `.a.b |= \"frog\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {b: frog}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: {b: apple}}`,\n\t\texpression: `.a.b | (. |= \"frog\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a b], (!!str)::frog\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: {b: apple}}`,\n\t\texpression: `.a.b |= 5`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {b: 5}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: {b: apple}}`,\n\t\texpression: `.a.b |= 3.142`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {b: 3.142}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Update deeply selected results\",\n\t\tsubdescription: \"Note that the LHS is wrapped in brackets! This is to ensure we don't first filter out the yaml and then update the snippet.\",\n\t\tdocument: `{a: {b: apple, c: cactus}}`,\n\t\texpression: `(.a[] | select(. == \"apple\")) = \"frog\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {b: frog, c: cactus}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: {b: apple, c: cactus}}`,\n\t\texpression: `(.a.[] | select(. == \"apple\")) = \"frog\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {b: frog, c: cactus}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Update array values\",\n\t\tdocument: `[candy, apple, sandy]`,\n\t\texpression: `(.[] | select(. == \"*andy\")) = \"bogs\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[bogs, apple, bogs]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Update empty object\",\n\t\tdontFormatInputForDoc: true,\n\t\tdocument: `{}`,\n\t\texpression: `.a.b |= \"bogs\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a:\\n b: bogs\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Update node value that has an anchor\",\n\t\tsubdescription: \"Anchor will remain\",\n\t\tdontFormatInputForDoc: true,\n\t\tdocument: `a: &cool cat`,\n\t\texpression: `.a = \"dog\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: &cool dog\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Update empty object and array\",\n\t\tdontFormatInputForDoc: true,\n\t\tdocument: `{}`,\n\t\texpression: `.a.b.[0] |= \"bogs\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a:\\n b:\\n - bogs\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{}`,\n\t\texpression: `.a.b.[1].c |= \"bogs\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a:\\n b:\\n - null\\n - c: bogs\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Custom types are maintained by default\",\n\t\tdocument: \"a: !cat meow\\nb: !dog woof\",\n\t\texpression: `.a = .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !cat woof\\nb: !dog woof\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Custom types: clobber\",\n\t\tsubdescription: \"Use the `c` option to clobber custom tags\",\n\t\tdocument: \"a: !cat meow\\nb: !dog woof\",\n\t\texpression: `.a =c .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !dog woof\\nb: !dog woof\\n\",\n\t\t},\n\t},\n}\n\nfunc TestAssignOperatorScenarios(t *testing.T) {\n\tfor _, tt := range assignOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"assign-update\", assignOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_booleans.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n\t\"strings\"\n)\n\nfunc isTruthyNode(node *CandidateNode) bool {\n\tif node == nil {\n\t\treturn false\n\t}\n\tif node.Tag == \"!!null\" {\n\t\treturn false\n\t}\n\tif node.Kind == ScalarNode && node.Tag == \"!!bool\" {\n\t\t// yes/y/true/on\n\t\treturn (strings.EqualFold(node.Value, \"y\") ||\n\t\t\tstrings.EqualFold(node.Value, \"yes\") ||\n\t\t\tstrings.EqualFold(node.Value, \"on\") ||\n\t\t\tstrings.EqualFold(node.Value, \"true\"))\n\n\t}\n\treturn true\n}\n\nfunc getOwner(lhs *CandidateNode, rhs *CandidateNode) *CandidateNode {\n\towner := lhs\n\n\tif lhs == nil && rhs == nil {\n\t\towner = &CandidateNode{}\n\t} else if lhs == nil {\n\t\towner = rhs\n\t}\n\treturn owner\n}\n\nfunc returnRhsTruthy(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {\n\towner := getOwner(lhs, rhs)\n\trhsBool := isTruthyNode(rhs)\n\n\treturn createBooleanCandidate(owner, rhsBool), nil\n}\n\nfunc returnLHSWhen(targetBool bool) func(lhs *CandidateNode) (*CandidateNode, error) {\n\treturn func(lhs *CandidateNode) (*CandidateNode, error) {\n\t\tvar err error\n\t\tvar lhsBool bool\n\n\t\tif lhsBool = isTruthyNode(lhs); lhsBool != targetBool {\n\t\t\treturn nil, err\n\t\t}\n\t\towner := &CandidateNode{}\n\t\tif lhs != nil {\n\t\t\towner = lhs\n\t\t}\n\t\treturn createBooleanCandidate(owner, targetBool), nil\n\t}\n}\n\nfunc findBoolean(wantBool bool, d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, sequenceNode *CandidateNode) (bool, error) {\n\tfor _, node := range sequenceNode.Content {\n\n\t\tif expressionNode != nil {\n\t\t\t//need to evaluate the expression against the node\n\t\t\trhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(node), expressionNode)\n\t\t\tif err != nil {\n\t\t\t\treturn false, err\n\t\t\t}\n\t\t\tif rhs.MatchingNodes.Len() > 0 {\n\t\t\t\tnode = rhs.MatchingNodes.Front().Value.(*CandidateNode)\n\t\t\t} else {\n\t\t\t\t// no results found, ignore this entry\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\tif isTruthyNode(node) == wantBool {\n\t\t\treturn true, nil\n\t\t}\n\t}\n\treturn false, nil\n}\n\nfunc allOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tif candidate.Kind != SequenceNode {\n\t\t\treturn Context{}, fmt.Errorf(\"all only supports arrays, was %v\", candidate.Tag)\n\t\t}\n\t\tbooleanResult, err := findBoolean(false, d, context, expressionNode.RHS, candidate)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\tresult := createBooleanCandidate(candidate, !booleanResult)\n\t\tresults.PushBack(result)\n\t}\n\treturn context.ChildContext(results), nil\n}\n\nfunc anyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tif candidate.Kind != SequenceNode {\n\t\t\treturn Context{}, fmt.Errorf(\"any only supports arrays, was %v\", candidate.Tag)\n\t\t}\n\t\tbooleanResult, err := findBoolean(true, d, context, expressionNode.RHS, candidate)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\tresult := createBooleanCandidate(candidate, booleanResult)\n\t\tresults.PushBack(result)\n\t}\n\treturn context.ChildContext(results), nil\n}\n\nfunc orOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tprefs := crossFunctionPreferences{\n\t\tCalcWhenEmpty: true,\n\t\tCalculation: returnRhsTruthy,\n\t\tLhsResultValue: returnLHSWhen(true),\n\t}\n\treturn crossFunctionWithPrefs(d, context.ReadOnlyClone(), expressionNode, prefs)\n}\n\nfunc andOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tprefs := crossFunctionPreferences{\n\t\tCalcWhenEmpty: true,\n\t\tCalculation: returnRhsTruthy,\n\t\tLhsResultValue: returnLHSWhen(false),\n\t}\n\treturn crossFunctionWithPrefs(d, context.ReadOnlyClone(), expressionNode, prefs)\n}\n\nfunc notOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"notOperation\")\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tlog.Debug(\"notOperation checking %v\", candidate)\n\t\ttruthy := isTruthyNode(candidate)\n\t\tresult := createBooleanCandidate(candidate, !truthy)\n\t\tresults.PushBack(result)\n\t}\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_booleans_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar booleanOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"`or` example\",\n\t\texpression: `true or false`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"\\\"yes\\\" and \\\"no\\\" are strings\",\n\t\tsubdescription: \"In the yaml 1.2 standard, support for yes/no as booleans was dropped - they are now considered strings. See '10.2.1.2. Boolean' in https://yaml.org/spec/1.2.2/\",\n\t\tdocument: `[yes, no]`,\n\t\texpression: `.[] | tag`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!str)::!!str\\n\",\n\t\t\t\"D0, P[1], (!!str)::!!str\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"b: hi\",\n\t\texpression: `.a or .c`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"b: false\",\n\t\texpression: `.b or .c`,\n\t\texpected: []string{\n\t\t\t\"D0, P[b], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"b: hi\",\n\t\texpression: `select(.a or .b)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::b: hi\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"b: hi\",\n\t\texpression: `select((.a and .b) | not)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::b: hi\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"And should not run 2nd arg if first is false\",\n\t\texpression: `false and test(3)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Or should not run 2nd arg if first is true\",\n\t\texpression: `true or test(3)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"`and` example\",\n\t\texpression: `true and false`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: \"[{a: bird, b: dog}, {a: frog, b: bird}, {a: cat, b: fly}]\",\n\t\tdescription: \"Matching nodes with select, equals and or\",\n\t\texpression: `[.[] | select(.a == \"cat\" or .b == \"dog\")]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- {a: bird, b: dog}\\n- {a: cat, b: fly}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"`any` returns true if any boolean in a given array is true\",\n\t\tdocument: `[false, true]`,\n\t\texpression: \"any\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"`any` returns false for an empty array\",\n\t\tdocument: `[]`,\n\t\texpression: \"any\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"`any_c` returns true if any element in the array is true for the given condition.\",\n\t\tdocument: \"a: [rad, awesome]\\nb: [meh, whatever]\",\n\t\texpression: `.[] |= any_c(. == \"awesome\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: true\\nb: false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[{pet: cat}]`,\n\t\texpression: `any_c(.name == \"harry\") as $c | .`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[{pet: cat}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[{pet: cat}]`,\n\t\texpression: `any_c(.name == \"harry\") as $c | $c`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[{pet: cat}]`,\n\t\texpression: `all_c(.name == \"harry\") as $c | $c`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[false, false]`,\n\t\texpression: \"any\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"`all` returns true if all booleans in a given array are true\",\n\t\tdocument: `[true, true]`,\n\t\texpression: \"all\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[false, true]`,\n\t\texpression: \"all\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"`all` returns true for an empty array\",\n\t\tdocument: `[]`,\n\t\texpression: \"all\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"`all_c` returns true if all elements in the array are true for the given condition.\",\n\t\tdocument: \"a: [rad, awesome]\\nb: [meh, 12]\",\n\t\texpression: `.[] |= all_c(tag == \"!!str\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: true\\nb: false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `false or false`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: true, b: false}`,\n\t\texpression: `.[] or (false, true)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::true\\n\",\n\t\t\t\"D0, P[b], (!!bool)::false\\n\",\n\t\t\t\"D0, P[b], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: true, b: false}`,\n\t\texpression: `.[] and (false, true)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::false\\n\",\n\t\t\t\"D0, P[a], (!!bool)::true\\n\",\n\t\t\t\"D0, P[b], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{}`,\n\t\texpression: `(.a.b or .c) as $x | .`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{}`,\n\t\texpression: `(.a.b and .c) as $x | .`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Not true is false\",\n\t\texpression: `true | not`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Not false is true\",\n\t\texpression: `false | not`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"String values considered to be true\",\n\t\texpression: `\"cat\" | not`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Empty string value considered to be true\",\n\t\texpression: `\"\" | not`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Numbers are considered to be true\",\n\t\texpression: `1 | not`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Zero is considered to be true\",\n\t\texpression: `0 | not`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\n\t\tdescription: \"Null is considered to be false\",\n\t\texpression: `~ | not`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::true\\n\",\n\t\t},\n\t},\n}\n\nfunc TestBooleanOperatorScenarios(t *testing.T) {\n\tfor _, tt := range booleanOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"boolean-operators\", booleanOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_collect.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n)\n\nfunc collectTogether(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (*CandidateNode, error) {\n\tcollectedNode := &CandidateNode{Kind: SequenceNode, Tag: \"!!seq\"}\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tcollectExpResults, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tfor result := collectExpResults.MatchingNodes.Front(); result != nil; result = result.Next() {\n\t\t\tresultC := result.Value.(*CandidateNode)\n\t\t\tlog.Debugf(\"found this: %v\", NodeToString(resultC))\n\t\t\tcollectedNode.AddChild(resultC)\n\t\t}\n\t}\n\treturn collectedNode, nil\n}\n\nfunc collectOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"collectOperation\")\n\n\tif context.MatchingNodes.Len() == 0 {\n\t\tlog.Debugf(\"nothing to collect\")\n\t\tnode := &CandidateNode{Kind: SequenceNode, Tag: \"!!seq\", Value: \"[]\"}\n\t\treturn context.SingleChildContext(node), nil\n\t}\n\n\tvar evaluateAllTogether = true\n\tfor matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() {\n\t\tevaluateAllTogether = evaluateAllTogether && matchEl.Value.(*CandidateNode).EvaluateTogether\n\t\tif !evaluateAllTogether {\n\t\t\tbreak\n\t\t}\n\t}\n\n\tif evaluateAllTogether {\n\t\tlog.Debugf(\"collect together\")\n\t\tcollectedNode, err := collectTogether(d, context, expressionNode.RHS)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\treturn context.SingleChildContext(collectedNode), nil\n\n\t}\n\n\tvar results = list.New()\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\tcollectCandidate := candidate.CreateReplacement(SequenceNode, \"!!seq\", \"\")\n\n\t\tlog.Debugf(\"collect rhs: %v\", expressionNode.RHS.Operation.toString())\n\n\t\tcollectExpResults, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.RHS)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t\tfor result := collectExpResults.MatchingNodes.Front(); result != nil; result = result.Next() {\n\t\t\tresultC := result.Value.(*CandidateNode)\n\t\t\tlog.Debugf(\"found this: %v\", NodeToString(resultC))\n\t\t\tcollectCandidate.AddChild(resultC)\n\t\t}\n\t\tlog.Debugf(\"done collect rhs: %v\", expressionNode.RHS.Operation.toString())\n\n\t\tresults.PushBack(collectCandidate)\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_collect_object.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n)\n\n/*\n[Mike: cat, Bob: dog]\n[Thing: rabbit, peter: sam]\n\n==> cross multiply\n\n{Mike: cat, Thing: rabbit}\n{Mike: cat, peter: sam}\n...\n*/\n\nfunc collectObjectOperator(d *dataTreeNavigator, originalContext Context, _ *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"collectObjectOperation\")\n\n\tcontext := originalContext.WritableClone()\n\n\tif context.MatchingNodes.Len() == 0 {\n\t\tcandidate := &CandidateNode{Kind: MappingNode, Tag: \"!!map\", Value: \"{}\"}\n\t\tlog.Debugf(\"collectObjectOperation - starting with empty map\")\n\t\treturn context.SingleChildContext(candidate), nil\n\t}\n\tfirst := context.MatchingNodes.Front().Value.(*CandidateNode)\n\tvar rotated = make([]*list.List, len(first.Content))\n\n\tfor i := 0; i < len(first.Content); i++ {\n\t\trotated[i] = list.New()\n\t}\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidateNode := el.Value.(*CandidateNode)\n\t\tif len(candidateNode.Content) < len(first.Content) {\n\t\t\treturn Context{}, fmt.Errorf(\"CollectObject: mismatching node sizes; are you creating a map with mismatching key value pairs?\")\n\t\t}\n\n\t\tfor i := 0; i < len(first.Content); i++ {\n\t\t\tlog.Debugf(\"rotate[%v] = %v\", i, NodeToString(candidateNode.Content[i]))\n\t\t\tlog.Debugf(\"children:\\n%v\", NodeContentToString(candidateNode.Content[i], 0))\n\t\t\trotated[i].PushBack(candidateNode.Content[i])\n\t\t}\n\t}\n\tlog.Debugf(\"collectObjectOperation, length of rotated is %v\", len(rotated))\n\n\tnewObject := list.New()\n\tfor i := 0; i < len(first.Content); i++ {\n\t\tadditions, err := collect(d, context.ChildContext(list.New()), rotated[i])\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\t// we should reset the parents and keys of these top level nodes,\n\t\t// as they are new\n\t\tfor el := additions.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\t\taddition := el.Value.(*CandidateNode)\n\t\t\tadditionCopy := addition.Copy()\n\n\t\t\tadditionCopy.SetParent(nil)\n\t\t\tadditionCopy.Key = nil\n\n\t\t\tlog.Debugf(\"collectObjectOperation, adding result %v\", NodeToString(additionCopy))\n\n\t\t\tnewObject.PushBack(additionCopy)\n\t\t}\n\t}\n\n\treturn context.ChildContext(newObject), nil\n\n}\n\nfunc collect(d *dataTreeNavigator, context Context, remainingMatches *list.List) (Context, error) {\n\tif remainingMatches.Len() == 0 {\n\t\treturn context, nil\n\t}\n\n\tcandidate := remainingMatches.Remove(remainingMatches.Front()).(*CandidateNode)\n\tlog.Debugf(\"collectObjectOperation - collect %v\", NodeToString(candidate))\n\n\tsplatted, err := splat(context.SingleChildContext(candidate),\n\t\ttraversePreferences{DontFollowAlias: true, IncludeMapKeys: false})\n\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\n\tif context.MatchingNodes.Len() == 0 {\n\t\tlog.Debugf(\"collectObjectOperation - collect context is empty, next\")\n\t\treturn collect(d, splatted, remainingMatches)\n\t}\n\n\tnewAgg := list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\taggCandidate := el.Value.(*CandidateNode)\n\t\tfor splatEl := splatted.MatchingNodes.Front(); splatEl != nil; splatEl = splatEl.Next() {\n\t\t\tsplatCandidate := splatEl.Value.(*CandidateNode)\n\t\t\tlog.Debugf(\"collectObjectOperation; splatCandidate: %v\", NodeToString(splatCandidate))\n\t\t\tnewCandidate := aggCandidate.Copy()\n\t\t\tlog.Debugf(\"collectObjectOperation; aggCandidate: %v\", NodeToString(aggCandidate))\n\n\t\t\tnewCandidate, err = multiply(multiplyPreferences{AppendArrays: false})(d, context, newCandidate, splatCandidate)\n\n\t\t\tif err != nil {\n\t\t\t\treturn Context{}, err\n\t\t\t}\n\t\t\tnewAgg.PushBack(newCandidate)\n\t\t}\n\t}\n\treturn collect(d, context.ChildContext(newAgg), remainingMatches)\n\n}\n" }, { "path": "pkg/yqlib/operator_collect_object_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar collectObjectOperatorScenarios = []expressionScenario{\n\t{\n\t\tskipDoc: true,\n\t\texpression: `{\"name\": \"mike\"} | .name`,\n\t\texpected: []string{\n\t\t\t\"D0, P[name], (!!str)::mike\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `{\"c\": \"a\", \"b\", \"d\"}`,\n\t\texpectedError: \"CollectObject: mismatching node sizes; are you creating a map with mismatching key value pairs?\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `{\"person\": {\"names\": [\"mike\"]}} | .person.names[0]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[person names 0], (!!str)::mike\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[{name: cat}, {name: dog}]`,\n\t\texpression: `.[] | {.name: \"great\"}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::cat: great\\n\",\n\t\t\t\"D0, P[], (!!map)::dog: great\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"collect splat\",\n\t\tskipDoc: true,\n\t\tdocument: `[{name: cat}, {name: dog}]`,\n\t\texpression: `.[] | {.name: \"great\"}[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[cat], (!!str)::great\\n\",\n\t\t\t\"D0, P[dog], (!!str)::great\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `({} + {}) | (.b = 3)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::b: 3\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: []\",\n\t\texpression: `.a += [{\"key\": \"att2\", \"value\": \"val2\"}]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a:\\n - key: att2\\n value: val2\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"\",\n\t\texpression: `.a += {\"key\": \"att2\", \"value\": \"val2\"}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], ()::a:\\n key: att2\\n value: val2\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"\",\n\t\texpression: `.a += [0]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], ()::a:\\n - 0\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: `Collect empty object`,\n\t\tdocument: ``,\n\t\texpression: `{}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: `Wrap (prefix) existing object`,\n\t\tdocument: \"{name: Mike}\\n\",\n\t\texpression: `{\"wrap\": .}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::wrap: {name: Mike}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Two documents\",\n\t\tdocument: \"{name: Mike}\\n\",\n\t\tdocument2: \"{name: Bob}\\n\",\n\t\texpression: `{\"wrap\": .}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::wrap: {name: Mike}\\n\",\n\t\t\t\"D0, P[], (!!map)::wrap: {name: Bob}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"two embedded documents\",\n\t\tdocument: \"{name: Mike}\\n---\\n{name: Bob}\",\n\t\texpression: `{\"wrap\": .}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::wrap: {name: Mike}\\n\",\n\t\t\t\"D1, P[], (!!map)::wrap: {name: Bob}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{name: Mike, age: 32}`,\n\t\texpression: `{.name: .age}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::Mike: 32\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: `Using splat to create multiple objects`,\n\t\tdocument: `{name: Mike, pets: [cat, dog]}`,\n\t\texpression: `{.name: .pets.[]}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::Mike: cat\\n\",\n\t\t\t\"D0, P[], (!!map)::Mike: dog\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: `Working with multiple documents`,\n\t\tdontFormatInputForDoc: false,\n\t\tdocument: \"{name: Mike, pets: [cat, dog]}\\n---\\n{name: Rosey, pets: [monkey, sheep]}\",\n\t\texpression: `{.name: .pets.[]}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::Mike: cat\\n\",\n\t\t\t\"D0, P[], (!!map)::Mike: dog\\n\",\n\t\t\t\"D1, P[], (!!map)::Rosey: monkey\\n\",\n\t\t\t\"D1, P[], (!!map)::Rosey: sheep\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{name: Mike, pets: [cat, dog], food: [hotdog, burger]}`,\n\t\texpression: `{.name: .pets.[], \"f\":.food.[]}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::Mike: cat\\nf: hotdog\\n\",\n\t\t\t\"D0, P[], (!!map)::Mike: cat\\nf: burger\\n\",\n\t\t\t\"D0, P[], (!!map)::Mike: dog\\nf: hotdog\\n\",\n\t\t\t\"D0, P[], (!!map)::Mike: dog\\nf: burger\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"name: Mike\\npets:\\n cows:\\n - apl\\n - bba\",\n\t\tdocument2: \"name: Rosey\\npets:\\n sheep:\\n - frog\\n - meow\",\n\t\texpression: `{\"a\":.name, \"b\":.pets}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: Mike\\nb:\\n cows:\\n - apl\\n - bba\\n\",\n\t\t\t\"D0, P[], (!!map)::a: Rosey\\nb:\\n sheep:\\n - frog\\n - meow\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Creating yaml from scratch\",\n\t\tdocument: ``,\n\t\texpression: `{\"wrap\": \"frog\"}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::wrap: frog\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `{\"wrap\": \"frog\", \"bing\": \"bong\"}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::wrap: frog\\nbing: bong\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{name: Mike}`,\n\t\texpression: `{\"wrap\": .}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::wrap: {name: Mike}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{name: Mike}`,\n\t\texpression: `{\"wrap\": {\"further\": .}} | (.. style= \"flow\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{wrap: {further: {name: Mike}}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Creating yaml from scratch with multiple objects\",\n\t\texpression: `(.a.b = \"foo\") | (.d.e = \"bar\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], ()::a:\\n b: foo\\nd:\\n e: bar\\n\",\n\t\t},\n\t},\n}\n\nfunc TestCollectObjectOperatorScenarios(t *testing.T) {\n\tfor _, tt := range collectObjectOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"create-collect-into-object\", collectObjectOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_collect_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar collectOperatorScenarios = []expressionScenario{\n\t{\n\t\tskipDoc: true,\n\t\texpression: `[\"x\", \"y\"] | .[1]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[1], (!!str)::y\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: ``,\n\t\texpression: `.a += [0]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], ()::a:\\n - 0\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `[1,2,3]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- 1\\n- 2\\n- 3\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"update in collect\",\n\t\texpression: `[.a = \"cat\"]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- a: cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Collect empty\",\n\t\tdocument: ``,\n\t\texpression: `[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"{a: apple}\\n---\\n{b: frog}\",\n\t\texpression: `[.]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- {a: apple}\\n- {b: frog}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"with comments\",\n\t\tskipDoc: true,\n\t\tdocument: \"# abc\\n[{a: apple}]\\n\\n# xyz\\n\",\n\n\t\texpression: `[.[]]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- {a: apple}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: ``,\n\t\texpression: `[3]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- 3\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Collect single\",\n\t\tdocument: ``,\n\t\texpression: `[\"cat\"]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: ``,\n\t\tskipDoc: true,\n\t\texpression: `[true]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- true\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Collect many\",\n\t\tdocument: `{a: cat, b: dog}`,\n\t\texpression: `[.a, .b]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- cat\\n- dog\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: ``,\n\t\tskipDoc: true,\n\t\texpression: `collect(1)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- 1\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: `[1,2,3]`,\n\t\tskipDoc: true,\n\t\texpression: `[.[]]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- 1\\n- 2\\n- 3\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `[1,2][]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!int)::1\\n\",\n\t\t\t\"D0, P[1], (!!int)::2\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: `a: {b: [1,2,3]}`,\n\t\texpression: `[.a.b.[]]`,\n\t\tskipDoc: true,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- 1\\n- 2\\n- 3\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[{name: cat, thing: bor}, {name: dog}]`,\n\t\texpression: `.[] | [.name]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!seq)::- cat\\n\",\n\t\t\t\"D0, P[1], (!!seq)::- dog\\n\",\n\t\t},\n\t},\n}\n\nfunc TestCollectOperatorScenarios(t *testing.T) {\n\tfor _, tt := range collectOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"collect-into-array\", collectOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_column.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n)\n\nfunc columnOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"columnOperator\")\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tresult := candidate.CreateReplacement(ScalarNode, \"!!int\", fmt.Sprintf(\"%v\", candidate.Column))\n\t\tresults.PushBack(result)\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_column_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar columnOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Returns column of _value_ node\",\n\t\tdocument: \"a: cat\\nb: bob\",\n\t\texpression: `.b | column`,\n\t\texpected: []string{\n\t\t\t\"D0, P[b], (!!int)::4\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Returns column of _key_ node\",\n\t\tsubdescription: \"Pipe through the key operator to get the column of the key\",\n\t\tdocument: \"a: cat\\nb: bob\",\n\t\texpression: `.b | key | column`,\n\t\texpected: []string{\n\t\t\t\"D0, P[b], (!!int)::1\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"First column is 1\",\n\t\tdocument: \"a: cat\",\n\t\texpression: `.a | key | column`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!int)::1\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"No column data is 0\",\n\t\texpression: `{\"a\": \"new entry\"} | column`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!int)::0\\n\",\n\t\t},\n\t},\n}\n\nfunc TestColumnOperatorScenarios(t *testing.T) {\n\tfor _, tt := range columnOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"column\", columnOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_comments.go", "content": "package yqlib\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"container/list\"\n\t\"regexp\"\n)\n\ntype commentOpPreferences struct {\n\tLineComment bool\n\tHeadComment bool\n\tFootComment bool\n}\n\nfunc assignCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\n\tlog.Debugf(\"AssignComments operator!\")\n\n\tlhs, err := d.GetMatchingNodes(context, expressionNode.LHS)\n\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\n\tpreferences := expressionNode.Operation.Preferences.(commentOpPreferences)\n\n\tcomment := \"\"\n\tif !expressionNode.Operation.UpdateAssign {\n\t\trhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t\tif rhs.MatchingNodes.Front() != nil {\n\t\t\tcomment = rhs.MatchingNodes.Front().Value.(*CandidateNode).Value\n\t\t}\n\t}\n\n\tlog.Debugf(\"AssignComments comment is %v\", comment)\n\n\tfor el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\tlog.Debugf(\"AssignComments lhs %v\", NodeToString(candidate))\n\n\t\tif expressionNode.Operation.UpdateAssign {\n\t\t\trhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS)\n\t\t\tif err != nil {\n\t\t\t\treturn Context{}, err\n\t\t\t}\n\n\t\t\tif rhs.MatchingNodes.Front() != nil {\n\t\t\t\tcomment = rhs.MatchingNodes.Front().Value.(*CandidateNode).Value\n\t\t\t}\n\t\t}\n\n\t\tlog.Debugf(\"Setting comment of : %v\", candidate.GetKey())\n\t\tif preferences.LineComment {\n\t\t\tlog.Debugf(\"Setting line comment of : %v to %v\", candidate.GetKey(), comment)\n\t\t\tcandidate.LineComment = comment\n\t\t}\n\t\tif preferences.HeadComment {\n\t\t\tcandidate.HeadComment = comment\n\t\t\tcandidate.LeadingContent = \"\" // clobber the leading content, if there was any.\n\t\t}\n\t\tif preferences.FootComment {\n\t\t\tcandidate.FootComment = comment\n\t\t}\n\n\t}\n\treturn context, nil\n}\n\nfunc getCommentsOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tpreferences := expressionNode.Operation.Preferences.(commentOpPreferences)\n\tvar startCommentCharacterRegExp = regexp.MustCompile(`^# `)\n\tvar subsequentCommentCharacterRegExp = regexp.MustCompile(`\\n# `)\n\n\tlog.Debugf(\"GetComments operator!\")\n\tvar results = list.New()\n\n\tyamlPrefs := ConfiguredYamlPreferences.Copy()\n\tyamlPrefs.PrintDocSeparators = false\n\tyamlPrefs.UnwrapScalar = false\n\tyamlPrefs.ColorsEnabled = false\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tcomment := \"\"\n\t\tif preferences.LineComment {\n\t\t\tlog.Debugf(\"Reading line comment of : %v to %v\", candidate.GetKey(), candidate.LineComment)\n\t\t\tcomment = candidate.LineComment\n\t\t} else if preferences.HeadComment && candidate.LeadingContent != \"\" {\n\t\t\tvar chompRegexp = regexp.MustCompile(`\\n$`)\n\t\t\tvar output bytes.Buffer\n\t\t\tvar writer = bufio.NewWriter(&output)\n\t\t\tvar encoder = NewYamlEncoder(yamlPrefs)\n\t\t\tif err := encoder.PrintLeadingContent(writer, candidate.LeadingContent); err != nil {\n\t\t\t\treturn Context{}, err\n\t\t\t}\n\t\t\tif err := writer.Flush(); err != nil {\n\t\t\t\treturn Context{}, err\n\t\t\t}\n\t\t\tcomment = output.String()\n\t\t\tcomment = chompRegexp.ReplaceAllString(comment, \"\")\n\t\t} else if preferences.HeadComment {\n\t\t\tcomment = candidate.HeadComment\n\t\t} else if preferences.FootComment {\n\t\t\tcomment = candidate.FootComment\n\t\t}\n\t\tcomment = startCommentCharacterRegExp.ReplaceAllString(comment, \"\")\n\t\tcomment = subsequentCommentCharacterRegExp.ReplaceAllString(comment, \"\\n\")\n\n\t\tresult := candidate.CreateReplacement(ScalarNode, \"!!str\", comment)\n\t\tif candidate.IsMapKey {\n\t\t\tresult.IsMapKey = false\n\t\t\tresult.Key = candidate\n\t\t}\n\t\tresults.PushBack(result)\n\t}\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_comments_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar expectedWhereIsMyCommentMapKey = `D0, P[], (!!seq)::- p: \"\"\n isKey: false\n hc: \"\"\n lc: \"\"\n fc: \"\"\n- p: hello\n isKey: true\n hc: \"\"\n lc: hello-world-comment\n fc: \"\"\n- p: hello\n isKey: false\n hc: \"\"\n lc: \"\"\n fc: \"\"\n- p: hello.message\n isKey: true\n hc: \"\"\n lc: \"\"\n fc: \"\"\n- p: hello.message\n isKey: false\n hc: \"\"\n lc: \"\"\n fc: \"\"\n`\n\nvar expectedWhereIsMyCommentArray = `D0, P[], (!!seq)::- p: \"\"\n isKey: false\n hc: \"\"\n lc: \"\"\n fc: \"\"\n- p: name\n isKey: true\n hc: \"\"\n lc: \"\"\n fc: \"\"\n- p: name\n isKey: false\n hc: \"\"\n lc: \"\"\n fc: \"\"\n- p: name.0\n isKey: false\n hc: under-name-comment\n lc: \"\"\n fc: \"\"\n`\n\nvar commentOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Set line comment\",\n\t\tsubdescription: \"Set the comment on the key node for more reliability (see below).\",\n\t\tdocument: `a: cat`,\n\t\texpression: `.a line_comment=\"single\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: cat # single\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Set line comment of a maps/arrays\",\n\t\tsubdescription: \"For maps and arrays, you need to set the line comment on the _key_ node. This will also work for scalars.\",\n\t\tdocument: \"a:\\n b: things\",\n\t\texpression: `(.a | key) line_comment=\"single\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: # single\\n b: things\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: cat\\nb: dog\",\n\t\texpression: `.a line_comment=.b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: cat # dog\\nb: dog\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: cat\\n---\\na: dog\",\n\t\texpression: `.a line_comment |= documentIndex`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: cat # 0\\n\",\n\t\t\t\"D1, P[], (!!map)::a: dog # 1\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Use update assign to perform relative updates\",\n\t\tdocument: \"a: cat\\nb: dog\",\n\t\texpression: `.. line_comment |= .`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: cat # cat\\nb: dog # dog\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: cat\\nb: dog\",\n\t\texpression: `.. comments |= .`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: cat # cat\\n# cat\\n\\n# cat\\nb: dog # dog\\n# dog\\n\\n# dog\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Where is the comment - map key example\",\n\t\tsubdescription: \"The underlying yaml parser can assign comments in a document to surprising nodes. Use an expression like this to find where you comment is. 'p' indicates the path, 'isKey' is if the node is a map key (as opposed to a map value).\\nFrom this, you can see the 'hello-world-comment' is actually on the 'hello' key\",\n\t\tdocument: \"hello: # hello-world-comment\\n message: world\",\n\t\texpression: `[... | {\"p\": path | join(\".\"), \"isKey\": is_key, \"hc\": headComment, \"lc\": lineComment, \"fc\": footComment}]`,\n\t\texpected: []string{\n\t\t\texpectedWhereIsMyCommentMapKey,\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Retrieve comment - map key example\",\n\t\tsubdescription: \"From the previous example, we know that the comment is on the 'hello' _key_ as a lineComment\",\n\t\tdocument: \"hello: # hello-world-comment\\n message: world\",\n\t\texpression: `.hello | key | line_comment`,\n\t\texpected: []string{\n\t\t\t\"D0, P[hello], (!!str)::hello-world-comment\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Where is the comment - array example\",\n\t\tsubdescription: \"The underlying yaml parser can assign comments in a document to surprising nodes. Use an expression like this to find where you comment is. 'p' indicates the path, 'isKey' is if the node is a map key (as opposed to a map value).\\nFrom this, you can see the 'under-name-comment' is actually on the first child\",\n\t\tdocument: \"name:\\n # under-name-comment\\n - first-array-child\",\n\t\texpression: `[... | {\"p\": path | join(\".\"), \"isKey\": is_key, \"hc\": headComment, \"lc\": lineComment, \"fc\": footComment}]`,\n\t\texpected: []string{\n\t\t\texpectedWhereIsMyCommentArray,\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Retrieve comment - array example\",\n\t\tsubdescription: \"From the previous example, we know that the comment is on the first child as a headComment\",\n\t\tdocument: \"name:\\n # under-name-comment\\n - first-array-child\",\n\t\texpression: `.name[0] | headComment`,\n\t\texpected: []string{\n\t\t\t\"D0, P[name 0], (!!str)::under-name-comment\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Set head comment\",\n\t\tdocument: `a: cat`,\n\t\texpression: `. head_comment=\"single\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::# single\\na: cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Set head comment of a map entry\",\n\t\tdocument: \"f: foo\\na:\\n b: cat\",\n\t\texpression: `(.a | key) head_comment=\"single\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::f: foo\\n# single\\na:\\n b: cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Set foot comment, using an expression\",\n\t\tdocument: `a: cat`,\n\t\texpression: `. foot_comment=.a`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: cat\\n# cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Set foot comment, using an expression\",\n\t\tdocument: \"a: cat\\n\\n# hi\",\n\t\texpression: `. foot_comment=\"\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a: cat`,\n\t\texpression: `. foot_comment=.b.d`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a: cat`,\n\t\texpression: `. foot_comment|=.b.d`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Remove comment\",\n\t\tdocument: \"a: cat # comment\\nb: dog # leave this\",\n\t\texpression: `.a line_comment=\"\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: cat\\nb: dog # leave this\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Remove (strip) all comments\",\n\t\tsubdescription: \"Note the use of `...` to ensure key nodes are included.\",\n\t\tdocument: \"# hi\\n\\na: cat # comment\\n\\n# great\\n\\nb: # key comment\",\n\t\texpression: `... comments=\"\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: cat\\nb:\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Get line comment\",\n\t\tdocument: \"# welcome!\\n\\na: cat # meow\\n\\n# have a great day\",\n\t\texpression: `.a | line_comment`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!str)::meow\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Get head comment\",\n\t\tdontFormatInputForDoc: true,\n\t\tdocument: \"# welcome!\\n\\na: cat # meow\\n\\n# have a great day\",\n\t\texpression: `. | head_comment`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::welcome!\\n\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"strip trailing comment recurse all\",\n\t\tdocument: \"a: cat\\n\\n# haha\",\n\t\texpression: `... comments= \"\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"strip trailing comment recurse values\",\n\t\tdocument: \"a: cat\\n\\n# haha\",\n\t\texpression: `.. comments= \"\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Head comment with document split\",\n\t\tdontFormatInputForDoc: true,\n\t\tdocument: \"# welcome!\\n---\\n# bob\\na: cat # meow\\n\\n# have a great day\",\n\t\texpression: `head_comment`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::welcome!\\nbob\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Get foot comment\",\n\t\tdontFormatInputForDoc: true,\n\t\tdocument: \"# welcome!\\n\\na: cat # meow\\n\\n# have a great day\\n# no really\",\n\t\texpression: `. | foot_comment`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::have a great day\\nno really\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"leading spaces\",\n\t\tskipDoc: true,\n\t\tdocument: \" # hi\",\n\t\texpression: `.`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!null):: # hi\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"string spaces\",\n\t\tskipDoc: true,\n\t\tdocument: \"# hi\\ncat\\n\",\n\t\texpression: `.`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::# hi\\ncat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"leading spaces with new line\",\n\t\tskipDoc: true,\n\t\tdocument: \" # hi\\n\",\n\t\texpression: `.`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!null):: # hi\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"directive\",\n\t\tskipDoc: true,\n\t\tdocument: \"%YAML 1.1\\n# hi\\n\",\n\t\texpression: `.`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!null)::%YAML 1.1\\n# hi\\n\",\n\t\t},\n\t},\n}\n\nfunc TestCommentOperatorScenarios(t *testing.T) {\n\tfor _, tt := range commentOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"comment-operators\", commentOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_compare.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n\t\"strconv\"\n)\n\ntype compareTypePref struct {\n\tOrEqual bool\n\tGreater bool\n}\n\nfunc compareOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"compareOperator\")\n\tprefs := expressionNode.Operation.Preferences.(compareTypePref)\n\treturn crossFunction(d, context, expressionNode, compare(prefs), true)\n}\n\nfunc compare(prefs compareTypePref) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {\n\treturn func(_ *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {\n\t\tlog.Debugf(\"compare cross function\")\n\t\tif lhs == nil && rhs == nil {\n\t\t\towner := &CandidateNode{}\n\t\t\treturn createBooleanCandidate(owner, prefs.OrEqual), nil\n\t\t} else if lhs == nil {\n\t\t\tlog.Debugf(\"lhs nil, but rhs is not\")\n\t\t\treturn createBooleanCandidate(rhs, false), nil\n\t\t} else if rhs == nil {\n\t\t\tlog.Debugf(\"rhs nil, but rhs is not\")\n\t\t\treturn createBooleanCandidate(lhs, false), nil\n\t\t}\n\n\t\tswitch lhs.Kind {\n\t\tcase MappingNode:\n\t\t\treturn nil, fmt.Errorf(\"maps not yet supported for comparison\")\n\t\tcase SequenceNode:\n\t\t\treturn nil, fmt.Errorf(\"arrays not yet supported for comparison\")\n\t\tdefault:\n\t\t\tif rhs.Kind != ScalarNode {\n\t\t\t\treturn nil, fmt.Errorf(\"%v (%v) cannot be subtracted from %v\", rhs.Tag, rhs.GetNicePath(), lhs.Tag)\n\t\t\t}\n\t\t\ttarget := lhs.CopyWithoutContent()\n\t\t\tboolV, err := compareScalars(context, prefs, lhs, rhs)\n\n\t\t\treturn createBooleanCandidate(target, boolV), err\n\t\t}\n\t}\n}\n\nfunc compareDateTime(layout string, prefs compareTypePref, lhs *CandidateNode, rhs *CandidateNode) (bool, error) {\n\tlhsTime, err := parseDateTime(layout, lhs.Value)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\trhsTime, err := parseDateTime(layout, rhs.Value)\n\tif err != nil {\n\t\treturn false, err\n\t}\n\n\tif prefs.OrEqual && lhsTime.Equal(rhsTime) {\n\t\treturn true, nil\n\t}\n\tif prefs.Greater {\n\t\treturn lhsTime.After(rhsTime), nil\n\t}\n\treturn lhsTime.Before(rhsTime), nil\n\n}\n\nfunc compareScalars(context Context, prefs compareTypePref, lhs *CandidateNode, rhs *CandidateNode) (bool, error) {\n\tlhsTag := lhs.guessTagFromCustomType()\n\trhsTag := rhs.guessTagFromCustomType()\n\n\tisDateTime := lhs.Tag == \"!!timestamp\"\n\t// if the lhs is a string, it might be a timestamp in a custom format.\n\tif lhsTag == \"!!str\" {\n\t\t_, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value)\n\t\tisDateTime = err == nil\n\t}\n\tif isDateTime {\n\t\treturn compareDateTime(context.GetDateTimeLayout(), prefs, lhs, rhs)\n\t} else if lhsTag == \"!!int\" && rhsTag == \"!!int\" {\n\t\t_, lhsNum, err := parseInt64(lhs.Value)\n\t\tif err != nil {\n\t\t\treturn false, err\n\t\t}\n\t\t_, rhsNum, err := parseInt64(rhs.Value)\n\t\tif err != nil {\n\t\t\treturn false, err\n\t\t}\n\n\t\tif prefs.OrEqual && lhsNum == rhsNum {\n\t\t\treturn true, nil\n\t\t}\n\t\tif prefs.Greater {\n\t\t\treturn lhsNum > rhsNum, nil\n\t\t}\n\t\treturn lhsNum < rhsNum, nil\n\t} else if (lhsTag == \"!!int\" || lhsTag == \"!!float\") && (rhsTag == \"!!int\" || rhsTag == \"!!float\") {\n\t\tlhsNum, err := strconv.ParseFloat(lhs.Value, 64)\n\t\tif err != nil {\n\t\t\treturn false, err\n\t\t}\n\t\trhsNum, err := strconv.ParseFloat(rhs.Value, 64)\n\t\tif err != nil {\n\t\t\treturn false, err\n\t\t}\n\t\tif prefs.OrEqual && lhsNum == rhsNum {\n\t\t\treturn true, nil\n\t\t}\n\t\tif prefs.Greater {\n\t\t\treturn lhsNum > rhsNum, nil\n\t\t}\n\t\treturn lhsNum < rhsNum, nil\n\t} else if lhsTag == \"!!str\" && rhsTag == \"!!str\" {\n\t\tif prefs.OrEqual && lhs.Value == rhs.Value {\n\t\t\treturn true, nil\n\t\t}\n\t\tif prefs.Greater {\n\t\t\treturn lhs.Value > rhs.Value, nil\n\t\t}\n\t\treturn lhs.Value < rhs.Value, nil\n\t} else if lhsTag == \"!!null\" && rhsTag == \"!!null\" && prefs.OrEqual {\n\t\treturn true, nil\n\t} else if lhsTag == \"!!null\" || rhsTag == \"!!null\" {\n\t\treturn false, nil\n\t}\n\n\treturn false, fmt.Errorf(\"%v not yet supported for comparison\", lhs.Tag)\n}\n\nfunc superlativeByComparison(d *dataTreeNavigator, context Context, prefs compareTypePref) (Context, error) {\n\tfn := compare(prefs)\n\n\tvar results = list.New()\n\n\tfor seq := context.MatchingNodes.Front(); seq != nil; seq = seq.Next() {\n\t\tsplatted, err := splat(context.SingleChildContext(seq.Value.(*CandidateNode)), traversePreferences{})\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\tresult := splatted.MatchingNodes.Front()\n\t\tif result != nil {\n\t\t\tfor el := result.Next(); el != nil; el = el.Next() {\n\t\t\t\tcmp, err := fn(d, context, el.Value.(*CandidateNode), result.Value.(*CandidateNode))\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn Context{}, err\n\t\t\t\t}\n\t\t\t\tif isTruthyNode(cmp) {\n\t\t\t\t\tresult = el\n\t\t\t\t}\n\t\t\t}\n\t\t\tresults.PushBack(result.Value)\n\t\t}\n\t}\n\treturn context.ChildContext(results), nil\n}\n\nfunc minOperator(d *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tlog.Debug((\"Min\"))\n\treturn superlativeByComparison(d, context, compareTypePref{Greater: false})\n}\n\nfunc maxOperator(d *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tlog.Debug((\"Max\"))\n\treturn superlativeByComparison(d, context, compareTypePref{Greater: true})\n}\n" }, { "path": "pkg/yqlib/operator_contains.go", "content": "package yqlib\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n)\n\nfunc containsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\treturn crossFunction(d, context.ReadOnlyClone(), expressionNode, containsWithNodes, false)\n}\n\nfunc containsArrayElement(array *CandidateNode, item *CandidateNode) (bool, error) {\n\tfor index := 0; index < len(array.Content); index = index + 1 {\n\t\tcontainedInArray, err := contains(array.Content[index], item)\n\t\tif err != nil {\n\t\t\treturn false, err\n\t\t}\n\t\tif containedInArray {\n\t\t\treturn true, nil\n\t\t}\n\t}\n\treturn false, nil\n}\n\nfunc containsArray(lhs *CandidateNode, rhs *CandidateNode) (bool, error) {\n\tif rhs.Kind != SequenceNode {\n\t\treturn containsArrayElement(lhs, rhs)\n\t}\n\tfor index := 0; index < len(rhs.Content); index = index + 1 {\n\t\titemInArray, err := containsArrayElement(lhs, rhs.Content[index])\n\t\tif err != nil {\n\t\t\treturn false, err\n\t\t}\n\t\tif !itemInArray {\n\t\t\treturn false, nil\n\t\t}\n\t}\n\treturn true, nil\n}\n\nfunc containsObject(lhs *CandidateNode, rhs *CandidateNode) (bool, error) {\n\tif rhs.Kind != MappingNode {\n\t\treturn false, nil\n\t}\n\tfor index := 0; index < len(rhs.Content); index = index + 2 {\n\t\trhsKey := rhs.Content[index]\n\t\trhsValue := rhs.Content[index+1]\n\t\tlog.Debugf(\"Looking for %v in the lhs\", rhsKey.Value)\n\t\tlhsKeyIndex := findInArray(lhs, rhsKey)\n\t\tlog.Debugf(\"index is %v\", lhsKeyIndex)\n\t\tif lhsKeyIndex < 0 || lhsKeyIndex%2 != 0 {\n\t\t\treturn false, nil\n\t\t}\n\t\tlhsValue := lhs.Content[lhsKeyIndex+1]\n\t\tlog.Debugf(\"lhsValue is %v\", lhsValue.Value)\n\n\t\titemInArray, err := contains(lhsValue, rhsValue)\n\t\tlog.Debugf(\"rhsValue is %v\", rhsValue.Value)\n\t\tif err != nil {\n\t\t\treturn false, err\n\t\t}\n\t\tif !itemInArray {\n\t\t\treturn false, nil\n\t\t}\n\t}\n\treturn true, nil\n}\n\nfunc containsScalars(lhs *CandidateNode, rhs *CandidateNode) (bool, error) {\n\tif lhs.Tag == \"!!str\" {\n\t\treturn strings.Contains(lhs.Value, rhs.Value), nil\n\t}\n\treturn lhs.Value == rhs.Value, nil\n}\n\nfunc contains(lhs *CandidateNode, rhs *CandidateNode) (bool, error) {\n\tswitch lhs.Kind {\n\tcase MappingNode:\n\t\treturn containsObject(lhs, rhs)\n\tcase SequenceNode:\n\t\treturn containsArray(lhs, rhs)\n\tcase ScalarNode:\n\t\tif rhs.Kind != ScalarNode || lhs.Tag != rhs.Tag {\n\t\t\treturn false, nil\n\t\t}\n\t\tif lhs.Tag == \"!!null\" {\n\t\t\treturn rhs.Tag == \"!!null\", nil\n\t\t}\n\t\treturn containsScalars(lhs, rhs)\n\t}\n\n\treturn false, fmt.Errorf(\"%v not yet supported for contains\", lhs.Tag)\n}\n\nfunc containsWithNodes(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {\n\tif lhs.Kind != rhs.Kind {\n\t\treturn nil, fmt.Errorf(\"%v cannot check contained in %v\", rhs.Tag, lhs.Tag)\n\t}\n\n\tresult, err := contains(lhs, rhs)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn createBooleanCandidate(lhs, result), nil\n}\n" }, { "path": "pkg/yqlib/operator_contains_test.go", "content": "package yqlib\n\nimport \"testing\"\n\nvar containsOperatorScenarios = []expressionScenario{\n\t{\n\t\tskipDoc: true,\n\t\texpression: `null | contains(~)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `3 | contains(3)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `3 | contains(32)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Array contains array\",\n\t\tsubdescription: \"Array is equal or subset of\",\n\t\tdocument: `[\"foobar\", \"foobaz\", \"blarp\"]`,\n\t\texpression: `contains([\"baz\", \"bar\"])`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Array has a subset array\",\n\t\tsubdescription: \"Subtract the superset array from the subset, if there's anything left, it's not a subset\",\n\t\tdocument: `[\"foobar\", \"foobaz\", \"blarp\"]`,\n\t\texpression: `[\"baz\", \"bar\"] - . | length == 0`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `[\"dog\", \"cat\", \"giraffe\"] | contains([\"camel\"])`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Object included in array\",\n\t\tdocument: `{\"foo\": 12, \"bar\":[1,2,{\"barp\":12, \"blip\":13}]}`,\n\t\texpression: `contains({\"bar\": [{\"barp\": 12}]})`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Object not included in array\",\n\t\tdocument: `{\"foo\": 12, \"bar\":[1,2,{\"barp\":12, \"blip\":13}]}`,\n\t\texpression: `contains({\"foo\": 12, \"bar\": [{\"barp\": 15}]})`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"String contains substring\",\n\t\tdocument: `\"foobar\"`,\n\t\texpression: `contains(\"bar\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"String equals string\",\n\t\tdocument: `\"meow\"`,\n\t\texpression: `contains(\"meow\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::true\\n\",\n\t\t},\n\t},\n}\n\nfunc TestContainsOperatorScenarios(t *testing.T) {\n\tfor _, tt := range containsOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"contains\", containsOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_create_map.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n)\n\nfunc createMapOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"createMapOperation\")\n\n\t//each matchingNodes entry should turn into a sequence of keys to create.\n\t//then collect object should do a cross function of the same index sequence for all matches.\n\n\tsequences := list.New()\n\n\tif context.MatchingNodes.Len() > 0 {\n\n\t\tfor matchingNodeEl := context.MatchingNodes.Front(); matchingNodeEl != nil; matchingNodeEl = matchingNodeEl.Next() {\n\t\t\tmatchingNode := matchingNodeEl.Value.(*CandidateNode)\n\t\t\tsequenceNode, err := sequenceFor(d, context, matchingNode, expressionNode)\n\t\t\tif err != nil {\n\t\t\t\treturn Context{}, err\n\t\t\t}\n\t\t\tsequences.PushBack(sequenceNode)\n\t\t}\n\t} else {\n\t\tsequenceNode, err := sequenceFor(d, context, nil, expressionNode)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\tsequences.PushBack(sequenceNode)\n\t}\n\n\tnode := listToNodeSeq(sequences)\n\n\treturn context.SingleChildContext(node), nil\n\n}\n\nfunc sequenceFor(d *dataTreeNavigator, context Context, matchingNode *CandidateNode, expressionNode *ExpressionNode) (*CandidateNode, error) {\n\tvar document uint\n\tvar filename string\n\tvar fileIndex int\n\n\tvar matches = list.New()\n\n\tif matchingNode != nil {\n\t\tdocument = matchingNode.GetDocument()\n\t\tfilename = matchingNode.GetFilename()\n\t\tfileIndex = matchingNode.GetFileIndex()\n\t\tmatches.PushBack(matchingNode)\n\t}\n\n\tlog.Debugf(\"**********sequenceFor %v\", NodeToString(matchingNode))\n\n\tmapPairs, err := crossFunction(d, context.ChildContext(matches), expressionNode,\n\t\tfunc(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {\n\t\t\tnode := &CandidateNode{Kind: MappingNode, Tag: \"!!map\"}\n\n\t\t\tlog.Debugf(\"**********adding key %v and value %v\", NodeToString(lhs), NodeToString(rhs))\n\n\t\t\tnode.AddKeyValueChild(lhs, rhs)\n\n\t\t\tnode.document = document\n\t\t\tnode.fileIndex = fileIndex\n\t\t\tnode.filename = filename\n\n\t\t\treturn node, nil\n\t\t}, false)\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tinnerList := listToNodeSeq(mapPairs.MatchingNodes)\n\tinnerList.Style = FlowStyle\n\tinnerList.document = document\n\tinnerList.fileIndex = fileIndex\n\tinnerList.filename = filename\n\treturn innerList, nil\n}\n\n// NOTE: here the document index gets dropped so we\n// no longer know where the node originates from.\nfunc listToNodeSeq(list *list.List) *CandidateNode {\n\tnode := CandidateNode{Kind: SequenceNode, Tag: \"!!seq\"}\n\tfor entry := list.Front(); entry != nil; entry = entry.Next() {\n\t\tentryCandidate := entry.Value.(*CandidateNode)\n\t\tlog.Debugf(\"Collecting %v into sequence\", NodeToString(entryCandidate))\n\t\tnode.AddChild(entryCandidate)\n\t}\n\treturn &node\n}\n" }, { "path": "pkg/yqlib/operator_create_map_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar createMapOperatorScenarios = []expressionScenario{\n\t{\n\t\tdocument: ``,\n\t\texpression: `\"frog\": \"jumps\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- [{frog: jumps}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"sets key properly\",\n\t\texpression: `(\"frog\": \"jumps\") | .[0][0] | .frog`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0 0 frog], (!!str)::jumps\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"sets key properly on map\",\n\t\texpression: `{\"frog\": \"jumps\"} | .frog`,\n\t\texpected: []string{\n\t\t\t\"D0, P[frog], (!!str)::jumps\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: `{name: Mike, pets: [cat, dog]}`,\n\t\texpression: `(.name: .pets.[]) | .[0][0] | ..`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0 0], (!!map)::Mike: cat\\n\",\n\t\t\t\"D0, P[0 0 Mike], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"check path of nested child\",\n\t\tdocument: \"pets:\\n cows: value\",\n\t\texpression: `(\"b\":.pets) | .[0][0] | .b.cows`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0 0 b cows], (!!str)::value\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: `{name: Mike, age: 32}`,\n\t\texpression: `.name: .age`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- [{Mike: 32}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: `{name: Mike, pets: [cat, dog]}`,\n\t\texpression: `.name: .pets.[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- [{Mike: cat}, {Mike: dog}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: `{name: Mike, pets: [cat, dog], food: [hotdog, burger]}`,\n\t\texpression: `.name: .pets.[], \"f\":.food.[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- [{Mike: cat}, {Mike: dog}]\\n\",\n\t\t\t\"D0, P[], (!!seq)::- [{f: hotdog}, {f: burger}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: \"{name: Mike, pets: [cat, dog], food: [hotdog, burger]}\\n---\\n{name: Fred, pets: [mouse], food: [pizza, onion, apple]}\",\n\t\texpression: `.name: .pets.[], \"f\":.food.[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- [{Mike: cat}, {Mike: dog}]\\n- [{Fred: mouse}]\\n\",\n\t\t\t\"D0, P[], (!!seq)::- [{f: hotdog}, {f: burger}]\\n- [{f: pizza}, {f: onion}, {f: apple}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: `{name: Mike, pets: {cows: [apl, bba]}}`,\n\t\texpression: `\"a\":.name, \"b\":.pets`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- [{a: Mike}]\\n\",\n\t\t\t\"D0, P[], (!!seq)::- [{b: {cows: [apl, bba]}}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: `{name: Mike}`,\n\t\texpression: `\"wrap\": .`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- [{wrap: {name: Mike}}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: \"{name: Mike}\\n---\\n{name: Bob}\",\n\t\texpression: `\"wrap\": .`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- [{wrap: {name: Mike}}]\\n- [{wrap: {name: Bob}}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: \"{name: Mike}\\n---\\n{name: Bob}\",\n\t\texpression: `\"wrap\": ., .name: \"great\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- [{wrap: {name: Mike}}]\\n- [{wrap: {name: Bob}}]\\n\",\n\t\t\t\"D0, P[], (!!seq)::- [{Mike: great}]\\n- [{Bob: great}]\\n\",\n\t\t},\n\t},\n}\n\nfunc TestCreateMapOperatorScenarios(t *testing.T) {\n\tfor _, tt := range createMapOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n}\n" }, { "path": "pkg/yqlib/operator_datetime.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"errors\"\n\t\"fmt\"\n\t\"strconv\"\n\t\"time\"\n)\n\nfunc getStringParameter(parameterName string, d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (string, error) {\n\tresult, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode)\n\n\tif err != nil {\n\t\treturn \"\", err\n\t} else if result.MatchingNodes.Len() == 0 {\n\t\treturn \"\", fmt.Errorf(\"could not find %v for format_time\", parameterName)\n\t}\n\n\treturn result.MatchingNodes.Front().Value.(*CandidateNode).Value, nil\n}\n\nfunc withDateTimeFormat(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tif expressionNode.RHS.Operation.OperationType == blockOpType || expressionNode.RHS.Operation.OperationType == unionOpType {\n\t\tlayout, err := getStringParameter(\"layout\", d, context, expressionNode.RHS.LHS)\n\t\tif err != nil {\n\t\t\treturn Context{}, fmt.Errorf(\"could not get date time format: %w\", err)\n\t\t}\n\t\tcontext.SetDateTimeLayout(layout)\n\t\treturn d.GetMatchingNodes(context, expressionNode.RHS.RHS)\n\n\t}\n\treturn Context{}, errors.New(`must provide a date time format string and an expression, e.g. with_dtf(\"Monday, 02-Jan-06 at 3:04PM MST\"; )`)\n\n}\n\n// for unit tests\nvar Now = time.Now\n\nfunc nowOp(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\n\tnode := &CandidateNode{\n\t\tTag: \"!!timestamp\",\n\t\tKind: ScalarNode,\n\t\tValue: Now().Format(time.RFC3339),\n\t}\n\n\treturn context.SingleChildContext(node), nil\n\n}\n\nfunc parseDateTime(layout string, datestring string) (time.Time, error) {\n\n\tparsedTime, err := time.Parse(layout, datestring)\n\tif err != nil && layout == time.RFC3339 {\n\t\t// try parsing the date time with only the date\n\t\treturn time.Parse(\"2006-01-02\", datestring)\n\t}\n\treturn parsedTime, err\n\n}\n\nfunc formatDateTime(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tformat, err := getStringParameter(\"format\", d, context, expressionNode.RHS)\n\tlayout := context.GetDateTimeLayout()\n\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\tparsedTime, err := parseDateTime(layout, candidate.Value)\n\t\tif err != nil {\n\t\t\treturn Context{}, fmt.Errorf(\"could not parse datetime of [%v]: %w\", candidate.GetNicePath(), err)\n\t\t}\n\t\tformattedTimeStr := parsedTime.Format(format)\n\n\t\tnode, errorReading := parseSnippet(formattedTimeStr)\n\t\tif errorReading != nil {\n\t\t\tlog.Debugf(\"could not parse %v - lets just leave it as a string: %w\", formattedTimeStr, errorReading)\n\t\t\tnode = &CandidateNode{\n\t\t\t\tKind: ScalarNode,\n\t\t\t\tTag: \"!!str\",\n\t\t\t\tValue: formattedTimeStr,\n\t\t\t}\n\t\t}\n\t\tnode.Parent = candidate.Parent\n\t\tnode.Key = candidate.Key\n\t\tresults.PushBack(node)\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n\nfunc tzOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\ttimezoneStr, err := getStringParameter(\"timezone\", d, context, expressionNode.RHS)\n\tlayout := context.GetDateTimeLayout()\n\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\tvar results = list.New()\n\n\ttimezone, err := time.LoadLocation(timezoneStr)\n\tif err != nil {\n\t\treturn Context{}, fmt.Errorf(\"could not load tz [%v]: %w\", timezoneStr, err)\n\t}\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\tparsedTime, err := parseDateTime(layout, candidate.Value)\n\t\tif err != nil {\n\t\t\treturn Context{}, fmt.Errorf(\"could not parse datetime of [%v] using layout [%v]: %w\", candidate.GetNicePath(), layout, err)\n\t\t}\n\t\ttzTime := parsedTime.In(timezone)\n\n\t\tresults.PushBack(candidate.CreateReplacement(ScalarNode, candidate.Tag, tzTime.Format(layout)))\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n\nfunc parseUnixTime(unixTime string) (time.Time, error) {\n\tseconds, err := strconv.ParseFloat(unixTime, 64)\n\n\tif err != nil {\n\t\treturn time.Now(), err\n\t}\n\n\treturn time.UnixMilli(int64(seconds * 1000)), nil\n}\n\nfunc fromUnixOp(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\tactualTag := candidate.guessTagFromCustomType()\n\n\t\tif actualTag != \"!!int\" && actualTag != \"!!float\" {\n\t\t\treturn Context{}, fmt.Errorf(\"from_unix only works on numbers, found %v instead\", candidate.Tag)\n\t\t}\n\n\t\tparsedTime, err := parseUnixTime(candidate.Value)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t\tnode := candidate.CreateReplacement(ScalarNode, \"!!timestamp\", parsedTime.Format(time.RFC3339))\n\n\t\tresults.PushBack(node)\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n\nfunc toUnixOp(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\n\tlayout := context.GetDateTimeLayout()\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\tparsedTime, err := parseDateTime(layout, candidate.Value)\n\t\tif err != nil {\n\t\t\treturn Context{}, fmt.Errorf(\"could not parse datetime of [%v] using layout [%v]: %w\", candidate.GetNicePath(), layout, err)\n\t\t}\n\n\t\tresults.PushBack(candidate.CreateReplacement(ScalarNode, \"!!int\", fmt.Sprintf(\"%v\", parsedTime.Unix())))\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_datetime_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar dateTimeOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Format: from standard RFC3339 format\",\n\t\tsubdescription: \"Providing a single parameter assumes a standard RFC3339 datetime format. If the target format is not a valid yaml datetime format, the result will be a string tagged node.\",\n\t\tdocument: `a: 2001-12-15T02:59:43.1Z`,\n\t\texpression: `.a |= format_datetime(\"Monday, 02-Jan-06 at 3:04PM\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: Saturday, 15-Dec-01 at 2:59AM\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Format: from custom date time\",\n\t\tsubdescription: \"Use with_dtf to set a custom datetime format for parsing.\",\n\t\tdocument: `a: Saturday, 15-Dec-01 at 2:59AM`,\n\t\texpression: `.a |= with_dtf(\"Monday, 02-Jan-06 at 3:04PM\"; format_datetime(\"2006-01-02\"))`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: 2001-12-15\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Format: get the day of the week\",\n\t\tdocument: `a: 2001-12-15`,\n\t\texpression: `.a | format_datetime(\"Monday\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!str)::Saturday\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Now\",\n\t\tdocument: \"a: cool\",\n\t\texpression: `.updated = now`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: cool\\nupdated: 2021-05-19T01:02:03Z\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"From Unix\",\n\t\tsubdescription: \"Converts from unix time. Note, you don't have to pipe through the tz operator :)\",\n\t\texpression: `1675301929 | from_unix | tz(\"UTC\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!timestamp)::2023-02-02T01:38:49Z\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"To Unix\",\n\t\tsubdescription: \"Converts to unix time\",\n\t\texpression: `now | to_unix`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!int)::1621386123\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Timezone: from standard RFC3339 format\",\n\t\tsubdescription: \"Returns a new datetime in the specified timezone. Specify standard IANA Time Zone format or 'utc', 'local'. When given a single parameter, this assumes the datetime is in RFC3339 format.\",\n\n\t\tdocument: \"a: cool\",\n\t\texpression: `.updated = (now | tz(\"Australia/Sydney\"))`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: cool\\nupdated: 2021-05-19T11:02:03+10:00\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Timezone: with custom format\",\n\t\tsubdescription: \"Specify standard IANA Time Zone format or 'utc', 'local'\",\n\t\tdocument: \"a: Saturday, 15-Dec-01 at 2:59AM GMT\",\n\t\texpression: `.a |= with_dtf(\"Monday, 02-Jan-06 at 3:04PM MST\"; tz(\"Australia/Sydney\"))`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: Saturday, 15-Dec-01 at 1:59PM AEDT\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Add and tz custom format\",\n\t\tsubdescription: \"Specify standard IANA Time Zone format or 'utc', 'local'\",\n\t\tdocument: \"a: Saturday, 15-Dec-01 at 2:59AM GMT\",\n\t\texpression: `.a |= with_dtf(\"Monday, 02-Jan-06 at 3:04PM MST\"; tz(\"Australia/Sydney\"))`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: Saturday, 15-Dec-01 at 1:59PM AEDT\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Date addition\",\n\t\tdocument: `a: 2021-01-01T00:00:00Z`,\n\t\texpression: `.a += \"3h10m\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: 2021-01-01T03:10:00Z\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Date subtraction\",\n\t\tsubdescription: \"You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/datetime#date-time-formattings) for more information.\",\n\t\tdocument: `a: 2021-01-01T03:10:00Z`,\n\t\texpression: `.a -= \"3h10m\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: 2021-01-01T00:00:00Z\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Date addition - custom format\",\n\t\tdocument: `a: Saturday, 15-Dec-01 at 2:59AM GMT`,\n\t\texpression: `with_dtf(\"Monday, 02-Jan-06 at 3:04PM MST\"; .a += \"3h1m\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: Saturday, 15-Dec-01 at 6:00AM GMT\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Date script with custom format\",\n\t\tsubdescription: \"You can embed full expressions in with_dtf if needed.\",\n\t\tdocument: `a: Saturday, 15-Dec-01 at 2:59AM GMT`,\n\t\texpression: `with_dtf(\"Monday, 02-Jan-06 at 3:04PM MST\"; .a = (.a + \"3h1m\" | tz(\"Australia/Perth\")))`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: Saturday, 15-Dec-01 at 2:00PM AWST\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"allow comma\",\n\t\tskipDoc: true,\n\t\tdocument: \"a: Saturday, 15-Dec-01 at 2:59AM GMT\",\n\t\texpression: `.a |= with_dtf(\"Monday, 02-Jan-06 at 3:04PM MST\", tz(\"Australia/Sydney\"))`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: Saturday, 15-Dec-01 at 1:59PM AEDT\\n\",\n\t\t},\n\t},\n}\n\nfunc TestDatetimeOperatorScenarios(t *testing.T) {\n\tfor _, tt := range dateTimeOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"datetime\", dateTimeOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_delete.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n)\n\nfunc deleteChildOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tnodesToDelete, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS)\n\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\t//need to iterate backwards to ensure correct indices when deleting multiple\n\tfor el := nodesToDelete.MatchingNodes.Back(); el != nil; el = el.Prev() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\tif candidate.Parent == nil {\n\t\t\t// must be a top level thing, delete it\n\t\t\treturn removeFromContext(context, candidate)\n\t\t}\n\t\tlog.Debugf(\"processing deletion of candidate %v\", NodeToString(candidate))\n\n\t\tparentNode := candidate.Parent\n\n\t\tcandidatePath := candidate.GetPath()\n\t\tchildPath := candidatePath[len(candidatePath)-1]\n\n\t\tswitch parentNode.Kind {\n\t\tcase MappingNode:\n\t\t\tdeleteFromMap(candidate.Parent, childPath)\n\t\tcase SequenceNode:\n\t\t\tdeleteFromArray(candidate.Parent, childPath)\n\t\tdefault:\n\t\t\treturn Context{}, fmt.Errorf(\"cannot delete nodes from parent of tag %v\", parentNode.Tag)\n\t\t}\n\t}\n\treturn context, nil\n}\n\nfunc removeFromContext(context Context, candidate *CandidateNode) (Context, error) {\n\tnewResults := list.New()\n\tfor item := context.MatchingNodes.Front(); item != nil; item = item.Next() {\n\t\tnodeInContext := item.Value.(*CandidateNode)\n\t\tif nodeInContext != candidate {\n\t\t\tnewResults.PushBack(nodeInContext)\n\t\t} else {\n\t\t\tlog.Info(\"Need to delete this %v\", NodeToString(nodeInContext))\n\t\t}\n\t}\n\treturn context.ChildContext(newResults), nil\n}\n\nfunc deleteFromMap(node *CandidateNode, childPath interface{}) {\n\tlog.Debug(\"deleteFromMap\")\n\tcontents := node.Content\n\tnewContents := make([]*CandidateNode, 0)\n\n\tfor index := 0; index < len(contents); index = index + 2 {\n\t\tkey := contents[index]\n\t\tvalue := contents[index+1]\n\n\t\tshouldDelete := key.Value == childPath\n\n\t\tlog.Debugf(\"shouldDelete %v? %v == %v = %v\", NodeToString(value), key.Value, childPath, shouldDelete)\n\n\t\tif !shouldDelete {\n\t\t\tnewContents = append(newContents, key, value)\n\t\t}\n\t}\n\tnode.Content = newContents\n}\n\nfunc deleteFromArray(node *CandidateNode, childPath interface{}) {\n\tlog.Debug(\"deleteFromArray\")\n\tcontents := node.Content\n\tnewContents := make([]*CandidateNode, 0)\n\n\tfor index := 0; index < len(contents); index = index + 1 {\n\t\tvalue := contents[index]\n\n\t\tshouldDelete := fmt.Sprintf(\"%v\", index) == fmt.Sprintf(\"%v\", childPath)\n\n\t\tif !shouldDelete {\n\t\t\tvalue.Key.Value = fmt.Sprintf(\"%v\", len(newContents))\n\t\t\tnewContents = append(newContents, value)\n\t\t}\n\t}\n\tnode.Content = newContents\n}\n" }, { "path": "pkg/yqlib/operator_delete_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar deleteOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Delete entry in map\",\n\t\tdocument: `{a: cat, b: dog}`,\n\t\texpression: `del(.b)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: cat}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Delete nested entry in map\",\n\t\tdocument: `{a: {a1: fred, a2: frood}}`,\n\t\texpression: `del(.a.a1)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {a2: frood}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: {a1: fred, a2: frood}}`,\n\t\texpression: `.a | del(.a1)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!map)::{a2: frood}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"delete whole document\",\n\t\tdocument2: `a: slow`,\n\t\tdocument: `a: fast`,\n\t\texpression: `del(select(.a == \"fast\"))`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: slow\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a: [1,2,3]`,\n\t\texpression: `.a | del(.[1])`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!seq)::[1, 3]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[0, {a: cat, b: dog}]`,\n\t\texpression: `.[1] | del(.a)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[1], (!!map)::{b: dog}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[{a: cat, b: dog}]`,\n\t\texpression: `.[0] | del(.a)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!map)::{b: dog}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[{a: {b: thing, c: frog}}]`,\n\t\texpression: `.[0].a | del(.b)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0 a], (!!map)::{c: frog}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[{a: {b: thing, c: frog}}]`,\n\t\texpression: `.[0] | del(.a.b)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!map)::{a: {c: frog}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: [0, {b: thing, c: frog}]}`,\n\t\texpression: `.a[1] | del(.b)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a 1], (!!map)::{c: frog}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: [0, {b: thing, c: frog}]}`,\n\t\texpression: `.a | del(.[1].b)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!seq)::[0, {c: frog}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: {a1: fred, a2: frood}}`,\n\t\texpression: `del(.. | select(.==\"frood\"))`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {a1: fred}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Delete entry in array\",\n\t\tdocument: `[1,2,3]`,\n\t\texpression: `del(.[1])`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[1, 3]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a: [1,2,3]`,\n\t\texpression: `del(.a[])`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: []\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Delete entry appended to an array\",\n\t\tdocument: `[1,2]`,\n\t\texpression: `. += [3] | del(.[2])`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[1, 2]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Delete entry after sorting an array\",\n\t\tdocument: `[3,2,1]`,\n\t\texpression: `sort | del(.[2])`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[1, 2]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Delete entry after reversing an array\",\n\t\tdocument: `[1,2,3]`,\n\t\texpression: `reverse | del(.[2])`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[3, 2]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Delete entry after shuffling an array\",\n\t\tdocument: `[1,2,3]`,\n\t\texpression: `shuffle | del(.[2])`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[3, 1]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Delete entry from keys array\",\n\t\tdocument: `{\"a\": 1, \"b\": 2, \"c\": 3}`,\n\t\texpression: `keys | del(.[] | select(.==\"b\"))`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- \\\"a\\\"\\n- \\\"c\\\"\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Delete entry after flattening an array\",\n\t\tdocument: `[1,[2],[[3]]]`,\n\t\texpression: `flatten | del(.[2])`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[1, 2]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a: [10,x,10, 10, x, 10]`,\n\t\texpression: `del(.a[] | select(. == 10))`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: [x, x]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a: null`,\n\t\texpression: `del(..)`,\n\t\texpected: []string{},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a: {thing1: yep, thing2: cool, thing3: hi, b: {thing1: cool, great: huh}}`,\n\t\texpression: `del(..)`,\n\t\texpected: []string{},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a: {thing1: yep, thing2: cool, thing3: hi, b: {thing1: cool, great: huh}}`,\n\t\texpression: `del(.. | select(tag == \"!!map\") | (.b.thing1,.thing2))`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: {thing1: yep, thing3: hi, b: {great: huh}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Delete nested entry in array\",\n\t\tdocument: `[{a: cat, b: dog}]`,\n\t\texpression: `del(.[0].a)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[{b: dog}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Delete no matches\",\n\t\tdocument: `{a: cat, b: dog}`,\n\t\texpression: `del(.c)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: cat, b: dog}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Delete matching entries\",\n\t\tdocument: `{a: cat, b: dog, c: bat}`,\n\t\texpression: `del( .[] | select(. == \"*at\") )`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{b: dog}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Recursively delete matching keys\",\n\t\tdocument: `{a: {name: frog, b: {name: blog, age: 12}}}`,\n\t\texpression: `del(.. | select(has(\"name\")).name)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {b: {age: 12}}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Repeatedly delete the first element of a list\",\n\t\tdocument: `a: [0, 1, 2, 3]`,\n\t\texpression: `del(.a[0]) | del(.a[0])`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: [2, 3]\\n\",\n\t\t},\n\t},\n}\n\nfunc TestDeleteOperatorScenarios(t *testing.T) {\n\tfor _, tt := range deleteOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"delete\", deleteOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_divide.go", "content": "package yqlib\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n)\n\nfunc divideOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"Divide operator\")\n\n\treturn crossFunction(d, context.ReadOnlyClone(), expressionNode, divide, false)\n}\n\nfunc divide(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {\n\tif lhs.Tag == \"!!null\" {\n\t\treturn nil, fmt.Errorf(\"%v (%v) cannot be divided by %v (%v)\", lhs.Tag, lhs.GetNicePath(), rhs.Tag, rhs.GetNicePath())\n\t}\n\n\ttarget := lhs.CopyWithoutContent()\n\n\tif lhs.Kind == ScalarNode && rhs.Kind == ScalarNode {\n\t\tif err := divideScalars(target, lhs, rhs); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else {\n\t\treturn nil, fmt.Errorf(\"%v (%v) cannot be divided by %v (%v)\", lhs.Tag, lhs.GetNicePath(), rhs.Tag, rhs.GetNicePath())\n\t}\n\n\treturn target, nil\n}\n\nfunc divideScalars(target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error {\n\tlhsTag := lhs.Tag\n\trhsTag := rhs.guessTagFromCustomType()\n\tlhsIsCustom := false\n\tif !strings.HasPrefix(lhsTag, \"!!\") {\n\t\t// custom tag - we have to have a guess\n\t\tlhsTag = lhs.guessTagFromCustomType()\n\t\tlhsIsCustom = true\n\t}\n\n\tif lhsTag == \"!!str\" && rhsTag == \"!!str\" {\n\t\ttKind, tTag, res := split(lhs.Value, rhs.Value)\n\t\ttarget.Kind = tKind\n\t\ttarget.Tag = tTag\n\t\ttarget.AddChildren(res)\n\t} else if (lhsTag == \"!!int\" || lhsTag == \"!!float\") && (rhsTag == \"!!int\" || rhsTag == \"!!float\") {\n\t\ttarget.Kind = ScalarNode\n\t\ttarget.Style = lhs.Style\n\n\t\tlhsNum, err := strconv.ParseFloat(lhs.Value, 64)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\trhsNum, err := strconv.ParseFloat(rhs.Value, 64)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tquotient := lhsNum / rhsNum\n\t\tif lhsIsCustom {\n\t\t\ttarget.Tag = lhs.Tag\n\t\t} else {\n\t\t\ttarget.Tag = \"!!float\"\n\t\t}\n\t\ttarget.Value = fmt.Sprintf(\"%v\", quotient)\n\t} else {\n\t\treturn fmt.Errorf(\"%v cannot be divided by %v\", lhsTag, rhsTag)\n\t}\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/operator_divide_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar divideOperatorScenarios = []expressionScenario{\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[{a: foo_bar, b: _}, {a: 4, b: 2}]`,\n\t\texpression: \".[] | .a / .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[0 a], (!!seq)::- foo\\n- bar\\n\",\n\t\t\t\"D0, P[1 a], (!!float)::2\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{}`,\n\t\texpression: \"(.a / .b) as $x | .\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"String split\",\n\t\tdocument: `{a: cat_meow, b: _}`,\n\t\texpression: `.c = .a / .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: cat_meow, b: _, c: [cat, meow]}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Number division\",\n\t\tsubdescription: \"The result during division is calculated as a float\",\n\t\tdocument: `{a: 12, b: 2.5}`,\n\t\texpression: `.a = .a / .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: 4.8, b: 2.5}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Number division by zero\",\n\t\tsubdescription: \"Dividing by zero results in +Inf or -Inf\",\n\t\tdocument: `{a: 1, b: -1}`,\n\t\texpression: `.a = .a / 0 | .b = .b / 0`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: !!float +Inf, b: !!float -Inf}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Custom types: that are really strings\",\n\t\tdocument: \"a: !horse cat_meow\\nb: !goat _\",\n\t\texpression: `.a = .a / .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !horse\\n - cat\\n - meow\\nb: !goat _\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Custom types: that are really numbers\",\n\t\tdocument: \"a: !horse 1.2\\nb: !goat 2.3\",\n\t\texpression: `.a = .a / .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !horse 0.5217391304347826\\nb: !goat 2.3\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: 2\\nb: !goat 2.3\",\n\t\texpression: `.a = .a / .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: 0.8695652173913044\\nb: !goat 2.3\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Custom types: that are really ints\",\n\t\tdocument: \"a: !horse 2\\nb: !goat 3\",\n\t\texpression: `.a = .a / .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !horse 0.6666666666666666\\nb: !goat 3\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Keep anchors\",\n\t\tdocument: \"a: &horse [1]\",\n\t\texpression: `.a[1] = .a[0] / 2`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: &horse [1, 0.5]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Divide int by string\",\n\t\tdocument: \"a: 123\\nb: '2'\",\n\t\texpression: `.a / .b`,\n\t\texpectedError: \"!!int cannot be divided by !!str\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Divide string by int\",\n\t\tdocument: \"a: 2\\nb: '123'\",\n\t\texpression: `.b / .a`,\n\t\texpectedError: \"!!str cannot be divided by !!int\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Divide map by int\",\n\t\tdocument: \"a: {\\\"a\\\":1}\\nb: 2\",\n\t\texpression: `.a / .b`,\n\t\texpectedError: \"!!map (a) cannot be divided by !!int (b)\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Divide array by str\",\n\t\tdocument: \"a: [1,2]\\nb: '2'\",\n\t\texpression: `.a / .b`,\n\t\texpectedError: \"!!seq (a) cannot be divided by !!str (b)\",\n\t},\n}\n\nfunc TestDivideOperatorScenarios(t *testing.T) {\n\tfor _, tt := range divideOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"divide\", divideOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_document_index.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n)\n\nfunc getDocumentIndexOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tscalar := candidate.CreateReplacement(ScalarNode, \"!!int\", fmt.Sprintf(\"%v\", candidate.GetDocument()))\n\t\tresults.PushBack(scalar)\n\t}\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_document_index_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar documentIndexScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Retrieve a document index\",\n\t\tdocument: \"a: cat\\n---\\na: frog\\n\",\n\t\texpression: `.a | document_index`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!int)::0\\n\",\n\t\t\t\"D1, P[a], (!!int)::1\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Retrieve a document index, shorthand\",\n\t\tdocument: \"a: cat\\n---\\na: frog\\n\",\n\t\texpression: `.a | di`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!int)::0\\n\",\n\t\t\t\"D1, P[a], (!!int)::1\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Filter by document index\",\n\t\tdocument: \"a: cat\\n---\\na: frog\\n\",\n\t\texpression: `select(document_index == 1)`,\n\t\texpected: []string{\n\t\t\t\"D1, P[], (!!map)::a: frog\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Filter by document index shorthand\",\n\t\tdocument: \"a: cat\\n---\\na: frog\\n\",\n\t\texpression: `select(di == 1)`,\n\t\texpected: []string{\n\t\t\t\"D1, P[], (!!map)::a: frog\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Print Document Index with matches\",\n\t\tdocument: \"a: cat\\n---\\na: frog\\n\",\n\t\texpression: `.a | ({\"match\": ., \"doc\": document_index})`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::match: cat\\ndoc: 0\\n\",\n\t\t\t\"D1, P[], (!!map)::match: frog\\ndoc: 1\\n\",\n\t\t},\n\t},\n}\n\nfunc TestDocumentIndexScenarios(t *testing.T) {\n\tfor _, tt := range documentIndexScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"document-index\", documentIndexScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_encoder_decoder.go", "content": "package yqlib\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"container/list\"\n\t\"errors\"\n\t\"regexp\"\n\t\"strings\"\n)\n\nfunc configureEncoder(format *Format, indent int) Encoder {\n\n\tswitch format {\n\tcase JSONFormat:\n\t\tprefs := ConfiguredJSONPreferences.Copy()\n\t\tprefs.Indent = indent\n\t\tprefs.ColorsEnabled = false\n\t\tprefs.UnwrapScalar = false\n\t\treturn NewJSONEncoder(prefs)\n\tcase YamlFormat:\n\t\tvar prefs = ConfiguredYamlPreferences.Copy()\n\t\tprefs.Indent = indent\n\t\tprefs.ColorsEnabled = false\n\t\treturn NewYamlEncoder(prefs)\n\tcase XMLFormat:\n\t\tvar xmlPrefs = ConfiguredXMLPreferences.Copy()\n\t\txmlPrefs.Indent = indent\n\t\treturn NewXMLEncoder(xmlPrefs)\n\t}\n\treturn format.EncoderFactory()\n}\n\nfunc encodeToString(candidate *CandidateNode, prefs encoderPreferences) (string, error) {\n\tvar output bytes.Buffer\n\tlog.Debug(\"printing with indent: %v\", prefs.indent)\n\n\tencoder := configureEncoder(prefs.format, prefs.indent)\n\tif encoder == nil {\n\t\treturn \"\", errors.New(\"no support for output format\")\n\t}\n\n\tprinter := NewPrinter(encoder, NewSinglePrinterWriter(bufio.NewWriter(&output)))\n\terr := printer.PrintResults(candidate.AsList())\n\treturn output.String(), err\n}\n\ntype encoderPreferences struct {\n\tformat *Format\n\tindent int\n}\n\n/* encodes object as yaml string */\nvar chomper = regexp.MustCompile(\"\\n+$\")\n\nfunc encodeOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tpreferences := expressionNode.Operation.Preferences.(encoderPreferences)\n\tvar results = list.New()\n\n\thasOnlyOneNewLine := regexp.MustCompile(\"[^\\n].*\\n$\")\n\tendWithNewLine := regexp.MustCompile(\".*\\n$\")\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tstringValue, err := encodeToString(candidate, preferences)\n\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t\t// remove trailing newlines if needed.\n\t\t// check if we originally decoded this path, and the original thing had a single line.\n\t\toriginalList := context.GetVariable(\"decoded: \" + candidate.GetKey())\n\t\tif originalList != nil && originalList.Len() > 0 && hasOnlyOneNewLine.MatchString(stringValue) {\n\n\t\t\toriginal := originalList.Front().Value.(*CandidateNode)\n\t\t\t// original block did not have a newline at the end, get rid of this one too\n\t\t\tif !endWithNewLine.MatchString(original.Value) {\n\t\t\t\tstringValue = chomper.ReplaceAllString(stringValue, \"\")\n\t\t\t}\n\t\t}\n\n\t\t// dont print a newline when printing json on a single line.\n\t\tif (preferences.format == JSONFormat && preferences.indent == 0) ||\n\t\t\tpreferences.format == CSVFormat ||\n\t\t\tpreferences.format == TSVFormat {\n\t\t\tstringValue = chomper.ReplaceAllString(stringValue, \"\")\n\t\t}\n\n\t\tresults.PushBack(candidate.CreateReplacement(ScalarNode, \"!!str\", stringValue))\n\t}\n\treturn context.ChildContext(results), nil\n}\n\ntype decoderPreferences struct {\n\tformat *Format\n}\n\n/* takes a string and decodes it back into an object */\nfunc decodeOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\n\tpreferences := expressionNode.Operation.Preferences.(decoderPreferences)\n\n\tdecoder := preferences.format.DecoderFactory()\n\tif decoder == nil {\n\t\treturn Context{}, errors.New(\"no support for input format\")\n\t}\n\n\tvar results = list.New()\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\tcontext.SetVariable(\"decoded: \"+candidate.GetKey(), candidate.AsList())\n\n\t\tlog.Debugf(\"got: [%v]\", candidate.Value)\n\n\t\terr := decoder.Init(strings.NewReader(candidate.Value))\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t\tnode, errorReading := decoder.Decode()\n\t\tif errorReading != nil {\n\t\t\treturn Context{}, errorReading\n\t\t}\n\t\tnode.Key = candidate.Key\n\t\tnode.Parent = candidate.Parent\n\n\t\tresults.PushBack(node)\n\t}\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_encoder_decoder_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar prefix = \"D0, P[], (!!map)::a:\\n cool:\\n bob: dylan\\n\"\n\nvar encoderDecoderOperatorScenarios = []expressionScenario{\n\t{\n\t\trequiresFormat: \"json\",\n\t\tdescription: \"Encode value as json string\",\n\t\tdocument: `{a: {cool: \"thing\"}}`,\n\t\texpression: `.b = (.a | to_json)`,\n\t\texpected: []string{\n\t\t\t`D0, P[], (!!map)::{a: {cool: \"thing\"}, b: \"{\\n \\\"cool\\\": \\\"thing\\\"\\n}\\n\"}\n`,\n\t\t},\n\t},\n\t{\n\t\trequiresFormat: \"json\",\n\t\tdescription: \"Encode value as json string, on one line\",\n\t\tsubdescription: \"Pass in a 0 indent to print json on a single line.\",\n\t\tdocument: `{a: {cool: \"thing\"}}`,\n\t\texpression: `.b = (.a | to_json(0))`,\n\t\texpected: []string{\n\t\t\t`D0, P[], (!!map)::{a: {cool: \"thing\"}, b: '{\"cool\":\"thing\"}'}\n`,\n\t\t},\n\t},\n\t{\n\t\trequiresFormat: \"json\",\n\t\tdescription: \"Encode value as json string, on one line shorthand\",\n\t\tsubdescription: \"Pass in a 0 indent to print json on a single line.\",\n\t\tdocument: `{a: {cool: \"thing\"}}`,\n\t\texpression: `.b = (.a | @json)`,\n\t\texpected: []string{\n\t\t\t`D0, P[], (!!map)::{a: {cool: \"thing\"}, b: '{\"cool\":\"thing\"}'}\n`,\n\t\t},\n\t},\n\t{\n\t\trequiresFormat: \"json\",\n\t\tdescription: \"Decode a json encoded string\",\n\t\tsubdescription: \"Keep in mind JSON is a subset of YAML. If you want idiomatic yaml, pipe through the style operator to clear out the JSON styling.\",\n\t\tdocument: `a: '{\"cool\":\"thing\"}'`,\n\t\texpression: `.a | from_json | ... style=\"\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!map)::cool: thing\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: {cool: \"thing\"}}`,\n\t\texpression: `.b = (.a | to_props)`,\n\t\texpected: []string{\n\t\t\t`D0, P[], (!!map)::{a: {cool: \"thing\"}, b: \"cool = thing\\n\"}\n`,\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Encode value as props string\",\n\t\tdocument: `{a: {cool: \"thing\"}}`,\n\t\texpression: `.b = (.a | @props)`,\n\t\texpected: []string{\n\t\t\t`D0, P[], (!!map)::{a: {cool: \"thing\"}, b: \"cool = thing\\n\"}\n`,\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Decode props encoded string\",\n\t\tdocument: `a: \"cats=great\\ndogs=cool as well\"`,\n\t\texpression: `.a |= @propsd`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a:\\n cats: great\\n dogs: cool as well\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Decode csv encoded string\",\n\t\tdocument: `a: \"cats,dogs\\ngreat,cool as well\"`,\n\t\texpression: `.a |= @csvd`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a:\\n - cats: great\\n dogs: cool as well\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Decode tsv encoded string\",\n\t\tdocument: `a: \"cats\tdogs\\ngreat\tcool as well\"`,\n\t\texpression: `.a |= @tsvd`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a:\\n - cats: great\\n dogs: cool as well\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a:\\n cool:\\n bob: dylan\",\n\t\texpression: `.b = (.a | @yaml)`,\n\t\texpected: []string{\n\t\t\tprefix + \"b: |\\n cool:\\n bob: dylan\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Encode value as yaml string\",\n\t\tsubdescription: \"Indent defaults to 2\",\n\t\tdocument: \"a:\\n cool:\\n bob: dylan\",\n\t\texpression: `.b = (.a | to_yaml)`,\n\t\texpected: []string{\n\t\t\tprefix + \"b: |\\n cool:\\n bob: dylan\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Encode value as yaml string, with custom indentation\",\n\t\tsubdescription: \"You can specify the indentation level as the first parameter.\",\n\t\tdocument: \"a:\\n cool:\\n bob: dylan\",\n\t\texpression: `.b = (.a | to_yaml(8))`,\n\t\texpected: []string{\n\t\t\tprefix + \"b: |\\n cool:\\n bob: dylan\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: {cool: \"thing\"}}`,\n\t\texpression: `.b = (.a | to_yaml)`,\n\t\texpected: []string{\n\t\t\t`D0, P[], (!!map)::{a: {cool: \"thing\"}, b: \"{cool: \\\"thing\\\"}\\n\"}\n`,\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Decode a yaml encoded string\",\n\t\tdocument: `a: \"foo: bar\"`,\n\t\texpression: `.b = (.a | from_yaml)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: \\\"foo: bar\\\"\\nb:\\n foo: bar\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Update a multiline encoded yaml string\",\n\t\tdontFormatInputForDoc: true,\n\t\tdocument: \"a: |\\n foo: bar\\n baz: dog\\n\",\n\t\texpression: `.a |= (from_yaml | .foo = \"cat\" | to_yaml)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: |\\n foo: cat\\n baz: dog\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdontFormatInputForDoc: true,\n\t\tdocument: \"a: |-\\n foo: bar\\n baz: dog\\n\",\n\t\texpression: `.a |= (from_yaml | .foo = \"cat\" | to_yaml)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: |-\\n foo: cat\\n baz: dog\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Update a single line encoded yaml string\",\n\t\tdontFormatInputForDoc: true,\n\t\tdocument: \"a: 'foo: bar'\",\n\t\texpression: `.a |= (from_yaml | .foo = \"cat\" | to_yaml)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: 'foo: cat'\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Encode array of scalars as csv string\",\n\t\tsubdescription: \"Scalars are strings, numbers and booleans.\",\n\t\tdocument: `[cat, \"thing1,thing2\", true, 3.40]`,\n\t\texpression: `@csv`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::cat,\\\"thing1,thing2\\\",true,3.40\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Encode array of arrays as csv string\",\n\t\tdocument: `[[cat, \"thing1,thing2\", true, 3.40], [dog, thing3, false, 12]]`,\n\t\texpression: `@csv`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::cat,\\\"thing1,thing2\\\",true,3.40\\ndog,thing3,false,12\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Encode array of arrays as tsv string\",\n\t\tsubdescription: \"Scalars are strings, numbers and booleans.\",\n\t\tdocument: `[[cat, \"thing1,thing2\", true, 3.40], [dog, thing3, false, 12]]`,\n\t\texpression: `@tsv`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::cat\\tthing1,thing2\\ttrue\\t3.40\\ndog\\tthing3\\tfalse\\t12\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdontFormatInputForDoc: true,\n\t\tdocument: \"a: \\\"foo: bar\\\"\",\n\t\texpression: `.a |= (from_yaml | .foo = {\"a\": \"frog\"} | to_yaml)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: \\\"foo:\\\\n a: frog\\\"\\n\",\n\t\t},\n\t},\n\t{\n\t\trequiresFormat: \"xml\",\n\t\tdescription: \"Encode value as xml string\",\n\t\tdocument: `{a: {cool: {foo: \"bar\", +@id: hi}}}`,\n\t\texpression: `.a | to_xml`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!str)::\\n bar\\n\\n\\n\",\n\t\t},\n\t},\n\t{\n\t\trequiresFormat: \"xml\",\n\t\tdescription: \"Encode value as xml string on a single line\",\n\t\tdocument: `{a: {cool: {foo: \"bar\", +@id: hi}}}`,\n\t\texpression: `.a | @xml`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!str)::bar\\n\\n\",\n\t\t},\n\t},\n\t{\n\t\trequiresFormat: \"xml\",\n\t\tdescription: \"Encode value as xml string with custom indentation\",\n\t\tdocument: `{a: {cool: {foo: \"bar\", +@id: hi}}}`,\n\t\texpression: `{\"cat\": .a | to_xml(1)}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::cat: |\\n \\n bar\\n \\n\",\n\t\t},\n\t},\n\t{\n\t\trequiresFormat: \"xml\",\n\t\tdescription: \"Decode a xml encoded string\",\n\t\tdocument: `a: \"bar\"`,\n\t\texpression: `.b = (.a | from_xml)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: \\\"bar\\\"\\nb:\\n foo: bar\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Encode a string to base64\",\n\t\tdocument: \"coolData: a special string\",\n\t\texpression: \".coolData | @base64\",\n\t\texpected: []string{\n\t\t\t\"D0, P[coolData], (!!str)::YSBzcGVjaWFsIHN0cmluZw==\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Encode a yaml document to base64\",\n\t\tsubdescription: \"Pipe through @yaml first to convert to a string, then use @base64 to encode it.\",\n\t\tdocument: \"a: apple\",\n\t\texpression: \"@yaml | @base64\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::YTogYXBwbGUK\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Encode a string to uri\",\n\t\tdocument: \"coolData: this has & special () characters *\",\n\t\texpression: \".coolData | @uri\",\n\t\texpected: []string{\n\t\t\t\"D0, P[coolData], (!!str)::this+has+%26+special+%28%29+characters+%2A\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Decode a URI to a string\",\n\t\tdocument: \"this+has+%26+special+%28%29+characters+%2A\",\n\t\texpression: \"@urid\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::this has & special () characters *\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Encode a string to sh\",\n\t\tsubdescription: \"Sh/Bash friendly string\",\n\t\tdocument: \"coolData: strings with spaces and a 'quote'\",\n\t\texpression: \".coolData | @sh\",\n\t\texpected: []string{\n\t\t\t\"D0, P[coolData], (!!str)::strings' with spaces and a '\\\\'quote\\\\'\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Encode a string to sh\",\n\t\tsubdescription: \"Watch out for stray '' (empty strings)\",\n\t\tdocument: \"coolData: \\\"'starts, contains more '' and ends with a quote'\\\"\",\n\t\texpression: \".coolData | @sh\",\n\t\texpected: []string{\n\t\t\t\"D0, P[coolData], (!!str)::\\\\'starts,' contains more '\\\\'\\\\'' and ends with a quote'\\\\'\\n\",\n\t\t},\n\t\tskipDoc: true,\n\t},\n\t{\n\t\tdescription: \"Decode a base64 encoded string\",\n\t\tsubdescription: \"Decoded data is assumed to be a string.\",\n\t\tdocument: \"coolData: V29ya3Mgd2l0aCBVVEYtMTYg8J+Yig==\",\n\t\texpression: \".coolData | @base64d\",\n\t\texpected: []string{\n\t\t\t\"D0, P[coolData], (!!str)::Works with UTF-16 😊\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Decode a base64 encoded yaml document\",\n\t\tsubdescription: \"Pipe through `from_yaml` to parse the decoded base64 string as a yaml document.\",\n\t\tdocument: \"coolData: YTogYXBwbGUK\",\n\t\texpression: \".coolData |= (@base64d | from_yaml)\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::coolData:\\n a: apple\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"empty base64 decode\",\n\t\tskipDoc: true,\n\t\texpression: `\"\" | @base64d`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"base64 missing padding test\",\n\t\tskipDoc: true,\n\t\texpression: `\"Y2F0cw\" | @base64d`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::cats\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"base64 missing padding test\",\n\t\tskipDoc: true,\n\t\texpression: `\"cats\" | @base64 | @base64d`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::cats\\n\",\n\t\t},\n\t},\n\t{\n\t\trequiresFormat: \"xml\",\n\t\tdescription: \"empty xml decode\",\n\t\tskipDoc: true,\n\t\texpression: `\"\" | @xmld`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!null)::\\n\",\n\t\t},\n\t},\n}\n\nfunc TestEncoderDecoderOperatorScenarios(t *testing.T) {\n\tfor _, tt := range encoderDecoderOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"encode-decode\", encoderDecoderOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_entries.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n)\n\nfunc entrySeqFor(key *CandidateNode, value *CandidateNode) *CandidateNode {\n\tvar keyKey = &CandidateNode{Kind: ScalarNode, Tag: \"!!str\", Value: \"key\"}\n\tvar valueKey = &CandidateNode{Kind: ScalarNode, Tag: \"!!str\", Value: \"value\"}\n\tcandidate := &CandidateNode{Kind: MappingNode, Tag: \"!!map\"}\n\tcandidate.AddKeyValueChild(keyKey, key)\n\tcandidate.AddKeyValueChild(valueKey, value)\n\treturn candidate\n}\n\nfunc toEntriesFromMap(candidateNode *CandidateNode) *CandidateNode {\n\tvar sequence = candidateNode.CreateReplacementWithComments(SequenceNode, \"!!seq\", 0)\n\n\tvar contents = candidateNode.Content\n\tfor index := 0; index < len(contents); index = index + 2 {\n\t\tkey := contents[index]\n\t\tvalue := contents[index+1]\n\n\t\tsequence.AddChild(entrySeqFor(key, value))\n\t}\n\treturn sequence\n}\n\nfunc toEntriesfromSeq(candidateNode *CandidateNode) *CandidateNode {\n\tvar sequence = candidateNode.CreateReplacementWithComments(SequenceNode, \"!!seq\", 0)\n\n\tvar contents = candidateNode.Content\n\tfor index := 0; index < len(contents); index = index + 1 {\n\t\tkey := &CandidateNode{Kind: ScalarNode, Tag: \"!!int\", Value: fmt.Sprintf(\"%v\", index)}\n\t\tvalue := contents[index]\n\n\t\tsequence.AddChild(entrySeqFor(key, value))\n\t}\n\treturn sequence\n}\n\nfunc toEntriesOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tvar results = list.New()\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\tswitch candidate.Kind {\n\t\tcase MappingNode:\n\t\t\tresults.PushBack(toEntriesFromMap(candidate))\n\n\t\tcase SequenceNode:\n\t\t\tresults.PushBack(toEntriesfromSeq(candidate))\n\t\tdefault:\n\t\t\tif candidate.Tag != \"!!null\" {\n\t\t\t\treturn Context{}, fmt.Errorf(\"%v has no keys\", candidate.Tag)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n\nfunc parseEntry(candidateNode *CandidateNode, position int) (*CandidateNode, *CandidateNode, error) {\n\tprefs := traversePreferences{DontAutoCreate: true}\n\n\tkeyResults, err := traverseMap(Context{}, candidateNode, createStringScalarNode(\"key\"), prefs, false)\n\n\tif err != nil {\n\t\treturn nil, nil, err\n\t} else if keyResults.Len() != 1 {\n\t\treturn nil, nil, fmt.Errorf(\"expected to find one 'key' entry but found %v in position %v\", keyResults.Len(), position)\n\t}\n\n\tvalueResults, err := traverseMap(Context{}, candidateNode, createStringScalarNode(\"value\"), prefs, false)\n\n\tif err != nil {\n\t\treturn nil, nil, err\n\t} else if valueResults.Len() != 1 {\n\t\treturn nil, nil, fmt.Errorf(\"expected to find one 'value' entry but found %v in position %v\", valueResults.Len(), position)\n\t}\n\n\treturn keyResults.Front().Value.(*CandidateNode), valueResults.Front().Value.(*CandidateNode), nil\n\n}\n\nfunc fromEntries(candidateNode *CandidateNode) (*CandidateNode, error) {\n\tvar node = candidateNode.CopyWithoutContent()\n\n\tvar contents = candidateNode.Content\n\n\tfor index := 0; index < len(contents); index = index + 1 {\n\t\tkey, value, err := parseEntry(contents[index], index)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tnode.AddKeyValueChild(key, value)\n\t}\n\tnode.Kind = MappingNode\n\tnode.Tag = \"!!map\"\n\treturn node, nil\n}\n\nfunc fromEntriesOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tvar results = list.New()\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\tswitch candidate.Kind {\n\t\tcase SequenceNode:\n\t\t\tmapResult, err := fromEntries(candidate)\n\t\t\tif err != nil {\n\t\t\t\treturn Context{}, err\n\t\t\t}\n\t\t\tresults.PushBack(mapResult)\n\t\tdefault:\n\t\t\treturn Context{}, fmt.Errorf(\"from entries only runs against arrays\")\n\t\t}\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n\nfunc withEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\n\t//to_entries on the context\n\ttoEntries, err := toEntriesOperator(d, context, expressionNode)\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\n\tvar results = list.New()\n\n\tfor el := toEntries.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\t//run expression against entries\n\t\t// splat toEntries and pipe it into Rhs\n\t\tsplatted, err := splat(context.SingleChildContext(candidate), traversePreferences{})\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t\tnewResults := list.New()\n\n\t\tfor itemEl := splatted.MatchingNodes.Front(); itemEl != nil; itemEl = itemEl.Next() {\n\t\t\tresult, err := d.GetMatchingNodes(splatted.SingleChildContext(itemEl.Value.(*CandidateNode)), expressionNode.RHS)\n\t\t\tif err != nil {\n\t\t\t\treturn Context{}, err\n\t\t\t}\n\t\t\tnewResults.PushBackList(result.MatchingNodes)\n\t\t}\n\n\t\tselfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}\n\t\tcollected, err := collectTogether(d, splatted.ChildContext(newResults), selfExpression)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\tlog.Debug(\"candidate %v\", NodeToString(candidate))\n\t\tlog.Debug(\"candidate leading content: %v\", candidate.LeadingContent)\n\t\tcollected.LeadingContent = candidate.LeadingContent\n\t\tlog.Debug(\"candidate FootComment: [%v]\", candidate.FootComment)\n\n\t\tcollected.HeadComment = candidate.HeadComment\n\t\tcollected.FootComment = candidate.FootComment\n\n\t\tlog.Debugf(\"collected %v\", collected.LeadingContent)\n\n\t\tfromEntries, err := fromEntriesOperator(d, context.SingleChildContext(collected), expressionNode)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\tresults.PushBackList(fromEntries.MatchingNodes)\n\n\t}\n\n\t//from_entries on the result\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_entries_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar entriesOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"to_entries splat\",\n\t\tskipDoc: true,\n\t\tdocument: `{a: 1, b: 2}`,\n\t\texpression: `to_entries[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!map)::key: a\\nvalue: 1\\n\",\n\t\t\t\"D0, P[1], (!!map)::key: b\\nvalue: 2\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"to_entries, delete key\",\n\t\tskipDoc: true,\n\t\tdocument: `{a: 1, b: 2}`,\n\t\texpression: `to_entries | map(del(.key))`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- value: 1\\n- value: 2\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"to_entries Map\",\n\t\tdocument: `{a: 1, b: 2}`,\n\t\texpression: `to_entries`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- key: a\\n value: 1\\n- key: b\\n value: 2\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"to_entries Array\",\n\t\tdocument: `[a, b]`,\n\t\texpression: `to_entries`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- key: 0\\n value: a\\n- key: 1\\n value: b\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"to_entries null\",\n\t\tdocument: `null`,\n\t\texpression: `to_entries`,\n\t\texpected: []string{},\n\t},\n\t{\n\t\tdescription: \"from_entries map\",\n\t\tdocument: `{a: 1, b: 2}`,\n\t\texpression: `to_entries | from_entries`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: 1\\nb: 2\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"from_entries with numeric key indices\",\n\t\tsubdescription: \"from_entries always creates a map, even for numeric keys\",\n\t\tdocument: `[a,b]`,\n\t\texpression: `to_entries | from_entries`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::0: a\\n1: b\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Use with_entries to update keys\",\n\t\tdocument: `{a: 1, b: 2}`,\n\t\t// expression: `to_entries | with(.[]; .key |= \"KEY_\" + .) | from_entries`,\n\t\texpression: `with_entries(.key |= \"KEY_\" + .)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::KEY_a: 1\\nKEY_b: 2\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Use with_entries to update keys recursively\",\n\t\tdocument: `{a: 1, b: {b_a: nested, b_b: thing}}`,\n\t\texpression: `(.. | select(tag==\"!!map\")) |= with_entries(.key |= \"KEY_\" + .)`,\n\t\tsubdescription: \"We use (.. | select(tag=\\\"map\\\")) to find all the maps in the doc, then |= to update each one of those maps. In the update, with_entries is used.\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{KEY_a: 1, KEY_b: {KEY_b_a: nested, KEY_b_b: thing}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Use with_entries to update keys comment\",\n\t\tdocument: `{a: 1, b: 2}`,\n\t\texpression: `with_entries(.key headComment= .value)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::# 1\\na: 1\\n# 2\\nb: 2\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Custom sort map keys\",\n\t\tsubdescription: \"Use to_entries to convert to an array of key/value pairs, sort the array using sort/sort_by/etc, and convert it back.\",\n\t\tdocument: `{a: 1, c: 3, b: 2}`,\n\t\texpression: `to_entries | sort_by(.key) | reverse | from_entries`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::c: 3\\nb: 2\\na: 1\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: 1, b: 2}`,\n\t\tdocument2: `{c: 1, d: 2}`,\n\t\texpression: `with_entries(.key |= \"KEY_\" + .)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::KEY_a: 1\\nKEY_b: 2\\n\",\n\t\t\t\"D0, P[], (!!map)::KEY_c: 1\\nKEY_d: 2\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[{a: 1, b: 2}, {c: 1, d: 2}]`,\n\t\texpression: `.[] | with_entries(.key |= \"KEY_\" + .)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::KEY_a: 1\\nKEY_b: 2\\n\",\n\t\t\t\"D0, P[], (!!map)::KEY_c: 1\\nKEY_d: 2\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Use with_entries to filter the map\",\n\t\tdocument: `{a: { b: bird }, c: { d: dog }}`,\n\t\texpression: `with_entries(select(.value | has(\"b\")))`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: {b: bird}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Use with_entries to filter the map; head comment\",\n\t\tskipDoc: true,\n\t\tdocument: \"# abc\\n{a: { b: bird }, c: { d: dog }}\\n# xyz\",\n\t\texpression: `with_entries(select(.value | has(\"b\")))`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::# abc\\na: {b: bird}\\n# xyz\\n\",\n\t\t},\n\t},\n}\n\nfunc TestEntriesOperatorScenarios(t *testing.T) {\n\tfor _, tt := range entriesOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"entries\", entriesOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_env.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n\t\"os\"\n\t\"strings\"\n\n\tparse \"github.com/a8m/envsubst/parse\"\n)\n\ntype envOpPreferences struct {\n\tStringValue bool\n\tNoUnset bool\n\tNoEmpty bool\n\tFailFast bool\n}\n\nfunc envOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tif ConfiguredSecurityPreferences.DisableEnvOps {\n\t\treturn Context{}, fmt.Errorf(\"env operations have been disabled\")\n\t}\n\tenvName := expressionNode.Operation.CandidateNode.Value\n\tlog.Debug(\"EnvOperator, env name:\", envName)\n\n\trawValue := os.Getenv(envName)\n\n\tpreferences := expressionNode.Operation.Preferences.(envOpPreferences)\n\n\tvar node *CandidateNode\n\tif preferences.StringValue {\n\t\tnode = &CandidateNode{\n\t\t\tKind: ScalarNode,\n\t\t\tTag: \"!!str\",\n\t\t\tValue: rawValue,\n\t\t}\n\t} else if rawValue == \"\" {\n\t\treturn Context{}, fmt.Errorf(\"value for env variable '%v' not provided in env()\", envName)\n\t} else {\n\t\tdecoder := NewYamlDecoder(ConfiguredYamlPreferences)\n\t\tif err := decoder.Init(strings.NewReader(rawValue)); err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\tvar err error\n\t\tnode, err = decoder.Decode()\n\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t}\n\tlog.Debug(\"ENV tag\", node.Tag)\n\tlog.Debug(\"ENV value\", node.Value)\n\tlog.Debug(\"ENV Kind\", node.Kind)\n\n\treturn context.SingleChildContext(node), nil\n}\n\nfunc envsubstOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tif ConfiguredSecurityPreferences.DisableEnvOps {\n\t\treturn Context{}, fmt.Errorf(\"env operations have been disabled\")\n\t}\n\tvar results = list.New()\n\tpreferences := envOpPreferences{}\n\tif expressionNode.Operation.Preferences != nil {\n\t\tpreferences = expressionNode.Operation.Preferences.(envOpPreferences)\n\t}\n\n\tparser := parse.New(\"string\", os.Environ(),\n\t\t&parse.Restrictions{NoUnset: preferences.NoUnset, NoEmpty: preferences.NoEmpty})\n\n\tif preferences.FailFast {\n\t\tparser.Mode = parse.Quick\n\t} else {\n\t\tparser.Mode = parse.AllErrors\n\t}\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tnode := el.Value.(*CandidateNode)\n\t\tif node.Tag != \"!!str\" {\n\t\t\tlog.Warning(\"EnvSubstOperator, env name:\", node.Tag, node.Value)\n\t\t\treturn Context{}, fmt.Errorf(\"cannot substitute with %v, can only substitute strings. Hint: Most often you'll want to use '|=' over '=' for this operation\", node.Tag)\n\t\t}\n\n\t\tvalue, err := parser.Parse(node.Value)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\tresult := node.CreateReplacement(ScalarNode, \"!!str\", value)\n\t\tresults.PushBack(result)\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_env_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar envOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Read string environment variable\",\n\t\tskipDoc: true,\n\t\tenvironmentVariables: map[string]string{\"myenv\": \"[cat,dog]\"},\n\t\texpression: `env(myenv)[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!str)::cat\\n\",\n\t\t\t\"D0, P[1], (!!str)::dog\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Read string environment variable\",\n\t\tenvironmentVariables: map[string]string{\"myenv\": \"cat meow\"},\n\t\texpression: `.a = env(myenv)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], ()::a: cat meow\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Read boolean environment variable\",\n\t\tenvironmentVariables: map[string]string{\"myenv\": \"true\"},\n\t\texpression: `.a = env(myenv)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], ()::a: true\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Read numeric environment variable\",\n\t\tenvironmentVariables: map[string]string{\"myenv\": \"12\"},\n\t\texpression: `.a = env(myenv)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], ()::a: 12\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Read yaml environment variable\",\n\t\tenvironmentVariables: map[string]string{\"myenv\": \"{b: fish}\"},\n\t\texpression: `.a = env(myenv)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], ()::a: {b: fish}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Read boolean environment variable as a string\",\n\t\tenvironmentVariables: map[string]string{\"myenv\": \"true\"},\n\t\texpression: `.a = strenv(myenv)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], ()::a: \\\"true\\\"\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Read numeric environment variable as a string\",\n\t\tenvironmentVariables: map[string]string{\"myenv\": \"12\"},\n\t\texpression: `.a = strenv(myenv)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], ()::a: \\\"12\\\"\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Dynamically update a path from an environment variable\",\n\t\tsubdescription: \"The env variable can be any valid yq expression.\",\n\t\tdocument: `{a: {b: [{name: dog}, {name: cat}]}}`,\n\t\tenvironmentVariables: map[string]string{\"pathEnv\": \".a.b[0].name\", \"valueEnv\": \"moo\"},\n\t\texpression: `eval(strenv(pathEnv)) = strenv(valueEnv)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {b: [{name: moo}, {name: cat}]}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Dynamic key lookup with environment variable\",\n\t\tenvironmentVariables: map[string]string{\"myenv\": \"cat\"},\n\t\tdocument: `{cat: meow, dog: woof}`,\n\t\texpression: `.[env(myenv)]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[cat], (!!str)::meow\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Replace strings with envsubst\",\n\t\tenvironmentVariables: map[string]string{\"myenv\": \"cat\"},\n\t\texpression: `\"the ${myenv} meows\" | envsubst`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::the cat meows\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Replace strings with envsubst, missing variables\",\n\t\texpression: `\"the ${myenvnonexisting} meows\" | envsubst`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::the meows\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Replace strings with envsubst(nu), missing variables\",\n\t\tsubdescription: \"(nu) not unset, will fail if there are unset (missing) variables\",\n\t\texpression: `\"the ${myenvnonexisting} meows\" | envsubst(nu)`,\n\t\texpectedError: \"variable ${myenvnonexisting} not set\",\n\t},\n\t{\n\t\tdescription: \"Replace strings with envsubst(ne), missing variables\",\n\t\tsubdescription: \"(ne) not empty, only validates set variables\",\n\t\texpression: `\"the ${myenvnonexisting} meows\" | envsubst(ne)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::the meows\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Replace strings with envsubst(ne), empty variable\",\n\t\tsubdescription: \"(ne) not empty, will fail if a references variable is empty\",\n\t\tenvironmentVariables: map[string]string{\"myenv\": \"\"},\n\t\texpression: `\"the ${myenv} meows\" | envsubst(ne)`,\n\t\texpectedError: \"variable ${myenv} set but empty\",\n\t},\n\t{\n\t\tdescription: \"Replace strings with envsubst, missing variables with defaults\",\n\t\texpression: `\"the ${myenvnonexisting-dog} meows\" | envsubst`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::the dog meows\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Replace strings with envsubst(nu), missing variables with defaults\",\n\t\tsubdescription: \"Having a default specified skips over the missing variable.\",\n\t\texpression: `\"the ${myenvnonexisting-dog} meows\" | envsubst(nu)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::the dog meows\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Replace strings with envsubst(ne), missing variables with defaults\",\n\t\tsubdescription: \"Fails, because the variable is explicitly set to blank.\",\n\t\tenvironmentVariables: map[string]string{\"myEmptyEnv\": \"\"},\n\t\texpression: `\"the ${myEmptyEnv-dog} meows\" | envsubst(ne)`,\n\t\texpectedError: \"variable ${myEmptyEnv} set but empty\",\n\t},\n\t{\n\t\tdescription: \"Replace string environment variable in document\",\n\t\tenvironmentVariables: map[string]string{\"myenv\": \"cat meow\"},\n\t\tdocument: \"{v: \\\"${myenv}\\\"}\",\n\t\texpression: `.v |= envsubst`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{v: \\\"cat meow\\\"}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"(Default) Return all envsubst errors\",\n\t\tsubdescription: \"By default, all errors are returned at once.\",\n\t\texpression: `\"the ${notThere} ${alsoNotThere}\" | envsubst(nu)`,\n\t\texpectedError: \"variable ${notThere} not set\\nvariable ${alsoNotThere} not set\",\n\t},\n\t{\n\t\tdescription: \"Fail fast, return the first envsubst error (and abort)\",\n\t\texpression: `\"the ${notThere} ${alsoNotThere}\" | envsubst(nu,ff)`,\n\t\texpectedError: \"variable ${notThere} not set\",\n\t},\n\t{\n\t\tdescription: \"with header/footer\",\n\t\tskipDoc: true,\n\t\tenvironmentVariables: map[string]string{\"myenv\": \"cat meow\"},\n\t\tdocument: \"# abc\\n{v: \\\"${myenv}\\\"}\\n# xyz\\n\",\n\t\texpression: `(.. | select(tag == \"!!str\")) |= envsubst`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::# abc\\n{v: \\\"cat meow\\\"}\\n# xyz\\n\",\n\t\t},\n\t},\n}\n\nfunc TestEnvOperatorScenarios(t *testing.T) {\n\tfor _, tt := range envOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"env-variable-operators\", envOperatorScenarios)\n}\n\nvar envOperatorSecurityDisabledScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"env() operation fails when security is enabled\",\n\t\tsubdescription: \"Use `--security-disable-env-ops` to disable env operations for security.\",\n\t\texpression: `env(\"MYENV\")`,\n\t\texpectedError: \"env operations have been disabled\",\n\t},\n\t{\n\t\tdescription: \"strenv() operation fails when security is enabled\",\n\t\tsubdescription: \"Use `--security-disable-env-ops` to disable env operations for security.\",\n\t\texpression: `strenv(\"MYENV\")`,\n\t\texpectedError: \"env operations have been disabled\",\n\t},\n\t{\n\t\tdescription: \"envsubst() operation fails when security is enabled\",\n\t\tsubdescription: \"Use `--security-disable-env-ops` to disable env operations for security.\",\n\t\texpression: `\"value: ${MYENV}\" | envsubst`,\n\t\texpectedError: \"env operations have been disabled\",\n\t},\n}\n\nfunc TestEnvOperatorSecurityDisabledScenarios(t *testing.T) {\n\t// Save original security preferences\n\toriginalDisableEnvOps := ConfiguredSecurityPreferences.DisableEnvOps\n\tdefer func() {\n\t\tConfiguredSecurityPreferences.DisableEnvOps = originalDisableEnvOps\n\t}()\n\n\t// Test that env() fails when DisableEnvOps is true\n\tConfiguredSecurityPreferences.DisableEnvOps = true\n\n\tfor _, tt := range envOperatorSecurityDisabledScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tappendOperatorDocumentScenario(t, \"env-variable-operators\", envOperatorSecurityDisabledScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_equals.go", "content": "package yqlib\n\nfunc equalsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"equalsOperation\")\n\treturn crossFunction(d, context, expressionNode, isEquals(false), true)\n}\n\nfunc isEquals(flip bool) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {\n\treturn func(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {\n\t\tvalue := false\n\t\tlog.Debugf(\"isEquals cross function\")\n\t\tif lhs == nil && rhs == nil {\n\t\t\tlog.Debugf(\"both are nil\")\n\t\t\towner := &CandidateNode{}\n\t\t\treturn createBooleanCandidate(owner, !flip), nil\n\t\t} else if lhs == nil {\n\t\t\tlog.Debugf(\"lhs nil, but rhs is not\")\n\t\t\tvalue := rhs.Tag == \"!!null\"\n\t\t\tif flip {\n\t\t\t\tvalue = !value\n\t\t\t}\n\t\t\treturn createBooleanCandidate(rhs, value), nil\n\t\t} else if rhs == nil {\n\t\t\tlog.Debugf(\"lhs not nil, but rhs is\")\n\t\t\tvalue := lhs.Tag == \"!!null\"\n\t\t\tif flip {\n\t\t\t\tvalue = !value\n\t\t\t}\n\t\t\treturn createBooleanCandidate(lhs, value), nil\n\t\t}\n\n\t\tif lhs.Tag == \"!!null\" {\n\t\t\tvalue = (rhs.Tag == \"!!null\")\n\t\t} else if lhs.Kind == ScalarNode && rhs.Kind == ScalarNode {\n\t\t\tvalue = matchKey(lhs.Value, rhs.Value)\n\t\t}\n\t\tlog.Debugf(\"%v == %v ? %v\", NodeToString(lhs), NodeToString(rhs), value)\n\t\tif flip {\n\t\t\tvalue = !value\n\t\t}\n\t\treturn createBooleanCandidate(lhs, value), nil\n\t}\n}\n\nfunc notEqualsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"notEqualsOperator\")\n\treturn crossFunction(d, context.ReadOnlyClone(), expressionNode, isEquals(true), true)\n}\n" }, { "path": "pkg/yqlib/operator_equals_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar equalsOperatorScenarios = []expressionScenario{\n\t{\n\t\tskipDoc: true,\n\t\texpression: \".a == .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\texpression: `(.k | length) == 0`,\n\t\tskipDoc: true,\n\t\texpected: []string{\n\t\t\t\"D0, P[k], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a: cat`,\n\t\texpression: \".a == .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a: cat`,\n\t\texpression: \".b == .a\",\n\t\texpected: []string{\n\t\t\t\"D0, P[b], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"cat\",\n\t\tdocument2: \"dog\",\n\t\texpression: \"select(fi==0) == select(fi==1)\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"{}\",\n\t\texpression: \"(.a == .b) as $x | .\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"{}\",\n\t\texpression: \".a == .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"{}\",\n\t\texpression: \"(.a != .b) as $x | .\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"{}\",\n\t\texpression: \".a != .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"{a: {b: 10}}\",\n\t\texpression: \"select(.c != null)\",\n\t\texpected: []string{},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"{a: {b: 10}}\",\n\t\texpression: \"select(.d == .c)\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {b: 10}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"{a: {b: 10}}\",\n\t\texpression: \"select(null == .c)\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {b: 10}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"{a: { b: {things: \\\"\\\"}, f: [1], g: [] }}\",\n\t\texpression: \".. | select(. == \\\"\\\")\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a b things], (!!str)::\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Match string\",\n\t\tdocument: `[cat,goat,dog]`,\n\t\texpression: `.[] | (. == \"*at\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!bool)::true\\n\",\n\t\t\t\"D0, P[1], (!!bool)::true\\n\",\n\t\t\t\"D0, P[2], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Don't match string\",\n\t\tdocument: `[cat,goat,dog]`,\n\t\texpression: `.[] | (. != \"*at\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!bool)::false\\n\",\n\t\t\t\"D0, P[1], (!!bool)::false\\n\",\n\t\t\t\"D0, P[2], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Match number\",\n\t\tdocument: `[3, 4, 5]`,\n\t\texpression: `.[] | (. == 4)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!bool)::false\\n\",\n\t\t\t\"D0, P[1], (!!bool)::true\\n\",\n\t\t\t\"D0, P[2], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Don't match number\",\n\t\tdocument: `[3, 4, 5]`,\n\t\texpression: `.[] | (. != 4)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!bool)::true\\n\",\n\t\t\t\"D0, P[1], (!!bool)::false\\n\",\n\t\t\t\"D0, P[2], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a: { cat: {b: apple, c: whatever}, pat: {b: banana} }`,\n\t\texpression: `.a | (.[].b == \"apple\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a cat b], (!!bool)::true\\n\",\n\t\t\t\"D0, P[a pat b], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: ``,\n\t\texpression: `null == null`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Match nulls\",\n\t\tdocument: ``,\n\t\texpression: `null == ~`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Non existent key doesn't equal a value\",\n\t\tdocument: \"a: frog\",\n\t\texpression: `select(.b != \"thing\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: frog\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Two non existent keys are equal\",\n\t\tdocument: \"a: frog\",\n\t\texpression: `select(.b == .c)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: frog\\n\",\n\t\t},\n\t},\n}\n\nfunc TestEqualOperatorScenarios(t *testing.T) {\n\tfor _, tt := range equalsOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"equals\", equalsOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_error.go", "content": "package yqlib\n\nimport (\n\t\"errors\"\n)\n\nfunc errorOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\n\tlog.Debugf(\"errorOperation\")\n\n\trhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS)\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\terrorMessage := \"aborted\"\n\tif rhs.MatchingNodes.Len() > 0 {\n\t\terrorMessage = rhs.MatchingNodes.Front().Value.(*CandidateNode).Value\n\t}\n\treturn Context{}, errors.New(errorMessage)\n}\n" }, { "path": "pkg/yqlib/operator_error_test.go", "content": "package yqlib\n\nimport \"testing\"\n\nconst validationExpression = `\n\twith(env(numberOfCats); select(tag == \"!!int\") or error(\"numberOfCats is not a number :(\")) | \n\t.numPets = env(numberOfCats)\n`\n\nvar errorOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Validate a particular value\",\n\t\tdocument: `a: hello`,\n\t\texpression: `select(.a == \"howdy\") or error(\".a [\" + .a + \"] is not howdy!\")`,\n\t\texpectedError: \".a [hello] is not howdy!\",\n\t},\n\t{\n\t\tdescription: \"Validate the environment variable is a number - invalid\",\n\t\tenvironmentVariables: map[string]string{\"numberOfCats\": \"please\"},\n\t\texpression: `env(numberOfCats) | select(tag == \"!!int\") or error(\"numberOfCats is not a number :(\")`,\n\t\texpectedError: \"numberOfCats is not a number :(\",\n\t},\n\t{\n\t\tdescription: \"Validate the environment variable is a number - valid\",\n\t\tsubdescription: \"`with` can be a convenient way of encapsulating validation.\",\n\t\tenvironmentVariables: map[string]string{\"numberOfCats\": \"3\"},\n\t\tdocument: \"name: Bob\\nfavouriteAnimal: cat\\n\",\n\t\texpression: validationExpression,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::name: Bob\\nfavouriteAnimal: cat\\nnumPets: 3\\n\",\n\t\t},\n\t},\n}\n\nfunc TestErrorOperatorScenarios(t *testing.T) {\n\tfor _, tt := range errorOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"error\", errorOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_eval.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n)\n\nfunc evalOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"Eval\")\n\tpathExpStrResults, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS)\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\n\texpressions := make([]*ExpressionNode, pathExpStrResults.MatchingNodes.Len())\n\texpIndex := 0\n\t//parse every expression\n\tfor pathExpStrEntry := pathExpStrResults.MatchingNodes.Front(); pathExpStrEntry != nil; pathExpStrEntry = pathExpStrEntry.Next() {\n\t\texpressionStrCandidate := pathExpStrEntry.Value.(*CandidateNode)\n\n\t\texpressions[expIndex], err = ExpressionParser.ParseExpression(expressionStrCandidate.Value)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t\texpIndex++\n\t}\n\n\tresults := list.New()\n\n\tfor matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() {\n\t\tfor expIndex = 0; expIndex < len(expressions); expIndex++ {\n\t\t\tresult, err := d.GetMatchingNodes(context, expressions[expIndex])\n\t\t\tif err != nil {\n\t\t\t\treturn Context{}, err\n\t\t\t}\n\t\t\tresults.PushBackList(result.MatchingNodes)\n\t\t}\n\t}\n\n\treturn context.ChildContext(results), nil\n\n}\n" }, { "path": "pkg/yqlib/operator_eval_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar evalOperatorScenarios = []expressionScenario{\n\n\t{\n\t\tdescription: \"Dynamically evaluate a path\",\n\t\tdocument: `{pathExp: '.a.b[] | select(.name == \"cat\")', a: {b: [{name: dog}, {name: cat}]}}`,\n\t\texpression: `eval(.pathExp)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a b 1], (!!map)::{name: cat}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"eval splat\",\n\t\tskipDoc: true,\n\t\tdocument: `{pathExp: '.a.b[] | select(.name == \"cat\")', a: {b: [{name: dog}, {name: cat}]}}`,\n\t\texpression: `eval(.pathExp)[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a b 1 name], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Dynamically update a path from an environment variable\",\n\t\tsubdescription: \"The env variable can be any valid yq expression.\",\n\t\tdocument: `{a: {b: [{name: dog}, {name: cat}]}}`,\n\t\tenvironmentVariables: map[string]string{\"pathEnv\": \".a.b[0].name\", \"valueEnv\": \"moo\"},\n\t\texpression: `eval(strenv(pathEnv)) = strenv(valueEnv)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {b: [{name: moo}, {name: cat}]}}\\n\",\n\t\t},\n\t},\n}\n\nfunc TestEvalOperatorsScenarios(t *testing.T) {\n\tfor _, tt := range evalOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"eval\", evalOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_expression.go", "content": "package yqlib\n\ntype expressionOpPreferences struct {\n\texpression string\n}\n\nfunc expressionOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\n\tprefs := expressionNode.Operation.Preferences.(expressionOpPreferences)\n\texpNode, err := ExpressionParser.ParseExpression(prefs.expression)\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\n\treturn d.GetMatchingNodes(context, expNode)\n}\n" }, { "path": "pkg/yqlib/operator_file.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n)\n\nfunc getFilenameOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"GetFilename\")\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tresult := candidate.CreateReplacement(ScalarNode, \"!!str\", candidate.GetFilename())\n\t\tresults.PushBack(result)\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n\nfunc getFileIndexOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"GetFileIndex\")\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tresult := candidate.CreateReplacement(ScalarNode, \"!!int\", fmt.Sprintf(\"%v\", candidate.GetFileIndex()))\n\t\tresults.PushBack(result)\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_file_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar fileOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Get filename\",\n\t\tdocument: `{a: cat}`,\n\t\texpression: `filename`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::sample.yml\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Get file index\",\n\t\tdocument: `{a: cat}`,\n\t\texpression: `file_index`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!int)::0\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Get file indices of multiple documents\",\n\t\tdocument: `{a: cat}`,\n\t\tdocument2: `{a: cat}`,\n\t\texpression: `file_index`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!int)::0\\n\",\n\t\t\t\"D0, P[], (!!int)::1\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Get file index alias\",\n\t\tdocument: `{a: cat}`,\n\t\texpression: `fi`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!int)::0\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: cat\\nb: dog\",\n\t\texpression: `.. lineComment |= filename`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: cat # sample.yml\\nb: dog # sample.yml\\n\",\n\t\t},\n\t},\n}\n\nfunc TestFileOperatorsScenarios(t *testing.T) {\n\tfor _, tt := range fileOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"file-operators\", fileOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_filter.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n)\n\nfunc filterOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"filterOperation\")\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tchildren := context.SingleChildContext(candidate)\n\t\tsplatted, err := splat(children, traversePreferences{})\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\tfiltered, err := selectOperator(d, splatted, expressionNode)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t\tselfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}\n\t\tcollected, err := collectTogether(d, filtered, selfExpression)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\tcollected.Style = candidate.Style\n\t\tresults.PushBack(collected)\n\t}\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_filter_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar filterOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Filter array\",\n\t\tdocument: `[1,2,3]`,\n\t\texpression: `filter(. < 3)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[1, 2]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Filter array splat\",\n\t\tskipDoc: true,\n\t\tdocument: `[1,2,3]`,\n\t\texpression: `filter(. < 3)[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!int)::1\\n\",\n\t\t\t\"D0, P[1], (!!int)::2\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Filter map values\",\n\t\tdocument: `{c: {things: cool, frog: yes}, d: {things: hot, frog: false}}`,\n\t\texpression: `filter(.things == \"cool\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[{things: cool, frog: yes}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[1,2,3]`,\n\t\texpression: `filter(. > 1)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[2, 3]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Filter array to empty\",\n\t\tdocument: `[1,2,3]`,\n\t\texpression: `filter(. > 4)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Filter empty array\",\n\t\tdocument: `[]`,\n\t\texpression: `filter(. > 1)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[]\\n\",\n\t\t},\n\t},\n}\n\nfunc TestFilterOperatorScenarios(t *testing.T) {\n\tfor _, tt := range filterOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"filter\", filterOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_first.go", "content": "package yqlib\n\nimport \"container/list\"\n\nfunc firstOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tresults := list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\t// If no RHS expression is provided, simply return the first entry in candidate.Content\n\t\tif expressionNode == nil || expressionNode.RHS == nil {\n\t\t\tif len(candidate.Content) > 0 {\n\t\t\t\tresults.PushBack(candidate.Content[0])\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\t\tsplatted, err := splat(context.SingleChildContext(candidate), traversePreferences{})\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t\tfor splatEl := splatted.MatchingNodes.Front(); splatEl != nil; splatEl = splatEl.Next() {\n\t\t\tsplatCandidate := splatEl.Value.(*CandidateNode)\n\t\t\t// Create a new context for this splatted candidate\n\t\t\tsplatContext := context.SingleChildContext(splatCandidate)\n\t\t\t// Evaluate the RHS expression against this splatted candidate\n\t\t\trhs, err := d.GetMatchingNodes(splatContext, expressionNode.RHS)\n\t\t\tif err != nil {\n\t\t\t\treturn Context{}, err\n\t\t\t}\n\n\t\t\tincludeResult := false\n\n\t\t\tfor resultEl := rhs.MatchingNodes.Front(); resultEl != nil; resultEl = resultEl.Next() {\n\t\t\t\tresult := resultEl.Value.(*CandidateNode)\n\t\t\t\tincludeResult = isTruthyNode(result)\n\t\t\t\tif includeResult {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\tif includeResult {\n\t\t\t\tresults.PushBack(splatCandidate)\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t}\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_first_test.go", "content": "package yqlib\n\nimport \"testing\"\n\nvar firstOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"First matching element from array\",\n\t\tdocument: \"[{a: banana},{a: cat},{a: apple}]\",\n\t\texpression: `first(.a == \"cat\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[1], (!!map)::{a: cat}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"First matching element from array with multiple matches\",\n\t\tdocument: \"[{a: banana},{a: cat, b: firstCat},{a: apple},{a: cat, b: secondCat}]\",\n\t\texpression: `first(.a == \"cat\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[1], (!!map)::{a: cat, b: firstCat}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"First matching element from array with numeric condition\",\n\t\tdocument: \"[{a: 10},{a: 100},{a: 1},{a: 101}]\",\n\t\texpression: `first(.a > 50)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[1], (!!map)::{a: 100}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"First matching element from array with boolean condition\",\n\t\tdocument: \"[{a: false},{a: true, b: firstTrue},{a: false}, {a: true, b: secondTrue}]\",\n\t\texpression: `first(.a == true)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[1], (!!map)::{a: true, b: firstTrue}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"First matching element from array with null values\",\n\t\tdocument: \"[{a: null},{a: cat},{a: apple}]\",\n\t\texpression: `first(.a != null)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[1], (!!map)::{a: cat}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"First matching element from array with complex condition\",\n\t\tdocument: \"[{a: dog, b: 7},{a: cat, b: 3},{a: apple, b: 5}]\",\n\t\texpression: `first(.b > 4 and .b < 6)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[2], (!!map)::{a: apple, b: 5}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"First matching element from map\",\n\t\tdocument: \"x: {a: banana}\\ny: {a: cat}\\nz: {a: apple}\",\n\t\texpression: `first(.a == \"cat\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[y], (!!map)::{a: cat}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"First matching element from map with numeric condition\",\n\t\tdocument: \"x: {a: 10}\\ny: {a: 100}\\nz: {a: 101}\",\n\t\texpression: `first(.a > 50)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[y], (!!map)::{a: 100}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"First matching element from nested structure\",\n\t\tdocument: \"items: [{a: banana},{a: cat},{a: apple}]\",\n\t\texpression: `.items | first(.a == \"cat\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[items 1], (!!map)::{a: cat}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"First matching element with no matches\",\n\t\tdocument: \"[{a: banana},{a: cat},{a: apple}]\",\n\t\texpression: `first(.a == \"dog\")`,\n\t\texpected: []string{\n\t\t\t// No output expected when no matches\n\t\t},\n\t},\n\t{\n\t\tdescription: \"First matching element from empty array\",\n\t\tdocument: \"[]\",\n\t\texpression: `first(.a == \"cat\")`,\n\t\texpected: []string{\n\t\t\t// No output expected when array is empty\n\t\t},\n\t},\n\t{\n\t\tdescription: \"First matching element from scalar node\",\n\t\tdocument: \"hello\",\n\t\texpression: `first(. == \"hello\")`,\n\t\texpected: []string{\n\t\t\t// No output expected when node is scalar (no content to splat)\n\t\t},\n\t},\n\t{\n\t\tdescription: \"First matching element from null node\",\n\t\tdocument: \"null\",\n\t\texpression: `first(. == \"hello\")`,\n\t\texpected: []string{\n\t\t\t// No output expected when node is null (no content to splat)\n\t\t},\n\t},\n\t{\n\t\tdescription: \"First matching element with string condition\",\n\t\tdocument: \"[{a: banana},{a: cat},{a: apple}]\",\n\t\texpression: `first(.a | test(\"^c\"))`,\n\t\texpected: []string{\n\t\t\t\"D0, P[1], (!!map)::{a: cat}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"First matching element with length condition\",\n\t\tdocument: \"[{a: hi},{a: hello},{a: world}]\",\n\t\texpression: `first(.a | length > 4)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[1], (!!map)::{a: hello}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"First matching element from array of strings\",\n\t\tdocument: \"[banana, cat, apple]\",\n\t\texpression: `first(. == \"cat\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[1], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"First matching element from array of numbers\",\n\t\tdocument: \"[10, 100, 1]\",\n\t\texpression: `first(. > 50)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[1], (!!int)::100\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"First element with no filter from array\",\n\t\tdocument: \"[10, 100, 1]\",\n\t\texpression: `first`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!int)::10\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"First element with no filter from array of maps\",\n\t\tdocument: \"[{a: 10},{a: 100}]\",\n\t\texpression: `first`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!map)::{a: 10}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"No filter on empty array returns nothing\",\n\t\tskipDoc: true,\n\t\tdocument: \"[]\",\n\t\texpression: `first`,\n\t\texpected: []string{},\n\t},\n\t{\n\t\tdescription: \"No filter on scalar returns nothing\",\n\t\tskipDoc: true,\n\t\tdocument: \"hello\",\n\t\texpression: `first`,\n\t\texpected: []string{},\n\t},\n\t{\n\t\tdescription: \"No filter on null returns nothing\",\n\t\tskipDoc: true,\n\t\tdocument: \"null\",\n\t\texpression: `first`,\n\t\texpected: []string{},\n\t},\n}\n\nfunc TestFirstOperatorScenarios(t *testing.T) {\n\tfor _, tt := range firstOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"first\", firstOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_flatten.go", "content": "package yqlib\n\nimport (\n\t\"fmt\"\n)\n\ntype flattenPreferences struct {\n\tdepth int\n}\n\nfunc flatten(node *CandidateNode, depth int) {\n\tif depth == 0 {\n\t\treturn\n\t}\n\tif node.Kind != SequenceNode {\n\t\treturn\n\t}\n\tcontent := node.Content\n\tnewSeq := make([]*CandidateNode, 0)\n\n\tfor i := 0; i < len(content); i++ {\n\t\tif content[i].Kind == SequenceNode {\n\t\t\tflatten(content[i], depth-1)\n\t\t\tfor j := 0; j < len(content[i].Content); j++ {\n\t\t\t\tnewSeq = append(newSeq, content[i].Content[j])\n\t\t\t}\n\t\t} else {\n\t\t\tnewSeq = append(newSeq, content[i])\n\t\t}\n\t}\n\tnode.Content = make([]*CandidateNode, 0)\n\tnode.AddChildren(newSeq)\n}\n\nfunc flattenOp(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\n\tlog.Debugf(\"flatten Operator\")\n\tdepth := expressionNode.Operation.Preferences.(flattenPreferences).depth\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tif candidate.Kind != SequenceNode {\n\t\t\treturn Context{}, fmt.Errorf(\"only arrays are supported for flatten\")\n\t\t}\n\n\t\tflatten(candidate, depth)\n\n\t}\n\n\treturn context, nil\n\n}\n" }, { "path": "pkg/yqlib/operator_flatten_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar flattenOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Flatten\",\n\t\tsubdescription: \"Recursively flattens all arrays\",\n\t\tdocument: `[1, [2], [[3]]]`,\n\t\texpression: `flatten`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[1, 2, 3]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Flatten splat\",\n\t\tskipDoc: true,\n\t\tdocument: `[1, [2], [[3]]]`,\n\t\texpression: `flatten[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!int)::1\\n\",\n\t\t\t\"D0, P[1], (!!int)::2\\n\",\n\t\t\t\"D0, P[2], (!!int)::3\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Flatten with depth of one\",\n\t\tdocument: `[1, [2], [[3]]]`,\n\t\texpression: `flatten(1)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[1, 2, [3]]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Flatten with depth and splat\",\n\t\tskipDoc: true,\n\t\tdocument: `[1, [2], [[3]]]`,\n\t\texpression: `flatten(1)[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!int)::1\\n\",\n\t\t\t\"D0, P[1], (!!int)::2\\n\",\n\t\t\t\"D0, P[2], (!!seq)::[3]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Flatten empty array\",\n\t\tdocument: `[[]]`,\n\t\texpression: `flatten`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Flatten array of objects\",\n\t\tdocument: `[{foo: bar}, [{foo: baz}]]`,\n\t\texpression: `flatten`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[{foo: bar}, {foo: baz}]\\n\",\n\t\t},\n\t},\n}\n\nfunc TestFlattenOperatorScenarios(t *testing.T) {\n\tfor _, tt := range flattenOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"flatten\", flattenOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_group_by.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n\n\t\"github.com/elliotchance/orderedmap\"\n)\n\nfunc processIntoGroups(d *dataTreeNavigator, context Context, rhsExp *ExpressionNode, node *CandidateNode) (*orderedmap.OrderedMap, error) {\n\tvar newMatches = orderedmap.NewOrderedMap()\n\tfor _, child := range node.Content {\n\t\trhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(child), rhsExp)\n\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tkeyValue := \"null\"\n\n\t\tif rhs.MatchingNodes.Len() > 0 {\n\t\t\tfirst := rhs.MatchingNodes.Front()\n\t\t\tkeyCandidate := first.Value.(*CandidateNode)\n\t\t\tkeyValue = keyCandidate.Value\n\t\t}\n\n\t\tgroupList, exists := newMatches.Get(keyValue)\n\n\t\tif !exists {\n\t\t\tgroupList = list.New()\n\t\t\tnewMatches.Set(keyValue, groupList)\n\t\t}\n\t\tgroupList.(*list.List).PushBack(child)\n\t}\n\treturn newMatches, nil\n}\n\nfunc groupBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\n\tlog.Debugf(\"groupBy Operator\")\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\tif candidate.Kind != SequenceNode {\n\t\t\treturn Context{}, fmt.Errorf(\"only arrays are supported for group by\")\n\t\t}\n\n\t\tnewMatches, err := processIntoGroups(d, context, expressionNode.RHS, candidate)\n\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t\tresultNode := candidate.CreateReplacement(SequenceNode, \"!!seq\", \"\")\n\t\tfor groupEl := newMatches.Front(); groupEl != nil; groupEl = groupEl.Next() {\n\t\t\tgroupResultNode := &CandidateNode{Kind: SequenceNode, Tag: \"!!seq\"}\n\t\t\tgroupList := groupEl.Value.(*list.List)\n\t\t\tfor groupItem := groupList.Front(); groupItem != nil; groupItem = groupItem.Next() {\n\t\t\t\tgroupResultNode.AddChild(groupItem.Value.(*CandidateNode))\n\t\t\t}\n\n\t\t\tresultNode.AddChild(groupResultNode)\n\t\t}\n\n\t\tresults.PushBack(resultNode)\n\n\t}\n\n\treturn context.ChildContext(results), nil\n\n}\n" }, { "path": "pkg/yqlib/operator_group_by_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar groupByOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Group by field\",\n\t\tdocument: `[{foo: 1, bar: 10}, {foo: 3, bar: 100}, {foo: 1, bar: 1}]`,\n\t\texpression: `group_by(.foo)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- - {foo: 1, bar: 10}\\n - {foo: 1, bar: 1}\\n- - {foo: 3, bar: 100}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Group splat\",\n\t\tskipDoc: true,\n\t\tdocument: `[{foo: 1, bar: 10}, {foo: 3, bar: 100}, {foo: 1, bar: 1}]`,\n\t\texpression: `group_by(.foo)[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!seq)::- {foo: 1, bar: 10}\\n- {foo: 1, bar: 1}\\n\",\n\t\t\t\"D0, P[1], (!!seq)::- {foo: 3, bar: 100}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Group by field, with nulls\",\n\t\tdocument: `[{cat: dog}, {foo: 1, bar: 10}, {foo: 3, bar: 100}, {no: foo for you}, {foo: 1, bar: 1}]`,\n\t\texpression: `group_by(.foo)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- - {cat: dog}\\n - {no: foo for you}\\n- - {foo: 1, bar: 10}\\n - {foo: 1, bar: 1}\\n- - {foo: 3, bar: 100}\\n\",\n\t\t},\n\t},\n}\n\nfunc TestGroupByOperatorScenarios(t *testing.T) {\n\tfor _, tt := range groupByOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"group-by\", groupByOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_has.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"strconv\"\n)\n\nfunc hasOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\n\tlog.Debugf(\"hasOperation\")\n\tvar results = list.New()\n\n\trhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS)\n\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\n\twantedKey := \"null\"\n\twanted := &CandidateNode{Tag: \"!!null\"}\n\tif rhs.MatchingNodes.Len() != 0 {\n\t\twanted = rhs.MatchingNodes.Front().Value.(*CandidateNode)\n\t\twantedKey = wanted.Value\n\t}\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\tvar contents = candidate.Content\n\t\tswitch candidate.Kind {\n\t\tcase MappingNode:\n\t\t\tcandidateHasKey := false\n\t\t\tfor index := 0; index < len(contents) && !candidateHasKey; index = index + 2 {\n\t\t\t\tkey := contents[index]\n\t\t\t\tif key.Value == wantedKey {\n\t\t\t\t\tcandidateHasKey = true\n\t\t\t\t}\n\t\t\t}\n\t\t\tresults.PushBack(createBooleanCandidate(candidate, candidateHasKey))\n\t\tcase SequenceNode:\n\t\t\tcandidateHasKey := false\n\t\t\tif wanted.Tag == \"!!int\" {\n\t\t\t\tvar number, errParsingInt = strconv.ParseInt(wantedKey, 10, 64)\n\t\t\t\tif errParsingInt != nil {\n\t\t\t\t\treturn Context{}, errParsingInt\n\t\t\t\t}\n\t\t\t\tcandidateHasKey = int64(len(contents)) > number\n\t\t\t}\n\t\t\tresults.PushBack(createBooleanCandidate(candidate, candidateHasKey))\n\t\tdefault:\n\t\t\tresults.PushBack(createBooleanCandidate(candidate, false))\n\t\t}\n\t}\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_has_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar hasOperatorScenarios = []expressionScenario{\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a: hello`,\n\t\texpression: `has(\"a\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a: hello`,\n\t\texpression: `has(.b) as $c | .`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: hello\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a: hello`,\n\t\texpression: `has(.b)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Has map key\",\n\t\tdocument: `- a: \"yes\"\n- a: ~\n- a: \n- b: nope\n`,\n\t\texpression: `.[] | has(\"a\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!bool)::true\\n\",\n\t\t\t\"D0, P[1], (!!bool)::true\\n\",\n\t\t\t\"D0, P[2], (!!bool)::true\\n\",\n\t\t\t\"D0, P[3], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Select, checking for existence of deep paths\",\n\t\tsubdescription: \"Simply pipe in parent expressions into `has`\",\n\t\tdocument: \"- {a: {b: {c: cat}}}\\n- {a: {b: {d: dog}}}\",\n\t\texpression: `.[] | select(.a.b | has(\"c\"))`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!map)::{a: {b: {c: cat}}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdontFormatInputForDoc: true,\n\t\tdescription: \"Has array index\",\n\t\tdocument: `- []\n- [1]\n- [1, 2]\n- [1, null]\n- [1, 2, 3]\n`,\n\t\texpression: `.[] | has(1)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!bool)::false\\n\",\n\t\t\t\"D0, P[1], (!!bool)::false\\n\",\n\t\t\t\"D0, P[2], (!!bool)::true\\n\",\n\t\t\t\"D0, P[3], (!!bool)::true\\n\",\n\t\t\t\"D0, P[4], (!!bool)::true\\n\",\n\t\t},\n\t},\n}\n\nfunc TestHasOperatorScenarios(t *testing.T) {\n\tfor _, tt := range hasOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"has\", hasOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_keys.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n)\n\nfunc isKeyOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"isKeyOperator\")\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\tresults.PushBack(createBooleanCandidate(candidate, candidate.IsMapKey))\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n\nfunc getKeyOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"getKeyOperator\")\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\tif candidate.Key != nil {\n\t\t\tresults.PushBack(candidate.Key)\n\t\t}\n\t}\n\n\treturn context.ChildContext(results), nil\n\n}\n\nfunc keysOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"keysOperator\")\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\tvar targetNode *CandidateNode\n\t\tswitch candidate.Kind {\n\t\tcase MappingNode:\n\t\t\ttargetNode = getMapKeys(candidate)\n\t\tcase SequenceNode:\n\t\t\ttargetNode = getIndices(candidate)\n\t\tdefault:\n\t\t\treturn Context{}, fmt.Errorf(\"cannot get keys of %v, keys only works for maps and arrays\", candidate.Tag)\n\t\t}\n\n\t\tresults.PushBack(targetNode)\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n\nfunc getMapKeys(node *CandidateNode) *CandidateNode {\n\tcontents := make([]*CandidateNode, 0)\n\tfor index := 0; index < len(node.Content); index = index + 2 {\n\t\tcontents = append(contents, node.Content[index])\n\t}\n\n\tseq := &CandidateNode{Kind: SequenceNode, Tag: \"!!seq\"}\n\tseq.AddChildren(contents)\n\treturn seq\n}\n\nfunc getIndices(node *CandidateNode) *CandidateNode {\n\tvar contents = make([]*CandidateNode, len(node.Content))\n\n\tfor index := range node.Content {\n\t\tcontents[index] = createScalarNode(index, fmt.Sprintf(\"%v\", index))\n\t}\n\n\tseq := &CandidateNode{Kind: SequenceNode, Tag: \"!!seq\"}\n\tseq.AddChildren(contents)\n\treturn seq\n}\n" }, { "path": "pkg/yqlib/operator_keys_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar expectedIsKey = `D0, P[], (!!seq)::- p: \"\"\n isKey: false\n tag: '!!map'\n- p: a\n isKey: true\n tag: '!!str'\n- p: a\n isKey: false\n tag: '!!map'\n- p: a.b\n isKey: true\n tag: '!!str'\n- p: a.b\n isKey: false\n tag: '!!seq'\n- p: a.b.0\n isKey: false\n tag: '!!str'\n- p: a.c\n isKey: true\n tag: '!!str'\n- p: a.c\n isKey: false\n tag: '!!str'\n`\n\nvar keysOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Map keys\",\n\t\tdocument: `{dog: woof, cat: meow}`,\n\t\texpression: `keys`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- dog\\n- cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Map keys with splat\",\n\t\tskipDoc: true,\n\t\tdocument: `{dog: woof, cat: meow}`,\n\t\texpression: `keys[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!str)::dog\\n\",\n\t\t\t\"D0, P[1], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{}`,\n\t\texpression: `keys`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Array keys\",\n\t\tdocument: `[apple, banana]`,\n\t\texpression: `keys`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- 0\\n- 1\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[]`,\n\t\texpression: `keys`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Retrieve array key\",\n\t\tdocument: \"[1,2,3]\",\n\t\texpression: `.[1] | key`,\n\t\texpected: []string{\n\t\t\t\"D0, P[1], (!!int)::1\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Retrieve map key\",\n\t\tdocument: \"a: thing\",\n\t\texpression: `.a | key`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!str)::a\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"No key\",\n\t\tdocument: \"{}\",\n\t\texpression: `key`,\n\t\texpected: []string{},\n\t},\n\t{\n\t\tdescription: \"Update map key\",\n\t\tdocument: \"a:\\n x: 3\\n y: 4\",\n\t\texpression: `(.a.x | key) = \"meow\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a:\\n meow: 3\\n y: 4\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Get comment from map key\",\n\t\tdocument: \"a: \\n # comment on key\\n x: 3\\n y: 4\",\n\t\texpression: `.a.x | key | headComment`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a x], (!!str)::comment on key\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Check node is a key\",\n\t\tdocument: \"a: \\n b: [cat]\\n c: frog\\n\",\n\t\texpression: `[... | { \"p\": path | join(\".\"), \"isKey\": is_key, \"tag\": tag }]`,\n\t\texpected: []string{\n\t\t\texpectedIsKey,\n\t\t},\n\t},\n}\n\nfunc TestKeysOperatorScenarios(t *testing.T) {\n\tfor _, tt := range keysOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"keys\", keysOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_kind.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n)\n\nfunc kindToText(kind Kind) string {\n\tswitch kind {\n\tcase MappingNode:\n\t\treturn \"map\"\n\tcase SequenceNode:\n\t\treturn \"seq\"\n\tcase ScalarNode:\n\t\treturn \"scalar\"\n\tcase AliasNode:\n\t\treturn \"alias\"\n\tdefault:\n\t\treturn \"unknown\"\n\t}\n}\n\nfunc getKindOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"GetKindOperator\")\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tresult := candidate.CreateReplacement(ScalarNode, \"!!str\", kindToText(candidate.Kind))\n\t\tresults.PushBack(result)\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_kind_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar kindOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Get kind\",\n\t\tdocument: `{a: cat, b: 5, c: 3.2, e: true, f: [], g: {}, h: null}`,\n\t\texpression: `.. | kind`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::map\\n\",\n\t\t\t\"D0, P[a], (!!str)::scalar\\n\",\n\t\t\t\"D0, P[b], (!!str)::scalar\\n\",\n\t\t\t\"D0, P[c], (!!str)::scalar\\n\",\n\t\t\t\"D0, P[e], (!!str)::scalar\\n\",\n\t\t\t\"D0, P[f], (!!str)::seq\\n\",\n\t\t\t\"D0, P[g], (!!str)::map\\n\",\n\t\t\t\"D0, P[h], (!!str)::scalar\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Get kind, ignores custom tags\",\n\t\tsubdescription: \"Unlike tag, kind is not affected by custom tags.\",\n\t\tdocument: `{a: !!thing cat, b: !!foo {}, c: !!bar []}`,\n\t\texpression: `.. | kind`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::map\\n\",\n\t\t\t\"D0, P[a], (!!str)::scalar\\n\",\n\t\t\t\"D0, P[b], (!!str)::map\\n\",\n\t\t\t\"D0, P[c], (!!str)::seq\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Add comments only to scalars\",\n\t\tsubdescription: \"An example of how you can use kind\",\n\t\tdocument: \"a:\\n b: 5\\n c: 3.2\\ne: true\\nf: []\\ng: {}\\nh: null\",\n\t\texpression: `(.. | select(kind == \"scalar\")) line_comment = \"this is a scalar\"`,\n\t\texpected: []string{\"D0, P[], (!!map)::a:\\n b: 5 # this is a scalar\\n c: 3.2 # this is a scalar\\ne: true # this is a scalar\\nf: []\\ng: {}\\nh: null # this is a scalar\\n\"},\n\t},\n}\n\nfunc TestKindOperatorScenarios(t *testing.T) {\n\tfor _, tt := range kindOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"kind\", kindOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_length.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n)\n\nfunc lengthOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"lengthOperation\")\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tvar length int\n\t\tswitch candidate.Kind {\n\t\tcase ScalarNode:\n\t\t\tif candidate.Tag == \"!!null\" {\n\t\t\t\tlength = 0\n\t\t\t} else {\n\t\t\t\tlength = len(candidate.Value)\n\t\t\t}\n\t\tcase MappingNode:\n\t\t\tlength = len(candidate.Content) / 2\n\t\tcase SequenceNode:\n\t\t\tlength = len(candidate.Content)\n\t\tdefault:\n\t\t\tlength = 0\n\t\t}\n\n\t\tresult := candidate.CreateReplacement(ScalarNode, \"!!int\", fmt.Sprintf(\"%v\", length))\n\t\tresults.PushBack(result)\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_length_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar lengthOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"String length\",\n\t\tsubdescription: \"returns length of string\",\n\t\tdocument: `{a: cat}`,\n\t\texpression: `.a | length`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!int)::3\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"null length\",\n\t\tdocument: `{a: null}`,\n\t\texpression: `.a | length`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!int)::0\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: ~}`,\n\t\texpression: `.a | length`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!int)::0\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"# abc\\n{a: key no exist}\",\n\t\texpression: `.b | length`,\n\t\texpected: []string{\n\t\t\t\"D0, P[b], (!!int)::0\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Map length\",\n\t\tsubdescription: \"returns number of entries\",\n\t\tdocument: `{a: cat, c: dog}`,\n\t\texpression: `length`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!int)::2\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Array length\",\n\t\tsubdescription: \"returns number of elements\",\n\t\tdocument: `[2,4,6,8]`,\n\t\texpression: `length`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!int)::4\\n\",\n\t\t},\n\t},\n}\n\nfunc TestLengthOperatorScenarios(t *testing.T) {\n\tfor _, tt := range lengthOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"length\", lengthOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_line.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n)\n\nfunc lineOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"lineOperator\")\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tresult := candidate.CreateReplacement(ScalarNode, \"!!int\", fmt.Sprintf(\"%v\", candidate.Line))\n\t\tresults.PushBack(result)\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_line_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar lineOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Returns line of _value_ node\",\n\t\tdocument: \"a: cat\\nb:\\n c: cat\",\n\t\texpression: `.b | line`,\n\t\texpected: []string{\n\t\t\t\"D0, P[b], (!!int)::3\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Returns line of _key_ node\",\n\t\tsubdescription: \"Pipe through the key operator to get the line of the key\",\n\t\tdocument: \"a: cat\\nb:\\n c: cat\",\n\t\texpression: `.b | key | line`,\n\t\texpected: []string{\n\t\t\t\"D0, P[b], (!!int)::2\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"First line is 1\",\n\t\tdocument: \"a: cat\",\n\t\texpression: `.a | line`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!int)::1\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"No line data is 0\",\n\t\texpression: `{\"a\": \"new entry\"} | line`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!int)::0\\n\",\n\t\t},\n\t},\n}\n\nfunc TestLineOperatorScenarios(t *testing.T) {\n\tfor _, tt := range lineOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"line\", lineOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_load.go", "content": "package yqlib\n\nimport (\n\t\"bufio\"\n\t\"container/list\"\n\t\"fmt\"\n\t\"os\"\n)\n\nvar LoadYamlPreferences = YamlPreferences{\n\tLeadingContentPreProcessing: false,\n\tPrintDocSeparators: true,\n\tUnwrapScalar: true,\n\tEvaluateTogether: false,\n}\n\ntype loadPrefs struct {\n\tdecoder Decoder\n}\n\nfunc loadString(filename string) (*CandidateNode, error) {\n\t// ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory,\n\t// and ensuring that it's not possible to give a path to a file outside that directory.\n\n\tfilebytes, err := os.ReadFile(filename) // #nosec\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &CandidateNode{Kind: ScalarNode, Tag: \"!!str\", Value: string(filebytes)}, nil\n}\n\nfunc loadWithDecoder(filename string, decoder Decoder) (*CandidateNode, error) {\n\tif decoder == nil {\n\t\treturn nil, fmt.Errorf(\"could not load %s\", filename)\n\t}\n\n\tfile, err := os.Open(filename) // #nosec\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treader := bufio.NewReader(file)\n\n\tdocuments, err := readDocuments(reader, filename, 0, decoder)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif documents.Len() == 0 {\n\t\t// return null candidate\n\t\treturn &CandidateNode{Kind: ScalarNode, Tag: \"!!null\"}, nil\n\t} else if documents.Len() == 1 {\n\t\tcandidate := documents.Front().Value.(*CandidateNode)\n\t\treturn candidate, nil\n\n\t}\n\tsequenceNode := &CandidateNode{Kind: SequenceNode}\n\tfor doc := documents.Front(); doc != nil; doc = doc.Next() {\n\t\tsequenceNode.AddChild(doc.Value.(*CandidateNode))\n\t}\n\treturn sequenceNode, nil\n}\n\nfunc loadStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"loadString\")\n\tif ConfiguredSecurityPreferences.DisableFileOps {\n\t\treturn Context{}, fmt.Errorf(\"file operations have been disabled\")\n\t}\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\trhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\tif rhs.MatchingNodes.Front() == nil {\n\t\t\treturn Context{}, fmt.Errorf(\"filename expression returned nil\")\n\t\t}\n\t\tnameCandidateNode := rhs.MatchingNodes.Front().Value.(*CandidateNode)\n\n\t\tfilename := nameCandidateNode.Value\n\n\t\tcontentsCandidate, err := loadString(filename)\n\t\tif err != nil {\n\t\t\treturn Context{}, fmt.Errorf(\"failed to load %v: %w\", filename, err)\n\t\t}\n\n\t\tresults.PushBack(contentsCandidate)\n\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n\nfunc loadOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"loadOperator\")\n\tif ConfiguredSecurityPreferences.DisableFileOps {\n\t\treturn Context{}, fmt.Errorf(\"file operations have been disabled\")\n\t}\n\n\tloadPrefs := expressionNode.Operation.Preferences.(loadPrefs)\n\n\t// need to evaluate the 1st parameter against the context\n\t// and return the data accordingly.\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\trhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\tif rhs.MatchingNodes.Front() == nil {\n\t\t\treturn Context{}, fmt.Errorf(\"filename expression returned nil\")\n\t\t}\n\t\tnameCandidateNode := rhs.MatchingNodes.Front().Value.(*CandidateNode)\n\n\t\tfilename := nameCandidateNode.Value\n\n\t\tcontentsCandidate, err := loadWithDecoder(filename, loadPrefs.decoder)\n\t\tif err != nil {\n\t\t\treturn Context{}, fmt.Errorf(\"failed to load %v: %w\", filename, err)\n\t\t}\n\n\t\tresults.PushBack(contentsCandidate)\n\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_load_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar loadScenarios = []expressionScenario{\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Load empty file with a comment\",\n\t\texpression: `load(\"../../examples/empty.yaml\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!null)::# comment\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Load and splat\",\n\t\texpression: `load(\"../../examples/small.yaml\")[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Load and traverse\",\n\t\texpression: `load(\"../../examples/small.yaml\").a`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Load file with a header comment into an array\",\n\t\tdocument: `- \"../../examples/small.yaml\"`,\n\t\texpression: `.[] |= load(.)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- # comment\\n # about things\\n a: cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Load empty file with no comment\",\n\t\texpression: `load(\"../../examples/empty-no-comment.yaml\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!null)::\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Load multiple documents\",\n\t\texpression: `load(\"../../examples/multiple_docs_small.yaml\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], ()::- a: Easy! as one two three\\n- another:\\n document: here\\n- - 1\\n - 2\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Simple example\",\n\t\tdocument: `{myFile: \"../../examples/thing.yml\"}`,\n\t\texpression: `load(.myFile)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: apple is included\\nb: cool.\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Replace node with referenced file\",\n\t\tsubdescription: \"Note that you can modify the filename in the load operator if needed.\",\n\t\tdocument: `{something: {file: \"thing.yml\"}}`,\n\t\texpression: `.something |= load(\"../../examples/\" + .file)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{something: {a: apple is included, b: cool.}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Replace _all_ nodes with referenced file\",\n\t\tsubdescription: \"Recursively match all the nodes (`..`) and then filter the ones that have a 'file' attribute. \",\n\t\tdocument: `{something: {file: \"thing.yml\"}, over: {here: [{file: \"thing.yml\"}]}}`,\n\t\texpression: `(.. | select(has(\"file\"))) |= load(\"../../examples/\" + .file)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{something: {a: apple is included, b: cool.}, over: {here: [{a: apple is included, b: cool.}]}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Replace node with referenced file as string\",\n\t\tsubdescription: \"This will work for any text based file\",\n\t\tdocument: `{something: {file: \"thing.yml\"}}`,\n\t\texpression: `.something |= load_str(\"../../examples/\" + .file)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{something: \\\"a: apple is included\\\\nb: cool.\\\"}\\n\",\n\t\t},\n\t},\n\t{\n\t\trequiresFormat: \"xml\",\n\t\tdescription: \"Load from XML\",\n\t\tdocument: \"cool: things\",\n\t\texpression: `.more_stuff = load_xml(\"../../examples/small.xml\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::cool: things\\nmore_stuff:\\n this: is some xml\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Load from Properties\",\n\t\tdocument: \"cool: things\",\n\t\texpression: `.more_stuff = load_props(\"../../examples/small.properties\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::cool: things\\nmore_stuff:\\n this:\\n is: a properties file\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Merge from properties\",\n\t\tsubdescription: \"This can be used as a convenient way to update a yaml document\",\n\t\tdocument: \"this:\\n is: from yaml\\n cool: ay\\n\",\n\t\texpression: `. *= load_props(\"../../examples/small.properties\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::this:\\n is: a properties file\\n cool: ay\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Load from base64 encoded file\",\n\t\tdocument: \"cool: things\",\n\t\texpression: `.more_stuff = load_base64(\"../../examples/base64.txt\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::cool: things\\nmore_stuff: my secret chilli recipe is....\\n\",\n\t\t},\n\t},\n}\n\nfunc TestLoadScenarios(t *testing.T) {\n\tfor _, tt := range loadScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"load\", loadScenarios)\n}\n\nvar loadOperatorSecurityDisabledScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"load() operation fails when security is enabled\",\n\t\tsubdescription: \"Use `--security-disable-file-ops` to disable file operations for security.\",\n\t\texpression: `load(\"../../examples/thing.yml\")`,\n\t\texpectedError: \"file operations have been disabled\",\n\t},\n\t{\n\t\tdescription: \"load_str() operation fails when security is enabled\",\n\t\tsubdescription: \"Use `--security-disable-file-ops` to disable file operations for security.\",\n\t\texpression: `load_str(\"../../examples/thing.yml\")`,\n\t\texpectedError: \"file operations have been disabled\",\n\t},\n\t{\n\t\tdescription: \"load_xml() operation fails when security is enabled\",\n\t\tsubdescription: \"Use `--security-disable-file-ops` to disable file operations for security.\",\n\t\texpression: `load_xml(\"../../examples/small.xml\")`,\n\t\texpectedError: \"file operations have been disabled\",\n\t},\n\t{\n\t\tdescription: \"load_props() operation fails when security is enabled\",\n\t\tsubdescription: \"Use `--security-disable-file-ops` to disable file operations for security.\",\n\t\texpression: `load_props(\"../../examples/small.properties\")`,\n\t\texpectedError: \"file operations have been disabled\",\n\t},\n\t{\n\t\tdescription: \"load_base64() operation fails when security is enabled\",\n\t\tsubdescription: \"Use `--security-disable-file-ops` to disable file operations for security.\",\n\t\texpression: `load_base64(\"../../examples/base64.txt\")`,\n\t\texpectedError: \"file operations have been disabled\",\n\t},\n}\n\nfunc TestLoadOperatorSecurityDisabledScenarios(t *testing.T) {\n\t// Save original security preferences\n\toriginalDisableFileOps := ConfiguredSecurityPreferences.DisableFileOps\n\tdefer func() {\n\t\tConfiguredSecurityPreferences.DisableFileOps = originalDisableFileOps\n\t}()\n\n\t// Test that load operations fail when DisableFileOps is true\n\tConfiguredSecurityPreferences.DisableFileOps = true\n\n\tfor _, tt := range loadOperatorSecurityDisabledScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tappendOperatorDocumentScenario(t, \"load\", loadOperatorSecurityDisabledScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_map.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n)\n\nfunc mapValuesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\t//run expression against entries\n\t\t// splat toEntries and pipe it into Rhs\n\t\tsplatted, err := splat(context.SingleChildContext(candidate), traversePreferences{})\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t\tassignUpdateExp := &ExpressionNode{\n\t\t\tOperation: &Operation{OperationType: assignOpType, UpdateAssign: true},\n\t\t\tRHS: expressionNode.RHS,\n\t\t}\n\t\t_, err = assignUpdateOperator(d, splatted, assignUpdateExp)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t}\n\n\treturn context, nil\n}\n\nfunc mapOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\t//run expression against entries\n\t\t// splat toEntries and pipe it into Rhs\n\t\tsplatted, err := splat(context.SingleChildContext(candidate), traversePreferences{})\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\tif splatted.MatchingNodes.Len() == 0 {\n\t\t\tresults.PushBack(candidate.Copy())\n\t\t\tcontinue\n\t\t}\n\n\t\tresult, err := d.GetMatchingNodes(splatted, expressionNode.RHS)\n\t\tlog.Debug(\"expressionNode.Rhs %v\", expressionNode.RHS.Operation.OperationType)\n\t\tlog.Debug(\"result %v\", result)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t\tselfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}\n\t\tcollected, err := collectTogether(d, result, selfExpression)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\tcollected.Style = candidate.Style\n\n\t\tresults.PushBack(collected)\n\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_map_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar mapOperatorScenarios = []expressionScenario{\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[1,2,3]`,\n\t\tdocument2: `[5,6,7]`,\n\t\texpression: `map(. + 1)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[2, 3, 4]\\n\",\n\t\t\t\"D0, P[], (!!seq)::[6, 7, 8]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"mapping against an empty array should do nothing\",\n\t\tskipDoc: true,\n\t\tdocument: `[]`,\n\t\tdocument2: `[\"cat\"]`,\n\t\texpression: `map(3)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[]\\n\",\n\t\t\t\"D0, P[], (!!seq)::[3]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"mapping against an empty array should do nothing\",\n\t\tskipDoc: true,\n\t\tdocument: `[[], [5]]`,\n\t\texpression: `.[] |= map(3)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[[], [3]]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"mapping against an empty array should do nothing #2\",\n\t\tskipDoc: true,\n\t\tdocument: `[]`,\n\t\tdocument2: `[5]`,\n\t\texpression: `map(3 + .)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[]\\n\",\n\t\t\t\"D0, P[], (!!seq)::[8]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"mapping against an empty array should do nothing\",\n\t\tskipDoc: true,\n\t\tdocument: `[[], [5]]`,\n\t\texpression: `.[] |= map(3 + .)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[[], [8]]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `[] | map(. + 42)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[1,2]`,\n\t\texpression: `map(. + 1)[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!int)::2\\n\",\n\t\t\t\"D0, P[1], (!!int)::3\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Map array\",\n\t\tdocument: `[1,2,3]`,\n\t\texpression: `map(. + 1)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[2, 3, 4]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{}`,\n\t\tdocument2: `{b: 12}`,\n\t\texpression: `map_values(3)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{}\\n\",\n\t\t\t\"D0, P[], (!!map)::{b: 3}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{}`,\n\t\tdocument2: `{b: 12}`,\n\t\texpression: `map_values(3 + .)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{}\\n\",\n\t\t\t\"D0, P[], (!!map)::{b: 15}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: 1, b: 2, c: 3}`,\n\t\tdocument2: `{x: 10, y: 20, z: 30}`,\n\t\texpression: `map_values(. + 1)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: 2, b: 3, c: 4}\\n\",\n\t\t\t\"D0, P[], (!!map)::{x: 11, y: 21, z: 31}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"map values splat\",\n\t\tskipDoc: true,\n\t\tdocument: `{a: 1, b: 2}`,\n\t\texpression: `map_values(. + 1)[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!int)::2\\n\",\n\t\t\t\"D0, P[b], (!!int)::3\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Map object values\",\n\t\tdocument: `{a: 1, b: 2, c: 3}`,\n\t\texpression: `map_values(. + 1)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: 2, b: 3, c: 4}\\n\",\n\t\t},\n\t},\n}\n\nfunc TestMapOperatorScenarios(t *testing.T) {\n\tfor _, tt := range mapOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"map\", mapOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_modulo.go", "content": "package yqlib\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\t\"strconv\"\n\t\"strings\"\n)\n\nfunc moduloOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"Modulo operator\")\n\n\treturn crossFunction(d, context.ReadOnlyClone(), expressionNode, modulo, false)\n}\n\nfunc modulo(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {\n\tif lhs.Tag == \"!!null\" {\n\t\treturn nil, fmt.Errorf(\"%v (%v) cannot modulo by %v (%v)\", lhs.Tag, lhs.GetNicePath(), rhs.Tag, rhs.GetNicePath())\n\t}\n\n\ttarget := lhs.CopyWithoutContent()\n\n\tif lhs.Kind == ScalarNode && rhs.Kind == ScalarNode {\n\t\tif err := moduloScalars(target, lhs, rhs); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t} else {\n\t\treturn nil, fmt.Errorf(\"%v (%v) cannot modulo by %v (%v)\", lhs.Tag, lhs.GetNicePath(), rhs.Tag, rhs.GetNicePath())\n\t}\n\n\treturn target, nil\n}\n\nfunc moduloScalars(target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error {\n\tlhsTag := lhs.Tag\n\trhsTag := rhs.guessTagFromCustomType()\n\tlhsIsCustom := false\n\tif !strings.HasPrefix(lhsTag, \"!!\") {\n\t\t// custom tag - we have to have a guess\n\t\tlhsTag = lhs.guessTagFromCustomType()\n\t\tlhsIsCustom = true\n\t}\n\n\tif lhsTag == \"!!int\" && rhsTag == \"!!int\" {\n\t\ttarget.Kind = ScalarNode\n\t\ttarget.Style = lhs.Style\n\n\t\tformat, lhsNum, err := parseInt64(lhs.Value)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t_, rhsNum, err := parseInt64(rhs.Value)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif rhsNum == 0 {\n\t\t\treturn fmt.Errorf(\"cannot modulo by 0\")\n\t\t}\n\t\tremainder := lhsNum % rhsNum\n\n\t\ttarget.Tag = lhs.Tag\n\t\ttarget.Value = fmt.Sprintf(format, remainder)\n\t} else if (lhsTag == \"!!int\" || lhsTag == \"!!float\") && (rhsTag == \"!!int\" || rhsTag == \"!!float\") {\n\t\ttarget.Kind = ScalarNode\n\t\ttarget.Style = lhs.Style\n\n\t\tlhsNum, err := strconv.ParseFloat(lhs.Value, 64)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\trhsNum, err := strconv.ParseFloat(rhs.Value, 64)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tremainder := math.Mod(lhsNum, rhsNum)\n\t\tif lhsIsCustom {\n\t\t\ttarget.Tag = lhs.Tag\n\t\t} else {\n\t\t\ttarget.Tag = \"!!float\"\n\t\t}\n\t\ttarget.Value = fmt.Sprintf(\"%v\", remainder)\n\t} else {\n\t\treturn fmt.Errorf(\"%v cannot modulo by %v\", lhsTag, rhsTag)\n\t}\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/operator_modulo_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar moduloOperatorScenarios = []expressionScenario{\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[{a: 2.5, b: 2}, {a: 2, b: 0.75}]`,\n\t\texpression: \".[] | .a % .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[0 a], (!!float)::0.5\\n\",\n\t\t\t\"D0, P[1 a], (!!float)::0.5\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{}`,\n\t\texpression: \"(.a / .b) as $x | .\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Number modulo - int\",\n\t\tsubdescription: \"If the lhs and rhs are ints then the expression will be calculated with ints.\",\n\t\tdocument: `{a: 13, b: 2}`,\n\t\texpression: `.a = .a % .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: 1, b: 2}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Number modulo - float\",\n\t\tsubdescription: \"If the lhs or rhs are floats then the expression will be calculated with floats.\",\n\t\tdocument: `{a: 12, b: 2.5}`,\n\t\texpression: `.a = .a % .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: !!float 2, b: 2.5}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Number modulo - int by zero\",\n\t\tsubdescription: \"If the lhs is an int and rhs is a 0 the result is an error.\",\n\t\tdocument: `{a: 1, b: 0}`,\n\t\texpression: `.a = .a % .b`,\n\t\texpectedError: \"cannot modulo by 0\",\n\t},\n\t{\n\t\tdescription: \"Number modulo - float by zero\",\n\t\tsubdescription: \"If the lhs is a float and rhs is a 0 the result is NaN.\",\n\t\tdocument: `{a: 1.1, b: 0}`,\n\t\texpression: `.a = .a % .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: !!float NaN, b: 0}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Custom types: that are really numbers\",\n\t\tdocument: \"a: !horse 333.975\\nb: !goat 299.2\",\n\t\texpression: `.a = .a % .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !horse 34.775000000000034\\nb: !goat 299.2\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: 2\\nb: !goat 2.3\",\n\t\texpression: `.a = .a % .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !!float 2\\nb: !goat 2.3\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Keep anchors\",\n\t\tdocument: \"a: &horse [1]\",\n\t\texpression: `.a[1] = .a[0] % 2`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: &horse [1, 1]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Modulo int by string\",\n\t\tdocument: \"a: 123\\nb: '2'\",\n\t\texpression: `.a % .b`,\n\t\texpectedError: \"!!int cannot modulo by !!str\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Modulo string by int\",\n\t\tdocument: \"a: 2\\nb: '123'\",\n\t\texpression: `.b % .a`,\n\t\texpectedError: \"!!str cannot modulo by !!int\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Modulo map by int\",\n\t\tdocument: \"a: {\\\"a\\\":1}\\nb: 2\",\n\t\texpression: `.a % .b`,\n\t\texpectedError: \"!!map (a) cannot modulo by !!int (b)\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Modulo array by str\",\n\t\tdocument: \"a: [1,2]\\nb: '2'\",\n\t\texpression: `.a % .b`,\n\t\texpectedError: \"!!seq (a) cannot modulo by !!str (b)\",\n\t},\n}\n\nfunc TestModuloOperatorScenarios(t *testing.T) {\n\tfor _, tt := range moduloOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"modulo\", moduloOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_multiply.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n)\n\ntype multiplyPreferences struct {\n\tAppendArrays bool\n\tDeepMergeArrays bool\n\tTraversePrefs traversePreferences\n\tAssignPrefs assignPreferences\n}\n\nfunc createMultiplyOp(prefs interface{}) func(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode {\n\treturn func(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode {\n\t\treturn &ExpressionNode{Operation: &Operation{OperationType: multiplyOpType, Preferences: prefs},\n\t\t\tLHS: lhs,\n\t\t\tRHS: rhs}\n\t}\n}\n\nfunc multiplyAssignOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tvar multiplyPrefs = expressionNode.Operation.Preferences\n\n\treturn compoundAssignFunction(d, context, expressionNode, createMultiplyOp(multiplyPrefs))\n}\n\nfunc multiplyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"MultiplyOperator\")\n\treturn crossFunction(d, context.ReadOnlyClone(), expressionNode, multiply(expressionNode.Operation.Preferences.(multiplyPreferences)), false)\n}\n\nfunc getComments(lhs *CandidateNode, rhs *CandidateNode) (leadingContent string, headComment string, footComment string) {\n\tleadingContent = rhs.LeadingContent\n\theadComment = rhs.HeadComment\n\tfootComment = rhs.FootComment\n\tif lhs.HeadComment != \"\" || lhs.LeadingContent != \"\" {\n\t\theadComment = lhs.HeadComment\n\t\tleadingContent = lhs.LeadingContent\n\t}\n\n\tif lhs.FootComment != \"\" {\n\t\tfootComment = lhs.FootComment\n\t}\n\n\treturn leadingContent, headComment, footComment\n}\n\nfunc multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {\n\treturn func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {\n\t\t// need to do this before unWrapping the potential document node\n\t\tleadingContent, headComment, footComment := getComments(lhs, rhs)\n\t\tlog.Debugf(\"Multiplying LHS: %v\", NodeToString(lhs))\n\t\tlog.Debugf(\"- RHS: %v\", NodeToString(rhs))\n\n\t\tif rhs.Tag == \"!!null\" {\n\t\t\treturn lhs.Copy(), nil\n\t\t}\n\n\t\tif (lhs.Kind == MappingNode && rhs.Kind == MappingNode) ||\n\t\t\t(lhs.Tag == \"!!null\" && rhs.Kind == MappingNode) ||\n\t\t\t(lhs.Kind == SequenceNode && rhs.Kind == SequenceNode) ||\n\t\t\t(lhs.Tag == \"!!null\" && rhs.Kind == SequenceNode) {\n\n\t\t\tvar newBlank = lhs.Copy()\n\n\t\t\tnewBlank.LeadingContent = leadingContent\n\t\t\tnewBlank.HeadComment = headComment\n\t\t\tnewBlank.FootComment = footComment\n\n\t\t\treturn mergeObjects(d, context.WritableClone(), newBlank, rhs, preferences)\n\t\t}\n\t\treturn multiplyScalars(lhs, rhs)\n\t}\n}\n\nfunc multiplyScalars(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {\n\tlhsTag := lhs.Tag\n\trhsTag := rhs.guessTagFromCustomType()\n\tlhsIsCustom := false\n\tif !strings.HasPrefix(lhsTag, \"!!\") {\n\t\t// custom tag - we have to have a guess\n\t\tlhsTag = lhs.guessTagFromCustomType()\n\t\tlhsIsCustom = true\n\t}\n\n\tif lhsTag == \"!!int\" && rhsTag == \"!!int\" {\n\t\treturn multiplyIntegers(lhs, rhs)\n\t} else if (lhsTag == \"!!int\" || lhsTag == \"!!float\") && (rhsTag == \"!!int\" || rhsTag == \"!!float\") {\n\t\treturn multiplyFloats(lhs, rhs, lhsIsCustom)\n\t} else if (lhsTag == \"!!str\" && rhsTag == \"!!int\") || (lhsTag == \"!!int\" && rhsTag == \"!!str\") {\n\t\treturn repeatString(lhs, rhs)\n\t}\n\treturn nil, fmt.Errorf(\"cannot multiply %v with %v\", lhs.Tag, rhs.Tag)\n}\n\nfunc multiplyFloats(lhs *CandidateNode, rhs *CandidateNode, lhsIsCustom bool) (*CandidateNode, error) {\n\ttarget := lhs.CopyWithoutContent()\n\ttarget.Kind = ScalarNode\n\ttarget.Style = lhs.Style\n\tif lhsIsCustom {\n\t\ttarget.Tag = lhs.Tag\n\t} else {\n\t\ttarget.Tag = \"!!float\"\n\t}\n\n\tlhsNum, err := strconv.ParseFloat(lhs.Value, 64)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\trhsNum, err := strconv.ParseFloat(rhs.Value, 64)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttarget.Value = fmt.Sprintf(\"%v\", lhsNum*rhsNum)\n\treturn target, nil\n}\n\nfunc multiplyIntegers(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {\n\ttarget := lhs.CopyWithoutContent()\n\ttarget.Kind = ScalarNode\n\ttarget.Style = lhs.Style\n\ttarget.Tag = lhs.Tag\n\n\tformat, lhsNum, err := parseInt64(lhs.Value)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\t_, rhsNum, err := parseInt64(rhs.Value)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\ttarget.Value = fmt.Sprintf(format, lhsNum*rhsNum)\n\treturn target, nil\n}\n\nfunc repeatString(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {\n\tvar stringNode *CandidateNode\n\tvar intNode *CandidateNode\n\tif lhs.Tag == \"!!str\" {\n\t\tstringNode = lhs\n\t\tintNode = rhs\n\t} else {\n\t\tstringNode = rhs\n\t\tintNode = lhs\n\t}\n\ttarget := lhs.CopyWithoutContent()\n\ttarget.UpdateAttributesFrom(stringNode, assignPreferences{})\n\n\tcount, err := parseInt(intNode.Value)\n\tif err != nil {\n\t\treturn nil, err\n\t} else if count < 0 {\n\t\treturn nil, fmt.Errorf(\"cannot repeat string by a negative number (%v)\", count)\n\t} else if count > 10000000 {\n\t\treturn nil, fmt.Errorf(\"cannot repeat string by more than 100 million (%v)\", count)\n\t}\n\ttarget.Value = strings.Repeat(stringNode.Value, count)\n\n\treturn target, nil\n}\n\nfunc mergeObjects(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode, preferences multiplyPreferences) (*CandidateNode, error) {\n\tvar results = list.New()\n\n\t// only need to recurse the array if we are doing a deep merge\n\tprefs := recursiveDescentPreferences{RecurseArray: preferences.DeepMergeArrays,\n\t\tTraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: true, ExactKeyMatch: true}}\n\tlog.Debugf(\"merge - preferences.DeepMergeArrays %v\", preferences.DeepMergeArrays)\n\tlog.Debugf(\"merge - preferences.AppendArrays %v\", preferences.AppendArrays)\n\terr := recursiveDecent(results, context.SingleChildContext(rhs), prefs)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar pathIndexToStartFrom int\n\tif results.Front() != nil {\n\t\tpathIndexToStartFrom = len(results.Front().Value.(*CandidateNode).GetPath())\n\t\tlog.Debugf(\"pathIndexToStartFrom: %v\", pathIndexToStartFrom)\n\t}\n\n\tfor el := results.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\tlog.Debugf(\"going to applied assignment to LHS: %v with RHS: %v\", NodeToString(lhs), NodeToString(candidate))\n\n\t\tif candidate.Tag == \"!!merge\" {\n\t\t\tcontinue\n\t\t}\n\n\t\terr := applyAssignment(d, context, pathIndexToStartFrom, lhs, candidate, preferences)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tlog.Debugf(\"applied assignment to LHS: %v\", NodeToString(lhs))\n\t}\n\treturn lhs, nil\n}\n\nfunc applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode, preferences multiplyPreferences) error {\n\tshouldAppendArrays := preferences.AppendArrays\n\n\tlhsPath := rhs.GetPath()[pathIndexToStartFrom:]\n\tlog.Debugf(\"merge - lhsPath %v\", lhsPath)\n\n\tassignmentOp := &Operation{OperationType: assignAttributesOpType, Preferences: preferences.AssignPrefs}\n\tif shouldAppendArrays && rhs.Kind == SequenceNode {\n\t\tassignmentOp.OperationType = addAssignOpType\n\t\tlog.Debugf(\"merge - assignmentOp.OperationType = addAssignOpType\")\n\t} else if !preferences.DeepMergeArrays && rhs.Kind == SequenceNode ||\n\t\t(rhs.Kind == ScalarNode || rhs.Kind == AliasNode) {\n\t\tassignmentOp.OperationType = assignOpType\n\t\tassignmentOp.UpdateAssign = false\n\t\tlog.Debugf(\"merge - rhs.Kind == SequenceNode: %v\", rhs.Kind == SequenceNode)\n\t\tlog.Debugf(\"merge - rhs.Kind == ScalarNode: %v\", rhs.Kind == ScalarNode)\n\t\tlog.Debugf(\"merge - rhs.Kind == AliasNode: %v\", rhs.Kind == AliasNode)\n\t\tlog.Debugf(\"merge - assignmentOp.OperationType = assignOpType, no updateassign\")\n\t} else {\n\t\tlog.Debugf(\"merge - assignmentOp := &Operation{OperationType: assignAttributesOpType}\")\n\t}\n\trhsOp := &Operation{OperationType: referenceOpType, CandidateNode: rhs}\n\n\tassignmentOpNode := &ExpressionNode{\n\t\tOperation: assignmentOp,\n\t\tLHS: createTraversalTree(lhsPath, preferences.TraversePrefs, rhs.IsMapKey),\n\t\tRHS: &ExpressionNode{Operation: rhsOp},\n\t}\n\n\t_, err := d.GetMatchingNodes(context.SingleChildContext(lhs), assignmentOpNode)\n\n\treturn err\n}\n" }, { "path": "pkg/yqlib/operator_multiply_test.go", "content": "package yqlib\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\t\"testing\"\n)\n\nvar doc1 = `list:\n# Hi this is a comment.\n# Hello this is another comment.\n- \"abc\"`\n\nvar doc2 = `list2:\n# This is yet another comment.\n# Indeed this is yet another comment.\n- \"123\"`\n\nvar docExpected = `D0, P[], (!!map)::list:\n # Hi this is a comment.\n # Hello this is another comment.\n - \"abc\"\nlist2:\n # This is yet another comment.\n # Indeed this is yet another comment.\n - \"123\"\n`\n\nvar mergeArrayWithAnchors = `sample:\n- &a\n- <<: *a\n`\n\nvar mergeArraysObjectKeysText = `\nThis is a fairly complex expression - you can use it as is by providing the environment variables as seen in the example below.\n\nIt merges in the array provided in the second file into the first - matching on equal keys.\n\nExplanation:\n\nThe approach, at a high level, is to reduce into a merged map (keyed by the unique key)\nand then convert that back into an array.\n\nFirst the expression will create a map from the arrays keyed by the idPath, the unique field we want to merge by.\nThe reduce operator is merging '({}; . * $item )', so array elements with the matching key will be merged together.\n\nNext, we convert the map back to an array, using reduce again, concatenating all the map values together.\n\nFinally, we set the result of the merged array back into the first doc.\n\nThanks Kev from [stackoverflow](https://stackoverflow.com/a/70109529/1168223)\n`\n\nvar mergeExpression = `\n(\n (( (eval(strenv(originalPath)) + eval(strenv(otherPath))) | .[] | {(eval(strenv(idPath))): .}) as $item ireduce ({}; . * $item )) as $uniqueMap\n | ( $uniqueMap | to_entries | .[]) as $item ireduce([]; . + $item.value)\n) as $mergedArray\n| select(fi == 0) | (eval(strenv(originalPath))) = $mergedArray\n`\n\nvar docWithHeader = `# here\n\na: apple\n`\n\nvar nodeWithHeader = `node:\n # here\n a: apple\n`\n\nvar docNoComments = `b: banana\n`\n\nvar docWithFooter = `a: apple\n\n# footer\n`\n\nvar nodeWithFooter = `a: apple\n# footer`\n\nvar document = `a: &cat {name: cat}\nb: {name: dog}\nc: \n <<: *cat\n`\n\nvar mergeWithGlobA = `\n\"**cat\": things,\n\"meow**cat\": stuff\n`\n\nvar mergeWithGlobB = `\n\"**cat\": newThings,\n`\n\nvar multiplyOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"multiple should be readonly\",\n\t\tskipDoc: true,\n\t\tdocument: \"\",\n\t\texpression: \".x |= (root | (.a * .b))\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], ()::x: null\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"glob keys are treated as literals when merging\",\n\t\tskipDoc: true,\n\t\tdocument: mergeWithGlobA,\n\t\tdocument2: mergeWithGlobB,\n\t\texpression: `select(fi == 0) * select(fi == 1)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::\\n\\\"**cat\\\": newThings,\\n\\\"meow**cat\\\": stuff\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: mergeArrayWithAnchors,\n\t\texpression: `. * .`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::sample:\\n - &a\\n - !!merge <<: *a\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[[c], [b]]`,\n\t\texpression: `.[] | . *+ [\"a\"]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!seq)::[c, a]\\n\",\n\t\t\t\"D0, P[1], (!!seq)::[b, a]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: docWithHeader,\n\t\tdocument2: docNoComments,\n\t\texpression: `select(fi == 0) * select(fi == 1)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::# here\\n\\na: apple\\nb: banana\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: nodeWithHeader,\n\t\tdocument2: docNoComments,\n\t\texpression: `(select(fi == 0) | .node) * select(fi == 1)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[node], (!!map)::# here\\na: apple\\nb: banana\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: docNoComments,\n\t\tdocument2: docWithHeader,\n\t\texpression: `select(fi == 0) * select(fi == 1)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::# here\\n\\nb: banana\\na: apple\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: docNoComments,\n\t\tdocument2: nodeWithHeader,\n\t\texpression: `select(fi == 0) * (select(fi == 1) | .node)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::b: banana\\n# here\\na: apple\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: docWithFooter,\n\t\tdocument2: docNoComments,\n\t\texpression: `select(fi == 0) * select(fi == 1)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: apple\\nb: banana\\n# footer\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: nodeWithFooter,\n\t\tdocument2: docNoComments,\n\t\texpression: `select(fi == 0) * select(fi == 1)`,\n\t\texpected: []string{ // not sure why there's an extra newline *shrug*\n\t\t\t\"D0, P[], (!!map)::a: apple\\n# footer\\n\\nb: banana\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"doc 2 has footer\",\n\t\tskipDoc: true,\n\t\tdocument: docNoComments,\n\t\tdocument2: docWithFooter,\n\t\texpression: `select(fi == 0) * select(fi == 1)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::b: banana\\na: apple\\n# footer\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Multiply integers\",\n\t\tdocument: \"a: 3\\nb: 4\",\n\t\texpression: `.a *= .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: 12\\nb: 4\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Multiply string node X int\",\n\t\tdocument: docNoComments,\n\t\texpression: \".b * 4\",\n\t\texpected: []string{\n\t\t\tfmt.Sprintf(\"D0, P[b], (!!str)::%s\\n\", strings.Repeat(\"banana\", 4)),\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Multiply int X string node\",\n\t\tdocument: docNoComments,\n\t\texpression: \"4 * .b\",\n\t\texpected: []string{\n\t\t\tfmt.Sprintf(\"D0, P[], (!!str)::%s\\n\", strings.Repeat(\"banana\", 4)),\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Multiply string X int node\",\n\t\tdocument: `n: 4\n`,\n\t\texpression: `\"banana\" * .n`,\n\t\texpected: []string{\n\t\t\tfmt.Sprintf(\"D0, P[], (!!str)::%s\\n\", strings.Repeat(\"banana\", 4)),\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Multiply string X by negative int\",\n\t\tskipDoc: true,\n\t\tdocument: `n: -4`,\n\t\texpression: `\"banana\" * .n`,\n\t\texpectedError: \"cannot repeat string by a negative number (-4)\",\n\t},\n\t{\n\t\tdescription: \"Multiply string X by more than 100 million\",\n\t\t// very large string.repeats causes a panic\n\t\tskipDoc: true,\n\t\tdocument: `n: 100000001`,\n\t\texpression: `\"banana\" * .n`,\n\t\texpectedError: \"cannot repeat string by more than 100 million (100000001)\",\n\t},\n\t{\n\t\tdescription: \"Multiply int node X string\",\n\t\tdocument: `n: 4\n`,\n\t\texpression: `.n * \"banana\"`,\n\t\texpected: []string{\n\t\t\tfmt.Sprintf(\"D0, P[n], (!!str)::%s\\n\", strings.Repeat(\"banana\", 4)),\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: doc1,\n\t\tdocument2: doc2,\n\t\texpression: `select(fi == 0) * select(fi == 1)`,\n\t\texpected: []string{\n\t\t\tdocExpected,\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `.x = {\"things\": \"whatever\"} * {}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], ()::x:\\n things: whatever\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `.x = {} * {\"things\": \"whatever\"}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], ()::x:\\n things: whatever\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `3 * 4.5`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!float)::13.5\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `4.5 * 3`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!float)::13.5\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: {also: [1]}, b: {also: me}}`,\n\t\texpression: `. * {\"a\" : .b}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {also: me}, b: {also: me}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"# b\\nb:\\n # a\\n a: cat\",\n\t\texpression: \"{} * .\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::# b\\nb:\\n # a\\n a: cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"# b\\nb:\\n # a\\n a: cat\",\n\t\texpression: \". * {}\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::# b\\nb:\\n # a\\n a: cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: &a { b: &b { c: &c cat } } }`,\n\t\texpression: `{} * .`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: &a\\n b: &b\\n c: &c cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: 2, b: 5}`,\n\t\tdocument2: `{a: 3, b: 10}`,\n\t\texpression: `.a * .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!int)::10\\n\",\n\t\t\t\"D0, P[a], (!!int)::20\\n\",\n\t\t\t\"D0, P[a], (!!int)::15\\n\",\n\t\t\t\"D0, P[a], (!!int)::30\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: 2}`,\n\t\tdocument2: `{b: 10}`,\n\t\texpression: `select(fi ==0) * select(fi==1)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: 2, b: 10}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `{} * {\"cat\":\"dog\"}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::cat: dog\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: {also: me}, b: {also: [1]}}`,\n\t\texpression: `. * {\"a\":.b}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {also: [1]}, b: {also: [1]}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Merge objects together, returning merged result only\",\n\t\tdocument: `{a: {field: me, fieldA: cat}, b: {field: {g: wizz}, fieldB: dog}}`,\n\t\texpression: `.a * .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!map)::{field: {g: wizz}, fieldA: cat, fieldB: dog}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Merge objects together, returning parent object\",\n\t\tdocument: `{a: {field: me, fieldA: cat}, b: {field: {g: wizz}, fieldB: dog}}`,\n\t\texpression: `. * {\"a\":.b}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {field: {g: wizz}, fieldA: cat, fieldB: dog}, b: {field: {g: wizz}, fieldB: dog}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: {also: {g: wizz}}, b: {also: me}}`,\n\t\texpression: `. * {\"a\":.b}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {also: me}, b: {also: me}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: {also: {g: wizz}}, b: {also: [1]}}`,\n\t\texpression: `. * {\"a\":.b}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {also: [1]}, b: {also: [1]}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: {also: [1]}, b: {also: {g: wizz}}}`,\n\t\texpression: `. * {\"a\":.b}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {also: {g: wizz}}, b: {also: {g: wizz}}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: {things: great}, b: {also: me}}`,\n\t\texpression: `. * {\"a\": .b}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {things: great, also: me}, b: {also: me}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Merge keeps style of LHS\",\n\t\tdontFormatInputForDoc: true,\n\t\tdocument: \"a: {things: great}\\nb:\\n also: \\\"me\\\"\",\n\t\texpression: `. * {\"a\":.b}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: {things: great, also: \\\"me\\\"}\\nb:\\n also: \\\"me\\\"\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Merge arrays\",\n\t\tdocument: `{a: [1,2,3], b: [3,4,5]}`,\n\t\texpression: `. * {\"a\":.b}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: [3, 4, 5], b: [3, 4, 5]}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: [1], b: [2]}`,\n\t\texpression: `.a *+ .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!seq)::[1, 2]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Merge, only existing fields\",\n\t\tdocument: `{a: {thing: one, cat: frog}, b: {missing: two, thing: two}}`,\n\t\texpression: `.a *? .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!map)::{thing: two, cat: frog}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Merge, only new fields\",\n\t\tdocument: `{a: {thing: one, cat: frog}, b: {missing: two, thing: two}}`,\n\t\texpression: `.a *n .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!map)::{thing: one, cat: frog, missing: two}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: [{thing: one}], b: [{missing: two, thing: two}]}`,\n\t\texpression: `.a *?d .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!seq)::[{thing: two}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: [{thing: one}], b: [{missing: two, thing: two}]}`,\n\t\texpression: `.a *nd .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!seq)::[{thing: one, missing: two}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: {array: [1]}, b: {}}`,\n\t\texpression: `.b *+ .a`,\n\t\texpected: []string{\n\t\t\t\"D0, P[b], (!!map)::array: [1]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Merge, appending arrays\",\n\t\tdocument: `{a: {array: [1, 2, animal: dog], value: coconut}, b: {array: [3, 4, animal: cat], value: banana}}`,\n\t\texpression: `.a *+ .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!map)::{array: [1, 2, {animal: dog}, 3, 4, {animal: cat}], value: banana}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Merge, only existing fields, appending arrays\",\n\t\tdocument: `{a: {thing: [1,2]}, b: {thing: [3,4], another: [1]}}`,\n\t\texpression: `.a *?+ .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!map)::{thing: [1, 2, 3, 4]}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Merge, only new fields, appending arrays\",\n\t\tsubdescription: \"Append (+) with (n) has no effect.\",\n\t\tskipDoc: true,\n\t\tdocument: `{a: {thing: [1,2]}, b: {thing: [3,4], another: [1]}}`,\n\t\texpression: `.a *n+ .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!map)::{thing: [1, 2], another: [1]}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Merge, deeply merging arrays\",\n\t\tsubdescription: \"Merging arrays deeply means arrays are merged like objects, with indices as their key. In this case, we merge the first item in the array and do nothing with the second.\",\n\t\tdocument: `{a: [{name: fred, age: 12}, {name: bob, age: 32}], b: [{name: fred, age: 34}]}`,\n\t\texpression: `.a *d .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!seq)::[{name: fred, age: 34}, {name: bob, age: 32}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Merge arrays of objects together, matching on a key\",\n\t\tsubdescription: mergeArraysObjectKeysText,\n\t\tdocument: `{myArray: [{a: apple, b: appleB}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB}], something: else}`,\n\t\tdocument2: `newArray: [{a: banana, c: bananaC}, {a: apple, b: appleB2}, {a: dingo, c: dingoC}]`,\n\t\tenvironmentVariables: map[string]string{\"originalPath\": \".myArray\", \"otherPath\": \".newArray\", \"idPath\": \".a\"},\n\t\texpression: mergeExpression,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{myArray: [{a: apple, b: appleB2}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB, c: bananaC}, {a: dingo, c: dingoC}], something: else}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Merge to prefix an element\",\n\t\tdocument: `{a: cat, b: dog}`,\n\t\texpression: `. * {\"a\": {\"c\": .a}}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {c: cat}, b: dog}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Merge with simple aliases\",\n\t\tdocument: `{a: &cat {c: frog}, b: {f: *cat}, c: {g: thongs}}`,\n\t\texpression: `.c * .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[c], (!!map)::{g: thongs, f: *cat}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Merge copies anchor names\",\n\t\tdocument: `{a: {c: &cat frog}, b: {f: *cat}, c: {g: thongs}}`,\n\t\texpression: `.c * .a`,\n\t\texpected: []string{\n\t\t\t\"D0, P[c], (!!map)::{g: thongs, c: &cat frog}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Merge with merge anchors\",\n\t\tdocument: mergeDocSample,\n\t\texpression: `.foobar * .foobarList`,\n\t\texpected: []string{\n\t\t\t\"D0, P[foobar], (!!map)::c: foobarList_c\\n!!merge <<: [*foo, *bar]\\nthing: foobar_thing\\nb: foobarList_b\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: document,\n\t\texpression: `.b * .c`,\n\t\texpected: []string{\n\t\t\t\"D0, P[b], (!!map)::{name: dog, <<: *cat}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Custom types: that are really numbers\",\n\t\tsubdescription: \"When custom tags are encountered, yq will try to decode the underlying type.\",\n\t\tdocument: \"a: !horse 2\\nb: !goat 3\",\n\t\texpression: \".a = .a * .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !horse 6\\nb: !goat 3\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Custom types: that are really numbers\",\n\t\tdocument: \"a: !horse 2.5\\nb: !goat 3.5\",\n\t\texpression: \".a = .a * .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !horse 8.75\\nb: !goat 3.5\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Custom types: that are really numbers\",\n\t\tdocument: \"a: 2\\nb: !goat 3.5\",\n\t\texpression: \".a = .a * .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !!float 7\\nb: !goat 3.5\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Custom types: that are really arrays\",\n\t\tdocument: \"a: !horse [1,2]\\nb: !goat [3]\",\n\t\texpression: \".a = .a * .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !horse [3]\\nb: !goat [3]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Custom types: that are really maps\",\n\t\tsubdescription: \"Custom tags will be maintained.\",\n\t\tdocument: \"a: !horse {cat: meow}\\nb: !goat {dog: woof}\",\n\t\texpression: \".a = .a * .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !horse {cat: meow, dog: woof}\\nb: !goat {dog: woof}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Custom types: clobber tags\",\n\t\tsubdescription: \"Use the `c` option to clobber custom tags. Note that the second tag is now used.\",\n\t\tdocument: \"a: !horse {cat: meow}\\nb: !goat {dog: woof}\",\n\t\texpression: \".a *=c .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !goat {cat: meow, dog: woof}\\nb: !goat {dog: woof}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Custom types: clobber tags - *=\",\n\t\tsubdescription: \"Use the `c` option to clobber custom tags - on both the `=` and `*` operator. Note that the second tag is now used.\",\n\t\tdocument: \"a: !horse {cat: meow}\\nb: !goat {dog: woof}\",\n\t\texpression: \".a =c .a *c .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !goat {cat: meow, dog: woof}\\nb: !goat {dog: woof}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Custom types: dont clobber tags - *=\",\n\t\tsubdescription: \"Use the `c` option to clobber custom tags - on both the `=` and `*` operator. Note that the second tag is now used.\",\n\t\tdocument: \"a: !horse {cat: meow}\\nb: !goat {dog: woof}\",\n\t\texpression: \".a *= .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !horse {cat: meow, dog: woof}\\nb: !goat {dog: woof}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Custom types: that are really maps\",\n\t\tdocument: \"a: {cat: !horse meow}\\nb: {cat: 5}\",\n\t\texpression: \".a = .a * .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: {cat: !horse 5}\\nb: {cat: 5}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Relative merge, new fields only\",\n\t\tdocument: \"a: {a: original}\\n\",\n\t\texpression: `.a *=n load(\"../../examples/thing.yml\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: {a: original, b: cool.}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Relative merge\",\n\t\tdocument: \"a: {a: original}\\n\",\n\t\texpression: `.a *= load(\"../../examples/thing.yml\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: {a: apple is included, b: cool.}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Merging a null with a map\",\n\t\texpression: `null * {\"some\": \"thing\"}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::some: thing\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Merging a map with null\",\n\t\texpression: `{\"some\": \"thing\"} * null`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::some: thing\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Merging a null with an array\",\n\t\texpression: `null * [\"some\"]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- some\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Merging an array with null\",\n\t\texpression: `[\"some\"] * null`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- some\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `null * null`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!null)::null\\n\",\n\t\t},\n\t},\n}\n\nfunc TestMultiplyOperatorScenarios(t *testing.T) {\n\tfor _, tt := range multiplyOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"multiply-merge\", multiplyOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_omit.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"strconv\"\n)\n\nfunc omitMap(original *CandidateNode, indices *CandidateNode) *CandidateNode {\n\tfilteredContent := make([]*CandidateNode, 0, max(0, len(original.Content)-len(indices.Content)*2))\n\n\tfor index := 0; index < len(original.Content); index += 2 {\n\t\tpos := findInArray(indices, original.Content[index])\n\t\tif pos < 0 {\n\t\t\tclonedKey := original.Content[index].Copy()\n\t\t\tclonedValue := original.Content[index+1].Copy()\n\t\t\tfilteredContent = append(filteredContent, clonedKey, clonedValue)\n\t\t}\n\t}\n\tresult := original.CopyWithoutContent()\n\tresult.AddChildren(filteredContent)\n\treturn result\n}\n\nfunc omitSequence(original *CandidateNode, indices *CandidateNode) *CandidateNode {\n\tfilteredContent := make([]*CandidateNode, 0, max(0, len(original.Content)-len(indices.Content)))\n\n\tfor index := 0; index < len(original.Content); index++ {\n\t\tpos := findInArray(indices, createScalarNode(index, strconv.Itoa(index)))\n\t\tif pos < 0 {\n\t\t\tfilteredContent = append(filteredContent, original.Content[index].Copy())\n\t\t}\n\t}\n\tresult := original.CopyWithoutContent()\n\tresult.AddChildren(filteredContent)\n\treturn result\n}\n\nfunc omitOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"Omit\")\n\n\tcontextIndicesToOmit, err := d.GetMatchingNodes(context, expressionNode.RHS)\n\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\tindicesToOmit := &CandidateNode{}\n\tif contextIndicesToOmit.MatchingNodes.Len() > 0 {\n\t\tindicesToOmit = contextIndicesToOmit.MatchingNodes.Front().Value.(*CandidateNode)\n\t}\n\tif len(indicesToOmit.Content) == 0 {\n\t\tlog.Debugf(\"No omit indices specified\")\n\t\treturn context, nil\n\t}\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tnode := el.Value.(*CandidateNode)\n\n\t\tvar replacement *CandidateNode\n\n\t\tswitch node.Kind {\n\t\tcase MappingNode:\n\t\t\treplacement = omitMap(node, indicesToOmit)\n\t\tcase SequenceNode:\n\t\t\treplacement = omitSequence(node, indicesToOmit)\n\t\tdefault:\n\t\t\tlog.Debugf(\"Omit from type %v (%v) is noop\", node.Tag, node.GetNicePath())\n\t\t\treturn context, nil\n\t\t}\n\t\treplacement.LeadingContent = node.LeadingContent\n\t\tresults.PushBack(replacement)\n\t}\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_omit_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar omitOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Omit keys from map\",\n\t\tsubdescription: \"Note that non existent keys are skipped.\",\n\t\tdocument: \"myMap: {cat: meow, dog: bark, thing: hamster, hamster: squeak}\\n\",\n\t\texpression: `.myMap |= omit([\"hamster\", \"cat\", \"goat\"])`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::myMap: {dog: bark, thing: hamster}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Omit splat\",\n\t\tskipDoc: true,\n\t\tdocument: \"{cat: meow, dog: bark, hamster: squeak}\\n\",\n\t\texpression: `omit([\"dog\"])[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[cat], (!!str)::meow\\n\",\n\t\t\t\"D0, P[hamster], (!!str)::squeak\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Omit keys from map\",\n\t\tskipDoc: true,\n\t\tdocument: \"!things myMap: {cat: meow, dog: bark, thing: hamster, hamster: squeak}\\n\",\n\t\texpression: `.myMap |= omit([\"hamster\", \"cat\", \"goat\"])`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::!things myMap: {dog: bark, thing: hamster}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Omit keys from map with comments\",\n\t\tskipDoc: true,\n\t\tdocument: \"# abc\\nmyMap: {cat: meow, dog: bark, thing: hamster, hamster: squeak}\\n# xyz\\n\",\n\t\texpression: `.myMap |= omit([\"hamster\", \"cat\", \"goat\"])`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::# abc\\nmyMap: {dog: bark, thing: hamster}\\n# xyz\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Omit indices from array\",\n\t\tsubdescription: \"Note that non existent indices are skipped.\",\n\t\tdocument: `[cat, leopard, lion]`,\n\t\texpression: `omit([2, 0, 734, -5])`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[leopard]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Omit indices from array with comments\",\n\t\tskipDoc: true,\n\t\tdocument: \"# abc\\n[cat, leopard, lion]\\n# xyz\",\n\t\texpression: `omit([2, 0, 734, -5])`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::# abc\\n[leopard]\\n# xyz\\n\",\n\t\t},\n\t},\n}\n\nfunc TestOmitOperatorScenarios(t *testing.T) {\n\tfor _, tt := range omitOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"omit\", omitOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_parent.go", "content": "package yqlib\n\nimport \"container/list\"\n\ntype parentOpPreferences struct {\n\tLevel int\n}\n\nfunc getParentsOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"getParentsOperator\")\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tparentsList := &CandidateNode{Kind: SequenceNode, Tag: \"!!seq\"}\n\t\tparent := candidate.Parent\n\t\tfor parent != nil {\n\t\t\tparentsList.AddChild(parent)\n\t\t\tparent = parent.Parent\n\t\t}\n\t\tresults.PushBack(parentsList)\n\t}\n\n\treturn context.ChildContext(results), nil\n\n}\n\nfunc getParentOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"getParentOperator\")\n\n\tvar results = list.New()\n\n\tprefs := expressionNode.Operation.Preferences.(parentOpPreferences)\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\t// Handle negative levels: count total parents first\n\t\tlevelsToGoUp := prefs.Level\n\t\tif prefs.Level < 0 {\n\t\t\t// Count all parents\n\t\t\ttotalParents := 0\n\t\t\ttemp := candidate.Parent\n\t\t\tfor temp != nil {\n\t\t\t\ttotalParents++\n\t\t\t\ttemp = temp.Parent\n\t\t\t}\n\t\t\t// Convert negative index to positive\n\t\t\t// -1 means last parent (root), -2 means second to last, etc.\n\t\t\tlevelsToGoUp = totalParents + prefs.Level + 1\n\t\t\tif levelsToGoUp < 0 {\n\t\t\t\tlevelsToGoUp = 0\n\t\t\t}\n\t\t}\n\n\t\tcurrentLevel := 0\n\t\tfor currentLevel < levelsToGoUp && candidate != nil {\n\t\t\tlog.Debugf(\"currentLevel: %v, desired: %v\", currentLevel, levelsToGoUp)\n\t\t\tlog.Debugf(\"candidate: %v\", NodeToString(candidate))\n\t\t\tcandidate = candidate.Parent\n\t\t\tcurrentLevel++\n\t\t}\n\n\t\tlog.Debugf(\"found candidate: %v\", NodeToString(candidate))\n\t\tif candidate != nil {\n\t\t\tresults.PushBack(candidate)\n\t\t}\n\t}\n\n\treturn context.ChildContext(results), nil\n\n}\n" }, { "path": "pkg/yqlib/operator_parent_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar parentOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Simple example\",\n\t\tdocument: `a: {nested: cat}`,\n\t\texpression: `.a.nested | parent`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!map)::{nested: cat}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Parent of nested matches\",\n\t\tdocument: `{a: {fruit: apple, name: bob}, b: {fruit: banana, name: sam}}`,\n\t\texpression: `.. | select(. == \"banana\") | parent`,\n\t\texpected: []string{\n\t\t\t\"D0, P[b], (!!map)::{fruit: banana, name: sam}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Get parent attribute\",\n\t\tdocument: `{a: {fruit: apple, name: bob}, b: {fruit: banana, name: sam}}`,\n\t\texpression: `.. | select(. == \"banana\") | parent.name`,\n\t\texpected: []string{\n\t\t\t\"D0, P[b name], (!!str)::sam\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Get parents\",\n\t\tsubdescription: \"Match all parents\",\n\t\tdocument: \"{a: {b: {c: cat} } }\",\n\t\texpression: `.a.b.c | parents`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- {c: cat}\\n- {b: {c: cat}}\\n- {a: {b: {c: cat}}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Get the top (root) parent\",\n\t\tsubdescription: \"Use negative numbers to get the top parents. You can think of this as indexing into the 'parents' array above\",\n\t\tdocument: \"a:\\n b:\\n c: cat\\n\",\n\t\texpression: `.a.b.c | parent(-1)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a:\\n b:\\n c: cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Root\",\n\t\tsubdescription: \"Alias for parent(-1), returns the top level parent. This is usually the document node.\",\n\t\tdocument: \"a:\\n b:\\n c: cat\\n\",\n\t\texpression: `.a.b.c | root`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a:\\n b:\\n c: cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"boundary negative\",\n\t\tskipDoc: true,\n\t\tdocument: \"a:\\n b:\\n c: cat\\n\",\n\t\texpression: `.a.b.c | parent(-3)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a b], (!!map)::c: cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"large negative\",\n\t\tskipDoc: true,\n\t\tdocument: \"a:\\n b:\\n c: cat\\n\",\n\t\texpression: `.a.b.c | parent(-10)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a b c], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"parent zero\",\n\t\tskipDoc: true,\n\t\tdocument: \"a:\\n b:\\n c: cat\\n\",\n\t\texpression: `.a.b.c | parent(0)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a b c], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"large positive\",\n\t\tskipDoc: true,\n\t\tdocument: \"a:\\n b:\\n c: cat\\n\",\n\t\texpression: `.a.b.c | parent(10)`,\n\t\texpected: []string{},\n\t},\n\t{\n\t\tdescription: \"N-th parent\",\n\t\tsubdescription: \"You can optionally supply the number of levels to go up for the parent, the default being 1.\",\n\t\tdocument: \"a:\\n b:\\n c: cat\\n\",\n\t\texpression: `.a.b.c | parent(2)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!map)::b:\\n c: cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"N-th parent - another level\",\n\t\tdocument: \"a:\\n b:\\n c: cat\\n\",\n\t\texpression: `.a.b.c | parent(3)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a:\\n b:\\n c: cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"N-th negative\",\n\t\tsubdescription: \"Similarly, use negative numbers to index backwards from the parents array\",\n\t\tdocument: \"a:\\n b:\\n c: cat\\n\",\n\t\texpression: `.a.b.c | parent(-2)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!map)::b:\\n c: cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"No parent\",\n\t\tdocument: `{}`,\n\t\texpression: `parent`,\n\t\texpected: []string{},\n\t},\n}\n\nfunc TestParentOperatorScenarios(t *testing.T) {\n\tfor _, tt := range parentOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"parent\", parentOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_path.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n)\n\nfunc createPathNodeFor(pathElement interface{}) *CandidateNode {\n\tswitch pathElement := pathElement.(type) {\n\tcase string:\n\t\treturn &CandidateNode{Kind: ScalarNode, Value: pathElement, Tag: \"!!str\"}\n\tdefault:\n\t\treturn &CandidateNode{Kind: ScalarNode, Value: fmt.Sprintf(\"%v\", pathElement), Tag: \"!!int\"}\n\t}\n}\n\nfunc getPathArrayFromNode(funcName string, node *CandidateNode) ([]interface{}, error) {\n\tif node.Kind != SequenceNode {\n\t\treturn nil, fmt.Errorf(\"%v: expected path array, but got %v instead\", funcName, node.Tag)\n\t}\n\n\tpath := make([]interface{}, len(node.Content))\n\n\tfor i, childNode := range node.Content {\n\t\tswitch childNode.Tag {\n\t\tcase \"!!str\":\n\t\t\tpath[i] = childNode.Value\n\t\tcase \"!!int\":\n\t\t\tnumber, err := parseInt(childNode.Value)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, fmt.Errorf(\"%v: could not parse %v as an int: %w\", funcName, childNode.Value, err)\n\t\t\t}\n\t\t\tpath[i] = number\n\t\tdefault:\n\t\t\treturn nil, fmt.Errorf(\"%v: expected either a !!str or !!int in the path, found %v instead\", funcName, childNode.Tag)\n\t\t}\n\n\t}\n\treturn path, nil\n}\n\n// SETPATH(pathArray; value)\nfunc setPathOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"SetPath\")\n\n\tif expressionNode.RHS.Operation.OperationType != blockOpType {\n\t\treturn Context{}, fmt.Errorf(\"SETPATH must be given a block (;), got %v instead\", expressionNode.RHS.Operation.OperationType.Type)\n\t}\n\n\tlhsPathContext, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS.LHS)\n\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\n\tif lhsPathContext.MatchingNodes.Len() != 1 {\n\t\treturn Context{}, fmt.Errorf(\"SETPATH: expected single path but found %v results instead\", lhsPathContext.MatchingNodes.Len())\n\t}\n\tlhsValue := lhsPathContext.MatchingNodes.Front().Value.(*CandidateNode)\n\n\tlhsPath, err := getPathArrayFromNode(\"SETPATH\", lhsValue)\n\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\n\tlhsTraversalTree := createTraversalTree(lhsPath, traversePreferences{}, false)\n\n\tassignmentOp := &Operation{OperationType: assignOpType}\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\ttargetContextValue, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS.RHS)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t\tif targetContextValue.MatchingNodes.Len() != 1 {\n\t\t\treturn Context{}, fmt.Errorf(\"SETPATH: expected single value on RHS but found %v\", targetContextValue.MatchingNodes.Len())\n\t\t}\n\n\t\trhsOp := &Operation{OperationType: referenceOpType, CandidateNode: targetContextValue.MatchingNodes.Front().Value.(*CandidateNode)}\n\n\t\tassignmentOpNode := &ExpressionNode{\n\t\t\tOperation: assignmentOp,\n\t\t\tLHS: lhsTraversalTree,\n\t\t\tRHS: &ExpressionNode{Operation: rhsOp},\n\t\t}\n\n\t\t_, err = d.GetMatchingNodes(context.SingleChildContext(candidate), assignmentOpNode)\n\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t}\n\treturn context, nil\n}\n\nfunc delPathsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"delPaths\")\n\t// single RHS expression that returns an array of paths (array of arrays)\n\n\tpathArraysContext, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS)\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\tif pathArraysContext.MatchingNodes.Len() != 1 {\n\t\treturn Context{}, fmt.Errorf(\"DELPATHS: expected single value but found %v\", pathArraysContext.MatchingNodes.Len())\n\t}\n\tpathArraysNode := pathArraysContext.MatchingNodes.Front().Value.(*CandidateNode)\n\n\tif pathArraysNode.Tag != \"!!seq\" {\n\t\treturn Context{}, fmt.Errorf(\"DELPATHS: expected a sequence of sequences, but found %v\", pathArraysNode.Tag)\n\t}\n\n\tupdatedContext := context\n\n\tfor i, child := range pathArraysNode.Content {\n\n\t\tif child.Tag != \"!!seq\" {\n\t\t\treturn Context{}, fmt.Errorf(\"DELPATHS: expected entry [%v] to be a sequence, but its a %v. Note that delpaths takes an array of path arrays, e.g. [[\\\"a\\\", \\\"b\\\"]]\", i, child.Tag)\n\t\t}\n\t\tchildPath, err := getPathArrayFromNode(\"DELPATHS\", child)\n\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t\tchildTraversalExp := createTraversalTree(childPath, traversePreferences{}, false)\n\t\tdeleteChildOp := &Operation{OperationType: deleteChildOpType}\n\n\t\tdeleteChildOpNode := &ExpressionNode{\n\t\t\tOperation: deleteChildOp,\n\t\t\tRHS: childTraversalExp,\n\t\t}\n\n\t\tupdatedContext, err = d.GetMatchingNodes(updatedContext, deleteChildOpNode)\n\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t}\n\n\treturn updatedContext, nil\n\n}\n\nfunc getPathOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"GetPath\")\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tnode := candidate.CreateReplacement(SequenceNode, \"!!seq\", \"\")\n\n\t\tpath := candidate.GetPath()\n\n\t\tcontent := make([]*CandidateNode, len(path))\n\t\tfor pathIndex := 0; pathIndex < len(path); pathIndex++ {\n\t\t\tpath := path[pathIndex]\n\t\t\tcontent[pathIndex] = createPathNodeFor(path)\n\t\t}\n\t\tnode.AddChildren(content)\n\t\tresults.PushBack(node)\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_path_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar documentToPrune = `\nparentA: bob\nparentB:\n child1: i am child1\n child2: i am child2\nparentC:\n child1: me child1\n child2: me child2\n`\n\nvar pathOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Map path\",\n\t\tdocument: `{a: {b: cat}}`,\n\t\texpression: `.a.b | path`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a b], (!!seq)::- a\\n- b\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Map path\",\n\t\tdocument: `{a: {b: cat}}`,\n\t\tskipDoc: true,\n\t\texpression: `.a.b | path[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a b 0], (!!str)::a\\n\",\n\t\t\t\"D0, P[a b 1], (!!str)::b\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a:\n b:\n c:\n - 0\n - 1\n - 2\n - 3`,\n\t\texpression: `.a.b.c.[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a b c 0], (!!int)::0\\n\",\n\t\t\t\"D0, P[a b c 1], (!!int)::1\\n\",\n\t\t\t\"D0, P[a b c 2], (!!int)::2\\n\",\n\t\t\t\"D0, P[a b c 3], (!!int)::3\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Get map key\",\n\t\tdocument: `{a: {b: cat}}`,\n\t\texpression: `.a.b | path | .[-1]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a b 1], (!!str)::b\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Array path\",\n\t\tdocument: `{a: [cat, dog]}`,\n\t\texpression: `.a.[] | select(. == \"dog\") | path`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a 1], (!!seq)::- a\\n- 1\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Get array index\",\n\t\tdocument: `{a: [cat, dog]}`,\n\t\texpression: `.a.[] | select(. == \"dog\") | path | .[-1]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a 1 1], (!!int)::1\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Print path and value\",\n\t\tdocument: `{a: [cat, dog, frog]}`,\n\t\texpression: `.a[] | select(. == \"*og\") | [{\"path\":path, \"value\":.}]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a 1], (!!seq)::- path:\\n - a\\n - 1\\n value: dog\\n\",\n\t\t\t\"D0, P[a 2], (!!seq)::- path:\\n - a\\n - 2\\n value: frog\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Set path\",\n\t\tdocument: `{a: {b: cat}}`,\n\t\texpression: `setpath([\"a\", \"b\"]; \"things\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {b: things}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Set on empty document\",\n\t\texpression: `setpath([\"a\", \"b\"]; \"things\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], ()::a:\\n b: things\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Set path to prune deep paths\",\n\t\tsubdescription: \"Like pick but recursive. This uses `ireduce` to deeply set the selected paths into an empty object.\",\n\t\tdocument: documentToPrune,\n\t\texpression: \"(.parentB.child2, .parentC.child1) as $i\\n ireduce({}; setpath($i | path; $i))\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::parentB:\\n child2: i am child2\\nparentC:\\n child1: me child1\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Set array path\",\n\t\tdocument: `a: [cat, frog]`,\n\t\texpression: `setpath([\"a\", 0]; \"things\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: [things, frog]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Set array path empty\",\n\t\texpression: `setpath([\"a\", 0]; \"things\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], ()::a:\\n - things\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Delete path\",\n\t\tsubdescription: \"Notice delpaths takes an _array_ of paths.\",\n\t\tdocument: `{a: {b: cat, c: dog, d: frog}}`,\n\t\texpression: `delpaths([[\"a\", \"c\"], [\"a\", \"d\"]])`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {b: cat}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Delete array path\",\n\t\tdocument: `a: [cat, frog]`,\n\t\texpression: `delpaths([[\"a\", 0]])`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: [frog]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Delete splat\",\n\t\tskipDoc: true,\n\t\tdocument: `a: [cat, frog]`,\n\t\texpression: `delpaths([[\"a\", 0]])[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!seq)::[frog]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Delete - wrong parameter\",\n\t\tsubdescription: \"delpaths does not work with a single path array\",\n\t\tdocument: `a: [cat, frog]`,\n\t\texpression: `delpaths([\"a\", 0])`,\n\t\texpectedError: \"DELPATHS: expected entry [0] to be a sequence, but its a !!str. Note that delpaths takes an array of path arrays, e.g. [[\\\"a\\\", \\\"b\\\"]]\",\n\t},\n}\n\nfunc TestPathOperatorsScenarios(t *testing.T) {\n\tfor _, tt := range pathOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"path\", pathOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_pick.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n)\n\nfunc pickMap(original *CandidateNode, indices *CandidateNode) *CandidateNode {\n\n\tfilteredContent := make([]*CandidateNode, 0)\n\tfor index := 0; index < len(indices.Content); index = index + 1 {\n\t\tkeyToFind := indices.Content[index]\n\n\t\tindexInMap := findKeyInMap(original, keyToFind)\n\t\tif indexInMap > -1 {\n\t\t\tclonedKey := original.Content[indexInMap].Copy()\n\t\t\tclonedValue := original.Content[indexInMap+1].Copy()\n\t\t\tfilteredContent = append(filteredContent, clonedKey, clonedValue)\n\t\t}\n\t}\n\n\tnewNode := original.CopyWithoutContent()\n\tnewNode.AddChildren(filteredContent)\n\n\treturn newNode\n}\n\nfunc pickSequence(original *CandidateNode, indices *CandidateNode) (*CandidateNode, error) {\n\n\tfilteredContent := make([]*CandidateNode, 0)\n\tfor index := 0; index < len(indices.Content); index = index + 1 {\n\t\tindexInArray, err := parseInt(indices.Content[index].Value)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"cannot index array with %v\", indices.Content[index].Value)\n\t\t}\n\n\t\tif indexInArray > -1 && indexInArray < len(original.Content) {\n\t\t\tfilteredContent = append(filteredContent, original.Content[indexInArray].Copy())\n\t\t}\n\t}\n\n\tnewNode := original.CopyWithoutContent()\n\tnewNode.AddChildren(filteredContent)\n\n\treturn newNode, nil\n}\n\nfunc pickOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"Pick\")\n\n\tcontextIndicesToPick, err := d.GetMatchingNodes(context, expressionNode.RHS)\n\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\tindicesToPick := &CandidateNode{}\n\tif contextIndicesToPick.MatchingNodes.Len() > 0 {\n\t\tindicesToPick = contextIndicesToPick.MatchingNodes.Front().Value.(*CandidateNode)\n\t}\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tnode := el.Value.(*CandidateNode)\n\n\t\tvar replacement *CandidateNode\n\t\tswitch node.Kind {\n\t\tcase MappingNode:\n\t\t\treplacement = pickMap(node, indicesToPick)\n\t\tcase SequenceNode:\n\t\t\treplacement, err = pickSequence(node, indicesToPick)\n\t\t\tif err != nil {\n\t\t\t\treturn Context{}, err\n\t\t\t}\n\n\t\tdefault:\n\t\t\treturn Context{}, fmt.Errorf(\"cannot pick indices from type %v (%v)\", node.Tag, node.GetNicePath())\n\t\t}\n\n\t\treplacement.LeadingContent = node.LeadingContent\n\t\tresults.PushBack(replacement)\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_pick_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar pickOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Pick keys from map\",\n\t\tsubdescription: \"Note that the order of the keys matches the pick order and non existent keys are skipped.\",\n\t\tdocument: \"myMap: {cat: meow, dog: bark, thing: hamster, hamster: squeak}\\n\",\n\t\texpression: `.myMap |= pick([\"hamster\", \"cat\", \"goat\"])`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::myMap: {hamster: squeak, cat: meow}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Pick keys from map, included all the keys\",\n\t\tsubdescription: \"We create a map of the picked keys plus all the current keys, and run that through unique\",\n\t\tdocument: \"myMap: {cat: meow, dog: bark, thing: hamster, hamster: squeak}\\n\",\n\t\texpression: `.myMap |= pick( ([\"thing\"] + keys) | unique)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::myMap: {thing: hamster, cat: meow, dog: bark, hamster: squeak}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Pick splat\",\n\t\tskipDoc: true,\n\t\tdocument: \"{cat: meow, dog: bark, thing: hamster, hamster: squeak}\\n\",\n\t\texpression: `pick([\"hamster\", \"cat\"])[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[hamster], (!!str)::squeak\\n\",\n\t\t\t\"D0, P[cat], (!!str)::meow\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Pick keys from map\",\n\t\tskipDoc: true,\n\t\tdocument: \"!things myMap: {cat: meow, dog: bark, thing: hamster, hamster: squeak}\\n\",\n\t\texpression: `.myMap |= pick([\"hamster\", \"cat\", \"goat\"])`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::!things myMap: {hamster: squeak, cat: meow}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Pick keys from map with comments\",\n\t\tskipDoc: true,\n\t\tdocument: \"# abc\\nmyMap: {cat: meow, dog: bark, thing: hamster, hamster: squeak}\\n# xyz\\n\",\n\t\texpression: `.myMap |= pick([\"hamster\", \"cat\", \"goat\"])`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::# abc\\nmyMap: {hamster: squeak, cat: meow}\\n# xyz\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Pick indices from array\",\n\t\tsubdescription: \"Note that the order of the indices matches the pick order and non existent indices are skipped.\",\n\t\tdocument: `[cat, leopard, lion]`,\n\t\texpression: `pick([2, 0, 734, -5])`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[lion, cat]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Pick indices from array with comments\",\n\t\tskipDoc: true,\n\t\tdocument: \"# abc\\n[cat, leopard, lion]\\n# xyz\",\n\t\texpression: `pick([2, 0, 734, -5])`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::# abc\\n[lion, cat]\\n# xyz\\n\",\n\t\t},\n\t},\n}\n\nfunc TestPickOperatorScenarios(t *testing.T) {\n\tfor _, tt := range pickOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"pick\", pickOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_pipe.go", "content": "package yqlib\n\nfunc pipeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\n\tif expressionNode.LHS.Operation.OperationType == assignVariableOpType {\n\t\treturn variableLoop(d, context, expressionNode)\n\t}\n\tlhs, err := d.GetMatchingNodes(context, expressionNode.LHS)\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\trhsContext := context.ChildContext(lhs.MatchingNodes)\n\trhs, err := d.GetMatchingNodes(rhsContext, expressionNode.RHS)\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\treturn context.ChildContext(rhs.MatchingNodes), nil\n}\n" }, { "path": "pkg/yqlib/operator_pipe_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar pipeOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Simple Pipe\",\n\t\tdocument: `{a: {b: cat}}`,\n\t\texpression: `.a | .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a b], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Multiple updates\",\n\t\tdocument: `{a: cow, b: sheep, c: same}`,\n\t\texpression: `.a = \"cat\" | .b = \"dog\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: cat, b: dog, c: same}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Don't pass readonly context\",\n\t\texpression: `(3 + 4) | ({} | .b = \"dog\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::b: dog\\n\",\n\t\t},\n\t},\n}\n\nfunc TestPipeOperatorScenarios(t *testing.T) {\n\tfor _, tt := range pipeOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"pipe\", pipeOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_pivot.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n)\n\nfunc getUniqueElementTag(seq *CandidateNode) (string, error) {\n\tswitch l := len(seq.Content); l {\n\tcase 0:\n\t\treturn \"\", nil\n\tdefault:\n\t\tresult := seq.Content[0].Tag\n\t\tfor i := 1; i < l; i++ {\n\t\t\tt := seq.Content[i].Tag\n\t\t\tif t != result {\n\t\t\t\treturn \"\", fmt.Errorf(\"sequence contains elements of %v and %v types\", result, t)\n\t\t\t}\n\t\t}\n\t\treturn result, nil\n\t}\n}\n\nvar nullNodeFactory = func() *CandidateNode { return createScalarNode(nil, \"\") }\n\nfunc pad[E any](array []E, length int, factory func() E) []E {\n\tsz := len(array)\n\tif sz >= length {\n\t\treturn array\n\t}\n\tpad := make([]E, length-sz)\n\tfor i := 0; i < len(pad); i++ {\n\t\tpad[i] = factory()\n\t}\n\treturn append(array, pad...)\n}\n\nfunc pivotSequences(seq *CandidateNode) *CandidateNode {\n\tsz := len(seq.Content)\n\tif sz == 0 {\n\t\treturn seq\n\t}\n\tm := make(map[int][]*CandidateNode)\n\n\tfor i := 0; i < sz; i++ {\n\t\trow := seq.Content[i]\n\t\tfor j := 0; j < len(row.Content); j++ {\n\t\t\te := m[j]\n\t\t\tif e == nil {\n\t\t\t\te = make([]*CandidateNode, 0, sz)\n\t\t\t}\n\t\t\tm[j] = append(pad(e, i, nullNodeFactory), row.Content[j])\n\t\t}\n\t}\n\tresult := CandidateNode{Kind: SequenceNode}\n\n\tfor i := 0; i < len(m); i++ {\n\t\te := CandidateNode{Kind: SequenceNode}\n\t\te.AddChildren(pad(m[i], sz, nullNodeFactory))\n\t\tresult.AddChild(&e)\n\t}\n\treturn &result\n}\n\nfunc pivotMaps(seq *CandidateNode) *CandidateNode {\n\tsz := len(seq.Content)\n\tif sz == 0 {\n\t\treturn &CandidateNode{Kind: MappingNode}\n\t}\n\tm := make(map[string][]*CandidateNode)\n\tkeys := make([]string, 0)\n\n\tfor i := 0; i < sz; i++ {\n\t\trow := seq.Content[i]\n\t\tfor j := 0; j < len(row.Content); j += 2 {\n\t\t\tk := row.Content[j].Value\n\t\t\tv := row.Content[j+1]\n\t\t\te := m[k]\n\t\t\tif e == nil {\n\t\t\t\tkeys = append(keys, k)\n\t\t\t\te = make([]*CandidateNode, 0, sz)\n\t\t\t}\n\t\t\tm[k] = append(pad(e, i, nullNodeFactory), v)\n\t\t}\n\t}\n\tresult := CandidateNode{Kind: MappingNode}\n\tfor _, k := range keys {\n\t\tpivotRow := CandidateNode{Kind: SequenceNode}\n\t\tpivotRow.AddChildren(\n\t\t\tpad(m[k], sz, nullNodeFactory))\n\t\tresult.AddKeyValueChild(createScalarNode(k, k), &pivotRow)\n\t}\n\treturn &result\n}\n\nfunc pivotOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tlog.Debug(\"Pivot\")\n\tresults := list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tif candidate.Tag != \"!!seq\" {\n\t\t\treturn Context{}, fmt.Errorf(\"cannot pivot node of type %v\", candidate.Tag)\n\t\t}\n\t\ttag, err := getUniqueElementTag(candidate)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\tvar pivot *CandidateNode\n\t\tswitch tag {\n\t\tcase \"!!seq\":\n\t\t\tpivot = pivotSequences(candidate)\n\t\tcase \"!!map\":\n\t\t\tpivot = pivotMaps(candidate)\n\t\tdefault:\n\t\t\treturn Context{}, fmt.Errorf(\"can only pivot elements of !!seq or !!map types, received %v\", tag)\n\t\t}\n\t\tresults.PushBack(pivot)\n\t}\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_pivot_test.go", "content": "package yqlib\n\nimport \"testing\"\n\nvar pivotOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Pivot a sequence of sequences\",\n\t\tdocument: \"[[foo, bar, baz], [sis, boom, bah]]\\n\",\n\t\texpression: `pivot`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], ()::- - foo\\n - sis\\n- - bar\\n - boom\\n- - baz\\n - bah\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Pivot splat\",\n\t\tskipDoc: true,\n\t\tdocument: \"[[foo, bar], [sis, boom]]\\n\",\n\t\texpression: `pivot[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], ()::- foo\\n- sis\\n\",\n\t\t\t\"D0, P[1], ()::- bar\\n- boom\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Pivot sequence of heterogeneous sequences\",\n\t\tsubdescription: `Missing values are \"padded\" to null.`,\n\t\tdocument: \"[[foo, bar, baz], [sis, boom, bah, blah]]\\n\",\n\t\texpression: `pivot`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], ()::- - foo\\n - sis\\n- - bar\\n - boom\\n- - baz\\n - bah\\n- -\\n - blah\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Pivot sequence of maps\",\n\t\tdocument: \"[{foo: a, bar: b, baz: c}, {foo: x, bar: y, baz: z}]\\n\",\n\t\texpression: `pivot`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], ()::foo:\\n - a\\n - x\\nbar:\\n - b\\n - y\\nbaz:\\n - c\\n - z\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Pivot sequence of heterogeneous maps\",\n\t\tsubdescription: `Missing values are \"padded\" to null.`,\n\t\tdocument: \"[{foo: a, bar: b, baz: c}, {foo: x, bar: y, baz: z, what: ever}]\\n\",\n\t\texpression: `pivot`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], ()::foo:\\n - a\\n - x\\nbar:\\n - b\\n - y\\nbaz:\\n - c\\n - z\\nwhat:\\n -\\n - ever\\n\",\n\t\t},\n\t},\n}\n\nfunc TestPivotOperatorScenarios(t *testing.T) {\n\tfor _, tt := range pivotOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"pivot\", pivotOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_recursive_descent.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n)\n\ntype recursiveDescentPreferences struct {\n\tTraversePreferences traversePreferences\n\tRecurseArray bool\n}\n\nfunc recursiveDescentOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tvar results = list.New()\n\n\tpreferences := expressionNode.Operation.Preferences.(recursiveDescentPreferences)\n\terr := recursiveDecent(results, context, preferences)\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n\nfunc recursiveDecent(results *list.List, context Context, preferences recursiveDescentPreferences) error {\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\tlog.Debugf(\"added %v\", NodeToString(candidate))\n\t\tresults.PushBack(candidate)\n\n\t\tif candidate.Kind != AliasNode && len(candidate.Content) > 0 &&\n\t\t\t(preferences.RecurseArray || candidate.Kind != SequenceNode) {\n\n\t\t\tchildren, err := splat(context.SingleChildContext(candidate), preferences.TraversePreferences)\n\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\terr = recursiveDecent(results, children, preferences)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/operator_recursive_descent_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar recursiveDescentOperatorScenarios = []expressionScenario{\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{}`,\n\t\texpression: `..`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{}`,\n\t\texpression: `...`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[]`,\n\t\texpression: `..`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[]`,\n\t\texpression: `...`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `cat`,\n\t\texpression: `..`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `cat`,\n\t\texpression: `...`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Recurse map (values only)\",\n\t\tdocument: `{a: frog}`,\n\t\texpression: `..`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: frog}\\n\",\n\t\t\t\"D0, P[a], (!!str)::frog\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Recursively find nodes with keys\",\n\t\tsubdescription: \"Note that this example has wrapped the expression in `[]` to show that there are two matches returned. You do not have to wrap in `[]` in your path expression.\",\n\t\tdocument: `{a: {name: frog, b: {name: blog, age: 12}}}`,\n\t\texpression: `[.. | select(has(\"name\"))]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- {name: frog, b: {name: blog, age: 12}}\\n- {name: blog, age: 12}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Recursively find nodes with values\",\n\t\tdocument: `{a: {nameA: frog, b: {nameB: frog, age: 12}}}`,\n\t\texpression: `.. | select(. == \"frog\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a nameA], (!!str)::frog\\n\",\n\t\t\t\"D0, P[a b nameB], (!!str)::frog\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Recurse map (values and keys)\",\n\t\tsubdescription: \"Note that the map key appears in the results\",\n\t\tdocument: `{a: frog}`,\n\t\texpression: `...`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: frog}\\n\",\n\t\t\t\"D0, P[a], (!!str)::a\\n\",\n\t\t\t\"D0, P[a], (!!str)::frog\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: {b: apple}}`,\n\t\texpression: `..`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {b: apple}}\\n\",\n\t\t\t\"D0, P[a], (!!map)::{b: apple}\\n\",\n\t\t\t\"D0, P[a b], (!!str)::apple\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: {b: apple}}`,\n\t\texpression: `...`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: {b: apple}}\\n\",\n\t\t\t\"D0, P[a], (!!str)::a\\n\",\n\t\t\t\"D0, P[a], (!!map)::{b: apple}\\n\",\n\t\t\t\"D0, P[a b], (!!str)::b\\n\",\n\t\t\t\"D0, P[a b], (!!str)::apple\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[1,2,3]`,\n\t\texpression: `..`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[1, 2, 3]\\n\",\n\t\t\t\"D0, P[0], (!!int)::1\\n\",\n\t\t\t\"D0, P[1], (!!int)::2\\n\",\n\t\t\t\"D0, P[2], (!!int)::3\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[1,2,3]`,\n\t\texpression: `...`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[1, 2, 3]\\n\",\n\t\t\t\"D0, P[0], (!!int)::1\\n\",\n\t\t\t\"D0, P[1], (!!int)::2\\n\",\n\t\t\t\"D0, P[2], (!!int)::3\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[{a: cat},2,true]`,\n\t\texpression: `..`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[{a: cat}, 2, true]\\n\",\n\t\t\t\"D0, P[0], (!!map)::{a: cat}\\n\",\n\t\t\t\"D0, P[0 a], (!!str)::cat\\n\",\n\t\t\t\"D0, P[1], (!!int)::2\\n\",\n\t\t\t\"D0, P[2], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[{a: cat},2,true]`,\n\t\texpression: `...`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[{a: cat}, 2, true]\\n\",\n\t\t\t\"D0, P[0], (!!map)::{a: cat}\\n\",\n\t\t\t\"D0, P[0 a], (!!str)::a\\n\",\n\t\t\t\"D0, P[0 a], (!!str)::cat\\n\",\n\t\t\t\"D0, P[1], (!!int)::2\\n\",\n\t\t\t\"D0, P[2], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Aliases are not traversed\",\n\t\tdocument: `{a: &cat {c: frog}, b: *cat}`,\n\t\texpression: `[..]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- {a: &cat {c: frog}, b: *cat}\\n- &cat {c: frog}\\n- frog\\n- *cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: &cat {c: frog}, b: *cat}`,\n\t\texpression: `...`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: &cat {c: frog}, b: *cat}\\n\",\n\t\t\t\"D0, P[a], (!!str)::a\\n\",\n\t\t\t\"D0, P[a], (!!map)::&cat {c: frog}\\n\",\n\t\t\t\"D0, P[a c], (!!str)::c\\n\",\n\t\t\t\"D0, P[a c], (!!str)::frog\\n\",\n\t\t\t\"D0, P[b], (!!str)::b\\n\",\n\t\t\t\"D0, P[b], (alias)::*cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Merge docs are not traversed\",\n\t\tdocument: mergeDocSample,\n\t\texpression: `.foobar | [..]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[foobar], (!!seq)::- c: foobar_c\\n !!merge <<: *foo\\n thing: foobar_thing\\n- foobar_c\\n- *foo\\n- foobar_thing\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: mergeDocSample,\n\t\texpression: `.foobar | [...]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[foobar], (!!seq)::- c: foobar_c\\n !!merge <<: *foo\\n thing: foobar_thing\\n- c\\n- foobar_c\\n- !!merge <<\\n- *foo\\n- thing\\n- foobar_thing\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: mergeDocSample,\n\t\texpression: `.foobarList | ..`,\n\t\texpected: []string{\n\t\t\t\"D0, P[foobarList], (!!map)::b: foobarList_b\\n!!merge <<: [*foo, *bar]\\nc: foobarList_c\\n\",\n\t\t\t\"D0, P[foobarList b], (!!str)::foobarList_b\\n\",\n\t\t\t\"D0, P[foobarList <<], (!!seq)::[*foo, *bar]\\n\",\n\t\t\t\"D0, P[foobarList << 0], (alias)::*foo\\n\",\n\t\t\t\"D0, P[foobarList << 1], (alias)::*bar\\n\",\n\t\t\t\"D0, P[foobarList c], (!!str)::foobarList_c\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: mergeDocSample,\n\t\texpression: `.foobarList | ...`,\n\t\texpected: []string{\n\t\t\t\"D0, P[foobarList], (!!map)::b: foobarList_b\\n!!merge <<: [*foo, *bar]\\nc: foobarList_c\\n\",\n\t\t\t\"D0, P[foobarList b], (!!str)::b\\n\",\n\t\t\t\"D0, P[foobarList b], (!!str)::foobarList_b\\n\",\n\t\t\t\"D0, P[foobarList <<], (!!merge)::<<\\n\",\n\t\t\t\"D0, P[foobarList <<], (!!seq)::[*foo, *bar]\\n\",\n\t\t\t\"D0, P[foobarList << 0], (alias)::*foo\\n\",\n\t\t\t\"D0, P[foobarList << 1], (alias)::*bar\\n\",\n\t\t\t\"D0, P[foobarList c], (!!str)::c\\n\",\n\t\t\t\"D0, P[foobarList c], (!!str)::foobarList_c\\n\",\n\t\t},\n\t},\n}\n\nfunc TestRecursiveDescentOperatorScenarios(t *testing.T) {\n\tfor _, tt := range recursiveDescentOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"recursive-descent-glob\", recursiveDescentOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_reduce.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n)\n\nfunc reduceOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"reduceOp\")\n\t//.a as $var reduce (0; . + $var)\n\t//lhs is the assignment operator\n\t//rhs is the reduce block\n\t// '.' refers to the current accumulator, initialised to 0\n\t// $var references a single element from the .a\n\n\t//ensure lhs is actually an assignment\n\t//and rhs is a block (empty)\n\tif expressionNode.LHS.Operation.OperationType != assignVariableOpType {\n\t\treturn Context{}, fmt.Errorf(\"reduce must be given a variables assignment, got %v instead\", expressionNode.LHS.Operation.OperationType.Type)\n\t} else if expressionNode.RHS.Operation.OperationType != blockOpType {\n\t\treturn Context{}, fmt.Errorf(\"reduce must be given a block, got %v instead\", expressionNode.RHS.Operation.OperationType.Type)\n\t}\n\n\tarrayExpNode := expressionNode.LHS.LHS\n\tarray, err := d.GetMatchingNodes(context, arrayExpNode)\n\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\n\tvariableName := expressionNode.LHS.RHS.Operation.StringValue\n\n\tinitExp := expressionNode.RHS.LHS\n\n\taccum, err := d.GetMatchingNodes(context, initExp)\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\n\tlog.Debugf(\"with variable %v\", variableName)\n\n\tblockExp := expressionNode.RHS.RHS\n\tfor el := array.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tlog.Debugf(\"REDUCING WITH %v\", NodeToString(candidate))\n\t\tl := list.New()\n\t\tl.PushBack(candidate)\n\t\taccum.SetVariable(variableName, l)\n\n\t\taccum, err = d.GetMatchingNodes(accum, blockExp)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t}\n\n\treturn accum, nil\n}\n" }, { "path": "pkg/yqlib/operator_reduce_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar reduceOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Sum numbers\",\n\t\tdocument: `[10,2, 5, 3]`,\n\t\texpression: `.[] as $item ireduce (0; . + $item)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!int)::20\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Merge all yaml files together\",\n\t\tdocument: `a: cat`,\n\t\tdocument2: `b: dog`,\n\t\texpression: `. as $item ireduce ({}; . * $item )`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: cat\\nb: dog\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Convert an array to an object\",\n\t\tdocument: `[{name: Cathy, has: apples},{name: Bob, has: bananas}]`,\n\t\texpression: `.[] as $item ireduce ({}; .[$item | .name] = ($item | .has) )`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::Cathy: apples\\nBob: bananas\\n\",\n\t\t},\n\t},\n}\n\nfunc TestReduceOperatorScenarios(t *testing.T) {\n\tfor _, tt := range reduceOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"reduce\", reduceOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_reverse.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n)\n\nfunc reverseOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tresults := list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\tif candidate.Kind != SequenceNode {\n\t\t\treturn context, fmt.Errorf(\"node at path [%v] is not an array (it's a %v)\", candidate.GetNicePath(), candidate.Tag)\n\t\t}\n\n\t\treverseList := candidate.CreateReplacementWithComments(SequenceNode, \"!!seq\", candidate.Style)\n\t\treverseContent := make([]*CandidateNode, len(candidate.Content))\n\n\t\tfor i, originalNode := range candidate.Content {\n\t\t\treverseContent[len(candidate.Content)-i-1] = originalNode\n\t\t}\n\t\treverseList.AddChildren(reverseContent)\n\t\tresults.PushBack(reverseList)\n\n\t}\n\n\treturn context.ChildContext(results), nil\n\n}\n" }, { "path": "pkg/yqlib/operator_reverse_test.go", "content": "package yqlib\n\nimport \"testing\"\n\nvar reverseOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Reverse\",\n\t\tdocument: \"[1, 2, 3]\",\n\t\texpression: `reverse`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[3, 2, 1]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Reverse\",\n\t\tskipDoc: true,\n\t\tdocument: \"[1, 2]\",\n\t\texpression: `reverse[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!int)::2\\n\",\n\t\t\t\"D0, P[1], (!!int)::1\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"[]\",\n\t\texpression: `reverse`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"[1]\",\n\t\texpression: `reverse`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[1]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"[1,2]\",\n\t\texpression: `reverse`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[2, 1]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Sort descending by string field\",\n\t\tsubdescription: \"Use sort with reverse to sort in descending order.\",\n\t\tdocument: \"[{a: banana},{a: cat},{a: apple}]\",\n\t\texpression: `sort_by(.a) | reverse`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[{a: cat}, {a: banana}, {a: apple}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Sort descending by string field, with comments\",\n\t\tskipDoc: true,\n\t\tdocument: \"# abc\\n[{a: banana},{a: cat},{a: apple}]\\n# xyz\",\n\t\texpression: `sort_by(.a) | reverse`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::# abc\\n[{a: cat}, {a: banana}, {a: apple}]\\n# xyz\\n\",\n\t\t},\n\t},\n}\n\nfunc TestReverseOperatorScenarios(t *testing.T) {\n\tfor _, tt := range reverseOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"reverse\", reverseOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_select.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n)\n\nfunc selectOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\n\tlog.Debugf(\"selectOperation\")\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\trhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS)\n\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t\t// find any truthy node\n\t\tincludeResult := false\n\n\t\tfor resultEl := rhs.MatchingNodes.Front(); resultEl != nil; resultEl = resultEl.Next() {\n\t\t\tresult := resultEl.Value.(*CandidateNode)\n\t\t\tincludeResult = isTruthyNode(result)\n\t\t\tif includeResult {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\tif includeResult {\n\t\t\tresults.PushBack(candidate)\n\t\t}\n\t}\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_select_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar selectOperatorScenarios = []expressionScenario{\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `cat: pants`,\n\t\texpression: `select(.nope) | key + \" why though?\"`,\n\t\texpected: []string{},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `cat`,\n\t\texpression: `select(false, true)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `cat`,\n\t\texpression: `select(true, false)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `cat`,\n\t\texpression: `select(false)`,\n\t\texpected: []string{},\n\t},\n\t{\n\t\tdescription: \"Select elements from array using wildcard prefix\",\n\t\tdocument: `[cat,goat,dog]`,\n\t\texpression: `.[] | select(. == \"*at\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!str)::cat\\n\",\n\t\t\t\"D0, P[1], (!!str)::goat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Select elements from array using wildcard suffix\",\n\t\tdocument: `[go-kart,goat,dog]`,\n\t\texpression: `.[] | select(. == \"go*\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!str)::go-kart\\n\",\n\t\t\t\"D0, P[1], (!!str)::goat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Select elements from array using wildcard prefix and suffix\",\n\t\tdocument: `[ago, go, meow, going]`,\n\t\texpression: `.[] | select(. == \"*go*\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!str)::ago\\n\",\n\t\t\t\"D0, P[1], (!!str)::go\\n\",\n\t\t\t\"D0, P[3], (!!str)::going\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Select elements from array with regular expression\",\n\t\tsubdescription: \"See more regular expression examples under the [`string` operator docs](https://mikefarah.gitbook.io/yq/operators/string-operators).\",\n\t\tdocument: `[this_0, not_this, nor_0_this, thisTo_4]`,\n\t\texpression: `.[] | select(test(\"[a-zA-Z]+_[0-9]$\"))`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!str)::this_0\\n\",\n\t\t\t\"D0, P[3], (!!str)::thisTo_4\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: hello\",\n\t\tdocument2: \"b: world\",\n\t\texpression: `select(.a == \"hello\" or .b == \"world\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: hello\\n\",\n\t\t\t\"D0, P[], (!!map)::b: world\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"select splat\",\n\t\tskipDoc: true,\n\t\tdocument: \"a: hello\",\n\t\tdocument2: \"b: world\",\n\t\texpression: `select(.a == \"hello\" or .b == \"world\")[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!str)::hello\\n\",\n\t\t\t\"D0, P[b], (!!str)::world\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"select does not update the map\",\n\t\tskipDoc: true,\n\t\tdocument: `[{animal: cat, legs: {cool: true}}, {animal: fish}]`,\n\t\texpression: `(.[] | select(.legs.cool == true).canWalk) = true | (.[] | .alive.things) = \"yes\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[{animal: cat, legs: {cool: true}, canWalk: true, alive: {things: yes}}, {animal: fish, alive: {things: yes}}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[hot, fot, dog]`,\n\t\texpression: `.[] | select(. == \"*at\")`,\n\t\texpected: []string{},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a: [cat,goat,dog]`,\n\t\texpression: `.a.[] | select(. == \"*at\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a 0], (!!str)::cat\\n\",\n\t\t\t\"D0, P[a 1], (!!str)::goat\\n\"},\n\t},\n\t{\n\t\tdescription: \"Select items from a map\",\n\t\tdocument: `{ things: cat, bob: goat, horse: dog }`,\n\t\texpression: `.[] | select(. == \"cat\" or test(\"og$\"))`,\n\t\texpected: []string{\n\t\t\t\"D0, P[things], (!!str)::cat\\n\",\n\t\t\t\"D0, P[horse], (!!str)::dog\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Use select and with_entries to filter map keys\",\n\t\tdocument: `{name: bob, legs: 2, game: poker}`,\n\t\texpression: `with_entries(select(.key | test(\"ame$\")))`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::name: bob\\ngame: poker\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Select multiple items in a map and update\",\n\t\tsubdescription: \"Note the brackets around the entire LHS.\",\n\t\tdocument: `a: { things: cat, bob: goat, horse: dog }`,\n\t\texpression: `(.a.[] | select(. == \"cat\" or . == \"goat\")) |= \"rabbit\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: {things: rabbit, bob: rabbit, horse: dog}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a: { things: {include: true}, notMe: {include: false}, andMe: {include: fold} }`,\n\t\texpression: `.a.[] | select(.include)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a things], (!!map)::{include: true}\\n\",\n\t\t\t\"D0, P[a andMe], (!!map)::{include: fold}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[cat,~,dog]`,\n\t\texpression: `.[] | select(. == ~)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[1], (!!null)::~\\n\",\n\t\t},\n\t},\n}\n\nfunc TestSelectOperatorScenarios(t *testing.T) {\n\tfor _, tt := range selectOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"select\", selectOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_self.go", "content": "package yqlib\n\nfunc selfOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\treturn context, nil\n}\n" }, { "path": "pkg/yqlib/operator_shuffle.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n\t\"math/rand\"\n)\n\nfunc shuffleOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\n\t// ignore CWE-338 gosec issue of not using crypto/rand\n\t// this is just to shuffle an array rather generating a\n\t// secret or something that needs proper rand.\n\tmyRand := rand.New(rand.NewSource(Now().UnixNano())) // #nosec\n\n\tresults := list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\tif candidate.Kind != SequenceNode {\n\t\t\treturn context, fmt.Errorf(\"node at path [%v] is not an array (it's a %v)\", candidate.GetNicePath(), candidate.Tag)\n\t\t}\n\n\t\tresult := candidate.Copy()\n\n\t\ta := result.Content\n\n\t\tmyRand.Shuffle(len(a), func(i, j int) {\n\t\t\ta[i], a[j] = a[j], a[i]\n\t\t\toldIndex := a[i].Key.Value\n\t\t\ta[i].Key.Value = a[j].Key.Value\n\t\t\ta[j].Key.Value = oldIndex\n\t\t})\n\n\t\tresults.PushBack(result)\n\t}\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_shuffle_test.go", "content": "package yqlib\n\nimport \"testing\"\n\nvar shuffleOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Shuffle array\",\n\t\tdocument: \"[1, 2, 3, 4, 5]\",\n\t\texpression: `shuffle`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[5, 2, 4, 1, 3]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Shuffle array\",\n\t\tskipDoc: true,\n\t\tdocument: \"[1, 2, 3]\",\n\t\texpression: `shuffle[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!int)::3\\n\",\n\t\t\t\"D0, P[1], (!!int)::1\\n\",\n\t\t\t\"D0, P[2], (!!int)::2\\n\",\n\t\t},\n\t},\n\n\t{\n\t\tdescription: \"Shuffle array in place\",\n\t\tdocument: \"cool: [1, 2, 3, 4, 5]\",\n\t\texpression: `.cool |= shuffle`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::cool: [5, 2, 4, 1, 3]\\n\",\n\t\t},\n\t},\n}\n\nfunc TestShuffleByOperatorScenarios(t *testing.T) {\n\tfor _, tt := range shuffleOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"shuffle\", shuffleOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_slice.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n)\n\nfunc getSliceNumber(d *dataTreeNavigator, context Context, node *CandidateNode, expressionNode *ExpressionNode) (int, error) {\n\tresult, err := d.GetMatchingNodes(context.SingleChildContext(node), expressionNode)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tif result.MatchingNodes.Len() != 1 {\n\t\treturn 0, fmt.Errorf(\"expected to find 1 number, got %v instead\", result.MatchingNodes.Len())\n\t}\n\treturn parseInt(result.MatchingNodes.Front().Value.(*CandidateNode).Value)\n}\n\nfunc sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\n\tlog.Debug(\"slice array operator!\")\n\tlog.Debug(\"lhs: %v\", expressionNode.LHS.Operation.toString())\n\tlog.Debug(\"rhs: %v\", expressionNode.RHS.Operation.toString())\n\n\tresults := list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tlhsNode := el.Value.(*CandidateNode)\n\n\t\tfirstNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.LHS)\n\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\trelativeFirstNumber := firstNumber\n\t\tif relativeFirstNumber < 0 {\n\t\t\trelativeFirstNumber = len(lhsNode.Content) + firstNumber\n\t\t}\n\n\t\tsecondNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.RHS)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t\trelativeSecondNumber := secondNumber\n\t\tif relativeSecondNumber < 0 {\n\t\t\trelativeSecondNumber = len(lhsNode.Content) + secondNumber\n\t\t} else if relativeSecondNumber > len(lhsNode.Content) {\n\t\t\trelativeSecondNumber = len(lhsNode.Content)\n\t\t}\n\n\t\tlog.Debug(\"calculateIndicesToTraverse: slice from %v to %v\", relativeFirstNumber, relativeSecondNumber)\n\n\t\tvar newResults []*CandidateNode\n\t\tfor i := relativeFirstNumber; i < relativeSecondNumber; i++ {\n\t\t\tnewResults = append(newResults, lhsNode.Content[i])\n\t\t}\n\n\t\tsliceArrayNode := lhsNode.CreateReplacement(SequenceNode, lhsNode.Tag, \"\")\n\t\tsliceArrayNode.AddChildren(newResults)\n\t\tresults.PushBack(sliceArrayNode)\n\n\t}\n\n\t// result is now the context that has the nodes we need to put back into a sequence.\n\t//what about multiple arrays in the context? I think we need to create an array for each one\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_slice_test.go", "content": "package yqlib\n\nimport \"testing\"\n\nvar sliceArrayScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Slicing arrays\",\n\t\tdocument: `[cat, dog, frog, cow]`,\n\t\texpression: `.[1:3]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- dog\\n- frog\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Slicing arrays - without the first number\",\n\t\tsubdescription: \"Starts from the start of the array\",\n\t\tdocument: `[cat, dog, frog, cow]`,\n\t\texpression: `.[:2]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- cat\\n- dog\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Slicing arrays - without the second number\",\n\t\tsubdescription: \"Finishes at the end of the array\",\n\t\tdocument: `[cat, dog, frog, cow]`,\n\t\texpression: `.[2:]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- frog\\n- cow\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Slicing arrays - use negative numbers to count backwards from the end\",\n\t\tdocument: `[cat, dog, frog, cow]`,\n\t\texpression: `.[1:-1]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- dog\\n- frog\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Inserting into the middle of an array\",\n\t\tsubdescription: \"using an expression to find the index\",\n\t\tdocument: `[cat, dog, frog, cow]`,\n\t\texpression: `(.[] | select(. == \"dog\") | key + 1) as $pos | .[0:($pos)] + [\"rabbit\"] + .[$pos:]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- cat\\n- dog\\n- rabbit\\n- frog\\n- cow\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[[cat, dog, frog, cow], [apple, banana, grape, mango]]`,\n\t\texpression: `.[] | .[1:3]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!seq)::- dog\\n- frog\\n\",\n\t\t\t\"D0, P[1], (!!seq)::- banana\\n- grape\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"second index beyond array clamps\",\n\t\tdocument: `[cat]`,\n\t\texpression: `.[:3]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"first index beyond array returns nothing\",\n\t\tdocument: `[cat]`,\n\t\texpression: `.[3:]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[[cat, dog, frog, cow], [apple, banana, grape, mango]]`,\n\t\texpression: `.[] | .[-2:-1]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!seq)::- frog\\n\",\n\t\t\t\"D0, P[1], (!!seq)::- grape\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[cat1, cat2, cat3, cat4, cat5, cat6, cat7, cat8, cat9, cat10, cat11]`,\n\t\texpression: `.[10:11]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- cat11\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[cat1, cat2, cat3, cat4, cat5, cat6, cat7, cat8, cat9, cat10, cat11]`,\n\t\texpression: `.[-11:-10]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- cat1\\n\",\n\t\t},\n\t},\n}\n\nfunc TestSliceOperatorScenarios(t *testing.T) {\n\tfor _, tt := range sliceArrayScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"slice-array\", sliceArrayScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_sort.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n)\n\nfunc sortOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tselfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}\n\texpressionNode.RHS = selfExpression\n\treturn sortByOperator(d, context, expressionNode)\n}\n\n// context represents the current matching nodes in the expression pipeline\n// expressionNode is your current expression (sort_by)\nfunc sortByOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\n\tresults := list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\tvar sortableArray sortableNodeArray\n\n\t\tif candidate.CanVisitValues() {\n\t\t\tsortableArray = make(sortableNodeArray, 0)\n\t\t\tvisitor := func(valueNode *CandidateNode) error {\n\t\t\t\tcompareContext, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(valueNode), expressionNode.RHS)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tsortableNode := sortableNode{Node: valueNode, CompareContext: compareContext, dateTimeLayout: context.GetDateTimeLayout()}\n\t\t\t\tsortableArray = append(sortableArray, sortableNode)\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tif err := candidate.VisitValues(visitor); err != nil {\n\t\t\t\treturn context, err\n\t\t\t}\n\t\t} else {\n\t\t\treturn context, fmt.Errorf(\"node at path [%v] is not an array or map (it's a %v)\", candidate.GetNicePath(), candidate.Tag)\n\t\t}\n\n\t\tsort.Stable(sortableArray)\n\n\t\tsortedList := candidate.CopyWithoutContent()\n\t\tswitch candidate.Kind {\n\t\tcase MappingNode:\n\t\t\tfor _, sortedNode := range sortableArray {\n\t\t\t\tsortedList.AddKeyValueChild(sortedNode.Node.Key, sortedNode.Node)\n\t\t\t}\n\t\tcase SequenceNode:\n\t\t\tfor _, sortedNode := range sortableArray {\n\t\t\t\tsortedList.AddChild(sortedNode.Node)\n\t\t\t}\n\t\t}\n\n\t\t// convert array of value nodes back to map\n\t\tresults.PushBack(sortedList)\n\t}\n\treturn context.ChildContext(results), nil\n}\n\ntype sortableNode struct {\n\tNode *CandidateNode\n\tCompareContext Context\n\tdateTimeLayout string\n}\n\ntype sortableNodeArray []sortableNode\n\nfunc (a sortableNodeArray) Len() int { return len(a) }\nfunc (a sortableNodeArray) Swap(i, j int) { a[i], a[j] = a[j], a[i] }\n\nfunc (a sortableNodeArray) Less(i, j int) bool {\n\tlhsContext := a[i].CompareContext\n\trhsContext := a[j].CompareContext\n\n\trhsEl := rhsContext.MatchingNodes.Front()\n\n\tfor lhsEl := lhsContext.MatchingNodes.Front(); lhsEl != nil && rhsEl != nil; lhsEl = lhsEl.Next() {\n\t\tlhs := lhsEl.Value.(*CandidateNode)\n\t\trhs := rhsEl.Value.(*CandidateNode)\n\n\t\tresult := a.compare(lhs, rhs, a[i].dateTimeLayout)\n\n\t\tif result < 0 {\n\t\t\treturn true\n\t\t} else if result > 0 {\n\t\t\treturn false\n\t\t}\n\n\t\trhsEl = rhsEl.Next()\n\t}\n\treturn lhsContext.MatchingNodes.Len() < rhsContext.MatchingNodes.Len()\n}\n\nfunc (a sortableNodeArray) compare(lhs *CandidateNode, rhs *CandidateNode, dateTimeLayout string) int {\n\tlhsTag := lhs.Tag\n\trhsTag := rhs.Tag\n\n\tif !strings.HasPrefix(lhsTag, \"!!\") {\n\t\t// custom tag - we have to have a guess\n\t\tlhsTag = lhs.guessTagFromCustomType()\n\t}\n\n\tif !strings.HasPrefix(rhsTag, \"!!\") {\n\t\t// custom tag - we have to have a guess\n\t\trhsTag = rhs.guessTagFromCustomType()\n\t}\n\n\tisDateTime := lhsTag == \"!!timestamp\" && rhsTag == \"!!timestamp\"\n\tlayout := dateTimeLayout\n\t// if the lhs is a string, it might be a timestamp in a custom format.\n\tif lhsTag == \"!!str\" && layout != time.RFC3339 {\n\t\t_, errLhs := parseDateTime(layout, lhs.Value)\n\t\t_, errRhs := parseDateTime(layout, rhs.Value)\n\t\tisDateTime = errLhs == nil && errRhs == nil\n\t}\n\n\tif lhsTag == \"!!null\" && rhsTag != \"!!null\" {\n\t\treturn -1\n\t} else if lhsTag != \"!!null\" && rhsTag == \"!!null\" {\n\t\treturn 1\n\t} else if lhsTag == \"!!bool\" && rhsTag != \"!!bool\" {\n\t\treturn -1\n\t} else if lhsTag != \"!!bool\" && rhsTag == \"!!bool\" {\n\t\treturn 1\n\t} else if lhsTag == \"!!bool\" && rhsTag == \"!!bool\" {\n\t\tlhsTruthy := isTruthyNode(lhs)\n\n\t\trhsTruthy := isTruthyNode(rhs)\n\t\tif lhsTruthy == rhsTruthy {\n\t\t\treturn 0\n\t\t} else if lhsTruthy {\n\t\t\treturn 1\n\t\t}\n\t\treturn -1\n\t} else if isDateTime {\n\t\tlhsTime, err := parseDateTime(layout, lhs.Value)\n\t\tif err != nil {\n\t\t\tlog.Warningf(\"Could not parse time %v with layout %v for sort, sorting by string instead: %w\", lhs.Value, layout, err)\n\t\t\treturn strings.Compare(lhs.Value, rhs.Value)\n\t\t}\n\t\trhsTime, err := parseDateTime(layout, rhs.Value)\n\t\tif err != nil {\n\t\t\tlog.Warningf(\"Could not parse time %v with layout %v for sort, sorting by string instead: %w\", rhs.Value, layout, err)\n\t\t\treturn strings.Compare(lhs.Value, rhs.Value)\n\t\t}\n\t\tif lhsTime.Equal(rhsTime) {\n\t\t\treturn 0\n\t\t} else if lhsTime.Before(rhsTime) {\n\t\t\treturn -1\n\t\t}\n\n\t\treturn 1\n\t} else if lhsTag == \"!!int\" && rhsTag == \"!!int\" {\n\t\t_, lhsNum, err := parseInt64(lhs.Value)\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t\t_, rhsNum, err := parseInt64(rhs.Value)\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t\treturn int(lhsNum - rhsNum)\n\t} else if (lhsTag == \"!!int\" || lhsTag == \"!!float\") && (rhsTag == \"!!int\" || rhsTag == \"!!float\") {\n\t\tlhsNum, err := strconv.ParseFloat(lhs.Value, 64)\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t\trhsNum, err := strconv.ParseFloat(rhs.Value, 64)\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t\tif lhsNum == rhsNum {\n\t\t\treturn 0\n\t\t} else if lhsNum < rhsNum {\n\t\t\treturn -1\n\t\t}\n\n\t\treturn 1\n\t}\n\n\treturn strings.Compare(lhs.Value, rhs.Value)\n}\n" }, { "path": "pkg/yqlib/operator_sort_keys.go", "content": "package yqlib\n\nimport (\n\t\"sort\"\n)\n\nfunc sortKeysOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\trhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t\tfor childEl := rhs.MatchingNodes.Front(); childEl != nil; childEl = childEl.Next() {\n\t\t\tnode := childEl.Value.(*CandidateNode)\n\t\t\tif node.Kind == MappingNode {\n\t\t\t\tsortKeys(node)\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\treturn Context{}, err\n\t\t\t}\n\t\t}\n\n\t}\n\treturn context, nil\n}\n\nfunc sortKeys(node *CandidateNode) {\n\tkeys := make([]string, len(node.Content)/2)\n\tkeyBucket := map[string]*CandidateNode{}\n\tvalueBucket := map[string]*CandidateNode{}\n\tvar contents = node.Content\n\tfor index := 0; index < len(contents); index = index + 2 {\n\t\tkey := contents[index]\n\t\tvalue := contents[index+1]\n\t\tkeys[index/2] = key.Value\n\t\tkeyBucket[key.Value] = key\n\t\tvalueBucket[key.Value] = value\n\t}\n\tsort.Strings(keys)\n\tsortedContent := make([]*CandidateNode, len(node.Content))\n\tfor index := 0; index < len(keys); index = index + 1 {\n\t\tkeyString := keys[index]\n\t\tsortedContent[index*2] = keyBucket[keyString]\n\t\tsortedContent[1+(index*2)] = valueBucket[keyString]\n\t}\n\n\t// re-arranging children, no need to update their parent\n\t// relationship\n\tnode.Content = sortedContent\n}\n" }, { "path": "pkg/yqlib/operator_sort_keys_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar sortKeysOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Sort keys of map\",\n\t\tdocument: `{c: frog, a: blah, b: bing}`,\n\t\texpression: `sort_keys(.)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: blah, b: bing, c: frog}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Sort keys of map\",\n\t\tskipDoc: true,\n\t\tdocument: `{c: frog, a: zoo}`,\n\t\texpression: `sort_keys(.)[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!str)::zoo\\n\",\n\t\t\t\"D0, P[c], (!!str)::frog\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{c: frog}`,\n\t\texpression: `sort_keys(.d)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{c: frog}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Sort keys recursively\",\n\t\tsubdescription: \"Note the array elements are left unsorted, but maps inside arrays are sorted\",\n\t\tdocument: `{bParent: {c: dog, array: [3,1,2]}, aParent: {z: donkey, x: [{c: yum, b: delish}, {b: ew, a: apple}]}}`,\n\t\texpression: `sort_keys(..)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{aParent: {x: [{b: delish, c: yum}, {a: apple, b: ew}], z: donkey}, bParent: {array: [3, 1, 2], c: dog}}\\n\",\n\t\t},\n\t},\n}\n\nfunc TestSortKeysOperatorScenarios(t *testing.T) {\n\tfor _, tt := range sortKeysOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"sort-keys\", sortKeysOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_sort_test.go", "content": "package yqlib\n\nimport \"testing\"\n\nvar sortByOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Sort by string field\",\n\t\tdocument: \"[{a: banana},{a: cat},{a: apple}]\",\n\t\texpression: `sort_by(.a)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[{a: apple}, {a: banana}, {a: cat}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Sort by string field\",\n\t\tskipDoc: true,\n\t\tdocument: \"[{a: banana},{a: apple}]\",\n\t\texpression: `sort_by(.a)[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!map)::{a: apple}\\n\",\n\t\t\t\"D0, P[1], (!!map)::{a: banana}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Sort by with null\",\n\t\tskipDoc: true,\n\t\tdocument: \"[{a: banana},null,{a: apple}]\",\n\t\texpression: `sort_by(.a)[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!null)::null\\n\",\n\t\t\t\"D0, P[1], (!!map)::{a: apple}\\n\",\n\t\t\t\"D0, P[2], (!!map)::{a: banana}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Sort by multiple fields\",\n\t\tdocument: \"[{a: dog},{a: cat, b: banana},{a: cat, b: apple}]\",\n\t\texpression: `sort_by(.a, .b)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[{a: cat, b: apple}, {a: cat, b: banana}, {a: dog}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Sort by multiple fields\",\n\t\tskipDoc: true,\n\t\tdocument: \"[{a: dog, b: good},{a: cat, c: things},{a: cat, b: apple}]\",\n\t\texpression: `sort_by(.a, .b)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[{a: cat, c: things}, {a: cat, b: apple}, {a: dog, b: good}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Sort by multiple fields\",\n\t\tskipDoc: true,\n\t\tdocument: \"[{a: dog, b: 0.1},{a: cat, b: 0.01},{a: cat, b: 0.001}]\",\n\t\texpression: `sort_by(.a, .b)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[{a: cat, b: 0.001}, {a: cat, b: 0.01}, {a: dog, b: 0.1}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Sort descending by string field\",\n\t\tsubdescription: \"Use sort with reverse to sort in descending order.\",\n\t\tdocument: \"[{a: banana},{a: cat},{a: apple}]\",\n\t\texpression: `sort_by(.a) | reverse`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[{a: cat}, {a: banana}, {a: apple}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Sort array in place\",\n\t\tdocument: \"cool: [{a: banana},{a: cat},{a: apple}]\",\n\t\texpression: `.cool |= sort_by(.a)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::cool: [{a: apple}, {a: banana}, {a: cat}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Sort array of objects by key\",\n\t\tsubdescription: \"Note that you can give sort_by complex expressions, not just paths\",\n\t\tdocument: \"cool: [{b: banana},{a: banana},{c: banana}]\",\n\t\texpression: `.cool |= sort_by(keys | .[0])`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::cool: [{a: banana}, {b: banana}, {c: banana}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Sort a map\",\n\t\tsubdescription: \"Sorting a map, by default this will sort by the values\",\n\t\tdocument: \"y: b\\nz: a\\nx: c\\n\",\n\t\texpression: `sort`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::z: a\\ny: b\\nx: c\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Sort a map by keys\",\n\t\tsubdescription: \"Use sort_by to sort a map using a custom function\",\n\t\tdocument: \"Y: b\\nz: a\\nx: c\\n\",\n\t\texpression: `sort_by(key | downcase)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::x: c\\nY: b\\nz: a\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Sort is stable\",\n\t\tsubdescription: \"Note the order of the elements in unchanged when equal in sorting.\",\n\t\tdocument: \"[{a: banana, b: 1}, {a: banana, b: 2}, {a: banana, b: 3}, {a: banana, b: 4}]\",\n\t\texpression: `sort_by(.a)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[{a: banana, b: 1}, {a: banana, b: 2}, {a: banana, b: 3}, {a: banana, b: 4}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Sort by numeric field\",\n\t\tdocument: \"[{a: 10},{a: 100},{a: 1}]\",\n\t\texpression: `sort_by(.a)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[{a: 1}, {a: 10}, {a: 100}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Sort by custom date field\",\n\t\tdocument: `[{a: \"12-Jun-2011\"},{a: \"23-Dec-2010\"},{a: \"10-Aug-2011\"}]`,\n\t\texpression: `with_dtf(\"02-Jan-2006\"; sort_by(.a))`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[{a: \\\"23-Dec-2010\\\"}, {a: \\\"12-Jun-2011\\\"}, {a: \\\"10-Aug-2011\\\"}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"[{a: 1.1},{a: 1.001},{a: 1.01}]\",\n\t\texpression: `sort_by(.a)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[{a: 1.001}, {a: 1.01}, {a: 1.1}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Sort, nulls come first\",\n\t\tdocument: \"[8,3,null,6, true, false, cat]\",\n\t\texpression: `sort`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[null, false, true, 3, 6, 8, cat]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Sort, nulls come first\",\n\t\tskipDoc: true,\n\t\tdocument: \"[8,null]\",\n\t\texpression: `sort[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!null)::null\\n\",\n\t\t\t\"D0, P[1], (!!int)::8\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"false before true\",\n\t\tdocument: \"[{a: false, b: 1}, {a: true, b: 2}, {a: false, b: 3}]\",\n\t\texpression: `sort_by(.a)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[{a: false, b: 1}, {a: false, b: 3}, {a: true, b: 2}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"head comment\",\n\t\tdocument: \"# abc\\n- def\\n# ghi\",\n\t\texpression: `sort`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::# abc\\n- def\\n# ghi\\n\",\n\t\t},\n\t},\n}\n\nfunc TestSortByOperatorScenarios(t *testing.T) {\n\tfor _, tt := range sortByOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"sort\", sortByOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_split_document.go", "content": "package yqlib\n\nfunc splitDocumentOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"splitDocumentOperator\")\n\n\tvar index uint\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tcandidate.SetDocument(index)\n\t\tcandidate.SetParent(nil)\n\t\tindex = index + 1\n\t}\n\n\treturn context, nil\n}\n" }, { "path": "pkg/yqlib/operator_split_document_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar splitDocOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Split empty\",\n\t\tdocument: ``,\n\t\texpression: `split_doc`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!null)::\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Split array\",\n\t\tdocument: `[{a: cat}, {b: dog}]`,\n\t\texpression: `.[] | split_doc`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!map)::{a: cat}\\n\",\n\t\t\t\"D1, P[1], (!!map)::{b: dog}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Split splat\",\n\t\tskipDoc: true,\n\t\tdocument: `[{a: cat}, {b: dog}]`,\n\t\texpression: `.[] | split_doc[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0 a], (!!str)::cat\\n\",\n\t\t\t\"D1, P[1 b], (!!str)::dog\\n\",\n\t\t},\n\t},\n}\n\nfunc TestSplitDocOperatorScenarios(t *testing.T) {\n\tfor _, tt := range splitDocOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"split-into-documents\", splitDocOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_strings.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n\t\"regexp\"\n\t\"strings\"\n)\n\nvar StringInterpolationEnabled = true\n\ntype changeCasePrefs struct {\n\tToUpperCase bool\n}\n\nfunc encodeToYamlString(node *CandidateNode) (string, error) {\n\tencoderPrefs := encoderPreferences{\n\t\tformat: YamlFormat,\n\t\tindent: ConfiguredYamlPreferences.Indent,\n\t}\n\tresult, err := encodeToString(node, encoderPrefs)\n\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\treturn chomper.ReplaceAllString(result, \"\"), nil\n}\n\nfunc evaluate(d *dataTreeNavigator, context Context, expStr string) (string, error) {\n\texp, err := ExpressionParser.ParseExpression(expStr)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tresult, err := d.GetMatchingNodes(context, exp)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\tif result.MatchingNodes.Len() == 0 {\n\t\treturn \"\", nil\n\t}\n\tnode := result.MatchingNodes.Front().Value.(*CandidateNode)\n\tif node.Kind != ScalarNode {\n\t\treturn encodeToYamlString(node)\n\t}\n\treturn node.Value, nil\n}\n\nfunc interpolate(d *dataTreeNavigator, context Context, str string) (string, error) {\n\tvar sb strings.Builder\n\tvar expSb strings.Builder\n\tinExpression := false\n\tnestedBracketsCounter := 0\n\trunes := []rune(str)\n\tfor i := 0; i < len(runes); i++ {\n\t\tchar := runes[i]\n\t\tif !inExpression {\n\t\t\tif char == '\\\\' && i < len(runes)-1 {\n\t\t\t\tswitch runes[i+1] {\n\t\t\t\tcase '(':\n\t\t\t\t\tinExpression = true\n\t\t\t\t\t// skip the lparen\n\t\t\t\t\ti++\n\t\t\t\t\tcontinue\n\t\t\t\tcase '\\\\':\n\t\t\t\t\t// skip the escaped backslash\n\t\t\t\t\ti++\n\t\t\t\tdefault:\n\t\t\t\t\tlog.Debugf(\"Ignoring non-escaping backslash @ %v[%d]\", str, i)\n\t\t\t\t}\n\t\t\t}\n\t\t\tsb.WriteRune(char)\n\t\t} else { // we are in an expression\n\t\t\tif char == ')' {\n\t\t\t\tif nestedBracketsCounter == 0 {\n\t\t\t\t\t// finished the expression!\n\t\t\t\t\tlog.Debugf(\"Expression is :%v\", expSb.String())\n\t\t\t\t\tvalue, err := evaluate(d, context, expSb.String())\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn \"\", err\n\t\t\t\t\t}\n\t\t\t\t\tinExpression = false\n\t\t\t\t\texpSb = strings.Builder{} // reset this\n\n\t\t\t\t\tsb.WriteString(value)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tnestedBracketsCounter--\n\t\t\t} else if char == '(' {\n\t\t\t\tnestedBracketsCounter++\n\t\t\t} else if char == '\\\\' && i < len(runes)-1 {\n\t\t\t\tswitch esc := runes[i+1]; esc {\n\t\t\t\tcase ')', '\\\\':\n\t\t\t\t\t// write escaped character\n\t\t\t\t\texpSb.WriteRune(esc)\n\t\t\t\t\ti++\n\t\t\t\t\tcontinue\n\t\t\t\tdefault:\n\t\t\t\t\tlog.Debugf(\"Ignoring non-escaping backslash @ %v[%d]\", str, i)\n\t\t\t\t}\n\t\t\t}\n\t\t\texpSb.WriteRune(char)\n\t\t}\n\t}\n\tif inExpression {\n\t\tlog.Warning(\"unclosed interpolation string, skipping interpolation\")\n\t\treturn str, nil\n\t}\n\treturn sb.String(), nil\n}\n\nfunc stringInterpolationOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tif !StringInterpolationEnabled {\n\t\treturn context.SingleChildContext(\n\t\t\tcreateScalarNode(expressionNode.Operation.StringValue, expressionNode.Operation.StringValue),\n\t\t), nil\n\t}\n\tif context.MatchingNodes.Len() == 0 {\n\t\tvalue, err := interpolate(d, context, expressionNode.Operation.StringValue)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\tnode := createScalarNode(value, value)\n\t\treturn context.SingleChildContext(node), nil\n\t}\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tvalue, err := interpolate(d, context.SingleChildContext(candidate), expressionNode.Operation.StringValue)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\tnode := createScalarNode(value, value)\n\t\tresults.PushBack(node)\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n\nfunc trimSpaceOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tresults := list.New()\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tnode := el.Value.(*CandidateNode)\n\n\t\tif node.guessTagFromCustomType() != \"!!str\" {\n\t\t\treturn Context{}, fmt.Errorf(\"cannot trim %v, can only operate on strings. \", node.Tag)\n\t\t}\n\n\t\tnewStringNode := node.CreateReplacement(ScalarNode, node.Tag, strings.TrimSpace(node.Value))\n\t\tnewStringNode.Style = node.Style\n\t\tresults.PushBack(newStringNode)\n\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n\nfunc toStringOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tresults := list.New()\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tnode := el.Value.(*CandidateNode)\n\t\tvar newStringNode *CandidateNode\n\t\tif node.Tag == \"!!str\" {\n\t\t\tnewStringNode = node.CreateReplacement(ScalarNode, \"!!str\", node.Value)\n\t\t} else if node.Kind == ScalarNode {\n\t\t\tnewStringNode = node.CreateReplacement(ScalarNode, \"!!str\", node.Value)\n\t\t\tnewStringNode.Style = DoubleQuotedStyle\n\t\t} else {\n\t\t\tresult, err := encodeToYamlString(node)\n\t\t\tif err != nil {\n\t\t\t\treturn Context{}, err\n\t\t\t}\n\t\t\tnewStringNode = node.CreateReplacement(ScalarNode, \"!!str\", result)\n\t\t\tnewStringNode.Style = DoubleQuotedStyle\n\t\t}\n\t\tnewStringNode.Tag = \"!!str\"\n\t\tresults.PushBack(newStringNode)\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n\nfunc changeCaseOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tresults := list.New()\n\tprefs := expressionNode.Operation.Preferences.(changeCasePrefs)\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tnode := el.Value.(*CandidateNode)\n\n\t\tif node.guessTagFromCustomType() != \"!!str\" {\n\t\t\treturn Context{}, fmt.Errorf(\"cannot change case with %v, can only operate on strings. \", node.Tag)\n\t\t}\n\n\t\tvalue := \"\"\n\t\tif prefs.ToUpperCase {\n\t\t\tvalue = strings.ToUpper(node.Value)\n\t\t} else {\n\t\t\tvalue = strings.ToLower(node.Value)\n\t\t}\n\t\tnewStringNode := node.CreateReplacement(ScalarNode, node.Tag, value)\n\t\tnewStringNode.Style = node.Style\n\t\tresults.PushBack(newStringNode)\n\n\t}\n\n\treturn context.ChildContext(results), nil\n\n}\n\nfunc getSubstituteParameters(d *dataTreeNavigator, block *ExpressionNode, context Context) (string, string, error) {\n\tregEx := \"\"\n\treplacementText := \"\"\n\n\tregExNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), block.LHS)\n\tif err != nil {\n\t\treturn \"\", \"\", err\n\t}\n\tif regExNodes.MatchingNodes.Front() != nil {\n\t\tregEx = regExNodes.MatchingNodes.Front().Value.(*CandidateNode).Value\n\t}\n\n\tlog.Debug(\"regEx %v\", regEx)\n\n\treplacementNodes, err := d.GetMatchingNodes(context, block.RHS)\n\tif err != nil {\n\t\treturn \"\", \"\", err\n\t}\n\tif replacementNodes.MatchingNodes.Front() != nil {\n\t\treplacementText = replacementNodes.MatchingNodes.Front().Value.(*CandidateNode).Value\n\t}\n\n\treturn regEx, replacementText, nil\n}\n\nfunc substitute(original string, regex *regexp.Regexp, replacement string) (Kind, string, string) {\n\treplacedString := regex.ReplaceAllString(original, replacement)\n\treturn ScalarNode, \"!!str\", replacedString\n}\n\nfunc substituteStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\t//rhs block operator\n\t//lhs of block = regex\n\t//rhs of block = replacement expression\n\tblock := expressionNode.RHS\n\n\tregExStr, replacementText, err := getSubstituteParameters(d, block, context)\n\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\n\tregEx, err := regexp.Compile(regExStr)\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tnode := el.Value.(*CandidateNode)\n\t\tif node.guessTagFromCustomType() != \"!!str\" {\n\t\t\treturn Context{}, fmt.Errorf(\"cannot substitute with %v, can only substitute strings. Hint: Most often you'll want to use '|=' over '=' for this operation\", node.Tag)\n\t\t}\n\n\t\tresult := node.CreateReplacement(substitute(node.Value, regEx, replacementText))\n\t\tresults.PushBack(result)\n\t}\n\n\treturn context.ChildContext(results), nil\n\n}\n\nfunc addMatch(original []*CandidateNode, match string, offset int, name string) []*CandidateNode {\n\n\tnewContent := append(original,\n\t\tcreateScalarNode(\"string\", \"string\"))\n\n\tif offset < 0 {\n\t\t// offset of -1 means there was no match, force a null value like jq\n\t\tnewContent = append(newContent,\n\t\t\tcreateScalarNode(nil, \"null\"),\n\t\t)\n\t} else {\n\t\tnewContent = append(newContent,\n\t\t\tcreateScalarNode(match, match),\n\t\t)\n\t}\n\n\tnewContent = append(newContent,\n\t\tcreateScalarNode(\"offset\", \"offset\"),\n\t\tcreateScalarNode(offset, fmt.Sprintf(\"%v\", offset)),\n\t\tcreateScalarNode(\"length\", \"length\"),\n\t\tcreateScalarNode(len(match), fmt.Sprintf(\"%v\", len(match))))\n\n\tif name != \"\" {\n\t\tnewContent = append(newContent,\n\t\t\tcreateScalarNode(\"name\", \"name\"),\n\t\t\tcreateScalarNode(name, name),\n\t\t)\n\t}\n\treturn newContent\n}\n\ntype matchPreferences struct {\n\tGlobal bool\n}\n\nfunc getMatches(matchPrefs matchPreferences, regEx *regexp.Regexp, value string) ([][]string, [][]int) {\n\tvar allMatches [][]string\n\tvar allIndices [][]int\n\n\tif matchPrefs.Global {\n\t\tallMatches = regEx.FindAllStringSubmatch(value, -1)\n\t\tallIndices = regEx.FindAllStringSubmatchIndex(value, -1)\n\t} else {\n\t\tallMatches = [][]string{regEx.FindStringSubmatch(value)}\n\t\tallIndices = [][]int{regEx.FindStringSubmatchIndex(value)}\n\t}\n\n\tlog.Debug(\"allMatches, %v\", allMatches)\n\treturn allMatches, allIndices\n}\n\nfunc match(matchPrefs matchPreferences, regEx *regexp.Regexp, candidate *CandidateNode, value string, results *list.List) {\n\tsubNames := regEx.SubexpNames()\n\tallMatches, allIndices := getMatches(matchPrefs, regEx, value)\n\n\t// if all matches just has an empty array in it,\n\t// then nothing matched\n\tif len(allMatches) > 0 && len(allMatches[0]) == 0 {\n\t\treturn\n\t}\n\n\tfor i, matches := range allMatches {\n\t\tcapturesListNode := &CandidateNode{Kind: SequenceNode}\n\t\tmatch, submatches := matches[0], matches[1:]\n\t\tfor j, submatch := range submatches {\n\t\t\tcaptureNode := &CandidateNode{Kind: MappingNode}\n\t\t\tcaptureNode.AddChildren(addMatch(captureNode.Content, submatch, allIndices[i][2+j*2], subNames[j+1]))\n\t\t\tcapturesListNode.AddChild(captureNode)\n\t\t}\n\n\t\tnode := candidate.CreateReplacement(MappingNode, \"!!map\", \"\")\n\t\tnode.AddChildren(addMatch(node.Content, match, allIndices[i][0], \"\"))\n\t\tnode.AddKeyValueChild(createScalarNode(\"captures\", \"captures\"), capturesListNode)\n\t\tresults.PushBack(node)\n\n\t}\n\n}\n\nfunc capture(matchPrefs matchPreferences, regEx *regexp.Regexp, candidate *CandidateNode, value string, results *list.List) {\n\tsubNames := regEx.SubexpNames()\n\tallMatches, allIndices := getMatches(matchPrefs, regEx, value)\n\n\t// if all matches just has an empty array in it,\n\t// then nothing matched\n\tif len(allMatches) > 0 && len(allMatches[0]) == 0 {\n\t\treturn\n\t}\n\n\tfor i, matches := range allMatches {\n\t\tcapturesNode := candidate.CreateReplacement(MappingNode, \"!!map\", \"\")\n\n\t\t_, submatches := matches[0], matches[1:]\n\t\tfor j, submatch := range submatches {\n\n\t\t\tkeyNode := createScalarNode(subNames[j+1], subNames[j+1])\n\t\t\tvar valueNode *CandidateNode\n\n\t\t\toffset := allIndices[i][2+j*2]\n\t\t\t// offset of -1 means there was no match, force a null value like jq\n\t\t\tif offset < 0 {\n\t\t\t\tvalueNode = createScalarNode(nil, \"null\")\n\t\t\t} else {\n\t\t\t\tvalueNode = createScalarNode(submatch, submatch)\n\t\t\t}\n\t\t\tcapturesNode.AddKeyValueChild(keyNode, valueNode)\n\t\t}\n\n\t\tresults.PushBack(capturesNode)\n\n\t}\n\n}\n\nfunc extractMatchArguments(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (*regexp.Regexp, matchPreferences, error) {\n\tregExExpNode := expressionNode.RHS\n\n\tmatchPrefs := matchPreferences{}\n\n\t// we got given parameters e.g. match(exp; params)\n\tif expressionNode.RHS.Operation.OperationType == blockOpType {\n\t\tblock := expressionNode.RHS\n\t\tregExExpNode = block.LHS\n\t\treplacementNodes, err := d.GetMatchingNodes(context, block.RHS)\n\t\tif err != nil {\n\t\t\treturn nil, matchPrefs, err\n\t\t}\n\t\tparamText := \"\"\n\t\tif replacementNodes.MatchingNodes.Front() != nil {\n\t\t\tparamText = replacementNodes.MatchingNodes.Front().Value.(*CandidateNode).Value\n\t\t}\n\t\tif strings.Contains(paramText, \"g\") {\n\t\t\tparamText = strings.ReplaceAll(paramText, \"g\", \"\")\n\t\t\tmatchPrefs.Global = true\n\t\t}\n\t\tif strings.Contains(paramText, \"i\") {\n\t\t\treturn nil, matchPrefs, fmt.Errorf(`'i' is not a valid option for match. To ignore case, use an expression like match(\"(?i)cat\")`)\n\t\t}\n\t\tif len(paramText) > 0 {\n\t\t\treturn nil, matchPrefs, fmt.Errorf(`unrecognised match params '%v', please see docs at https://mikefarah.gitbook.io/yq/operators/string-operators`, paramText)\n\t\t}\n\t}\n\n\tregExNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), regExExpNode)\n\tif err != nil {\n\t\treturn nil, matchPrefs, err\n\t}\n\tlog.Debug(NodesToString(regExNodes.MatchingNodes))\n\tregExStr := \"\"\n\tif regExNodes.MatchingNodes.Front() != nil {\n\t\tregExStr = regExNodes.MatchingNodes.Front().Value.(*CandidateNode).Value\n\t}\n\tlog.Debug(\"regEx %v\", regExStr)\n\tregEx, err := regexp.Compile(regExStr)\n\treturn regEx, matchPrefs, err\n}\n\nfunc matchOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tregEx, matchPrefs, err := extractMatchArguments(d, context, expressionNode)\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tnode := el.Value.(*CandidateNode)\n\t\tif node.guessTagFromCustomType() != \"!!str\" {\n\t\t\treturn Context{}, fmt.Errorf(\"cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation\", node.Tag)\n\t\t}\n\n\t\tmatch(matchPrefs, regEx, node, node.Value, results)\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n\nfunc captureOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tregEx, matchPrefs, err := extractMatchArguments(d, context, expressionNode)\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tnode := el.Value.(*CandidateNode)\n\t\tif node.guessTagFromCustomType() != \"!!str\" {\n\t\t\treturn Context{}, fmt.Errorf(\"cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation\", node.Tag)\n\t\t}\n\t\tcapture(matchPrefs, regEx, node, node.Value, results)\n\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n\nfunc testOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tregEx, _, err := extractMatchArguments(d, context, expressionNode)\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tnode := el.Value.(*CandidateNode)\n\t\tif node.guessTagFromCustomType() != \"!!str\" {\n\t\t\treturn Context{}, fmt.Errorf(\"cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation\", node.Tag)\n\t\t}\n\t\tmatches := regEx.FindStringSubmatch(node.Value)\n\t\tresults.PushBack(createBooleanCandidate(node, len(matches) > 0))\n\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n\nfunc joinStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"joinStringOperator\")\n\tjoinStr := \"\"\n\n\trhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS)\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\tif rhs.MatchingNodes.Front() != nil {\n\t\tjoinStr = rhs.MatchingNodes.Front().Value.(*CandidateNode).Value\n\t}\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tnode := el.Value.(*CandidateNode)\n\t\tif node.Kind != SequenceNode {\n\t\t\treturn Context{}, fmt.Errorf(\"cannot join with %v, can only join arrays of scalars\", node.Tag)\n\t\t}\n\t\tresult := node.CreateReplacement(join(node.Content, joinStr))\n\t\tresults.PushBack(result)\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n\nfunc join(content []*CandidateNode, joinStr string) (Kind, string, string) {\n\tvar stringsToJoin []string\n\tfor _, node := range content {\n\t\tstr := node.Value\n\t\tif node.Tag == \"!!null\" {\n\t\t\tstr = \"\"\n\t\t}\n\t\tstringsToJoin = append(stringsToJoin, str)\n\t}\n\n\treturn ScalarNode, \"!!str\", strings.Join(stringsToJoin, joinStr)\n}\n\nfunc splitStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"splitStringOperator\")\n\tsplitStr := \"\"\n\n\trhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS)\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\tif rhs.MatchingNodes.Front() != nil {\n\t\tsplitStr = rhs.MatchingNodes.Front().Value.(*CandidateNode).Value\n\t}\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tnode := el.Value.(*CandidateNode)\n\t\tif node.Tag == \"!!null\" {\n\t\t\tcontinue\n\t\t}\n\n\t\tif node.guessTagFromCustomType() != \"!!str\" {\n\t\t\treturn Context{}, fmt.Errorf(\"cannot split %v, can only split strings\", node.Tag)\n\t\t}\n\t\tkind, tag, content := split(node.Value, splitStr)\n\t\tresult := node.CreateReplacement(kind, tag, \"\")\n\t\tresult.AddChildren(content)\n\t\tresults.PushBack(result)\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n\nfunc split(value string, spltStr string) (Kind, string, []*CandidateNode) {\n\tvar contents []*CandidateNode\n\n\tif value != \"\" {\n\t\tlog.Debug(\"going to spltStr[%v]\", spltStr)\n\t\tvar newStrings = strings.Split(value, spltStr)\n\t\tcontents = make([]*CandidateNode, len(newStrings))\n\n\t\tfor index, str := range newStrings {\n\t\t\tcontents[index] = &CandidateNode{Kind: ScalarNode, Tag: \"!!str\", Value: str}\n\t\t}\n\t}\n\n\treturn SequenceNode, \"!!seq\", contents\n}\n" }, { "path": "pkg/yqlib/operator_strings_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar stringsOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Interpolation\",\n\t\tdocument: \"value: things\\nanother: stuff\",\n\t\texpression: `.message = \"I like \\(.value) and \\(.another)\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::value: things\\nanother: stuff\\nmessage: I like things and stuff\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Interpolation - not a string\",\n\t\tdocument: `value: {an: apple}`,\n\t\texpression: `.message = \"I like \\(.value)\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::value: {an: apple}\\nmessage: 'I like {an: apple}'\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Interpolation - just escape\",\n\t\texpression: `\"\\\\\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::\\\\\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Interpolation - nested\",\n\t\tdocument: `value: things`,\n\t\texpression: `\"Hi \\( (.value) )\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::Hi things\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Interpolation - don't\",\n\t\tdocument: `value: things`,\n\t\texpression: `\"Hi (.value)\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::Hi (.value)\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Interpolation - don't!\",\n\t\tdocument: `value: things`,\n\t\texpression: `\"Hi \\\\(.value)\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::Hi \\\\(.value)\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Interpolation - random close bracket\",\n\t\tdocument: `value: things`,\n\t\texpression: `\"Hi )\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::Hi )\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Interpolation - unclosed interpolation string\",\n\t\tdocument: `value: things`,\n\t\texpression: `\"Hi \\(\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::Hi \\\\(\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Interpolation - unclosed interpolation string due to escape\",\n\t\tdocument: `value: things`,\n\t\texpression: `\"Hi \\(\\)\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::Hi \\\\(\\\\)\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"To up (upper) case\",\n\t\tsubdescription: \"Works with unicode characters\",\n\t\tdocument: `água`,\n\t\texpression: \"upcase\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::ÁGUA\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `!camel água`,\n\t\texpression: \"upcase\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!camel)::ÁGUA\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"To down (lower) case\",\n\t\tsubdescription: \"Works with unicode characters\",\n\t\tdocument: `ÁgUA`,\n\t\texpression: \"downcase\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::água\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `!camel ÁgUA`,\n\t\texpression: \"downcase\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!camel)::água\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Join strings\",\n\t\tdocument: `[cat, meow, 1, null, true]`,\n\t\texpression: `join(\"; \")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::cat; meow; 1; ; true\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Trim strings\",\n\t\tdocument: `[\" cat\", \"dog \", \" cow cow \", horse]`,\n\t\texpression: `.[] | trim`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!str)::cat\\n\",\n\t\t\t\"D0, P[1], (!!str)::dog\\n\",\n\t\t\t\"D0, P[2], (!!str)::cow cow\\n\",\n\t\t\t\"D0, P[3], (!!str)::horse\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[!horse cat, !goat meow, !frog 1, null, true]`,\n\t\texpression: `join(\"; \")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::cat; meow; 1; ; true\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Match string\",\n\t\tdocument: `foo bar foo`,\n\t\texpression: `match(\"foo\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::string: foo\\noffset: 0\\nlength: 3\\ncaptures: []\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `!horse foo bar foo`,\n\t\texpression: `match(\"foo\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::string: foo\\noffset: 0\\nlength: 3\\ncaptures: []\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Match string, case insensitive\",\n\t\tdocument: `foo bar FOO`,\n\t\texpression: `[match(\"(?i)foo\"; \"g\")]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- string: foo\\n offset: 0\\n length: 3\\n captures: []\\n- string: FOO\\n offset: 8\\n length: 3\\n captures: []\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Match with global capture group\",\n\t\tdocument: `abc abc`,\n\t\texpression: `[match(\"(ab)(c)\"; \"g\")]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- string: abc\\n offset: 0\\n length: 3\\n captures:\\n - string: ab\\n offset: 0\\n length: 2\\n - string: c\\n offset: 2\\n length: 1\\n- string: abc\\n offset: 4\\n length: 3\\n captures:\\n - string: ab\\n offset: 4\\n length: 2\\n - string: c\\n offset: 6\\n length: 1\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Match with named capture groups\",\n\t\tdocument: `foo bar foo foo foo`,\n\t\texpression: `[match(\"foo (?Pbar)? foo\"; \"g\")]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- string: foo bar foo\\n offset: 0\\n length: 11\\n captures:\\n - string: bar\\n offset: 4\\n length: 3\\n name: bar123\\n- string: foo foo\\n offset: 12\\n length: 8\\n captures:\\n - string: null\\n offset: -1\\n length: 0\\n name: bar123\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Capture named groups into a map\",\n\t\tdocument: `xyzzy-14`,\n\t\texpression: `capture(\"(?P[a-z]+)-(?P[0-9]+)\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: xyzzy\\nn: \\\"14\\\"\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `!horse xyzzy-14`,\n\t\texpression: `capture(\"(?P[a-z]+)-(?P[0-9]+)\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: xyzzy\\nn: \\\"14\\\"\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Capture named groups into a map, with null\",\n\t\tdocument: `xyzzy-14`,\n\t\texpression: `capture(\"(?P[a-z]+)-(?P[0-9]+)(?Pbar)?\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: xyzzy\\nn: \\\"14\\\"\\nbar123: null\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Match without global flag\",\n\t\tdocument: `cat cat`,\n\t\texpression: `match(\"cat\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::string: cat\\noffset: 0\\nlength: 3\\ncaptures: []\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Match with global flag\",\n\t\tdocument: `cat cat`,\n\t\texpression: `[match(\"cat\"; \"g\")]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- string: cat\\n offset: 0\\n length: 3\\n captures: []\\n- string: cat\\n offset: 4\\n length: 3\\n captures: []\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `!horse cat cat`,\n\t\texpression: `[match(\"cat\"; \"g\")]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- string: cat\\n offset: 0\\n length: 3\\n captures: []\\n- string: cat\\n offset: 4\\n length: 3\\n captures: []\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"No match\",\n\t\tdocument: `dog`,\n\t\texpression: `match(\"cat\"; \"g\")`,\n\t\texpected: []string{},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"No match\",\n\t\texpression: `\"dog\" | match(\"cat\", \"g\")`,\n\t\texpected: []string{},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"No match\",\n\t\texpression: `\"dog\" | match(\"cat\")`,\n\t\texpected: []string{},\n\t},\n\t{\n\t\tdescription: \"Test using regex\",\n\t\tsubdescription: \"Like jq's equivalent, this works like match but only returns true/false instead of full match details\",\n\t\tdocument: `[\"cat\", \"dog\"]`,\n\t\texpression: `.[] | test(\"at\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!bool)::true\\n\",\n\t\t\t\"D0, P[1], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[!horse \"cat\", !cat \"dog\"]`,\n\t\texpression: `.[] | test(\"at\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!bool)::true\\n\",\n\t\t\t\"D0, P[1], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[\"cat*\", \"cat*\", \"cat\"]`,\n\t\texpression: `.[] | test(\"cat\\*\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!bool)::true\\n\",\n\t\t\t\"D0, P[1], (!!bool)::true\\n\",\n\t\t\t\"D0, P[2], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Substitute / Replace string\",\n\t\tsubdescription: \"This uses Golang's regex, described [here](https://github.com/google/re2/wiki/Syntax).\\nNote the use of `|=` to run in context of the current string value.\",\n\t\tdocument: `a: dogs are great`,\n\t\texpression: `.a |= sub(\"dogs\", \"cats\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: cats are great\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Substitute / Replace string with regex\",\n\t\tsubdescription: \"This uses Golang's regex, described [here](https://github.com/google/re2/wiki/Syntax).\\nNote the use of `|=` to run in context of the current string value.\",\n\t\tdocument: \"a: cat\\nb: heat\",\n\t\texpression: `.[] |= sub(\"(a)\", \"${1}r\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: cart\\nb: heart\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Custom types: that are really strings\",\n\t\tsubdescription: \"When custom tags are encountered, yq will try to decode the underlying type.\",\n\t\tdocument: \"a: !horse cat\\nb: !goat heat\",\n\t\texpression: `.[] |= sub(\"(a)\", \"${1}r\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !horse cart\\nb: !goat heart\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Split strings\",\n\t\tdocument: `\"cat; meow; 1; ; true\"`,\n\t\texpression: `split(\"; \")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- cat\\n- meow\\n- \\\"1\\\"\\n- \\\"\\\"\\n- \\\"true\\\"\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Split strings one match\",\n\t\tdocument: `\"word\"`,\n\t\texpression: `split(\"; \")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- word\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Split splat\",\n\t\tskipDoc: true,\n\t\tdocument: `\"word; cat\"`,\n\t\texpression: `split(\"; \")[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!str)::word\\n\",\n\t\t\t\"D0, P[1], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `!horse \"word\"`,\n\t\texpression: `split(\"; \")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- word\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `\"\"`,\n\t\texpression: `split(\"; \")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[]\\n\", // dont actually want this, just not to error\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `split(\"; \")`,\n\t\texpected: []string{},\n\t},\n\t{\n\t\tdescription: \"To string\",\n\t\tsubdescription: \"Note that you may want to force `yq` to leave scalar values wrapped by passing in `--unwrapScalar=false` or `-r=f`\",\n\t\tdocument: `[1, true, null, ~, cat, {an: object}, [array, 2]]`,\n\t\texpression: \".[] |= to_string\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[\\\"1\\\", \\\"true\\\", \\\"null\\\", \\\"~\\\", cat, \\\"{an: object}\\\", \\\"[array, 2]\\\"]\\n\",\n\t\t},\n\t},\n}\n\nfunc TestStringsOperatorScenarios(t *testing.T) {\n\tfor _, tt := range stringsOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"string-operators\", stringsOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_style.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n)\n\nfunc parseStyle(customStyle string) (Style, error) {\n\tif customStyle == \"tagged\" {\n\t\treturn TaggedStyle, nil\n\t} else if customStyle == \"double\" {\n\t\treturn DoubleQuotedStyle, nil\n\t} else if customStyle == \"single\" {\n\t\treturn SingleQuotedStyle, nil\n\t} else if customStyle == \"literal\" {\n\t\treturn LiteralStyle, nil\n\t} else if customStyle == \"folded\" {\n\t\treturn FoldedStyle, nil\n\t} else if customStyle == \"flow\" {\n\t\treturn FlowStyle, nil\n\t} else if customStyle != \"\" {\n\t\treturn 0, fmt.Errorf(\"unknown style %v\", customStyle)\n\t}\n\treturn 0, nil\n}\n\nfunc assignStyleOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\n\tlog.Debugf(\"AssignStyleOperator: %v\")\n\tvar style Style\n\tif !expressionNode.Operation.UpdateAssign {\n\t\trhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t\tif rhs.MatchingNodes.Front() != nil {\n\t\t\tstyle, err = parseStyle(rhs.MatchingNodes.Front().Value.(*CandidateNode).Value)\n\t\t\tif err != nil {\n\t\t\t\treturn Context{}, err\n\t\t\t}\n\t\t}\n\t}\n\n\tlhs, err := d.GetMatchingNodes(context, expressionNode.LHS)\n\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\n\tfor el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tlog.Debugf(\"Setting style of : %v\", NodeToString(candidate))\n\t\tif expressionNode.Operation.UpdateAssign {\n\t\t\trhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS)\n\t\t\tif err != nil {\n\t\t\t\treturn Context{}, err\n\t\t\t}\n\n\t\t\tif rhs.MatchingNodes.Front() != nil {\n\t\t\t\tstyle, err = parseStyle(rhs.MatchingNodes.Front().Value.(*CandidateNode).Value)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn Context{}, err\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tcandidate.Style = style\n\t}\n\n\treturn context, nil\n}\n\nfunc getStyleOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"GetStyleOperator\")\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tvar style string\n\t\tswitch candidate.Style {\n\t\tcase TaggedStyle:\n\t\t\tstyle = \"tagged\"\n\t\tcase DoubleQuotedStyle:\n\t\t\tstyle = \"double\"\n\t\tcase SingleQuotedStyle:\n\t\t\tstyle = \"single\"\n\t\tcase LiteralStyle:\n\t\t\tstyle = \"literal\"\n\t\tcase FoldedStyle:\n\t\t\tstyle = \"folded\"\n\t\tcase FlowStyle:\n\t\t\tstyle = \"flow\"\n\t\tcase 0:\n\t\t\tstyle = \"\"\n\t\tdefault:\n\t\t\tstyle = \"\"\n\t\t}\n\t\tresult := candidate.CreateReplacement(ScalarNode, \"!!str\", style)\n\t\tresults.PushBack(result)\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_style_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar styleOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Update and set style of a particular node (simple)\",\n\t\tdocument: `a: {b: thing, c: something}`,\n\t\texpression: `.a.b = \"new\" | .a.b style=\"double\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: {b: \\\"new\\\", c: something}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Update and set style of a particular node using path variables\",\n\t\tdocument: `a: {b: thing, c: something}`,\n\t\texpression: `with(.a.b ; . = \"new\" | . style=\"double\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: {b: \\\"new\\\", c: something}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Set tagged style\",\n\t\tdocument: `{a: cat, b: 5, c: 3.2, e: true, f: [1,2,3], g: { something: cool}}`,\n\t\texpression: `.. style=\"tagged\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::!!map\\na: !!str cat\\nb: !!int 5\\nc: !!float 3.2\\ne: !!bool true\\nf: !!seq\\n - !!int 1\\n - !!int 2\\n - !!int 3\\ng: !!map\\n something: !!str cool\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Set double quote style\",\n\t\tdocument: `{a: cat, b: 5, c: 3.2, e: true, f: [1,2,3], g: { something: cool}}`,\n\t\texpression: `.. style=\"double\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: \\\"cat\\\"\\nb: \\\"5\\\"\\nc: \\\"3.2\\\"\\ne: \\\"true\\\"\\nf:\\n - \\\"1\\\"\\n - \\\"2\\\"\\n - \\\"3\\\"\\ng:\\n something: \\\"cool\\\"\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Set double quote style on map keys too\",\n\t\tdocument: `{a: cat, b: 5, c: 3.2, e: true, f: [1,2,3], g: { something: cool}}`,\n\t\texpression: `... style=\"double\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::\\\"a\\\": \\\"cat\\\"\\n\\\"b\\\": \\\"5\\\"\\n\\\"c\\\": \\\"3.2\\\"\\n\\\"e\\\": \\\"true\\\"\\n\\\"f\\\":\\n - \\\"1\\\"\\n - \\\"2\\\"\\n - \\\"3\\\"\\n\\\"g\\\":\\n \\\"something\\\": \\\"cool\\\"\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"bing: &foo {x: z}\\na:\\n c: cat\\n <<: [*foo]\",\n\t\texpression: `(... | select(tag==\"!!str\")) style=\"single\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::'bing': &foo {'x': 'z'}\\n'a':\\n 'c': 'cat'\\n !!merge <<: [*foo]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Set single quote style\",\n\t\tdocument: `{a: cat, b: 5, c: 3.2, e: true, f: [1,2,3], g: { something: cool}}`,\n\t\texpression: `.. style=\"single\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: 'cat'\\nb: '5'\\nc: '3.2'\\ne: 'true'\\nf:\\n - '1'\\n - '2'\\n - '3'\\ng:\\n something: 'cool'\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Set literal quote style\",\n\t\tdocument: `{a: cat, b: 5, c: 3.2, e: true, f: [1,2,3], g: { something: cool}}`,\n\t\texpression: `.. style=\"literal\"`,\n\t\texpected: []string{\n\t\t\t`D0, P[], (!!map)::a: |-\n cat\nb: |-\n 5\nc: |-\n 3.2\ne: |-\n true\nf:\n - |-\n 1\n - |-\n 2\n - |-\n 3\ng:\n something: |-\n cool\n`,\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Set folded quote style\",\n\t\tdocument: `{a: cat, b: 5, c: 3.2, e: true, f: [1,2,3], g: { something: cool}}`,\n\t\texpression: `.. style=\"folded\"`,\n\t\texpected: []string{\n\t\t\t`D0, P[], (!!map)::a: >-\n cat\nb: >-\n 5\nc: >-\n 3.2\ne: >-\n true\nf:\n - >-\n 1\n - >-\n 2\n - >-\n 3\ng:\n something: >-\n cool\n`,\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Set flow quote style\",\n\t\tdocument: `{a: cat, b: 5, c: 3.2, e: true, f: [1,2,3], g: { something: cool}}`,\n\t\texpression: `.. style=\"flow\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: cat, b: 5, c: 3.2, e: true, f: [1, 2, 3], g: {something: cool}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Reset style - or pretty print\",\n\t\tsubdescription: \"Set empty (default) quote style, note the usage of `...` to match keys too. Note that there is a `--prettyPrint/-P` short flag for this.\",\n\t\tdontFormatInputForDoc: true,\n\t\tdocument: `{a: cat, \"b\": 5, 'c': 3.2, \"e\": true, f: [1,2,3], \"g\": { something: \"cool\"} }`,\n\t\texpression: `... style=\"\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: cat\\nb: 5\\nc: 3.2\\ne: true\\nf:\\n - 1\\n - 2\\n - 3\\ng:\\n something: cool\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Set style relatively with assign-update\",\n\t\tdocument: `{a: single, b: double}`,\n\t\texpression: `.[] style |= .`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: 'single', b: \\\"double\\\"}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: cat, b: double}`,\n\t\texpression: `.a style=.b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: \\\"cat\\\", b: double}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Read style\",\n\t\tdocument: `{a: \"cat\", b: 'thing'}`,\n\t\tdontFormatInputForDoc: true,\n\t\texpression: `.. | style`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::flow\\n\",\n\t\t\t\"D0, P[a], (!!str)::double\\n\",\n\t\t\t\"D0, P[b], (!!str)::single\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a: cat`,\n\t\texpression: `.. | style`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::\\n\",\n\t\t\t\"D0, P[a], (!!str)::\\n\",\n\t\t},\n\t},\n}\n\nfunc TestStyleOperatorScenarios(t *testing.T) {\n\tfor _, tt := range styleOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"style\", styleOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_subtract.go", "content": "package yqlib\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n)\n\nfunc createSubtractOp(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode {\n\treturn &ExpressionNode{Operation: &Operation{OperationType: subtractOpType},\n\t\tLHS: lhs,\n\t\tRHS: rhs}\n}\n\nfunc subtractAssignOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\treturn compoundAssignFunction(d, context, expressionNode, createSubtractOp)\n}\n\nfunc subtractOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"Subtract operator\")\n\n\treturn crossFunction(d, context.ReadOnlyClone(), expressionNode, subtract, false)\n}\n\nfunc subtractArray(lhs *CandidateNode, rhs *CandidateNode) []*CandidateNode {\n\tnewLHSArray := make([]*CandidateNode, 0)\n\n\tfor lindex := 0; lindex < len(lhs.Content); lindex = lindex + 1 {\n\t\tshouldInclude := true\n\t\tfor rindex := 0; rindex < len(rhs.Content) && shouldInclude; rindex = rindex + 1 {\n\t\t\tif recursiveNodeEqual(lhs.Content[lindex], rhs.Content[rindex]) {\n\t\t\t\tshouldInclude = false\n\t\t\t}\n\t\t}\n\t\tif shouldInclude {\n\t\t\tnewLHSArray = append(newLHSArray, lhs.Content[lindex])\n\t\t}\n\t}\n\treturn newLHSArray\n}\n\nfunc subtract(_ *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {\n\tif lhs.Tag == \"!!null\" {\n\t\treturn lhs.CopyAsReplacement(rhs), nil\n\t}\n\n\ttarget := lhs.CopyWithoutContent()\n\n\tswitch lhs.Kind {\n\tcase MappingNode:\n\t\treturn nil, fmt.Errorf(\"maps not yet supported for subtraction\")\n\tcase SequenceNode:\n\t\tif rhs.Kind != SequenceNode {\n\t\t\treturn nil, fmt.Errorf(\"%v (%v) cannot be subtracted from %v\", rhs.Tag, rhs.GetNicePath(), lhs.Tag)\n\t\t}\n\t\ttarget.Content = subtractArray(lhs, rhs)\n\tcase ScalarNode:\n\t\tif rhs.Kind != ScalarNode {\n\t\t\treturn nil, fmt.Errorf(\"%v (%v) cannot be subtracted from %v\", rhs.Tag, rhs.GetNicePath(), lhs.Tag)\n\t\t}\n\t\ttarget.Kind = ScalarNode\n\t\ttarget.Style = lhs.Style\n\t\tif err := subtractScalars(context, target, lhs, rhs); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\treturn target, nil\n}\n\nfunc subtractScalars(context Context, target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error {\n\tlhsTag := lhs.Tag\n\trhsTag := rhs.Tag\n\tlhsIsCustom := false\n\tif !strings.HasPrefix(lhsTag, \"!!\") {\n\t\t// custom tag - we have to have a guess\n\t\tlhsTag = lhs.guessTagFromCustomType()\n\t\tlhsIsCustom = true\n\t}\n\n\tif !strings.HasPrefix(rhsTag, \"!!\") {\n\t\t// custom tag - we have to have a guess\n\t\trhsTag = rhs.guessTagFromCustomType()\n\t}\n\n\tisDateTime := lhsTag == \"!!timestamp\"\n\t// if the lhs is a string, it might be a timestamp in a custom format.\n\tif lhsTag == \"!!str\" && context.GetDateTimeLayout() != time.RFC3339 {\n\t\t_, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value)\n\t\tisDateTime = err == nil\n\t}\n\n\tif isDateTime {\n\t\treturn subtractDateTime(context.GetDateTimeLayout(), target, lhs, rhs)\n\t} else if lhsTag == \"!!str\" {\n\t\treturn fmt.Errorf(\"strings cannot be subtracted\")\n\t} else if lhsTag == \"!!int\" && rhsTag == \"!!int\" {\n\t\tformat, lhsNum, err := parseInt64(lhs.Value)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t_, rhsNum, err := parseInt64(rhs.Value)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tresult := lhsNum - rhsNum\n\t\ttarget.Tag = lhs.Tag\n\t\ttarget.Value = fmt.Sprintf(format, result)\n\t} else if (lhsTag == \"!!int\" || lhsTag == \"!!float\") && (rhsTag == \"!!int\" || rhsTag == \"!!float\") {\n\t\tlhsNum, err := strconv.ParseFloat(lhs.Value, 64)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\trhsNum, err := strconv.ParseFloat(rhs.Value, 64)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tresult := lhsNum - rhsNum\n\t\tif lhsIsCustom {\n\t\t\ttarget.Tag = lhs.Tag\n\t\t} else {\n\t\t\ttarget.Tag = \"!!float\"\n\t\t}\n\t\ttarget.Value = fmt.Sprintf(\"%v\", result)\n\t} else {\n\t\treturn fmt.Errorf(\"%v cannot be added to %v\", lhs.Tag, rhs.Tag)\n\t}\n\n\treturn nil\n}\n\nfunc subtractDateTime(layout string, target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error {\n\tvar durationStr string\n\tif strings.HasPrefix(rhs.Value, \"-\") {\n\t\tdurationStr = rhs.Value[1:]\n\t} else {\n\t\tdurationStr = \"-\" + rhs.Value\n\t}\n\tduration, err := time.ParseDuration(durationStr)\n\n\tif err != nil {\n\t\treturn fmt.Errorf(\"unable to parse duration [%v]: %w\", rhs.Value, err)\n\t}\n\n\tcurrentTime, err := parseDateTime(layout, lhs.Value)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tnewTime := currentTime.Add(duration)\n\ttarget.Value = newTime.Format(layout)\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/operator_subtract_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar subtractOperatorScenarios = []expressionScenario{\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{}`,\n\t\texpression: \"(.a - .b) as $x | .\",\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"subtract sequence creates a new sequence\",\n\t\texpression: `[\"a\", \"b\"] as $f | {0:$f - [\"a\"], 1:$f}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::0:\\n - b\\n1:\\n - a\\n - b\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Array subtraction\",\n\t\texpression: `[1,2] - [2,3]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- 1\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `[2,1,2,2] - [2,3]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- 1\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Array subtraction with nested array\",\n\t\texpression: `[[1], 1, 2] - [[1], 3]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- 1\\n- 2\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `[[1], 1, [[[2]]]] - [[1], [[[3]]]]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- 1\\n- - - - 2\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Array subtraction with nested object\",\n\t\tsubdescription: `Note that order of the keys does not matter`,\n\t\tdocument: `[{a: b, c: d}, {a: b}]`,\n\t\texpression: `. - [{\"c\": \"d\", \"a\": \"b\"}]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[{a: b}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[{a: [1], c: d}, {a: [2], c: d}, {a: b}]`,\n\t\texpression: `. - [{\"c\": \"d\", \"a\": [1]}]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[{a: [2], c: d}, {a: b}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Number subtraction - float\",\n\t\tsubdescription: \"If the lhs or rhs are floats then the expression will be calculated with floats.\",\n\t\tdocument: `{a: 3, b: 4.5}`,\n\t\texpression: `.a = .a - .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: -1.5, b: 4.5}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Number subtraction - int\",\n\t\tsubdescription: \"If both the lhs and rhs are ints then the expression will be calculated with ints.\",\n\t\tdocument: `{a: 3, b: 4}`,\n\t\texpression: `.a = .a - .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: -1, b: 4}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Decrement numbers\",\n\t\tdocument: `{a: 3, b: 5}`,\n\t\texpression: `.[] -= 1`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: 2, b: 4}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Date subtraction\",\n\t\tsubdescription: \"You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.\",\n\t\tdocument: `a: 2021-01-01T03:10:00Z`,\n\t\texpression: `.a -= \"3h10m\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: 2021-01-01T00:00:00Z\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Date subtraction - only date\",\n\t\tskipDoc: true,\n\t\tdocument: `a: 2021-01-01`,\n\t\texpression: `.a -= \"24h\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: 2020-12-31T00:00:00Z\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Date subtraction - custom format\",\n\t\tsubdescription: \"Use with_dtf to specify your datetime format. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.\",\n\t\tdocument: `a: Saturday, 15-Dec-01 at 6:00AM GMT`,\n\t\texpression: `with_dtf(\"Monday, 02-Jan-06 at 3:04PM MST\", .a -= \"3h1m\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: Saturday, 15-Dec-01 at 2:59AM GMT\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Date subtraction - custom format\",\n\t\tsubdescription: \"You can subtract durations from dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.\",\n\t\tdocument: `a: !cat Saturday, 15-Dec-01 at 6:00AM GMT`,\n\t\texpression: `with_dtf(\"Monday, 02-Jan-06 at 3:04PM MST\", .a -= \"3h1m\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !cat Saturday, 15-Dec-01 at 2:59AM GMT\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Custom types: that are really numbers\",\n\t\tsubdescription: \"When custom tags are encountered, yq will try to decode the underlying type.\",\n\t\tdocument: \"a: !horse 2\\nb: !goat 1\",\n\t\texpression: `.a -= .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !horse 1\\nb: !goat 1\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Custom types: that are really floats\",\n\t\tsubdescription: \"When custom tags are encountered, yq will try to decode the underlying type.\",\n\t\tdocument: \"a: !horse 2.5\\nb: !goat 1.5\",\n\t\texpression: `.a - .b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!horse)::1\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Custom types: that are really maps\",\n\t\tdocument: `[!horse {a: b, c: d}, !goat {a: b}]`,\n\t\texpression: `. - [{\"c\": \"d\", \"a\": \"b\"}]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[!goat {a: b}]\\n\",\n\t\t},\n\t},\n}\n\nfunc TestSubtractOperatorScenarios(t *testing.T) {\n\tfor _, tt := range subtractOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"subtract\", subtractOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_tag.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n)\n\nfunc assignTagOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\n\tlog.Debugf(\"AssignTagOperator: %v\")\n\ttag := \"\"\n\n\tif !expressionNode.Operation.UpdateAssign {\n\t\trhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t\tif rhs.MatchingNodes.Front() != nil {\n\t\t\ttag = rhs.MatchingNodes.Front().Value.(*CandidateNode).Value\n\t\t}\n\t}\n\n\tlhs, err := d.GetMatchingNodes(context, expressionNode.LHS)\n\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\n\tfor el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tlog.Debugf(\"Setting tag of : %v\", candidate.GetKey())\n\t\tif expressionNode.Operation.UpdateAssign {\n\t\t\trhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS)\n\t\t\tif err != nil {\n\t\t\t\treturn Context{}, err\n\t\t\t}\n\n\t\t\tif rhs.MatchingNodes.Front() != nil {\n\t\t\t\ttag = rhs.MatchingNodes.Front().Value.(*CandidateNode).Value\n\t\t\t}\n\t\t}\n\t\tcandidate.Tag = tag\n\t}\n\n\treturn context, nil\n}\n\nfunc getTagOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"GetTagOperator\")\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tresult := candidate.CreateReplacement(ScalarNode, \"!!str\", candidate.Tag)\n\t\tresults.PushBack(result)\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_tag_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar tagOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"tag of key is not a key\",\n\t\tsubdescription: \"so it should have 'a' as the path\",\n\t\tskipDoc: true,\n\t\tdocument: \"a: frog\\n\",\n\t\texpression: `.a | key | tag`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!str)::!!str\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Get tag\",\n\t\tdocument: `{a: cat, b: 5, c: 3.2, e: true, f: []}`,\n\t\texpression: `.. | tag`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::!!map\\n\",\n\t\t\t\"D0, P[a], (!!str)::!!str\\n\",\n\t\t\t\"D0, P[b], (!!str)::!!int\\n\",\n\t\t\t\"D0, P[c], (!!str)::!!float\\n\",\n\t\t\t\"D0, P[e], (!!str)::!!bool\\n\",\n\t\t\t\"D0, P[f], (!!str)::!!seq\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"type is an alias for tag\",\n\t\tdocument: `{a: cat, b: 5, c: 3.2, e: true, f: []}`,\n\t\texpression: `.. | type`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::!!map\\n\",\n\t\t\t\"D0, P[a], (!!str)::!!str\\n\",\n\t\t\t\"D0, P[b], (!!str)::!!int\\n\",\n\t\t\t\"D0, P[c], (!!str)::!!float\\n\",\n\t\t\t\"D0, P[e], (!!str)::!!bool\\n\",\n\t\t\t\"D0, P[f], (!!str)::!!seq\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: cat, b: 5, c: 3.2, e: true, f: []}`,\n\t\texpression: `tag`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::!!map\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `32`,\n\t\texpression: `. tag= \"!!str\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::32\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Set custom tag\",\n\t\tdocument: `{a: str}`,\n\t\texpression: `.a tag = \"!!mikefarah\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: !!mikefarah str}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Set custom type\",\n\t\tdocument: `{a: str}`,\n\t\texpression: `.a type = \"!!mikefarah\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: !!mikefarah str}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Find numbers and convert them to strings\",\n\t\tdocument: `{a: cat, b: 5, c: 3.2, e: true}`,\n\t\texpression: `(.. | select(tag == \"!!int\")) tag= \"!!str\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: cat, b: \\\"5\\\", c: 3.2, e: true}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: \"!!frog\", b: \"!!customTag\"}`,\n\t\texpression: `.[] tag |= .`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{a: !!frog \\\"!!frog\\\", b: !!customTag \\\"!!customTag\\\"}\\n\",\n\t\t},\n\t},\n}\n\nfunc TestTagOperatorScenarios(t *testing.T) {\n\tfor _, tt := range tagOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"tag\", tagOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_to_number.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n\t\"strconv\"\n)\n\nfunc tryConvertToNumber(value string) (string, bool) {\n\t// try an int first\n\t_, _, err := parseInt64(value)\n\tif err == nil {\n\t\treturn \"!!int\", true\n\t}\n\t// try float\n\t_, floatErr := strconv.ParseFloat(value, 64)\n\n\tif floatErr == nil {\n\t\treturn \"!!float\", true\n\t}\n\treturn \"\", false\n\n}\n\nfunc toNumberOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"ToNumberOperator\")\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tif candidate.Kind != ScalarNode {\n\t\t\treturn Context{}, fmt.Errorf(\"cannot convert node at path %v of tag %v to number\", candidate.GetNicePath(), candidate.Tag)\n\t\t}\n\n\t\tif candidate.Tag == \"!!int\" || candidate.Tag == \"!!float\" {\n\t\t\t// it already is a number!\n\t\t\tresults.PushBack(candidate)\n\t\t} else {\n\t\t\ttag, converted := tryConvertToNumber(candidate.Value)\n\t\t\tif converted {\n\t\t\t\tresult := candidate.CreateReplacement(ScalarNode, tag, candidate.Value)\n\t\t\t\tresults.PushBack(result)\n\t\t\t} else {\n\t\t\t\treturn Context{}, fmt.Errorf(\"cannot convert node value [%v] at path %v of tag %v to number\", candidate.Value, candidate.GetNicePath(), candidate.Tag)\n\t\t\t}\n\n\t\t}\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_to_number_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar toNumberScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Converts strings to numbers\",\n\t\tdocument: `[\"3\", \"3.1\", \"-1e3\"]`,\n\t\texpression: `.[] | to_number`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!int)::3\\n\",\n\t\t\t\"D0, P[1], (!!float)::3.1\\n\",\n\t\t\t\"D0, P[2], (!!float)::-1e3\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Converts strings to numbers, with tonumber because jq\",\n\t\tdocument: `[\"3\", \"3.1\", \"-1e3\"]`,\n\t\texpression: `.[] | tonumber`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!int)::3\\n\",\n\t\t\t\"D0, P[1], (!!float)::3.1\\n\",\n\t\t\t\"D0, P[2], (!!float)::-1e3\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Doesn't change numbers\",\n\t\tdocument: `[3, 3.1, -1e3]`,\n\t\texpression: `.[] | to_number`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!int)::3\\n\",\n\t\t\t\"D0, P[1], (!!float)::3.1\\n\",\n\t\t\t\"D0, P[2], (!!float)::-1e3\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Cannot convert null\",\n\t\texpression: `.a.b | to_number`,\n\t\texpectedError: \"cannot convert node value [null] at path a.b of tag !!null to number\",\n\t},\n}\n\nfunc TestToNumberOperatorScenarios(t *testing.T) {\n\tfor _, tt := range toNumberScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"to_number\", toNumberScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_traverse_path.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n\t\"slices\"\n\n\t\"github.com/elliotchance/orderedmap\"\n)\n\ntype traversePreferences struct {\n\tDontFollowAlias bool\n\tIncludeMapKeys bool\n\tDontAutoCreate bool // by default, we automatically create entries on the fly.\n\tDontIncludeMapValues bool\n\tOptionalTraverse bool // e.g. .adf?\n\tExactKeyMatch bool // by default we let wild/glob patterns. Don't do that for merge though.\n}\n\nfunc splat(context Context, prefs traversePreferences) (Context, error) {\n\treturn traverseNodesWithArrayIndices(context, make([]*CandidateNode, 0), prefs)\n}\n\nfunc traversePathOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"traversePathOperator\")\n\tvar matches = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tnewNodes, err := traverse(context, el.Value.(*CandidateNode), expressionNode.Operation)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\tmatches.PushBackList(newNodes)\n\t}\n\n\treturn context.ChildContext(matches), nil\n}\n\nfunc traverse(context Context, matchingNode *CandidateNode, operation *Operation) (*list.List, error) {\n\tlog.Debug(\"Traversing %v\", NodeToString(matchingNode))\n\n\tif matchingNode.Tag == \"!!null\" && operation.Value != \"[]\" && !context.DontAutoCreate {\n\t\tlog.Debugf(\"Guessing kind\")\n\t\t// we must have added this automatically, lets guess what it should be now\n\t\tswitch operation.Value.(type) {\n\t\tcase int, int64:\n\t\t\tlog.Debugf(\"probably an array\")\n\t\t\tmatchingNode.Kind = SequenceNode\n\t\tdefault:\n\t\t\tlog.Debugf(\"probably a map\")\n\t\t\tmatchingNode.Kind = MappingNode\n\t\t}\n\t\tmatchingNode.Tag = \"\"\n\t}\n\n\tswitch matchingNode.Kind {\n\tcase MappingNode:\n\t\tlog.Debug(\"its a map with %v entries\", len(matchingNode.Content)/2)\n\t\treturn traverseMap(context, matchingNode, createStringScalarNode(operation.StringValue), operation.Preferences.(traversePreferences), false)\n\n\tcase SequenceNode:\n\t\tlog.Debug(\"its a sequence of %v things!\", len(matchingNode.Content))\n\t\treturn traverseArray(matchingNode, operation, operation.Preferences.(traversePreferences))\n\n\tcase AliasNode:\n\t\tlog.Debug(\"its an alias!\")\n\t\tmatchingNode = matchingNode.Alias\n\t\treturn traverse(context, matchingNode, operation)\n\tdefault:\n\t\treturn list.New(), nil\n\t}\n}\n\nfunc traverseArrayOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\t//lhs may update the variable context, we should pass that into the RHS\n\t// BUT we still return the original context back (see jq)\n\t// https://stedolan.github.io/jq/manual/#Variable/SymbolicBindingOperator:...as$identifier|...\n\n\tlog.Debugf(\"--traverseArrayOperator\")\n\n\tif expressionNode.RHS != nil && expressionNode.RHS.RHS != nil && expressionNode.RHS.RHS.Operation.OperationType == createMapOpType {\n\t\treturn sliceArrayOperator(d, context, expressionNode.RHS.RHS)\n\t}\n\n\tlhs, err := d.GetMatchingNodes(context, expressionNode.LHS)\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\n\t// rhs is a collect expression that will yield indices to retrieve of the arrays\n\n\trhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS)\n\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\tprefs := traversePreferences{}\n\n\tif expressionNode.Operation.Preferences != nil {\n\t\tprefs = expressionNode.Operation.Preferences.(traversePreferences)\n\t}\n\tvar indicesToTraverse = rhs.MatchingNodes.Front().Value.(*CandidateNode).Content\n\n\tlog.Debugf(\"indicesToTraverse %v\", len(indicesToTraverse))\n\n\t//now we traverse the result of the lhs against the indices we found\n\tresult, err := traverseNodesWithArrayIndices(lhs, indicesToTraverse, prefs)\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\treturn context.ChildContext(result.MatchingNodes), nil\n}\n\nfunc traverseNodesWithArrayIndices(context Context, indicesToTraverse []*CandidateNode, prefs traversePreferences) (Context, error) {\n\tvar matchingNodeMap = list.New()\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tnewNodes, err := traverseArrayIndices(context, candidate, indicesToTraverse, prefs)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\tmatchingNodeMap.PushBackList(newNodes)\n\t}\n\n\treturn context.ChildContext(matchingNodeMap), nil\n}\n\nfunc traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesToTraverse []*CandidateNode, prefs traversePreferences) (*list.List, error) { // call this if doc / alias like the other traverse\n\tif matchingNode.Tag == \"!!null\" {\n\t\tlog.Debugf(\"OperatorArrayTraverse got a null - turning it into an empty array\")\n\t\t// auto vivification\n\t\tmatchingNode.Tag = \"\"\n\t\tmatchingNode.Kind = SequenceNode\n\t\t//check that the indices are numeric, if not, then we should create an object\n\t\tif len(indicesToTraverse) != 0 && indicesToTraverse[0].Tag != \"!!int\" {\n\t\t\tmatchingNode.Kind = MappingNode\n\t\t}\n\t}\n\n\tswitch matchingNode.Kind {\n\tcase AliasNode:\n\t\tmatchingNode = matchingNode.Alias\n\t\treturn traverseArrayIndices(context, matchingNode, indicesToTraverse, prefs)\n\tcase SequenceNode:\n\t\treturn traverseArrayWithIndices(matchingNode, indicesToTraverse, prefs)\n\tcase MappingNode:\n\t\treturn traverseMapWithIndices(context, matchingNode, indicesToTraverse, prefs)\n\t}\n\tlog.Debugf(\"OperatorArrayTraverse skipping %v as its a %v\", matchingNode, matchingNode.Tag)\n\treturn list.New(), nil\n}\n\nfunc traverseMapWithIndices(context Context, candidate *CandidateNode, indices []*CandidateNode, prefs traversePreferences) (*list.List, error) {\n\tif len(indices) == 0 {\n\t\treturn traverseMap(context, candidate, createStringScalarNode(\"\"), prefs, true)\n\t}\n\n\tvar matchingNodeMap = list.New()\n\n\tfor _, indexNode := range indices {\n\t\tlog.Debug(\"traverseMapWithIndices: %v\", indexNode.Value)\n\t\tnewNodes, err := traverseMap(context, candidate, indexNode, prefs, false)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tmatchingNodeMap.PushBackList(newNodes)\n\t}\n\n\treturn matchingNodeMap, nil\n}\n\nfunc traverseArrayWithIndices(node *CandidateNode, indices []*CandidateNode, prefs traversePreferences) (*list.List, error) {\n\tlog.Debug(\"traverseArrayWithIndices\")\n\tvar newMatches = list.New()\n\tif len(indices) == 0 {\n\t\tlog.Debug(\"splatting\")\n\t\tvar index int\n\t\tfor index = 0; index < len(node.Content); index = index + 1 {\n\t\t\tnewMatches.PushBack(node.Content[index])\n\t\t}\n\t\treturn newMatches, nil\n\n\t}\n\n\tfor _, indexNode := range indices {\n\t\tlog.Debug(\"traverseArrayWithIndices: '%v'\", indexNode.Value)\n\t\tindex, err := parseInt(indexNode.Value)\n\t\tif err != nil && prefs.OptionalTraverse {\n\t\t\tcontinue\n\t\t}\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"cannot index array with '%v' (%w)\", indexNode.Value, err)\n\t\t}\n\t\tindexToUse := index\n\t\tcontentLength := len(node.Content)\n\t\tfor contentLength <= index {\n\t\t\tif contentLength == 0 {\n\t\t\t\t// default to nice yaml formatting\n\t\t\t\tnode.Style = 0\n\t\t\t}\n\n\t\t\tvalueNode := createScalarNode(nil, \"null\")\n\t\t\tnode.AddChild(valueNode)\n\t\t\tcontentLength = len(node.Content)\n\t\t}\n\n\t\tif indexToUse < 0 {\n\t\t\tindexToUse = contentLength + indexToUse\n\t\t}\n\n\t\tif indexToUse < 0 {\n\t\t\treturn nil, fmt.Errorf(\"index [%v] out of range, array size is %v\", index, contentLength)\n\t\t}\n\n\t\tnewMatches.PushBack(node.Content[indexToUse])\n\t}\n\treturn newMatches, nil\n}\n\nfunc keyMatches(key *CandidateNode, wantedKey string, exactKeyMatch bool) bool {\n\tif exactKeyMatch {\n\t\t// this is used for merge\n\t\treturn key.Value == wantedKey\n\t}\n\treturn matchKey(key.Value, wantedKey)\n}\n\nfunc traverseMap(context Context, matchingNode *CandidateNode, keyNode *CandidateNode, prefs traversePreferences, splat bool) (*list.List, error) {\n\tvar newMatches = orderedmap.NewOrderedMap()\n\terr := doTraverseMap(newMatches, matchingNode, keyNode.Value, prefs, splat)\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif !splat && !prefs.DontAutoCreate && !context.DontAutoCreate && newMatches.Len() == 0 {\n\t\tlog.Debugf(\"no matches, creating one for %v\", NodeToString(keyNode))\n\t\t//no matches, create one automagically\n\t\tvalueNode := matchingNode.CreateChild()\n\t\tvalueNode.Kind = ScalarNode\n\t\tvalueNode.Tag = \"!!null\"\n\t\tvalueNode.Value = \"null\"\n\n\t\tif len(matchingNode.Content) == 0 {\n\t\t\tmatchingNode.Style = 0\n\t\t}\n\n\t\tkeyNode, valueNode = matchingNode.AddKeyValueChild(keyNode, valueNode)\n\n\t\tif prefs.IncludeMapKeys {\n\t\t\tnewMatches.Set(keyNode.GetKey(), keyNode)\n\t\t}\n\t\tif !prefs.DontIncludeMapValues {\n\t\t\tnewMatches.Set(valueNode.GetKey(), valueNode)\n\t\t}\n\t}\n\n\tresults := list.New()\n\ti := 0\n\tfor el := newMatches.Front(); el != nil; el = el.Next() {\n\t\tresults.PushBack(el.Value)\n\t\ti++\n\t}\n\treturn results, nil\n}\n\nfunc doTraverseMap(newMatches *orderedmap.OrderedMap, node *CandidateNode, wantedKey string, prefs traversePreferences, splat bool) error {\n\t// value.Content is a concatenated array of key, value,\n\t// so keys are in the even indices, values in odd.\n\t// merge aliases are defined first, but we only want to traverse them\n\t// if we don't find a match directly on this node first.\n\n\tvar contents = node.Content\n\n\tif !prefs.DontFollowAlias {\n\t\tif ConfiguredYamlPreferences.FixMergeAnchorToSpec {\n\t\t\t// First evaluate merge keys to make explicit keys take precedence, following spec\n\t\t\t// We also iterate in reverse to make earlier merge keys take precedence,\n\t\t\t// although normally there's just one '<<'\n\t\t\tfor index := len(node.Content) - 2; index >= 0; index -= 2 {\n\t\t\t\tkeyNode := node.Content[index]\n\t\t\t\tvalueNode := node.Content[index+1]\n\t\t\t\tif keyNode.Tag == \"!!merge\" {\n\t\t\t\t\tlog.Debug(\"Merge anchor\")\n\t\t\t\t\terr := traverseMergeAnchor(newMatches, valueNode, wantedKey, prefs, splat)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tfor index := 0; index+1 < len(contents); index = index + 2 {\n\t\tkey := contents[index]\n\t\tvalue := contents[index+1]\n\n\t\t//skip the 'merge' tag, find a direct match first\n\t\tif key.Tag == \"!!merge\" && !prefs.DontFollowAlias && wantedKey != key.Value {\n\t\t\tif !ConfiguredYamlPreferences.FixMergeAnchorToSpec {\n\t\t\t\tlog.Debug(\"Merge anchor\")\n\t\t\t\tif showMergeAnchorToSpecWarning {\n\t\t\t\t\tlog.Warning(\"--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025. See https://mikefarah.gitbook.io/yq/operators/traverse-read for more details.\")\n\t\t\t\t\tshowMergeAnchorToSpecWarning = false\n\t\t\t\t}\n\t\t\t\terr := traverseMergeAnchor(newMatches, value, wantedKey, prefs, splat)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t}\n\t\t} else if splat || keyMatches(key, wantedKey, prefs.ExactKeyMatch) {\n\t\t\tlog.Debug(\"MATCHED\")\n\t\t\tif prefs.IncludeMapKeys {\n\t\t\t\tlog.Debug(\"including key\")\n\t\t\t\tkeyName := key.GetKey()\n\t\t\t\tif !newMatches.Set(keyName, key) {\n\t\t\t\t\tlog.Debug(\"overwriting existing key\")\n\t\t\t\t}\n\t\t\t}\n\t\t\tif !prefs.DontIncludeMapValues {\n\t\t\t\tlog.Debug(\"including value\")\n\t\t\t\tvalueName := value.GetKey()\n\t\t\t\tif !newMatches.Set(valueName, value) {\n\t\t\t\t\tlog.Debug(\"overwriting existing value\")\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc traverseMergeAnchor(newMatches *orderedmap.OrderedMap, merge *CandidateNode, wantedKey string, prefs traversePreferences, splat bool) error {\n\tif merge.Kind == AliasNode {\n\t\tmerge = merge.Alias\n\t}\n\tswitch merge.Kind {\n\tcase MappingNode:\n\t\treturn doTraverseMap(newMatches, merge, wantedKey, prefs, splat)\n\tcase SequenceNode:\n\t\tcontent := slices.All(merge.Content)\n\t\tif ConfiguredYamlPreferences.FixMergeAnchorToSpec {\n\t\t\t// Reverse to make earlier values take precedence, following spec\n\t\t\tcontent = slices.Backward(merge.Content)\n\t\t}\n\t\tfor _, childValue := range content {\n\t\t\tif childValue.Kind == AliasNode {\n\t\t\t\tchildValue = childValue.Alias\n\t\t\t}\n\t\t\tif childValue.Kind != MappingNode {\n\t\t\t\tlog.Debugf(\n\t\t\t\t\t\"can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got sequence containing %v\",\n\t\t\t\t\tchildValue.Tag)\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\terr := doTraverseMap(newMatches, childValue, wantedKey, prefs, splat)\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t\treturn nil\n\tdefault:\n\t\tlog.Debugf(\"can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got %v\", merge.Tag)\n\t\treturn nil\n\t}\n}\n\nfunc traverseArray(candidate *CandidateNode, operation *Operation, prefs traversePreferences) (*list.List, error) {\n\tlog.Debug(\"operation Value %v\", operation.Value)\n\tindices := []*CandidateNode{{Value: operation.StringValue}}\n\treturn traverseArrayWithIndices(candidate, indices, prefs)\n}\n" }, { "path": "pkg/yqlib/operator_traverse_path_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar mergeDocSample = `foo: &foo\n a: foo_a\n thing: foo_thing\n c: foo_c\n\nbar: &bar\n b: bar_b\n thing: bar_thing\n c: bar_c\n\nfoobarList:\n b: foobarList_b\n <<: [*foo,*bar]\n c: foobarList_c\n\nfoobar:\n c: foobar_c\n <<: *foo\n thing: foobar_thing\n`\n\nvar fixedTraversePathOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"FIXED: Traversing merge anchors with override\",\n\t\tsubdescription: \"Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour.\",\n\t\tdocument: mergeDocSample,\n\t\texpression: `.foobar.c`,\n\t\texpected: []string{\n\t\t\t\"D0, P[foobar c], (!!str)::foobar_c\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"FIXED: Traversing merge anchor lists\",\n\t\tsubdescription: \"Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Note that the keys earlier in the merge anchors sequence override later ones\",\n\t\tdocument: mergeDocSample,\n\t\texpression: `.foobarList.thing`,\n\t\texpected: []string{\n\t\t\t\"D0, P[foo thing], (!!str)::foo_thing\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"FIXED: Splatting merge anchors\",\n\t\tsubdescription: \"Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Note that the keys earlier in the merge anchors sequence override later ones\",\n\t\tdocument: mergeDocSample,\n\t\texpression: `.foobar[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[foo a], (!!str)::foo_a\\n\",\n\t\t\t\"D0, P[foobar thing], (!!str)::foobar_thing\\n\",\n\t\t\t\"D0, P[foobar c], (!!str)::foobar_c\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"FIXED: Splatting merge anchor lists\",\n\t\tsubdescription: \"Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Note that the keys earlier in the merge anchors sequence override later ones\",\n\t\tdocument: mergeDocSample,\n\t\texpression: `.foobarList[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[foobarList b], (!!str)::foobarList_b\\n\",\n\t\t\t\"D0, P[foo thing], (!!str)::foo_thing\\n\",\n\t\t\t\"D0, P[foobarList c], (!!str)::foobarList_c\\n\",\n\t\t\t\"D0, P[foo a], (!!str)::foo_a\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: mergeDocSample,\n\t\texpression: `.foobarList.b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[foobarList b], (!!str)::foobarList_b\\n\",\n\t\t},\n\t},\n}\n\nvar badTraversePathOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"LEGACY: Traversing merge anchors with override\",\n\t\tsubdescription: \"This is legacy behaviour, see --yaml-fix-merge-anchor-to-spec\",\n\t\tdocument: mergeDocSample,\n\t\texpression: `.foobar.c`,\n\t\texpected: []string{\n\t\t\t\"D0, P[foo c], (!!str)::foo_c\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"LEGACY: Traversing merge anchor lists\",\n\t\tsubdescription: \"Note that the later merge anchors override previous, \" +\n\t\t\t\"but this is legacy behaviour, see --yaml-fix-merge-anchor-to-spec\",\n\t\tdocument: mergeDocSample,\n\t\texpression: `.foobarList.thing`,\n\t\texpected: []string{\n\t\t\t\"D0, P[bar thing], (!!str)::bar_thing\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"LEGACY: Splatting merge anchors\",\n\t\tsubdescription: \"With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec\",\n\t\tdocument: mergeDocSample,\n\t\texpression: `.foobar[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[foo c], (!!str)::foo_c\\n\",\n\t\t\t\"D0, P[foo a], (!!str)::foo_a\\n\",\n\t\t\t\"D0, P[foobar thing], (!!str)::foobar_thing\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"LEGACY: Splatting merge anchor lists\",\n\t\tsubdescription: \"With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec\",\n\t\tdocument: mergeDocSample,\n\t\texpression: `.foobarList[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[bar b], (!!str)::bar_b\\n\",\n\t\t\t\"D0, P[foo a], (!!str)::foo_a\\n\",\n\t\t\t\"D0, P[bar thing], (!!str)::bar_thing\\n\",\n\t\t\t\"D0, P[foobarList c], (!!str)::foobarList_c\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tsubdescription: \"This is legacy behaviour, see --yaml-fix-merge-anchor-to-spec\",\n\t\tdocument: mergeDocSample,\n\t\texpression: `.foobarList.b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[bar b], (!!str)::bar_b\\n\",\n\t\t},\n\t},\n}\n\nvar traversePathOperatorScenarios = []expressionScenario{\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"strange map with key but no value\",\n\t\tdocument: \"!!null\\n-\",\n\t\texpression: \".x\",\n\t\texpected: []string{\n\t\t\t\"D0, P[x], (!!null)::null\\n\",\n\t\t},\n\t\tskipForGoccy: true, // throws an error instead, that's fine\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"access merge anchors\",\n\t\tdocument: \"foo: &foo {x: y}\\nbar:\\n <<: *foo\\n\",\n\t\texpression: `.bar[\"<<\"] | alias`,\n\t\texpected: []string{\n\t\t\t\"D0, P[bar <<], (!!str)::foo\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"dynamically set parent and key\",\n\t\texpression: `.a.b.c = 3 | .a.b.c`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a b c], (!!int)::3\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"dynamically set parent and key in array\",\n\t\texpression: `.a.b[0] = 3 | .a.b[0]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a b 0], (!!int)::3\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"dynamically set parent and key\",\n\t\texpression: `.a.b = [\"x\",\"y\"] | .a.b[1]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a b 1], (!!str)::y\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"splat empty map\",\n\t\tdocument: \"{}\",\n\t\texpression: \".[]\",\n\t\texpected: []string{},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[[1]]`,\n\t\texpression: `.[0][0]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0 0], (!!int)::1\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `.cat[\"12\"] = \"things\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], ()::cat:\\n \\\"12\\\": things\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `blah: {}`,\n\t\texpression: `.blah.cat = \"cool\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::blah:\\n cat: cool\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `blah: []`,\n\t\texpression: `.blah.0 = \"cool\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::blah:\\n - cool\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `b: cat`,\n\t\texpression: \".b\\n\",\n\t\texpected: []string{\n\t\t\t\"D0, P[b], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[[[1]]]`,\n\t\texpression: `.[0][0][0]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0 0 0], (!!int)::1\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `.[\"cat\"] = \"thing\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], ()::cat: thing\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Simple map navigation\",\n\t\tdocument: `{a: {b: apple}}`,\n\t\texpression: `.a`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!map)::{b: apple}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Splat\",\n\t\tsubdescription: \"Often used to pipe children into other operators\",\n\t\tdocument: `[{b: apple}, {c: banana}]`,\n\t\texpression: `.[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!map)::{b: apple}\\n\",\n\t\t\t\"D0, P[1], (!!map)::{c: banana}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Optional Splat\",\n\t\tsubdescription: \"Just like splat, but won't error if you run it against scalars\",\n\t\tdocument: `\"cat\"`,\n\t\texpression: `.[]`,\n\t\texpected: []string{},\n\t},\n\t{\n\t\tdescription: \"Special characters\",\n\t\tsubdescription: \"Use quotes with square brackets around path elements with special characters\",\n\t\tdocument: `{\"{}\": frog}`,\n\t\texpression: `.[\"{}\"]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[{}], (!!str)::frog\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Nested special characters\",\n\t\tdocument: `a: {\"key.withdots\": {\"another.key\": apple}}`,\n\t\texpression: `.a[\"key.withdots\"][\"another.key\"]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a key.withdots another.key], (!!str)::apple\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Keys with spaces\",\n\t\tsubdescription: \"Use quotes with square brackets around path elements with special characters\",\n\t\tdocument: `{\"red rabbit\": frog}`,\n\t\texpression: `.[\"red rabbit\"]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[red rabbit], (!!str)::frog\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{\"flying fox\": frog}`,\n\t\texpression: `.[\"flying fox\"]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[flying fox], (!!str)::frog\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `c: dog`,\n\t\texpression: `.[.a.b] as $x | .`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::c: dog\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Dynamic keys\",\n\t\tsubdescription: `Expressions within [] can be used to dynamically lookup / calculate keys`,\n\t\tdocument: `{b: apple, apple: crispy yum, banana: soft yum}`,\n\t\texpression: `.[.b]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[apple], (!!str)::crispy yum\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{b: apple, fruit: {apple: yum, banana: smooth}}`,\n\t\texpression: `.fruit[.b]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[fruit apple], (!!str)::yum\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Children don't exist\",\n\t\tsubdescription: \"Nodes are added dynamically while traversing\",\n\t\tdocument: `{c: banana}`,\n\t\texpression: `.a.b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a b], (!!null)::null\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Optional identifier\",\n\t\tsubdescription: \"Like jq, does not output an error when the yaml is not an array or object as expected\",\n\t\tdocument: `[1,2,3]`,\n\t\texpression: `.a?`,\n\t\texpected: []string{},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[[1,2,3], {a: frog}]`,\n\t\texpression: `.[] | .[\"a\"]?`,\n\t\texpected: []string{\"D0, P[1 a], (!!str)::frog\\n\"},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: ``,\n\t\texpression: `.[1].a`,\n\t\texpected: []string{\n\t\t\t\"D0, P[1 a], (!!null)::null\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{}`,\n\t\texpression: `.a[1]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a 1], (!!null)::null\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Wildcard matching\",\n\t\tdocument: `{a: {cat: apple, mad: things}}`,\n\t\texpression: `.a.\"*a*\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a cat], (!!str)::apple\\n\",\n\t\t\t\"D0, P[a mad], (!!str)::things\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: {cat: {b: 3}, mad: {b: 4}, fad: {c: t}}}`,\n\t\texpression: `.a.\"*a*\".b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a cat b], (!!int)::3\\n\",\n\t\t\t\"D0, P[a mad b], (!!int)::4\\n\",\n\t\t\t\"D0, P[a fad b], (!!null)::null\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: {cat: apple, mad: things}}`,\n\t\texpression: `.a | (.cat, .mad)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a cat], (!!str)::apple\\n\",\n\t\t\t\"D0, P[a mad], (!!str)::things\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: {cat: apple, mad: things}}`,\n\t\texpression: `.a | (.cat, .mad, .fad)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a cat], (!!str)::apple\\n\",\n\t\t\t\"D0, P[a mad], (!!str)::things\\n\",\n\t\t\t\"D0, P[a fad], (!!null)::null\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: {cat: apple, mad: things}}`,\n\t\texpression: `.a | (.cat, .mad, .fad) | select( (. == null) | not)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a cat], (!!str)::apple\\n\",\n\t\t\t\"D0, P[a mad], (!!str)::things\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Aliases\",\n\t\tdocument: `{a: &cat {c: frog}, b: *cat}`,\n\t\texpression: `.b`,\n\t\texpected: []string{\n\t\t\t\"D0, P[b], (alias)::*cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Traversing aliases with splat\",\n\t\tdocument: `{a: &cat {c: frog}, b: *cat}`,\n\t\texpression: `.b[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a c], (!!str)::frog\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Traversing aliases explicitly\",\n\t\tdocument: `{a: &cat {c: frog}, b: *cat}`,\n\t\texpression: `.b.c`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a c], (!!str)::frog\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Traversing arrays by index\",\n\t\tdocument: `[1,2,3]`,\n\t\texpression: `.[0]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!int)::1\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Traversing nested arrays by index\",\n\t\tdontFormatInputForDoc: true,\n\t\tdocument: `[[], [cat]]`,\n\t\texpression: `.[1][0]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[1 0], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Maps with numeric keys\",\n\t\tdocument: `{2: cat}`,\n\t\texpression: `.[2]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[2], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Maps with non existing numeric keys\",\n\t\tdocument: `{a: b}`,\n\t\texpression: `.[0]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!null)::null\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Merge anchor with inline map\",\n\t\tdocument: `{<<: {a: 42}}`,\n\t\texpression: `.a`,\n\t\texpected: []string{\n\t\t\t\"D0, P[<< a], (!!int)::42\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Merge anchor with sequence with inline map\",\n\t\tdocument: `{<<: [{a: 42}]}`,\n\t\texpression: `.a`,\n\t\texpected: []string{\n\t\t\t\"D0, P[<< 0 a], (!!int)::42\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Merge anchor with aliased sequence with inline map\",\n\t\tdocument: `{s: &s [{a: 42}], m: {<<: *s}}`,\n\t\texpression: `.m.a`,\n\t\texpected: []string{\n\t\t\t\"D0, P[s 0 a], (!!int)::42\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: mergeDocSample,\n\t\texpression: `.foobar`,\n\t\texpected: []string{\n\t\t\t\"D0, P[foobar], (!!map)::c: foobar_c\\n!!merge <<: *foo\\nthing: foobar_thing\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Traversing merge anchors\",\n\t\tdocument: mergeDocSample,\n\t\texpression: `.foobar.a`,\n\t\texpected: []string{\n\t\t\t\"D0, P[foo a], (!!str)::foo_a\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Traversing merge anchors with local override\",\n\t\tdocument: mergeDocSample,\n\t\texpression: `.foobar.thing`,\n\t\texpected: []string{\n\t\t\t\"D0, P[foobar thing], (!!str)::foobar_thing\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: mergeDocSample,\n\t\texpression: `.foobarList`,\n\t\texpected: []string{\n\t\t\t\"D0, P[foobarList], (!!map)::b: foobarList_b\\n!!merge <<: [*foo, *bar]\\nc: foobarList_c\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: mergeDocSample,\n\t\texpression: `.foobarList.a`,\n\t\texpected: []string{\n\t\t\t\"D0, P[foo a], (!!str)::foo_a\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: mergeDocSample,\n\t\texpression: `.foobarList.c`,\n\t\texpected: []string{\n\t\t\t\"D0, P[foobarList c], (!!str)::foobarList_c\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[a,b,c]`,\n\t\texpression: `.[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!str)::a\\n\",\n\t\t\t\"D0, P[1], (!!str)::b\\n\",\n\t\t\t\"D0, P[2], (!!str)::c\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[a,b,c]`,\n\t\texpression: `[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: [a,b,c]}`,\n\t\texpression: `.a[0]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a 0], (!!str)::a\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Select multiple indices\",\n\t\tdocument: `{a: [a,b,c]}`,\n\t\texpression: `.a[0, 2]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a 0], (!!str)::a\\n\",\n\t\t\t\"D0, P[a 2], (!!str)::c\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: [a,b,c]}`,\n\t\texpression: `.a[0, 2]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a 0], (!!str)::a\\n\",\n\t\t\t\"D0, P[a 2], (!!str)::c\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: [a,b,c]}`,\n\t\texpression: `.a[-1]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a 2], (!!str)::c\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: [a,b,c]}`,\n\t\texpression: `.a[-2]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a 1], (!!str)::b\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: [a,b,c]}`,\n\t\texpression: `.a[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a 0], (!!str)::a\\n\",\n\t\t\t\"D0, P[a 1], (!!str)::b\\n\",\n\t\t\t\"D0, P[a 2], (!!str)::c\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: [a,b,c]}`,\n\t\texpression: `.a[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a 0], (!!str)::a\\n\",\n\t\t\t\"D0, P[a 1], (!!str)::b\\n\",\n\t\t\t\"D0, P[a 2], (!!str)::c\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{a: [a,b,c]}`,\n\t\texpression: `.a | .[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a 0], (!!str)::a\\n\",\n\t\t\t\"D0, P[a 1], (!!str)::b\\n\",\n\t\t\t\"D0, P[a 2], (!!str)::c\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Duplicate keys\",\n\t\tsubdescription: \"outside merge anchor\",\n\t\tdocument: `{a: 1, a: 2}`,\n\t\texpression: `.a`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!int)::2\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Traversing map with invalid merge anchor should not fail\",\n\t\tsubdescription: \"Otherwise code cannot do anything with it\",\n\t\tdocument: `{a: 42, <<: 37}`,\n\t\texpression: `.a`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!int)::42\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Directly accessing invalid merge anchor should not fail\",\n\t\tdocument: `{<<: 37}`,\n\t\texpression: `.<<`,\n\t\texpected: []string{\n\t\t\t\"D0, P[<<], (!!int)::37\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"!!str << should not be treated as merge anchor\",\n\t\tdocument: `{!!str <<: {a: 37}}`,\n\t\texpression: `.a`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!null)::null\\n\",\n\t\t},\n\t},\n}\n\nfunc TestTraversePathOperatorScenarios(t *testing.T) {\n\tfor _, tt := range append(traversePathOperatorScenarios, badTraversePathOperatorScenarios...) {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"traverse-read\", append(traversePathOperatorScenarios, badTraversePathOperatorScenarios...))\n}\n\nfunc TestTraversePathOperatorAlignedToSpecScenarios(t *testing.T) {\n\tConfiguredYamlPreferences.FixMergeAnchorToSpec = true\n\tfor _, tt := range append(fixedTraversePathOperatorScenarios, traversePathOperatorScenarios...) {\n\t\ttestScenario(t, &tt)\n\t}\n\tappendOperatorDocumentScenario(t, \"traverse-read\", fixedTraversePathOperatorScenarios)\n\tConfiguredYamlPreferences.FixMergeAnchorToSpec = false\n}\n" }, { "path": "pkg/yqlib/operator_union.go", "content": "package yqlib\n\nimport \"container/list\"\n\nfunc unionOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debug(\"unionOperator--\")\n\tlog.Debug(\"unionOperator: context: %v\", NodesToString(context.MatchingNodes))\n\tlhs, err := d.GetMatchingNodes(context, expressionNode.LHS)\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\tlog.Debug(\"unionOperator: lhs: %v\", NodesToString(lhs.MatchingNodes))\n\tlog.Debug(\"unionOperator: rhs input: %v\", NodesToString(context.MatchingNodes))\n\tlog.Debug(\"unionOperator: rhs: %v\", expressionNode.RHS.Operation.toString())\n\trhs, err := d.GetMatchingNodes(context, expressionNode.RHS)\n\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\tlog.Debug(\"unionOperator: lhs: %v\", lhs.ToString())\n\tlog.Debug(\"unionOperator: rhs: %v\", rhs.ToString())\n\n\tresults := lhs.ChildContext(list.New())\n\tfor el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tnode := el.Value.(*CandidateNode)\n\t\tresults.MatchingNodes.PushBack(node)\n\t}\n\n\t// this can happen when both expressions modify the context\n\t// instead of creating their own.\n\t/// (.foo = \"bar\"), (.thing = \"cat\")\n\tif rhs.MatchingNodes != lhs.MatchingNodes {\n\n\t\tfor el := rhs.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\t\tnode := el.Value.(*CandidateNode)\n\t\t\tlog.Debug(\"union operator rhs: processing %v\", NodeToString(node))\n\n\t\t\tresults.MatchingNodes.PushBack(node)\n\t\t}\n\t}\n\tlog.Debug(\"union operator: all together: %v\", results.ToString())\n\treturn results, nil\n}\n" }, { "path": "pkg/yqlib/operator_union_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar unionOperatorScenarios = []expressionScenario{\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"{}\",\n\t\texpression: `(.a, .b.c) as $x | .`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"clone test\",\n\t\texpression: `\"abc\" as $a | [$a, \"cat\"]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- abc\\n- cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `(.foo = \"bar\"), (.toe = \"jam\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], ()::foo: bar\\ntoe: jam\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Combine scalars\",\n\t\texpression: `1, true, \"cat\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!int)::1\\n\",\n\t\t\t\"D0, P[], (!!bool)::true\\n\",\n\t\t\t\"D0, P[], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Combine selected paths\",\n\t\tdocument: `{a: fieldA, b: fieldB, c: fieldC}`,\n\t\texpression: `.a, .c`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!str)::fieldA\\n\",\n\t\t\t\"D0, P[c], (!!str)::fieldC\\n\",\n\t\t},\n\t},\n}\n\nfunc TestUnionOperatorScenarios(t *testing.T) {\n\tfor _, tt := range unionOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"union\", unionOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_unique.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n\n\t\"github.com/elliotchance/orderedmap\"\n)\n\nfunc unique(d *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tselfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}\n\tuniqueByExpression := &ExpressionNode{Operation: &Operation{OperationType: uniqueByOpType}, RHS: selfExpression}\n\treturn uniqueBy(d, context, uniqueByExpression)\n\n}\n\nfunc uniqueBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\n\tlog.Debugf(\"uniqueBy Operator\")\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\n\t\tif candidate.Kind != SequenceNode {\n\t\t\treturn Context{}, fmt.Errorf(\"only arrays are supported for unique\")\n\t\t}\n\n\t\tvar newMatches = orderedmap.NewOrderedMap()\n\t\tfor _, child := range candidate.Content {\n\t\t\trhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(child), expressionNode.RHS)\n\n\t\t\tif err != nil {\n\t\t\t\treturn Context{}, err\n\t\t\t}\n\n\t\t\tkeyValue, err := getUniqueKeyValue(rhs)\n\t\t\tif err != nil {\n\t\t\t\treturn Context{}, err\n\t\t\t}\n\n\t\t\t_, exists := newMatches.Get(keyValue)\n\n\t\t\tif !exists {\n\t\t\t\tnewMatches.Set(keyValue, child)\n\t\t\t}\n\t\t}\n\t\tresultNode := candidate.CreateReplacementWithComments(SequenceNode, \"!!seq\", candidate.Style)\n\t\tfor el := newMatches.Front(); el != nil; el = el.Next() {\n\t\t\tresultNode.AddChild(el.Value.(*CandidateNode))\n\t\t}\n\n\t\tresults.PushBack(resultNode)\n\t}\n\n\treturn context.ChildContext(results), nil\n\n}\n\nfunc getUniqueKeyValue(rhs Context) (string, error) {\n\tkeyValue := \"null\"\n\tvar err error\n\n\tif rhs.MatchingNodes.Len() > 0 {\n\t\tfirst := rhs.MatchingNodes.Front()\n\t\tkeyCandidate := first.Value.(*CandidateNode)\n\t\tkeyValue = keyCandidate.Value\n\t\tif keyCandidate.Kind != ScalarNode {\n\t\t\tkeyValue, err = encodeToString(keyCandidate, encoderPreferences{YamlFormat, 0})\n\t\t}\n\t}\n\treturn keyValue, err\n}\n" }, { "path": "pkg/yqlib/operator_unique_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar uniqueOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Unique array of scalars (string/numbers)\",\n\t\tsubdescription: \"Note that unique maintains the original order of the array.\",\n\t\tdocument: `[2,1,3,2]`,\n\t\texpression: `unique`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[2, 1, 3]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Unique splat\",\n\t\tskipDoc: true,\n\t\tdocument: `[2,1,2]`,\n\t\texpression: `unique[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!int)::2\\n\",\n\t\t\t\"D0, P[1], (!!int)::1\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Unique nulls\",\n\t\tsubdescription: \"Unique works on the node value, so it considers different representations of nulls to be different\",\n\t\tdocument: `[~,null, ~, null]`,\n\t\texpression: `unique`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[~, null]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Unique all nulls\",\n\t\tsubdescription: \"Run against the node tag to unique all the nulls\",\n\t\tdocument: `[~,null, ~, null]`,\n\t\texpression: `unique_by(tag)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[~]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Unique array objects\",\n\t\tdocument: `[{name: harry, pet: cat}, {name: billy, pet: dog}, {name: harry, pet: cat}]`,\n\t\texpression: `unique`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[{name: harry, pet: cat}, {name: billy, pet: dog}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Unique array of objects by a field\",\n\t\tdocument: `[{name: harry, pet: cat}, {name: billy, pet: dog}, {name: harry, pet: dog}]`,\n\t\texpression: `unique_by(.name)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[{name: harry, pet: cat}, {name: billy, pet: dog}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Unique array of arrays\",\n\t\tdocument: `[[cat,dog], [cat, sheep], [cat,dog]]`,\n\t\texpression: `unique`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[[cat, dog], [cat, sheep]]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[{name: harry, pet: cat}, {pet: fish}, {name: harry, pet: dog}]`,\n\t\texpression: `unique_by(.name)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[{name: harry, pet: cat}, {pet: fish}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"unique by splat\",\n\t\tskipDoc: true,\n\t\tdocument: `[{name: harry, pet: cat}, {pet: fish}, {name: harry, pet: dog}]`,\n\t\texpression: `unique_by(.name)[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!map)::{name: harry, pet: cat}\\n\",\n\t\t\t\"D0, P[1], (!!map)::{pet: fish}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[{name: harry, pet: cat}, {pet: fish}, {name: harry, pet: dog}]`,\n\t\texpression: `unique_by(.cat.dog)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[{name: harry, pet: cat}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"# abc\\n[{name: harry, pet: cat}, {pet: fish}, {name: harry, pet: dog}]\\n# xyz\",\n\t\texpression: `unique_by(.name)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::# abc\\n[{name: harry, pet: cat}, {pet: fish}]\\n# xyz\\n\",\n\t\t},\n\t\tskipForGoccy: true, // https://github.com/goccy/go-yaml/issues/757\n\t},\n}\n\nfunc TestUniqueOperatorScenarios(t *testing.T) {\n\tfor _, tt := range uniqueOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"unique\", uniqueOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_value.go", "content": "package yqlib\n\nimport \"container/list\"\n\nfunc referenceOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\treturn context.SingleChildContext(expressionNode.Operation.CandidateNode), nil\n}\n\nfunc valueOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debug(\"value = %v\", expressionNode.Operation.CandidateNode.Value)\n\tif context.MatchingNodes.Len() == 0 {\n\t\tclone := expressionNode.Operation.CandidateNode.Copy()\n\t\treturn context.SingleChildContext(clone), nil\n\t}\n\n\tvar results = list.New()\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tclone := expressionNode.Operation.CandidateNode.Copy()\n\t\tresults.PushBack(clone)\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n" }, { "path": "pkg/yqlib/operator_value_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar valueOperatorScenarios = []expressionScenario{\n\t{\n\t\tdocument: ``,\n\t\texpression: `1`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!int)::1\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: `[1,2,3]`,\n\t\texpression: `.[] | \"foo\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::foo\\n\",\n\t\t\t\"D0, P[], (!!str)::foo\\n\",\n\t\t\t\"D0, P[], (!!str)::foo\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: `[1,2,3]`,\n\t\texpression: `[.[] | \"foo\"] | .[0] = \"cat\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- cat\\n- foo\\n- foo\\n\",\n\t\t},\n\t},\n\t{\n\t\texpression: `\"foo\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::foo\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: ``,\n\t\texpression: `0x9f`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!int)::0x9f\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: ``,\n\t\texpression: `0x1A`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!int)::0x1A\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: ``,\n\t\texpression: `0x1A + 2`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!int)::0x1C\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: ``,\n\t\texpression: `0x12 * 2`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!int)::0x24\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: ``,\n\t\texpression: `0xF - 1`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!int)::0xE\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: ``,\n\t\texpression: `12`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!int)::12\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: ``,\n\t\texpression: `12 + 2`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!int)::14\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: ``,\n\t\texpression: `12 * 2`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!int)::24\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: ``,\n\t\texpression: `12 - 2`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!int)::10\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: ``,\n\t\texpression: `0X12`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!int)::0X12\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: ``,\n\t\texpression: `-1`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!int)::-1\\n\",\n\t\t},\n\t}, {\n\t\tdocument: ``,\n\t\texpression: `1.2`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!float)::1.2\\n\",\n\t\t},\n\t}, {\n\t\tdocument: ``,\n\t\texpression: `-5.2e11`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!float)::-5.2e11\\n\",\n\t\t},\n\t}, {\n\t\tdocument: ``,\n\t\texpression: `5e-10`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!float)::5e-10\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: ``,\n\t\texpression: `\"cat\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: ``,\n\t\texpression: `\"frog jumps\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::frog jumps\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: ``,\n\t\texpression: `\"1.3\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::1.3\\n\",\n\t\t},\n\t}, {\n\t\tdocument: ``,\n\t\texpression: `\"true\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!str)::true\\n\",\n\t\t},\n\t}, {\n\t\tdocument: ``,\n\t\texpression: `true`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::true\\n\",\n\t\t},\n\t}, {\n\t\tdocument: ``,\n\t\texpression: `false`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: ``,\n\t\texpression: `Null`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!null)::Null\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: ``,\n\t\texpression: `~`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!null)::~\\n\",\n\t\t},\n\t},\n}\n\nfunc TestValueOperatorScenarios(t *testing.T) {\n\tfor _, tt := range valueOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n}\n" }, { "path": "pkg/yqlib/operator_variables.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n)\n\nfunc getVariableOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tvariableName := expressionNode.Operation.StringValue\n\tlog.Debug(\"getVariableOperator %v\", variableName)\n\tresult := context.GetVariable(variableName)\n\tif result == nil {\n\t\tresult = list.New()\n\t}\n\treturn context.ChildContext(result), nil\n}\n\ntype assignVarPreferences struct {\n\tIsReference bool\n}\n\nfunc useWithPipe(_ *dataTreeNavigator, _ Context, _ *ExpressionNode) (Context, error) {\n\treturn Context{}, fmt.Errorf(\"must use variable with a pipe, e.g. `exp as $x | ...`\")\n}\n\n// variables are like loops in jq\n// https://stedolan.github.io/jq/manual/#Variable\nfunc variableLoop(d *dataTreeNavigator, context Context, originalExp *ExpressionNode) (Context, error) {\n\tlog.Debug(\"variable loop!\")\n\tresults := list.New()\n\tvar evaluateAllTogether = true\n\tfor matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() {\n\t\tevaluateAllTogether = evaluateAllTogether && matchEl.Value.(*CandidateNode).EvaluateTogether\n\t\tif !evaluateAllTogether {\n\t\t\tbreak\n\t\t}\n\t}\n\tif evaluateAllTogether {\n\t\treturn variableLoopSingleChild(d, context, originalExp)\n\t}\n\n\tfor el := context.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tresult, err := variableLoopSingleChild(d, context.SingleChildContext(el.Value.(*CandidateNode)), originalExp)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\tresults.PushBackList(result.MatchingNodes)\n\t}\n\treturn context.ChildContext(results), nil\n}\n\nfunc variableLoopSingleChild(d *dataTreeNavigator, context Context, originalExp *ExpressionNode) (Context, error) {\n\n\tvariableExp := originalExp.LHS\n\tlhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), variableExp.LHS)\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\tif variableExp.RHS.Operation.OperationType.Type != \"GET_VARIABLE\" {\n\t\treturn Context{}, fmt.Errorf(\"RHS of 'as' operator must be a variable name e.g. $foo\")\n\t}\n\tvariableName := variableExp.RHS.Operation.StringValue\n\n\tprefs := variableExp.Operation.Preferences.(assignVarPreferences)\n\n\tresults := list.New()\n\n\t// now we loop over lhs, set variable to each result and calculate originalExp.Rhs\n\tfor el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tlog.Debug(\"PROCESSING VARIABLE: \", NodeToString(el.Value.(*CandidateNode)))\n\t\tvar variableValue = list.New()\n\t\tif prefs.IsReference {\n\t\t\tvariableValue.PushBack(el.Value)\n\t\t} else {\n\t\t\tcandidateCopy := el.Value.(*CandidateNode).Copy()\n\t\t\tvariableValue.PushBack(candidateCopy)\n\t\t}\n\t\tnewContext := context.ChildContext(context.MatchingNodes)\n\t\tnewContext.SetVariable(variableName, variableValue)\n\n\t\trhs, err := d.GetMatchingNodes(newContext, originalExp.RHS)\n\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\tlog.Debug(\"PROCESSING VARIABLE DONE, got back: \", rhs.MatchingNodes.Len())\n\t\tresults.PushBackList(rhs.MatchingNodes)\n\t}\n\n\t// if there is no LHS - then I guess we just calculate originalExp.Rhs\n\tif lhs.MatchingNodes.Len() == 0 {\n\t\treturn d.GetMatchingNodes(context, originalExp.RHS)\n\t}\n\n\treturn context.ChildContext(results), nil\n\n}\n" }, { "path": "pkg/yqlib/operator_variables_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar variableOperatorScenarios = []expressionScenario{\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{}`,\n\t\texpression: `.a.b as $foo | .`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::{}\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `{}`,\n\t\texpression: `.a.b as $foo`,\n\t\texpectedError: \"must use variable with a pipe, e.g. `exp as $x | ...`\",\n\t},\n\t{\n\t\tdocument: \"a: [cat]\",\n\t\tskipDoc: true,\n\t\texpression: \"(.[] | {.name: .}) as $item | .\",\n\t\texpectedError: `cannot index array with 'name' (strconv.ParseInt: parsing \"name\": invalid syntax)`,\n\t},\n\t{\n\t\tdescription: \"Single value variable\",\n\t\tdocument: `a: cat`,\n\t\texpression: `.a as $foo | $foo`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!str)::cat\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Multi value variable\",\n\t\tdocument: `[cat, dog]`,\n\t\texpression: `.[] as $foo | $foo`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!str)::cat\\n\",\n\t\t\t\"D0, P[1], (!!str)::dog\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[1, 2]`,\n\t\texpression: `.[] | . as $f | select($f == 2)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[1], (!!int)::2\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `[1, 2]`,\n\t\texpression: `[.[] | . as $f | $f + 1]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- 2\\n- 3\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Using variables as a lookup\",\n\t\tsubdescription: \"Example taken from [jq](https://stedolan.github.io/jq/manual/#Variable/SymbolicBindingOperator:...as$identifier|...)\",\n\t\tdocument: `{\"posts\": [{\"title\": \"First post\", \"author\": \"anon\"},\n\t\t\t{\"title\": \"A well-written article\", \"author\": \"person1\"}],\n\t\"realnames\": {\"anon\": \"Anonymous Coward\",\n\t\t\t\t\t\"person1\": \"Person McPherson\"}}`,\n\t\texpression: `.realnames as $names | .posts[] | {\"title\":.title, \"author\": $names[.author]}`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::title: \\\"First post\\\"\\nauthor: \\\"Anonymous Coward\\\"\\n\",\n\t\t\t\"D0, P[], (!!map)::title: \\\"A well-written article\\\"\\nauthor: \\\"Person McPherson\\\"\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Using variables to swap values\",\n\t\tdocument: \"a: a_value\\nb: b_value\",\n\t\texpression: `.a as $x | .b as $y | .b = $x | .a = $y`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: b_value\\nb: a_value\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Use ref to reference a path repeatedly\",\n\t\tsubdescription: \"Note: You may find the `with` operator more useful.\",\n\t\tdocument: `a: {b: thing, c: something}`,\n\t\texpression: `.a.b ref $x | $x = \"new\" | $x style=\"double\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: {b: \\\"new\\\", c: something}\\n\",\n\t\t},\n\t},\n}\n\nfunc TestVariableOperatorScenarios(t *testing.T) {\n\tfor _, tt := range variableOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"variable-operators\", variableOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operator_with.go", "content": "package yqlib\n\nimport \"fmt\"\n\nfunc withOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {\n\tlog.Debugf(\"withOperator\")\n\t// with(path, exp)\n\n\tif expressionNode.RHS.Operation.OperationType != blockOpType {\n\t\treturn Context{}, fmt.Errorf(\"with must be given a block (;), got %v instead\", expressionNode.RHS.Operation.OperationType.Type)\n\t}\n\n\tpathExp := expressionNode.RHS.LHS\n\n\tupdateContext, err := d.GetMatchingNodes(context, pathExp)\n\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\n\tupdateExp := expressionNode.RHS.RHS\n\n\tfor el := updateContext.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\t_, err = d.GetMatchingNodes(updateContext.SingleChildContext(candidate), updateExp)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t}\n\n\treturn context, nil\n\n}\n" }, { "path": "pkg/yqlib/operator_with_test.go", "content": "package yqlib\n\nimport \"testing\"\n\nvar withOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Update and style\",\n\t\tdocument: `a: {deeply: {nested: value}}`,\n\t\texpression: `with(.a.deeply.nested; . = \"newValue\" | . style=\"single\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: {deeply: {nested: 'newValue'}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"with splat\",\n\t\tskipDoc: true,\n\t\tdocument: `a: {deeply: {nested: value}}`,\n\t\texpression: `with(.a.deeply.nested; . = \"newValue\" | . style=\"single\")[]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!map)::{deeply: {nested: 'newValue'}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Update multiple deeply nested properties\",\n\t\tdocument: `a: {deeply: {nested: value, other: thing}}`,\n\t\texpression: `with(.a.deeply; .nested = \"newValue\" | .other= \"newThing\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: {deeply: {nested: newValue, other: newThing}}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Update array elements relatively\",\n\t\tsubdescription: \"The second expression runs with each element of the array as it's contextual root. This allows you to make updates relative to the element.\",\n\t\tdocument: `myArray: [{a: apple},{a: banana}]`,\n\t\texpression: `with(.myArray[]; .b = .a + \" yum\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::myArray: [{a: apple, b: apple yum}, {a: banana, b: banana yum}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Update array elements relatively +=\",\n\t\tskipDoc: true,\n\t\tsubdescription: \"The second expression runs with each element of the array as it's contextual root. This allows you to make updates relative to the element.\",\n\t\tdocument: `myArray: [{a: apple},{a: banana}]`,\n\t\texpression: `with(.myArray[]; .a += .a)`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::myArray: [{a: appleapple}, {a: bananabanana}]\\n\",\n\t\t},\n\t},\n}\n\nfunc TestWithOperatorScenarios(t *testing.T) {\n\tfor _, tt := range withOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"with\", withOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operators.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"fmt\"\n\n\t\"github.com/jinzhu/copier\"\n\tlogging \"gopkg.in/op/go-logging.v1\"\n)\n\ntype operatorHandler func(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error)\n\ntype compoundCalculation func(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode\n\nfunc compoundAssignFunction(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation compoundCalculation) (Context, error) {\n\tlhs, err := d.GetMatchingNodes(context, expressionNode.LHS)\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\n\t// tricky logic when we are running *= with flags.\n\t// we have an op like: .a *=nc .b\n\t// which should roughly translate to .a =c .a *nc .b\n\t// note that the 'n' flag only applies to the multiple op, not the assignment\n\t// but the clobber flag applies to both!\n\n\tprefs := assignPreferences{}\n\tswitch typedPref := expressionNode.Operation.Preferences.(type) {\n\tcase assignPreferences:\n\t\tprefs = typedPref\n\tcase multiplyPreferences:\n\t\tprefs.ClobberCustomTags = typedPref.AssignPrefs.ClobberCustomTags\n\t}\n\n\tassignmentOp := &Operation{OperationType: assignOpType, Preferences: prefs}\n\n\tfor el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tcandidate := el.Value.(*CandidateNode)\n\t\tclone := candidate.Copy()\n\t\tvalueCopyExp := &ExpressionNode{Operation: &Operation{OperationType: referenceOpType, CandidateNode: clone}}\n\n\t\tvalueExpression := &ExpressionNode{Operation: &Operation{OperationType: referenceOpType, CandidateNode: candidate}}\n\n\t\tassignmentOpNode := &ExpressionNode{Operation: assignmentOp, LHS: valueExpression, RHS: calculation(valueCopyExp, expressionNode.RHS)}\n\n\t\t_, err = d.GetMatchingNodes(context, assignmentOpNode)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t}\n\treturn context, nil\n}\n\nfunc emptyOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {\n\tcontext.MatchingNodes = list.New()\n\treturn context, nil\n}\n\ntype crossFunctionCalculation func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error)\n\nfunc resultsForRHS(d *dataTreeNavigator, context Context, lhsCandidate *CandidateNode, prefs crossFunctionPreferences, rhsExp *ExpressionNode, results *list.List) error {\n\n\tif prefs.LhsResultValue != nil {\n\t\tresult, err := prefs.LhsResultValue(lhsCandidate)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t} else if result != nil {\n\t\t\tresults.PushBack(result)\n\t\t\treturn nil\n\t\t}\n\t}\n\n\trhs, err := d.GetMatchingNodes(context, rhsExp)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif prefs.CalcWhenEmpty && rhs.MatchingNodes.Len() == 0 {\n\t\tresultCandidate, err := prefs.Calculation(d, context, lhsCandidate, nil)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif resultCandidate != nil {\n\t\t\tresults.PushBack(resultCandidate)\n\t\t}\n\t\treturn nil\n\t}\n\n\tfor rightEl := rhs.MatchingNodes.Front(); rightEl != nil; rightEl = rightEl.Next() {\n\t\trhsCandidate := rightEl.Value.(*CandidateNode)\n\t\tif !log.IsEnabledFor(logging.DEBUG) {\n\t\t\tlog.Debugf(\"Applying lhs: %v, rhsCandidate, %v\", NodeToString(lhsCandidate), NodeToString(rhsCandidate))\n\t\t}\n\t\tresultCandidate, err := prefs.Calculation(d, context, lhsCandidate, rhsCandidate)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif resultCandidate != nil {\n\t\t\tresults.PushBack(resultCandidate)\n\t\t}\n\t}\n\treturn nil\n}\n\ntype crossFunctionPreferences struct {\n\tCalcWhenEmpty bool\n\t// if this returns a result node,\n\t// we wont bother calculating the RHS\n\tLhsResultValue func(*CandidateNode) (*CandidateNode, error)\n\tCalculation crossFunctionCalculation\n}\n\nfunc doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, prefs crossFunctionPreferences) (Context, error) {\n\tvar results = list.New()\n\tlhs, err := d.GetMatchingNodes(context, expressionNode.LHS)\n\tif err != nil {\n\t\treturn Context{}, err\n\t}\n\tlog.Debugf(\"crossFunction LHS len: %v\", lhs.MatchingNodes.Len())\n\n\tif prefs.CalcWhenEmpty && context.MatchingNodes.Len() > 0 && lhs.MatchingNodes.Len() == 0 {\n\t\terr := resultsForRHS(d, context, nil, prefs, expressionNode.RHS, results)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t}\n\n\tfor el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {\n\t\tlhsCandidate := el.Value.(*CandidateNode)\n\n\t\terr = resultsForRHS(d, context, lhsCandidate, prefs, expressionNode.RHS, results)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\n\t}\n\treturn context.ChildContext(results), nil\n}\n\nfunc crossFunction(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation crossFunctionCalculation, calcWhenEmpty bool) (Context, error) {\n\tprefs := crossFunctionPreferences{CalcWhenEmpty: calcWhenEmpty, Calculation: calculation}\n\treturn crossFunctionWithPrefs(d, context, expressionNode, prefs)\n}\n\nfunc crossFunctionWithPrefs(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, prefs crossFunctionPreferences) (Context, error) {\n\tvar results = list.New()\n\n\tvar evaluateAllTogether = true\n\tfor matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() {\n\t\tevaluateAllTogether = evaluateAllTogether && matchEl.Value.(*CandidateNode).EvaluateTogether\n\t\tif !evaluateAllTogether {\n\t\t\tbreak\n\t\t}\n\t}\n\n\tif evaluateAllTogether {\n\t\tlog.Debug(\"crossFunction evaluateAllTogether!\")\n\t\treturn doCrossFunc(d, context, expressionNode, prefs)\n\t}\n\n\tlog.Debug(\"crossFunction evaluate apart!\")\n\n\tfor matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() {\n\t\tinnerResults, err := doCrossFunc(d, context.SingleChildContext(matchEl.Value.(*CandidateNode)), expressionNode, prefs)\n\t\tif err != nil {\n\t\t\treturn Context{}, err\n\t\t}\n\t\tresults.PushBackList(innerResults.MatchingNodes)\n\t}\n\n\treturn context.ChildContext(results), nil\n}\n\nfunc createBooleanCandidate(owner *CandidateNode, value bool) *CandidateNode {\n\tvalString := \"true\"\n\tif !value {\n\t\tvalString = \"false\"\n\t}\n\tnoob := owner.CreateReplacement(ScalarNode, \"!!bool\", valString)\n\tif owner.IsMapKey {\n\t\tnoob.IsMapKey = false\n\t\tnoob.Key = owner\n\t}\n\n\treturn noob\n}\n\nfunc createTraversalTree(path []interface{}, traversePrefs traversePreferences, targetKey bool) *ExpressionNode {\n\tif len(path) == 0 {\n\t\treturn &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}\n\t} else if len(path) == 1 {\n\t\tlastPrefs := traversePrefs\n\t\tif targetKey {\n\t\t\terr := copier.Copy(&lastPrefs, traversePrefs)\n\t\t\tif err != nil {\n\t\t\t\tpanic(err)\n\t\t\t}\n\t\t\tlastPrefs.IncludeMapKeys = true\n\t\t\tlastPrefs.DontIncludeMapValues = true\n\t\t}\n\t\treturn &ExpressionNode{Operation: &Operation{OperationType: traversePathOpType, Preferences: lastPrefs, Value: path[0], StringValue: fmt.Sprintf(\"%v\", path[0])}}\n\t}\n\n\treturn &ExpressionNode{\n\t\tOperation: &Operation{OperationType: shortPipeOpType},\n\t\tLHS: createTraversalTree(path[0:1], traversePrefs, false),\n\t\tRHS: createTraversalTree(path[1:], traversePrefs, targetKey),\n\t}\n}\n" }, { "path": "pkg/yqlib/operators_compare_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar compareOperatorScenarios = []expressionScenario{\n\t// ints, not equal\n\t{\n\t\tdescription: \"Compare numbers (>)\",\n\t\tdocument: \"a: 5\\nb: 4\",\n\t\texpression: \".a > .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: \"(.k | length) >= 0\",\n\t\texpected: []string{\n\t\t\t\"D0, P[k], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: `\"2022-01-30T15:53:09Z\" > \"2020-01-30T15:53:09Z\"`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: 5\\nb: 4\",\n\t\texpression: \".a < .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Compare integers (>=)\",\n\t\tdocument: \"a: 5\\nb: 4\",\n\t\texpression: \".a >= .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: 5\\nb: 4\",\n\t\texpression: \".a <= .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::false\\n\",\n\t\t},\n\t},\n\n\t// ints, equal\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Compare equal numbers (>)\",\n\t\tdocument: \"a: 5\\nb: 5\",\n\t\texpression: \".a > .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: 5\\nb: 5\",\n\t\texpression: \".a < .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Compare equal numbers (>=)\",\n\t\tdocument: \"a: 5\\nb: 5\",\n\t\texpression: \".a >= .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: 5\\nb: 5\",\n\t\texpression: \".a <= .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::true\\n\",\n\t\t},\n\t},\n\n\t// floats, not equal\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: 5.2\\nb: 4.1\",\n\t\texpression: \".a > .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: 5.2\\nb: 4.1\",\n\t\texpression: \".a < .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: 5.2\\nb: 4.1\",\n\t\texpression: \".a >= .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: 5.5\\nb: 4.1\",\n\t\texpression: \".a <= .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::false\\n\",\n\t\t},\n\t},\n\n\t// floats, equal\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: 5.5\\nb: 5.5\",\n\t\texpression: \".a > .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: 5.5\\nb: 5.5\",\n\t\texpression: \".a < .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: 5.1\\nb: 5.1\",\n\t\texpression: \".a >= .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: 5.1\\nb: 5.1\",\n\t\texpression: \".a <= .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::true\\n\",\n\t\t},\n\t},\n\n\t// strings, not equal\n\t{\n\t\tdescription: \"Compare strings\",\n\t\tsubdescription: \"Compares strings by their bytecode.\",\n\t\tdocument: \"a: zoo\\nb: apple\",\n\t\texpression: \".a > .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: zoo\\nb: apple\",\n\t\texpression: \".a < .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: zoo\\nb: apple\",\n\t\texpression: \".a >= .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: zoo\\nb: apple\",\n\t\texpression: \".a <= .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::false\\n\",\n\t\t},\n\t},\n\n\t// strings, equal\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: cat\\nb: cat\",\n\t\texpression: \".a > .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: cat\\nb: cat\",\n\t\texpression: \".a < .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: cat\\nb: cat\",\n\t\texpression: \".a >= .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: cat\\nb: cat\",\n\t\texpression: \".a <= .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::true\\n\",\n\t\t},\n\t},\n\n\t// datetime, not equal\n\t{\n\t\tdescription: \"Compare date times\",\n\t\tsubdescription: \"You can compare date times. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.\",\n\t\tdocument: \"a: 2021-01-01T03:10:00Z\\nb: 2020-01-01T03:10:00Z\",\n\t\texpression: \".a > .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: 2021-01-01T03:10:00Z\\nb: 2020-01-01T03:10:00Z\",\n\t\texpression: \".a < .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: 2021-01-01T03:10:00Z\\nb: 2020-01-01T03:10:00Z\",\n\t\texpression: \".a >= .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: 2021-01-01T03:10:00Z\\nb: 2020-01-01T03:10:00Z\",\n\t\texpression: \".a <= .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::false\\n\",\n\t\t},\n\t},\n\n\t// datetime, equal\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: 2021-01-01T03:10:00Z\\nb: 2021-01-01T03:10:00Z\",\n\t\texpression: \".a > .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: 2021-01-01T03:10:00Z\\nb: 2021-01-01T03:10:00Z\",\n\t\texpression: \".a < .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: 2021-01-01T03:10:00Z\\nb: 2021-01-01T03:10:00Z\",\n\t\texpression: \".a >= .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: \"a: 2021-01-01T03:10:00Z\\nb: 2021-01-01T03:10:00Z\",\n\t\texpression: \".a <= .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t// both null\n\t{\n\t\tdescription: \"Both sides are null: > is false\",\n\t\texpression: \".a > .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: \".a < .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Both sides are null: >= is true\",\n\t\texpression: \".a >= .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::true\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\texpression: \".a <= .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::true\\n\",\n\t\t},\n\t},\n\n\t// one null\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"One side is null: > is false\",\n\t\tdocument: `a: 5`,\n\t\texpression: \".a > .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a: 5`,\n\t\texpression: \".a < .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"One side is null: >= is false\",\n\t\tdocument: `a: 5`,\n\t\texpression: \".a >= .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a: 5`,\n\t\texpression: \".a <= .b\",\n\t\texpected: []string{\n\t\t\t\"D0, P[a], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a: 5`,\n\t\texpression: \".b <= .a\",\n\t\texpected: []string{\n\t\t\t\"D0, P[b], (!!bool)::false\\n\",\n\t\t},\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdocument: `a: 5`,\n\t\texpression: \".b < .a\",\n\t\texpected: []string{\n\t\t\t\"D0, P[b], (!!bool)::false\\n\",\n\t\t},\n\t},\n}\n\nfunc TestCompareOperatorScenarios(t *testing.T) {\n\tfor _, tt := range compareOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"compare\", compareOperatorScenarios)\n}\n\nvar minOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Minimum int\",\n\t\tdocument: \"[99, 16, 12, 6, 66]\\n\",\n\t\texpression: `min`,\n\t\texpected: []string{\n\t\t\t\"D0, P[3], (!!int)::6\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Minimum string\",\n\t\tdocument: \"[foo, bar, baz]\\n\",\n\t\texpression: `min`,\n\t\texpected: []string{\n\t\t\t\"D0, P[1], (!!str)::bar\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Minimum of empty\",\n\t\tdocument: \"[]\\n\",\n\t\texpression: `min`,\n\t\texpected: []string{},\n\t},\n}\n\nfunc TestMinOperatorScenarios(t *testing.T) {\n\tfor _, tt := range minOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"min\", minOperatorScenarios)\n}\n\nvar maxOperatorScenarios = []expressionScenario{\n\t{\n\t\tdescription: \"Maximum int\",\n\t\tdocument: \"[99, 16, 12, 6, 66]\\n\",\n\t\texpression: `max`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!int)::99\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Maximum string\",\n\t\tdocument: \"[foo, bar, baz]\\n\",\n\t\texpression: `max`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!str)::foo\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Maximum of empty\",\n\t\tdocument: \"[]\\n\",\n\t\texpression: `max`,\n\t\texpected: []string{},\n\t},\n}\n\nfunc TestMaxOperatorScenarios(t *testing.T) {\n\tfor _, tt := range maxOperatorScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n\tdocumentOperatorScenarios(t, \"max\", maxOperatorScenarios)\n}\n" }, { "path": "pkg/yqlib/operators_test.go", "content": "package yqlib\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"container/list\"\n\t\"fmt\"\n\t\"io\"\n\t\"io/fs\"\n\t\"os\"\n\t\"sort\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n\tlogging \"gopkg.in/op/go-logging.v1\"\n)\n\ntype expressionScenario struct {\n\tdescription string\n\tsubdescription string\n\texplanation []string\n\tenvironmentVariables map[string]string\n\tdocument string\n\tdocument2 string\n\texpression string\n\texpected []string\n\tskipDoc bool\n\texpectedError string\n\tdontFormatInputForDoc bool // dont format input doc for documentation generation\n\trequiresFormat string\n\tskipForGoccy bool\n}\n\nvar goccyTesting = false\n\nvar testingDecoder = NewYamlDecoder(ConfiguredYamlPreferences)\n\nfunc TestMain(m *testing.M) {\n\tlogging.SetLevel(logging.WARNING, \"\")\n\tif os.Getenv(\"DEBUG\") == \"true\" {\n\t\tlogging.SetLevel(logging.DEBUG, \"\")\n\t}\n\tConfiguredYamlPreferences.ColorsEnabled = false\n\tConfiguredJSONPreferences.ColorsEnabled = false\n\n\tgoccyTesting = os.Getenv(\"GOCCY\") == \"true\"\n\n\tif goccyTesting {\n\t\ttestingDecoder = NewGoccyYAMLDecoder()\n\t}\n\n\tNow = func() time.Time {\n\t\treturn time.Date(2021, time.May, 19, 1, 2, 3, 4, time.UTC)\n\t}\n\tcode := m.Run()\n\tos.Exit(code)\n}\n\nfunc NewSimpleYamlPrinter(writer io.Writer, unwrapScalar bool, indent int, printDocSeparators bool) Printer {\n\tprefs := ConfiguredYamlPreferences.Copy()\n\tprefs.PrintDocSeparators = printDocSeparators\n\tprefs.UnwrapScalar = unwrapScalar\n\tprefs.Indent = indent\n\treturn NewPrinter(NewYamlEncoder(prefs), NewSinglePrinterWriter(writer))\n}\n\nfunc readDocument(content string, fakefilename string, fakeFileIndex int) (*list.List, error) {\n\treader := bufio.NewReader(strings.NewReader(content))\n\n\treturn readDocuments(reader, fakefilename, fakeFileIndex, testingDecoder)\n}\n\nfunc testScenario(t *testing.T, s *expressionScenario) {\n\tif s.skipForGoccy {\n\t\treturn\n\t}\n\tlog.Debugf(\"\\n\\ntesting scenario %v\", s.description)\n\tvar err error\n\tnode, err := getExpressionParser().ParseExpression(s.expression)\n\tif err != nil {\n\t\tt.Error(fmt.Errorf(\"Error parsing expression %v of %v: %w\", s.expression, s.description, err))\n\t\treturn\n\t}\n\tinputs := list.New()\n\n\tif s.document != \"\" {\n\t\tinputs, err = readDocument(s.document, \"sample.yml\", 0)\n\n\t\tif err != nil {\n\t\t\tt.Error(err, s.document, s.expression)\n\t\t\treturn\n\t\t}\n\n\t\tif s.document2 != \"\" {\n\t\t\tmoreInputs, err := readDocument(s.document2, \"another.yml\", 1)\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err, s.document2, s.expression)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tinputs.PushBackList(moreInputs)\n\t\t}\n\t} else {\n\t\tcandidateNode := &CandidateNode{\n\t\t\tdocument: 0,\n\t\t\tfilename: \"\",\n\t\t\tTag: \"!!null\",\n\t\t\tKind: ScalarNode,\n\t\t\tfileIndex: 0,\n\t\t}\n\t\tinputs.PushBack(candidateNode)\n\n\t}\n\n\tfor name, value := range s.environmentVariables {\n\t\tos.Setenv(name, value)\n\t}\n\n\tcontext, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: inputs}, node)\n\n\tif s.expectedError != \"\" {\n\t\tif err == nil {\n\t\t\tt.Errorf(\"Expected error '%v' but it worked!\", s.expectedError)\n\t\t} else {\n\t\t\ttest.AssertResultComplexWithContext(t, s.expectedError, err.Error(), fmt.Sprintf(\"desc: %v\\nexp: %v\\ndoc: %v\", s.description, s.expression, s.document))\n\t\t}\n\t\treturn\n\t}\n\n\tif s.requiresFormat != \"\" {\n\t\tformat := s.requiresFormat\n\t\tinputFormat, err := FormatFromString(format)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t\tif decoder := inputFormat.DecoderFactory(); decoder == nil {\n\t\t\tt.Skipf(\"no support for %s input format\", format)\n\t\t}\n\t\toutputFormat, err := FormatFromString(format)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t\tif encoder := configureEncoder(outputFormat, 4); encoder == nil {\n\t\t\tt.Skipf(\"no support for %s output format\", format)\n\t\t}\n\t}\n\tif err != nil {\n\t\tt.Error(fmt.Errorf(\"%w: %v: %v\", err, s.description, s.expression))\n\t\treturn\n\t}\n\ttest.AssertResultComplexWithContext(t, s.expected, resultsToString(t, context.MatchingNodes), fmt.Sprintf(\"desc: %v\\nexp: %v\\ndoc: %v\", s.description, s.expression, s.document))\n}\n\nfunc resultToString(t *testing.T, n *CandidateNode) string {\n\tvar valueBuffer bytes.Buffer\n\tlog.Debugf(\"printing result %v\", NodeToString(n))\n\tprinter := NewSimpleYamlPrinter(bufio.NewWriter(&valueBuffer), true, 4, true)\n\n\terr := printer.PrintResults(n.AsList())\n\tif err != nil {\n\t\tt.Error(err)\n\t\treturn \"\"\n\t}\n\n\ttag := n.Tag\n\tif n.Kind == AliasNode {\n\t\ttag = \"alias\"\n\t}\n\treturn fmt.Sprintf(`D%v, P%v, (%v)::%v`, n.GetDocument(), n.GetPath(), tag, valueBuffer.String())\n}\n\nfunc resultsToString(t *testing.T, results *list.List) []string {\n\tvar pretty = make([]string, 0)\n\n\tfor el := results.Front(); el != nil; el = el.Next() {\n\t\tn := el.Value.(*CandidateNode)\n\n\t\toutput := resultToString(t, n)\n\t\tpretty = append(pretty, output)\n\t}\n\treturn pretty\n}\n\nfunc writeOrPanic(w *bufio.Writer, text string) {\n\t_, err := w.WriteString(text)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n\nfunc copySnippet(source string, out *os.File) error {\n\t_, err := os.Stat(source)\n\tif os.IsNotExist(err) {\n\t\treturn nil\n\t}\n\tin, err := os.Open(source)\n\tif err != nil {\n\t\treturn err\n\t}\n\tdefer safelyCloseFile(in)\n\t_, err = io.Copy(out, in)\n\treturn err\n}\n\nfunc formatYaml(yaml string, filename string) string {\n\tvar output bytes.Buffer\n\tprinter := NewSimpleYamlPrinter(bufio.NewWriter(&output), true, 2, true)\n\n\tnode, err := getExpressionParser().ParseExpression(\".. style= \\\"\\\"\")\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tstreamEvaluator := NewStreamEvaluator()\n\t_, err = streamEvaluator.Evaluate(filename, strings.NewReader(yaml), node, printer, testingDecoder)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn output.String()\n}\n\ntype documentScenarioFunc func(t *testing.T, writer *bufio.Writer, scenario interface{})\n\nfunc documentScenarios(t *testing.T, folder string, title string, scenarios []interface{}, documentScenario documentScenarioFunc) {\n\tfilename := fmt.Sprintf(\"doc/%v/%v.md\", folder, title)\n\tf, err := os.Create(filename)\n\n\tif err != nil {\n\t\tt.Error(err)\n\t\treturn\n\t}\n\tdefer f.Close()\n\n\tsource := fmt.Sprintf(\"doc/%v/headers/%v.md\", folder, title)\n\terr = copySnippet(source, f)\n\tif err != nil {\n\t\tt.Error(err)\n\t\treturn\n\t}\n\n\terr = copySnippet(\"doc/notification-snippet.md\", f)\n\tif err != nil {\n\t\tt.Error(err)\n\t\treturn\n\t}\n\n\tw := bufio.NewWriter(f)\n\twriteOrPanic(w, \"\\n\")\n\n\tfor _, s := range scenarios {\n\t\tdocumentScenario(t, w, s)\n\t}\n\tw.Flush()\n}\n\nfunc appendOperatorDocumentScenario(t *testing.T, title string, scenarios []expressionScenario) {\n\tfilename := fmt.Sprintf(\"doc/%v/%v.md\", \"operators\", title)\n\tf, err := os.OpenFile(filename, os.O_CREATE|os.O_RDWR|os.O_APPEND, fs.ModeAppend)\n\tif err != nil {\n\t\tt.Error(err)\n\t\treturn\n\t}\n\tdefer f.Close()\n\tw := bufio.NewWriter(f)\n\tfor _, s := range scenarios {\n\t\tdocumentOperatorScenario(t, w, s)\n\t}\n\tw.Flush()\n\n}\n\nfunc documentOperatorScenarios(t *testing.T, title string, scenarios []expressionScenario) {\n\tgenericScenarios := make([]interface{}, len(scenarios))\n\tfor i, s := range scenarios {\n\t\tgenericScenarios[i] = s\n\t}\n\n\tdocumentScenarios(t, \"operators\", title, genericScenarios, documentOperatorScenario)\n}\n\nfunc documentOperatorScenario(t *testing.T, w *bufio.Writer, i interface{}) {\n\ts := i.(expressionScenario)\n\n\tif s.skipDoc {\n\t\treturn\n\t}\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\tformattedDoc, formattedDoc2 := documentInput(w, s)\n\n\twriteOrPanic(w, \"will output\\n\")\n\n\tdocumentOutput(t, w, s, formattedDoc, formattedDoc2)\n\n\tif len(s.explanation) > 0 {\n\t\twriteOrPanic(w, \"### Explanation:\\n\")\n\t\tfor _, text := range s.explanation {\n\t\t\twriteOrPanic(w, fmt.Sprintf(\"- %v\\n\", text))\n\t\t}\n\t\twriteOrPanic(w, \"\\n\")\n\t}\n}\n\nfunc documentInput(w *bufio.Writer, s expressionScenario) (string, string) {\n\tformattedDoc := \"\"\n\tformattedDoc2 := \"\"\n\tcommand := \"\"\n\n\tenvCommand := \"\"\n\n\tenvKeys := make([]string, 0, len(s.environmentVariables))\n\tfor k := range s.environmentVariables {\n\t\tenvKeys = append(envKeys, k)\n\t}\n\tsort.Strings(envKeys)\n\n\tfor _, name := range envKeys {\n\t\tvalue := s.environmentVariables[name]\n\t\tif envCommand == \"\" {\n\t\t\tenvCommand = fmt.Sprintf(\"%v=\\\"%v\\\" \", name, value)\n\t\t} else {\n\t\t\tenvCommand = fmt.Sprintf(\"%v %v=\\\"%v\\\" \", envCommand, name, value)\n\t\t}\n\t\tos.Setenv(name, value)\n\t}\n\n\tif s.document != \"\" {\n\t\tif s.dontFormatInputForDoc {\n\t\t\tformattedDoc = s.document + \"\\n\"\n\t\t} else {\n\t\t\tformattedDoc = formatYaml(s.document, \"sample.yml\")\n\t\t}\n\n\t\twriteOrPanic(w, \"Given a sample.yml file of:\\n\")\n\t\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v```\\n\", formattedDoc))\n\n\t\tfiles := \"sample.yml\"\n\n\t\tif s.document2 != \"\" {\n\t\t\tif s.dontFormatInputForDoc {\n\t\t\t\tformattedDoc2 = s.document2 + \"\\n\"\n\t\t\t} else {\n\t\t\t\tformattedDoc2 = formatYaml(s.document2, \"another.yml\")\n\t\t\t}\n\n\t\t\twriteOrPanic(w, \"And another sample another.yml file of:\\n\")\n\t\t\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v```\\n\", formattedDoc2))\n\t\t\tfiles = \"sample.yml another.yml\"\n\t\t\tcommand = \"eval-all \"\n\t\t}\n\n\t\twriteOrPanic(w, \"then\\n\")\n\n\t\tif s.expression != \"\" {\n\t\t\twriteOrPanic(w, fmt.Sprintf(\"```bash\\n%vyq %v'%v' %v\\n```\\n\", envCommand, command, strings.ReplaceAll(s.expression, \"'\", `'\\''`), files))\n\t\t} else {\n\t\t\twriteOrPanic(w, fmt.Sprintf(\"```bash\\n%vyq %v%v\\n```\\n\", envCommand, command, files))\n\t\t}\n\t} else {\n\t\twriteOrPanic(w, \"Running\\n\")\n\t\twriteOrPanic(w, fmt.Sprintf(\"```bash\\n%vyq %v--null-input '%v'\\n```\\n\", envCommand, command, s.expression))\n\t}\n\treturn formattedDoc, formattedDoc2\n}\n\nfunc documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formattedDoc string, formattedDoc2 string) {\n\tvar output bytes.Buffer\n\tvar err error\n\tprinter := NewSimpleYamlPrinter(bufio.NewWriter(&output), true, 2, true)\n\n\tnode, err := getExpressionParser().ParseExpression(s.expression)\n\tif err != nil {\n\t\tt.Error(fmt.Errorf(\"Error parsing expression %v of %v: %w\", s.expression, s.description, err))\n\t\treturn\n\t}\n\n\tinputs := list.New()\n\n\tif s.document != \"\" {\n\n\t\tinputs, err = readDocument(formattedDoc, \"sample.yml\", 0)\n\t\tif err != nil {\n\t\t\tt.Error(err, formattedDoc, \"exp: \"+s.expression)\n\t\t\treturn\n\t\t}\n\t\tif s.document2 != \"\" {\n\t\t\tmoreInputs, err := readDocument(formattedDoc2, \"another.yml\", 1)\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err, formattedDoc2, \"exp: \"+s.expression)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tinputs.PushBackList(moreInputs)\n\t\t}\n\t} else {\n\t\tcandidateNode := &CandidateNode{\n\t\t\tdocument: 0,\n\t\t\tfilename: \"\",\n\t\t\tTag: \"!!null\",\n\t\t\tKind: ScalarNode,\n\t\t\tfileIndex: 0,\n\t\t}\n\t\tinputs.PushBack(candidateNode)\n\n\t}\n\n\tcontext, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: inputs}, node)\n\n\tif s.expectedError != \"\" && err != nil {\n\t\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nError: %v\\n```\\n\\n\", err.Error()))\n\t\treturn\n\t} else if err != nil {\n\t\tt.Error(err, s.expression)\n\t\treturn\n\t}\n\n\terr = printer.PrintResults(context.MatchingNodes)\n\tif err != nil {\n\t\tt.Error(err, s.expression)\n\t}\n\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v```\\n\\n\", output.String()))\n}\n" }, { "path": "pkg/yqlib/printer.go", "content": "package yqlib\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"container/list\"\n\t\"fmt\"\n\t\"io\"\n\t\"regexp\"\n)\n\ntype Printer interface {\n\tPrintResults(matchingNodes *list.List) error\n\tPrintedAnything() bool\n\t//e.g. when given a front-matter doc, like jekyll\n\tSetAppendix(reader io.Reader)\n\tSetNulSepOutput(nulSepOutput bool)\n}\n\ntype resultsPrinter struct {\n\tencoder Encoder\n\tprinterWriter PrinterWriter\n\tfirstTimePrinting bool\n\tpreviousDocIndex uint\n\tpreviousFileIndex int\n\tprintedMatches bool\n\ttreeNavigator DataTreeNavigator\n\tappendixReader io.Reader\n\tnulSepOutput bool\n}\n\nfunc NewPrinter(encoder Encoder, printerWriter PrinterWriter) Printer {\n\treturn &resultsPrinter{\n\t\tencoder: encoder,\n\t\tprinterWriter: printerWriter,\n\t\tfirstTimePrinting: true,\n\t\ttreeNavigator: NewDataTreeNavigator(),\n\t\tnulSepOutput: false,\n\t}\n}\n\nfunc (p *resultsPrinter) SetNulSepOutput(nulSepOutput bool) {\n\tlog.Debug(\"Setting NUL separator output\")\n\n\tp.nulSepOutput = nulSepOutput\n}\n\nfunc (p *resultsPrinter) SetAppendix(reader io.Reader) {\n\tp.appendixReader = reader\n}\n\nfunc (p *resultsPrinter) PrintedAnything() bool {\n\treturn p.printedMatches\n}\n\nfunc (p *resultsPrinter) printNode(node *CandidateNode, writer io.Writer) error {\n\tp.printedMatches = p.printedMatches || (node.Tag != \"!!null\" &&\n\t\t(node.Tag != \"!!bool\" || node.Value != \"false\"))\n\treturn p.encoder.Encode(writer, node)\n}\n\nfunc removeLastEOL(b *bytes.Buffer) {\n\tdata := b.Bytes()\n\tn := len(data)\n\tif n >= 2 && data[n-2] == '\\r' && data[n-1] == '\\n' {\n\t\tb.Truncate(n - 2)\n\t} else if n >= 1 && (data[n-1] == '\\r' || data[n-1] == '\\n') {\n\t\tb.Truncate(n - 1)\n\t}\n}\n\nfunc (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {\n\tlog.Debug(\"PrintResults for %v matches\", matchingNodes.Len())\n\n\tif matchingNodes.Len() == 0 {\n\t\tlog.Debug(\"no matching results, nothing to print\")\n\t\treturn nil\n\t}\n\n\tif !p.encoder.CanHandleAliases() {\n\t\texplodeOp := Operation{OperationType: explodeOpType}\n\t\texplodeNode := ExpressionNode{Operation: &explodeOp}\n\t\tcontext, err := p.treeNavigator.GetMatchingNodes(Context{MatchingNodes: matchingNodes}, &explodeNode)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tmatchingNodes = context.MatchingNodes\n\t}\n\n\tif p.firstTimePrinting {\n\t\tnode := matchingNodes.Front().Value.(*CandidateNode)\n\t\tp.previousDocIndex = node.GetDocument()\n\t\tp.previousFileIndex = node.GetFileIndex()\n\t\tp.firstTimePrinting = false\n\t}\n\n\tfor el := matchingNodes.Front(); el != nil; el = el.Next() {\n\n\t\tmappedDoc := el.Value.(*CandidateNode)\n\t\tlog.Debug(\"print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v\", p.firstTimePrinting, p.previousDocIndex)\n\t\tlog.Debug(\"%v\", NodeToString(mappedDoc))\n\t\twriter, errorWriting := p.printerWriter.GetWriter(mappedDoc)\n\t\tif errorWriting != nil {\n\t\t\treturn errorWriting\n\t\t}\n\n\t\tcommentsStartWithSepExp := regexp.MustCompile(`^\\$yqDocSeparator\\$`)\n\t\tcommentStartsWithSeparator := commentsStartWithSepExp.MatchString(mappedDoc.LeadingContent)\n\n\t\tif (p.previousDocIndex != mappedDoc.GetDocument() || p.previousFileIndex != mappedDoc.GetFileIndex()) && !commentStartsWithSeparator {\n\t\t\tif err := p.encoder.PrintDocumentSeparator(writer); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\tvar destination io.Writer = writer\n\t\ttempBuffer := bytes.NewBuffer(nil)\n\t\tif p.nulSepOutput {\n\t\t\tdestination = tempBuffer\n\t\t}\n\n\t\tif err := p.encoder.PrintLeadingContent(destination, mappedDoc.LeadingContent); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tif err := p.printNode(mappedDoc, destination); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tif p.nulSepOutput {\n\t\t\tremoveLastEOL(tempBuffer)\n\t\t\ttempBufferBytes := tempBuffer.Bytes()\n\t\t\tif bytes.IndexByte(tempBufferBytes, 0) != -1 {\n\t\t\t\treturn fmt.Errorf(\n\t\t\t\t\t\"can't serialise value because it contains NUL char and you are using NUL separated output\",\n\t\t\t\t)\n\t\t\t}\n\t\t\tif _, err := writer.Write(tempBufferBytes); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif _, err := writer.Write([]byte{0}); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\tp.previousDocIndex = mappedDoc.GetDocument()\n\t\tif err := writer.Flush(); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tlog.Debugf(\"done printing results\")\n\t}\n\n\t// what happens if I remove output format check?\n\tif p.appendixReader != nil {\n\t\twriter, err := p.printerWriter.GetWriter(nil)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tlog.Debug(\"Piping appendix reader...\")\n\t\tbetterReader := bufio.NewReader(p.appendixReader)\n\t\t_, err = io.Copy(writer, betterReader)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := writer.Flush(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/printer_node_info.go", "content": "package yqlib\n\nimport (\n\t\"bufio\"\n\t\"container/list\"\n\t\"io\"\n\n\t\"go.yaml.in/yaml/v4\"\n)\n\ntype nodeInfoPrinter struct {\n\tprinterWriter PrinterWriter\n\tappendixReader io.Reader\n\tprintedMatches bool\n}\n\nfunc NewNodeInfoPrinter(printerWriter PrinterWriter) Printer {\n\treturn &nodeInfoPrinter{\n\t\tprinterWriter: printerWriter,\n\t}\n}\n\nfunc (p *nodeInfoPrinter) SetNulSepOutput(_ bool) {\n}\n\nfunc (p *nodeInfoPrinter) SetAppendix(reader io.Reader) {\n\tp.appendixReader = reader\n}\n\nfunc (p *nodeInfoPrinter) PrintedAnything() bool {\n\treturn p.printedMatches\n}\n\nfunc (p *nodeInfoPrinter) PrintResults(matchingNodes *list.List) error {\n\n\tfor el := matchingNodes.Front(); el != nil; el = el.Next() {\n\t\tmappedDoc := el.Value.(*CandidateNode)\n\t\twriter, errorWriting := p.printerWriter.GetWriter(mappedDoc)\n\t\tif errorWriting != nil {\n\t\t\treturn errorWriting\n\t\t}\n\t\tbytes, err := yaml.Marshal(mappedDoc.ConvertToNodeInfo())\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif _, err := writer.Write(bytes); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif _, err := writer.Write([]byte(\"\\n\")); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tp.printedMatches = true\n\t\tif err := writer.Flush(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif p.appendixReader != nil {\n\t\twriter, err := p.printerWriter.GetWriter(nil)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tlog.Debug(\"Piping appendix reader...\")\n\t\tbetterReader := bufio.NewReader(p.appendixReader)\n\t\t_, err = io.Copy(writer, betterReader)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := writer.Flush(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/printer_node_info_test.go", "content": "package yqlib\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"container/list\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nfunc TestNodeInfoPrinter_PrintResults(t *testing.T) {\n\t// Create a simple CandidateNode\n\tnode := &CandidateNode{\n\t\tKind: ScalarNode,\n\t\tStyle: DoubleQuotedStyle,\n\t\tTag: \"!!str\",\n\t\tValue: \"hello world\",\n\t\tLine: 5,\n\t\tColumn: 7,\n\t\tHeadComment: \"head\",\n\t\tLineComment: \"line\",\n\t\tFootComment: \"foot\",\n\t\tAnchor: \"anchor\",\n\t}\n\tlistNodes := list.New()\n\tlistNodes.PushBack(node)\n\n\tvar output bytes.Buffer\n\twriter := bufio.NewWriter(&output)\n\tprinter := NewNodeInfoPrinter(NewSinglePrinterWriter(writer))\n\terr := printer.PrintResults(listNodes)\n\twriter.Flush()\n\tif err != nil {\n\t\tt.Fatalf(\"PrintResults error: %v\", err)\n\t}\n\n\toutStr := output.String()\n\t// Check for key NodeInfo fields in YAML output using substring checks\n\ttest.AssertResult(t, true, strings.Contains(outStr, \"kind: ScalarNode\"))\n\ttest.AssertResult(t, true, strings.Contains(outStr, \"style: DoubleQuotedStyle\"))\n\ttest.AssertResult(t, true, strings.Contains(outStr, \"tag: '!!str'\"))\n\ttest.AssertResult(t, true, strings.Contains(outStr, \"value: hello world\"))\n\ttest.AssertResult(t, true, strings.Contains(outStr, \"line: 5\"))\n\ttest.AssertResult(t, true, strings.Contains(outStr, \"column: 7\"))\n\ttest.AssertResult(t, true, strings.Contains(outStr, \"headComment: head\"))\n\ttest.AssertResult(t, true, strings.Contains(outStr, \"lineComment: line\"))\n\ttest.AssertResult(t, true, strings.Contains(outStr, \"footComment: foot\"))\n\ttest.AssertResult(t, true, strings.Contains(outStr, \"anchor: anchor\"))\n}\n\nfunc TestNodeInfoPrinter_PrintedAnything_True(t *testing.T) {\n\tnode := &CandidateNode{\n\t\tKind: ScalarNode,\n\t\tTag: \"!!str\",\n\t\tValue: \"test\",\n\t}\n\tlistNodes := list.New()\n\tlistNodes.PushBack(node)\n\n\tvar output bytes.Buffer\n\twriter := bufio.NewWriter(&output)\n\tprinter := NewNodeInfoPrinter(NewSinglePrinterWriter(writer))\n\n\t// Before printing, should be false\n\ttest.AssertResult(t, false, printer.PrintedAnything())\n\n\terr := printer.PrintResults(listNodes)\n\twriter.Flush()\n\tif err != nil {\n\t\tt.Fatalf(\"PrintResults error: %v\", err)\n\t}\n\n\t// After printing, should be true\n\ttest.AssertResult(t, true, printer.PrintedAnything())\n}\n\nfunc TestNodeInfoPrinter_PrintedAnything_False(t *testing.T) {\n\tlistNodes := list.New()\n\n\tvar output bytes.Buffer\n\twriter := bufio.NewWriter(&output)\n\tprinter := NewNodeInfoPrinter(NewSinglePrinterWriter(writer))\n\n\terr := printer.PrintResults(listNodes)\n\twriter.Flush()\n\tif err != nil {\n\t\tt.Fatalf(\"PrintResults error: %v\", err)\n\t}\n\n\t// No nodes printed, should still be false\n\ttest.AssertResult(t, false, printer.PrintedAnything())\n}\n\nfunc TestNodeInfoPrinter_SetNulSepOutput(_ *testing.T) {\n\tvar output bytes.Buffer\n\twriter := bufio.NewWriter(&output)\n\tprinter := NewNodeInfoPrinter(NewSinglePrinterWriter(writer))\n\n\t// Should not panic or error\n\tprinter.SetNulSepOutput(true)\n\tprinter.SetNulSepOutput(false)\n}\n\nfunc TestNodeInfoPrinter_SetAppendix(t *testing.T) {\n\tnode := &CandidateNode{\n\t\tKind: ScalarNode,\n\t\tTag: \"!!str\",\n\t\tValue: \"test\",\n\t}\n\tlistNodes := list.New()\n\tlistNodes.PushBack(node)\n\n\tvar output bytes.Buffer\n\twriter := bufio.NewWriter(&output)\n\tprinter := NewNodeInfoPrinter(NewSinglePrinterWriter(writer))\n\n\tappendixText := \"This is appendix text\\n\"\n\tappendixReader := strings.NewReader(appendixText)\n\tprinter.SetAppendix(appendixReader)\n\n\terr := printer.PrintResults(listNodes)\n\twriter.Flush()\n\tif err != nil {\n\t\tt.Fatalf(\"PrintResults error: %v\", err)\n\t}\n\n\toutStr := output.String()\n\ttest.AssertResult(t, true, strings.Contains(outStr, \"test\"))\n\ttest.AssertResult(t, true, strings.Contains(outStr, appendixText))\n}\n\nfunc TestNodeInfoPrinter_MultipleNodes(t *testing.T) {\n\tnode1 := &CandidateNode{\n\t\tKind: ScalarNode,\n\t\tTag: \"!!str\",\n\t\tValue: \"first\",\n\t}\n\tnode2 := &CandidateNode{\n\t\tKind: ScalarNode,\n\t\tTag: \"!!str\",\n\t\tValue: \"second\",\n\t}\n\tlistNodes := list.New()\n\tlistNodes.PushBack(node1)\n\tlistNodes.PushBack(node2)\n\n\tvar output bytes.Buffer\n\twriter := bufio.NewWriter(&output)\n\tprinter := NewNodeInfoPrinter(NewSinglePrinterWriter(writer))\n\n\terr := printer.PrintResults(listNodes)\n\twriter.Flush()\n\tif err != nil {\n\t\tt.Fatalf(\"PrintResults error: %v\", err)\n\t}\n\n\toutStr := output.String()\n\ttest.AssertResult(t, true, strings.Contains(outStr, \"value: first\"))\n\ttest.AssertResult(t, true, strings.Contains(outStr, \"value: second\"))\n}\n\nfunc TestNodeInfoPrinter_SequenceNode(t *testing.T) {\n\tnode := &CandidateNode{\n\t\tKind: SequenceNode,\n\t\tTag: \"!!seq\",\n\t\tStyle: FlowStyle,\n\t}\n\tlistNodes := list.New()\n\tlistNodes.PushBack(node)\n\n\tvar output bytes.Buffer\n\twriter := bufio.NewWriter(&output)\n\tprinter := NewNodeInfoPrinter(NewSinglePrinterWriter(writer))\n\n\terr := printer.PrintResults(listNodes)\n\twriter.Flush()\n\tif err != nil {\n\t\tt.Fatalf(\"PrintResults error: %v\", err)\n\t}\n\n\toutStr := output.String()\n\ttest.AssertResult(t, true, strings.Contains(outStr, \"kind: SequenceNode\"))\n\ttest.AssertResult(t, true, strings.Contains(outStr, \"tag: '!!seq'\"))\n\ttest.AssertResult(t, true, strings.Contains(outStr, \"style: FlowStyle\"))\n}\n\nfunc TestNodeInfoPrinter_MappingNode(t *testing.T) {\n\tnode := &CandidateNode{\n\t\tKind: MappingNode,\n\t\tTag: \"!!map\",\n\t}\n\tlistNodes := list.New()\n\tlistNodes.PushBack(node)\n\n\tvar output bytes.Buffer\n\twriter := bufio.NewWriter(&output)\n\tprinter := NewNodeInfoPrinter(NewSinglePrinterWriter(writer))\n\n\terr := printer.PrintResults(listNodes)\n\twriter.Flush()\n\tif err != nil {\n\t\tt.Fatalf(\"PrintResults error: %v\", err)\n\t}\n\n\toutStr := output.String()\n\ttest.AssertResult(t, true, strings.Contains(outStr, \"kind: MappingNode\"))\n\ttest.AssertResult(t, true, strings.Contains(outStr, \"tag: '!!map'\"))\n}\n\nfunc TestNodeInfoPrinter_EmptyList(t *testing.T) {\n\tlistNodes := list.New()\n\n\tvar output bytes.Buffer\n\twriter := bufio.NewWriter(&output)\n\tprinter := NewNodeInfoPrinter(NewSinglePrinterWriter(writer))\n\n\terr := printer.PrintResults(listNodes)\n\twriter.Flush()\n\tif err != nil {\n\t\tt.Fatalf(\"PrintResults error: %v\", err)\n\t}\n\n\ttest.AssertResult(t, \"\", output.String())\n\ttest.AssertResult(t, false, printer.PrintedAnything())\n}\n" }, { "path": "pkg/yqlib/printer_test.go", "content": "package yqlib\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"container/list\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nvar multiDocSample = `a: banana\n---\na: apple\n---\na: coconut\n`\n\nvar multiDocSampleLeadingExpected = `# go cats\n---\na: banana\n---\na: apple\n---\n# cool\na: coconut\n`\n\nfunc nodeToList(candidate *CandidateNode) *list.List {\n\telMap := list.New()\n\telMap.PushBack(candidate)\n\treturn elMap\n}\n\nfunc TestPrinterMultipleDocsInSequenceOnly(t *testing.T) {\n\tvar output bytes.Buffer\n\tvar writer = bufio.NewWriter(&output)\n\tprinter := NewSimpleYamlPrinter(writer, true, 2, true)\n\n\tinputs, err := readDocuments(strings.NewReader(multiDocSample), \"sample.yml\", 0, NewYamlDecoder(ConfiguredYamlPreferences))\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tel := inputs.Front()\n\tsample1 := nodeToList(el.Value.(*CandidateNode))\n\n\tel = el.Next()\n\tsample2 := nodeToList(el.Value.(*CandidateNode))\n\n\tel = el.Next()\n\tsample3 := nodeToList(el.Value.(*CandidateNode))\n\n\terr = printer.PrintResults(sample1)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\terr = printer.PrintResults(sample2)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\terr = printer.PrintResults(sample3)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\twriter.Flush()\n\ttest.AssertResult(t, multiDocSample, output.String())\n}\n\nfunc TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {\n\tvar output bytes.Buffer\n\tvar writer = bufio.NewWriter(&output)\n\tprinter := NewSimpleYamlPrinter(writer, true, 2, true)\n\n\tinputs, err := readDocuments(strings.NewReader(multiDocSample), \"sample.yml\", 0, NewYamlDecoder(ConfiguredYamlPreferences))\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tel := inputs.Front()\n\tel.Value.(*CandidateNode).LeadingContent = \"# go cats\\n$yqDocSeparator$\\n\"\n\tsample1 := nodeToList(el.Value.(*CandidateNode))\n\n\tel = el.Next()\n\tel.Value.(*CandidateNode).LeadingContent = \"$yqDocSeparator$\\n\"\n\tsample2 := nodeToList(el.Value.(*CandidateNode))\n\n\tel = el.Next()\n\tel.Value.(*CandidateNode).LeadingContent = \"$yqDocSeparator$\\n# cool\\n\"\n\tsample3 := nodeToList(el.Value.(*CandidateNode))\n\n\terr = printer.PrintResults(sample1)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\terr = printer.PrintResults(sample2)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\terr = printer.PrintResults(sample3)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\twriter.Flush()\n\n\ttest.AssertResult(t, multiDocSampleLeadingExpected, output.String())\n}\n\nfunc TestPrinterMultipleFilesInSequence(t *testing.T) {\n\tvar output bytes.Buffer\n\tvar writer = bufio.NewWriter(&output)\n\tprinter := NewSimpleYamlPrinter(writer, true, 2, true)\n\n\tinputs, err := readDocuments(strings.NewReader(multiDocSample), \"sample.yml\", 0, NewYamlDecoder(ConfiguredYamlPreferences))\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tel := inputs.Front()\n\telNode := el.Value.(*CandidateNode)\n\telNode.document = 0\n\telNode.fileIndex = 0\n\tsample1 := nodeToList(elNode)\n\n\tel = el.Next()\n\telNode = el.Value.(*CandidateNode)\n\telNode.document = 0\n\telNode.fileIndex = 1\n\tsample2 := nodeToList(elNode)\n\n\tel = el.Next()\n\telNode = el.Value.(*CandidateNode)\n\telNode.document = 0\n\telNode.fileIndex = 2\n\tsample3 := nodeToList(elNode)\n\n\terr = printer.PrintResults(sample1)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\terr = printer.PrintResults(sample2)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\terr = printer.PrintResults(sample3)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\twriter.Flush()\n\ttest.AssertResult(t, multiDocSample, output.String())\n}\n\nfunc TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {\n\tvar output bytes.Buffer\n\tvar writer = bufio.NewWriter(&output)\n\tprinter := NewSimpleYamlPrinter(writer, true, 2, true)\n\n\tinputs, err := readDocuments(strings.NewReader(multiDocSample), \"sample.yml\", 0, NewYamlDecoder(ConfiguredYamlPreferences))\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tel := inputs.Front()\n\telNode := el.Value.(*CandidateNode)\n\telNode.document = 0\n\telNode.fileIndex = 0\n\telNode.LeadingContent = \"# go cats\\n$yqDocSeparator$\\n\"\n\tsample1 := nodeToList(elNode)\n\n\tel = el.Next()\n\telNode = el.Value.(*CandidateNode)\n\telNode.document = 0\n\telNode.fileIndex = 1\n\telNode.LeadingContent = \"$yqDocSeparator$\\n\"\n\tsample2 := nodeToList(elNode)\n\n\tel = el.Next()\n\telNode = el.Value.(*CandidateNode)\n\telNode.document = 0\n\telNode.fileIndex = 2\n\telNode.LeadingContent = \"$yqDocSeparator$\\n# cool\\n\"\n\tsample3 := nodeToList(elNode)\n\n\terr = printer.PrintResults(sample1)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\terr = printer.PrintResults(sample2)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\terr = printer.PrintResults(sample3)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\twriter.Flush()\n\ttest.AssertResult(t, multiDocSampleLeadingExpected, output.String())\n}\n\nfunc TestPrinterMultipleDocsInSinglePrint(t *testing.T) {\n\tvar output bytes.Buffer\n\tvar writer = bufio.NewWriter(&output)\n\tprinter := NewSimpleYamlPrinter(writer, true, 2, true)\n\n\tinputs, err := readDocuments(strings.NewReader(multiDocSample), \"sample.yml\", 0, NewYamlDecoder(ConfiguredYamlPreferences))\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\terr = printer.PrintResults(inputs)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\twriter.Flush()\n\ttest.AssertResult(t, multiDocSample, output.String())\n}\n\nfunc TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) {\n\tvar output bytes.Buffer\n\tvar writer = bufio.NewWriter(&output)\n\tprinter := NewSimpleYamlPrinter(writer, true, 2, true)\n\n\tinputs, err := readDocuments(strings.NewReader(multiDocSample), \"sample.yml\", 0, NewYamlDecoder(ConfiguredYamlPreferences))\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tinputs.Front().Value.(*CandidateNode).LeadingContent = \"# go cats\\n$yqDocSeparator$\\n\"\n\n\terr = printer.PrintResults(inputs)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\twriter.Flush()\n\texpected := `# go cats\n---\na: banana\n---\na: apple\n---\na: coconut\n`\n\ttest.AssertResult(t, expected, output.String())\n}\n\nfunc TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) {\n\tvar output bytes.Buffer\n\tvar writer = bufio.NewWriter(&output)\n\tprinter := NewSimpleYamlPrinter(writer, true, 2, true)\n\n\tinputs, err := readDocuments(strings.NewReader(multiDocSample), \"sample.yml\", 0, NewYamlDecoder(ConfiguredYamlPreferences))\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tinputs.Front().Value.(*CandidateNode).LeadingContent = \"$yqDocSeparator$\\n\"\n\terr = printer.PrintResults(inputs)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\twriter.Flush()\n\texpected := `---\na: banana\n---\na: apple\n---\na: coconut\n`\n\ttest.AssertResult(t, expected, output.String())\n}\n\nfunc TestPrinterScalarWithLeadingCont(t *testing.T) {\n\tvar output bytes.Buffer\n\tvar writer = bufio.NewWriter(&output)\n\tprinter := NewSimpleYamlPrinter(writer, true, 2, true)\n\n\tnode, err := getExpressionParser().ParseExpression(\".a\")\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tstreamEvaluator := NewStreamEvaluator()\n\t_, err = streamEvaluator.Evaluate(\"sample\", strings.NewReader(multiDocSample), node, printer, NewYamlDecoder(ConfiguredYamlPreferences))\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\twriter.Flush()\n\texpected := `banana\n---\napple\n---\ncoconut\n`\n\ttest.AssertResult(t, expected, output.String())\n}\n\nfunc TestPrinterMultipleDocsJson(t *testing.T) {\n\tvar output bytes.Buffer\n\tvar writer = bufio.NewWriter(&output)\n\t// note printDocSeparators is true, it should still not print document separators\n\t// when outputting JSON.\n\tprefs := ConfiguredJSONPreferences.Copy()\n\tprefs.Indent = 0\n\tencoder := NewJSONEncoder(prefs)\n\tif encoder == nil {\n\t\tt.Skipf(\"no support for %s output format\", \"json\")\n\t}\n\tprinter := NewPrinter(encoder, NewSinglePrinterWriter(writer))\n\n\tinputs, err := readDocuments(strings.NewReader(multiDocSample), \"sample.yml\", 0, NewYamlDecoder(ConfiguredYamlPreferences))\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tinputs.Front().Value.(*CandidateNode).LeadingContent = \"# ignore this\\n\"\n\n\terr = printer.PrintResults(inputs)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\texpected := `{\"a\":\"banana\"}\n{\"a\":\"apple\"}\n{\"a\":\"coconut\"}\n`\n\n\twriter.Flush()\n\ttest.AssertResult(t, expected, output.String())\n}\n\nfunc TestPrinterNulSeparator(t *testing.T) {\n\tvar output bytes.Buffer\n\tvar writer = bufio.NewWriter(&output)\n\tprinter := NewSimpleYamlPrinter(writer, true, 2, false)\n\tprinter.SetNulSepOutput(true)\n\tnode, err := getExpressionParser().ParseExpression(\".a\")\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tstreamEvaluator := NewStreamEvaluator()\n\t_, err = streamEvaluator.Evaluate(\"sample\", strings.NewReader(multiDocSample), node, printer, NewYamlDecoder(ConfiguredYamlPreferences))\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\twriter.Flush()\n\texpected := \"banana\\x00apple\\x00coconut\\x00\"\n\ttest.AssertResult(t, expected, output.String())\n}\n\nfunc TestPrinterNulSeparatorWithJson(t *testing.T) {\n\tvar output bytes.Buffer\n\tvar writer = bufio.NewWriter(&output)\n\t// note printDocSeparators is true, it should still not print document separators\n\t// when outputting JSON.\n\tprefs := ConfiguredJSONPreferences.Copy()\n\tprefs.Indent = 0\n\tencoder := NewJSONEncoder(prefs)\n\tif encoder == nil {\n\t\tt.Skipf(\"no support for %s output format\", \"json\")\n\t}\n\tprinter := NewPrinter(encoder, NewSinglePrinterWriter(writer))\n\tprinter.SetNulSepOutput(true)\n\n\tinputs, err := readDocuments(strings.NewReader(multiDocSample), \"sample.yml\", 0, NewYamlDecoder(ConfiguredYamlPreferences))\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\tinputs.Front().Value.(*CandidateNode).LeadingContent = \"# ignore this\\n\"\n\n\terr = printer.PrintResults(inputs)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\texpected := `{\"a\":\"banana\"}` + \"\\x00\" + `{\"a\":\"apple\"}` + \"\\x00\" + `{\"a\":\"coconut\"}` + \"\\x00\"\n\n\twriter.Flush()\n\ttest.AssertResult(t, expected, output.String())\n}\n\nfunc TestPrinterRootUnwrap(t *testing.T) {\n\tvar output bytes.Buffer\n\tvar writer = bufio.NewWriter(&output)\n\tprinter := NewSimpleYamlPrinter(writer, true, 2, false)\n\tnode, err := getExpressionParser().ParseExpression(\".\")\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tstreamEvaluator := NewStreamEvaluator()\n\t_, err = streamEvaluator.Evaluate(\"sample\", strings.NewReader(\"'a'\"), node, printer, NewYamlDecoder(ConfiguredYamlPreferences))\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\twriter.Flush()\n\texpected := `a\n`\n\ttest.AssertResult(t, expected, output.String())\n}\n\nfunc TestRemoveLastEOL(t *testing.T) {\n\t// Test with \\r\\n\n\tbuffer := bytes.NewBufferString(\"test\\r\\n\")\n\tremoveLastEOL(buffer)\n\ttest.AssertResult(t, \"test\", buffer.String())\n\n\t// Test with \\n only\n\tbuffer = bytes.NewBufferString(\"test\\n\")\n\tremoveLastEOL(buffer)\n\ttest.AssertResult(t, \"test\", buffer.String())\n\n\t// Test with \\r only\n\tbuffer = bytes.NewBufferString(\"test\\r\")\n\tremoveLastEOL(buffer)\n\ttest.AssertResult(t, \"test\", buffer.String())\n\n\t// Test with no EOL\n\tbuffer = bytes.NewBufferString(\"test\")\n\tremoveLastEOL(buffer)\n\ttest.AssertResult(t, \"test\", buffer.String())\n\n\t// Test with empty buffer\n\tbuffer = bytes.NewBufferString(\"\")\n\tremoveLastEOL(buffer)\n\ttest.AssertResult(t, \"\", buffer.String())\n\n\t// Test with multiple \\r\\n\n\tbuffer = bytes.NewBufferString(\"line1\\r\\nline2\\r\\n\")\n\tremoveLastEOL(buffer)\n\ttest.AssertResult(t, \"line1\\r\\nline2\", buffer.String())\n}\n\nfunc TestPrinterPrintedAnything(t *testing.T) {\n\tvar output bytes.Buffer\n\tvar writer = bufio.NewWriter(&output)\n\tprinter := NewSimpleYamlPrinter(writer, true, 2, true)\n\n\ttest.AssertResult(t, false, printer.PrintedAnything())\n\n\t// Print a scalar value\n\tnode := createStringScalarNode(\"test\")\n\tnodeList := nodeToList(node)\n\terr := printer.PrintResults(nodeList)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Should now be true\n\ttest.AssertResult(t, true, printer.PrintedAnything())\n}\n\nfunc TestPrinterNulSeparatorWithNullChar(t *testing.T) {\n\tvar output bytes.Buffer\n\tvar writer = bufio.NewWriter(&output)\n\tprinter := NewSimpleYamlPrinter(writer, true, 2, false)\n\tprinter.SetNulSepOutput(true)\n\n\t// Create a node with null character\n\tnode := createStringScalarNode(\"test\\x00value\")\n\tnodeList := nodeToList(node)\n\n\terr := printer.PrintResults(nodeList)\n\tif err == nil {\n\t\tt.Fatal(\"Expected error for null character in NUL separated output\")\n\t}\n\n\texpectedError := \"can't serialise value because it contains NUL char and you are using NUL separated output\"\n\tif err.Error() != expectedError {\n\t\tt.Fatalf(\"Expected error '%s', got '%s'\", expectedError, err.Error())\n\t}\n}\n\nfunc TestPrinterSetNulSepOutput(t *testing.T) {\n\tvar output bytes.Buffer\n\tvar writer = bufio.NewWriter(&output)\n\tprinter := NewSimpleYamlPrinter(writer, true, 2, false)\n\n\t// Test setting NUL separator output\n\tprinter.SetNulSepOutput(true)\n\ttest.AssertResult(t, true, true) // Placeholder assertion\n\n\tprinter.SetNulSepOutput(false)\n\t// Should also not cause errors\n\ttest.AssertResult(t, false, false) // Placeholder assertion\n}\n\nfunc TestPrinterSetAppendix(t *testing.T) {\n\tvar output bytes.Buffer\n\tvar writer = bufio.NewWriter(&output)\n\tprinter := NewSimpleYamlPrinter(writer, true, 2, true)\n\n\t// Test setting appendix\n\tappendix := strings.NewReader(\"appendix content\")\n\tprinter.SetAppendix(appendix)\n\ttest.AssertResult(t, true, true) // Placeholder assertion\n}\n" }, { "path": "pkg/yqlib/printer_writer.go", "content": "package yqlib\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"regexp\"\n)\n\ntype PrinterWriter interface {\n\tGetWriter(node *CandidateNode) (*bufio.Writer, error)\n}\n\ntype singlePrinterWriter struct {\n\tbufferedWriter *bufio.Writer\n}\n\nfunc NewSinglePrinterWriter(writer io.Writer) PrinterWriter {\n\treturn &singlePrinterWriter{\n\t\tbufferedWriter: bufio.NewWriter(writer),\n\t}\n}\n\nfunc (sp *singlePrinterWriter) GetWriter(_ *CandidateNode) (*bufio.Writer, error) {\n\treturn sp.bufferedWriter, nil\n}\n\ntype multiPrintWriter struct {\n\ttreeNavigator DataTreeNavigator\n\tnameExpression *ExpressionNode\n\textension string\n\tindex int\n}\n\nfunc NewMultiPrinterWriter(expression *ExpressionNode, format *Format) PrinterWriter {\n\textension := \"yml\"\n\n\tswitch format {\n\tcase JSONFormat:\n\t\textension = \"json\"\n\tcase PropertiesFormat:\n\t\textension = \"properties\"\n\t}\n\n\treturn &multiPrintWriter{\n\t\tnameExpression: expression,\n\t\textension: extension,\n\t\ttreeNavigator: NewDataTreeNavigator(),\n\t\tindex: 0,\n\t}\n}\n\nfunc (sp *multiPrintWriter) GetWriter(node *CandidateNode) (*bufio.Writer, error) {\n\tname := \"\"\n\n\tindexVariableNode := CandidateNode{Kind: ScalarNode, Tag: \"!!int\", Value: fmt.Sprintf(\"%v\", sp.index)}\n\n\tcontext := Context{MatchingNodes: node.AsList()}\n\tcontext.SetVariable(\"index\", indexVariableNode.AsList())\n\tresult, err := sp.treeNavigator.GetMatchingNodes(context, sp.nameExpression)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif result.MatchingNodes.Len() > 0 {\n\t\tname = result.MatchingNodes.Front().Value.(*CandidateNode).Value\n\t}\n\tvar extensionRegexp = regexp.MustCompile(`\\.[a-zA-Z0-9]+$`)\n\tif !extensionRegexp.MatchString(name) {\n\t\tname = fmt.Sprintf(\"%v.%v\", name, sp.extension)\n\t}\n\n\terr = os.MkdirAll(filepath.Dir(name), 0750)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tf, err := os.Create(name)\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tsp.index = sp.index + 1\n\n\treturn bufio.NewWriter(f), nil\n\n}\n" }, { "path": "pkg/yqlib/properties.go", "content": "package yqlib\n\ntype PropertiesPreferences struct {\n\tUnwrapScalar bool\n\tKeyValueSeparator string\n\tUseArrayBrackets bool\n}\n\nfunc NewDefaultPropertiesPreferences() PropertiesPreferences {\n\treturn PropertiesPreferences{\n\t\tUnwrapScalar: true,\n\t\tKeyValueSeparator: \" = \",\n\t\tUseArrayBrackets: false,\n\t}\n}\n\nfunc (p *PropertiesPreferences) Copy() PropertiesPreferences {\n\treturn PropertiesPreferences{\n\t\tUnwrapScalar: p.UnwrapScalar,\n\t\tKeyValueSeparator: p.KeyValueSeparator,\n\t\tUseArrayBrackets: p.UseArrayBrackets,\n\t}\n}\n\nvar ConfiguredPropertiesPreferences = NewDefaultPropertiesPreferences()\n" }, { "path": "pkg/yqlib/properties_test.go", "content": "package yqlib\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nconst propertiesWithCommentsOnMap = `this.thing = hi hi\n# important notes\n# about this value\nthis.value = cool\n`\n\nconst expectedPropertiesWithCommentsOnMapProps = `this.thing = hi hi\n\n# important notes\n# about this value\nthis.value = cool\n`\n\nconst expectedPropertiesWithCommentsOnMapYaml = `this:\n thing: hi hi\n # important notes\n # about this value\n value: cool\n`\n\nconst propertiesWithCommentInArray = `\nthis.array.0 = cat\n# important notes\n# about dogs\nthis.array.1 = dog\n`\n\nconst expectedPropertiesWithCommentInArrayProps = `this.array.0 = cat\n\n# important notes\n# about dogs\nthis.array.1 = dog\n`\n\nconst expectedPropertiesWithCommentInArrayYaml = `this:\n array:\n - cat\n # important notes\n # about dogs\n - dog\n`\n\nconst samplePropertiesYaml = `# block comments come through\nperson: # neither do comments on maps\n name: Mike Wazowski # comments on values appear\n pets: \n - cat # comments on array values appear\n - nested:\n - list entry\n food: [pizza] # comments on arrays do not\nemptyArray: []\nemptyMap: []\n`\n\nconst expectedPropertiesUnwrapped = `# block comments come through\n# comments on values appear\nperson.name = Mike Wazowski\n\n# comments on array values appear\nperson.pets.0 = cat\nperson.pets.1.nested.0 = list entry\nperson.food.0 = pizza\n`\n\nconst expectedPropertiesUnwrappedArrayBrackets = `# block comments come through\n# comments on values appear\nperson.name = Mike Wazowski\n\n# comments on array values appear\nperson.pets[0] = cat\nperson.pets[1].nested[0] = list entry\nperson.food[0] = pizza\n`\n\nconst expectedPropertiesUnwrappedCustomSeparator = `# block comments come through\n# comments on values appear\nperson.name :@ Mike Wazowski\n\n# comments on array values appear\nperson.pets.0 :@ cat\nperson.pets.1.nested.0 :@ list entry\nperson.food.0 :@ pizza\n`\n\nconst expectedPropertiesWrapped = `# block comments come through\n# comments on values appear\nperson.name = \"Mike Wazowski\"\n\n# comments on array values appear\nperson.pets.0 = cat\nperson.pets.1.nested.0 = \"list entry\"\nperson.food.0 = pizza\n`\n\nconst expectedUpdatedProperties = `# block comments come through\n# comments on values appear\nperson.name = Mike Wazowski\n\n# comments on array values appear\nperson.pets.0 = dog\nperson.pets.1.nested.0 = list entry\nperson.food.0 = pizza\n`\n\nconst expectedDecodedYaml = `person:\n # block comments come through\n # comments on values appear\n name: Mike Wazowski\n pets:\n # comments on array values appear\n - cat\n - nested:\n - list entry\n food:\n - pizza\n`\n\nconst expectedDecodedPersonYaml = `# block comments come through\n# comments on values appear\nname: Mike Wazowski\npets:\n # comments on array values appear\n - cat\n - nested:\n - list entry\nfood:\n - pizza\n`\n\nconst expectedPropertiesNoComments = `person.name = Mike Wazowski\nperson.pets.0 = cat\nperson.pets.1.nested.0 = list entry\nperson.food.0 = pizza\n`\n\nconst expectedPropertiesWithEmptyMapsAndArrays = `# block comments come through\n# comments on values appear\nperson.name = Mike Wazowski\n\n# comments on array values appear\nperson.pets.0 = cat\nperson.pets.1.nested.0 = list entry\nperson.food.0 = pizza\nemptyArray = \nemptyMap = \n`\n\nvar propertyScenarios = []formatScenario{\n\t{\n\t\tdescription: \"Encode properties\",\n\t\tsubdescription: \"Note that empty arrays and maps are not encoded by default.\",\n\t\tinput: samplePropertiesYaml,\n\t\texpected: expectedPropertiesUnwrapped,\n\t},\n\t{\n\t\tdescription: \"Encode properties with array brackets\",\n\t\tsubdescription: \"Declare the --properties-array-brackets flag to give array paths in brackets (e.g. SpringBoot).\",\n\t\tinput: samplePropertiesYaml,\n\t\texpected: expectedPropertiesUnwrappedArrayBrackets,\n\t\tscenarioType: \"encode-array-brackets\",\n\t},\n\t{\n\t\tdescription: \"Encode properties - custom separator\",\n\t\tsubdescription: \"Use the --properties-separator flag to specify your own key/value separator.\",\n\t\tinput: samplePropertiesYaml,\n\t\texpected: expectedPropertiesUnwrappedCustomSeparator,\n\t\tscenarioType: \"encode-custom-separator\",\n\t},\n\t{\n\t\tdescription: \"Encode properties: scalar encapsulation\",\n\t\tsubdescription: \"Note that string values with blank characters in them are encapsulated with double quotes\",\n\t\tinput: samplePropertiesYaml,\n\t\texpected: expectedPropertiesWrapped,\n\t\tscenarioType: \"encode-wrapped\",\n\t},\n\t{\n\t\tdescription: \"Encode properties: no comments\",\n\t\tinput: samplePropertiesYaml,\n\t\texpected: expectedPropertiesNoComments,\n\t\texpression: `... comments = \"\"`,\n\t},\n\t{\n\t\tdescription: \"Encode properties: include empty maps and arrays\",\n\t\tsubdescription: \"Use a yq expression to set the empty maps and sequences to your desired value.\",\n\t\texpression: `(.. | select( (tag == \"!!map\" or tag ==\"!!seq\") and length == 0)) = \"\"`,\n\t\tinput: samplePropertiesYaml,\n\t\texpected: expectedPropertiesWithEmptyMapsAndArrays,\n\t},\n\t{\n\t\tdescription: \"Decode properties\",\n\t\tinput: expectedPropertiesUnwrapped,\n\t\texpected: expectedDecodedYaml,\n\t\tscenarioType: \"decode\",\n\t},\n\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Decode properties - keeps key information\",\n\t\tinput: expectedPropertiesUnwrapped,\n\t\texpression: \".person.name | key\",\n\t\texpected: \"name\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Decode properties - keeps parent information\",\n\t\tinput: expectedPropertiesUnwrapped,\n\t\texpression: \".person.name | parent\",\n\t\texpected: expectedDecodedPersonYaml,\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Decode properties - keeps path information\",\n\t\tinput: expectedPropertiesUnwrapped,\n\t\texpression: \".person.name | path\",\n\t\texpected: \"- person\\n- name\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Decode properties: numbers\",\n\t\tsubdescription: \"All values are assumed to be strings when parsing properties, but you can use the `from_yaml` operator on all the strings values to autoparse into the correct type.\",\n\t\tinput: \"a.b = 10\",\n\t\texpression: \" (.. | select(tag == \\\"!!str\\\")) |= from_yaml\",\n\t\texpected: \"a:\\n b: 10\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Decode properties - array should be a map\",\n\t\tsubdescription: \"If you have a numeric map key in your property files, use array_to_map to convert them to maps.\",\n\t\tinput: `things.10 = mike`,\n\t\texpression: `.things |= array_to_map`,\n\t\texpected: \"things:\\n 10: mike\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"does not expand automatically\",\n\t\tskipDoc: true,\n\t\tinput: \"mike = ${dontExpand} this\",\n\t\texpected: \"mike: ${dontExpand} this\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"print scalar\",\n\t\tskipDoc: true,\n\t\tinput: \"mike = cat\",\n\t\texpression: \".mike\",\n\t\texpected: \"cat\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip\",\n\t\tinput: expectedPropertiesUnwrapped,\n\t\texpression: `.person.pets.0 = \"dog\"`,\n\t\texpected: expectedUpdatedProperties,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"comments on arrays roundtrip\",\n\t\tinput: propertiesWithCommentInArray,\n\t\texpected: expectedPropertiesWithCommentInArrayProps,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"comments on arrays decode\",\n\t\tinput: propertiesWithCommentInArray,\n\t\texpected: expectedPropertiesWithCommentInArrayYaml,\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"comments on map roundtrip\",\n\t\tinput: propertiesWithCommentsOnMap,\n\t\texpected: expectedPropertiesWithCommentsOnMapProps,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"comments on map decode\",\n\t\tinput: propertiesWithCommentsOnMap,\n\t\texpected: expectedPropertiesWithCommentsOnMapYaml,\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Empty doc\",\n\t\tskipDoc: true,\n\t\tinput: \"\",\n\t\texpected: \"\",\n\t\tscenarioType: \"decode\",\n\t},\n}\n\nfunc documentUnwrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.yml file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\n\texpression := s.expression\n\tprefs := NewDefaultPropertiesPreferences()\n\tuseArrayBracketsFlag := \"\"\n\tuseCustomSeparatorFlag := \"\"\n\tswitch s.scenarioType {\n\tcase \"encode-array-brackets\":\n\t\tuseArrayBracketsFlag = \" --properties-array-brackets\"\n\t\tprefs.UseArrayBrackets = true\n\tcase \"encode-custom-separator\":\n\t\tprefs.KeyValueSeparator = \" :@ \"\n\t\tuseCustomSeparatorFlag = ` --properties-separator=\" :@ \"`\n\t}\n\n\tif expression != \"\" {\n\t\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -o=props%v%v '%v' sample.yml\\n```\\n\", useArrayBracketsFlag, useCustomSeparatorFlag, expression))\n\t} else {\n\t\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -o=props%v%v sample.yml\\n```\\n\", useArrayBracketsFlag, useCustomSeparatorFlag))\n\t}\n\twriteOrPanic(w, \"will output\\n\")\n\tprefs.UnwrapScalar = true\n\n\twriteOrPanic(w, fmt.Sprintf(\"```properties\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(prefs))))\n}\n\nfunc documentWrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.yml file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\n\texpression := s.expression\n\n\tif expression != \"\" {\n\t\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -o=props --unwrapScalar=false '%v' sample.yml\\n```\\n\", expression))\n\t} else {\n\t\twriteOrPanic(w, \"```bash\\nyq -o=props --unwrapScalar=false sample.yml\\n```\\n\")\n\t}\n\twriteOrPanic(w, \"will output\\n\")\n\tprefs := ConfiguredPropertiesPreferences.Copy()\n\tprefs.UnwrapScalar = false\n\twriteOrPanic(w, fmt.Sprintf(\"```properties\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(prefs))))\n}\n\nfunc documentDecodePropertyScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.properties file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```properties\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\n\texpression := s.expression\n\tif expression != \"\" {\n\t\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -p=props '%v' sample.properties\\n```\\n\", expression))\n\t} else {\n\t\twriteOrPanic(w, \"```bash\\nyq -p=props sample.properties\\n```\\n\")\n\t}\n\n\twriteOrPanic(w, \"will output\\n\")\n\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(ConfiguredYamlPreferences))))\n}\n\nfunc documentRoundTripPropertyScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.properties file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```properties\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\n\texpression := s.expression\n\tif expression != \"\" {\n\t\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -p=props -o=props '%v' sample.properties\\n```\\n\", expression))\n\t} else {\n\t\twriteOrPanic(w, \"```bash\\nyq -p=props -o=props sample.properties\\n```\\n\")\n\t}\n\n\twriteOrPanic(w, \"will output\\n\")\n\n\twriteOrPanic(w, fmt.Sprintf(\"```properties\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(ConfiguredPropertiesPreferences))))\n}\n\nfunc documentPropertyScenario(_ *testing.T, w *bufio.Writer, i interface{}) {\n\ts := i.(formatScenario)\n\tif s.skipDoc {\n\t\treturn\n\t}\n\tswitch s.scenarioType {\n\tcase \"\", \"encode-array-brackets\", \"encode-custom-separator\":\n\t\tdocumentUnwrappedEncodePropertyScenario(w, s)\n\tcase \"decode\":\n\t\tdocumentDecodePropertyScenario(w, s)\n\tcase \"encode-wrapped\":\n\t\tdocumentWrappedEncodePropertyScenario(w, s)\n\tcase \"roundtrip\":\n\t\tdocumentRoundTripPropertyScenario(w, s)\n\n\tdefault:\n\t\tpanic(fmt.Sprintf(\"unhandled scenario type %q\", s.scenarioType))\n\t}\n}\n\nfunc TestPropertyScenarios(t *testing.T) {\n\tfor _, s := range propertyScenarios {\n\t\tswitch s.scenarioType {\n\t\tcase \"\":\n\t\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(ConfiguredPropertiesPreferences)), s.description)\n\t\tcase \"decode\":\n\t\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)\n\t\tcase \"encode-wrapped\":\n\t\t\tprefs := ConfiguredPropertiesPreferences.Copy()\n\t\t\tprefs.UnwrapScalar = false\n\t\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(prefs)), s.description)\n\t\tcase \"encode-array-brackets\":\n\t\t\tprefs := ConfiguredPropertiesPreferences.Copy()\n\t\t\tprefs.KeyValueSeparator = \" = \"\n\t\t\tprefs.UseArrayBrackets = true\n\t\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(prefs)), s.description)\n\t\tcase \"encode-custom-separator\":\n\t\t\tprefs := ConfiguredPropertiesPreferences.Copy()\n\t\t\tprefs.KeyValueSeparator = \" :@ \"\n\t\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(prefs)), s.description)\n\t\tcase \"roundtrip\":\n\t\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(ConfiguredPropertiesPreferences)), s.description)\n\n\t\tdefault:\n\t\t\tpanic(fmt.Sprintf(\"unhandled scenario type %q\", s.scenarioType))\n\t\t}\n\t}\n\tgenericScenarios := make([]interface{}, len(propertyScenarios))\n\tfor i, s := range propertyScenarios {\n\t\tgenericScenarios[i] = s\n\t}\n\tdocumentScenarios(t, \"usage\", \"properties\", genericScenarios, documentPropertyScenario)\n}\n" }, { "path": "pkg/yqlib/recipes_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n)\n\nvar bashEnvScript = `.[] |(\n\t( select(kind == \"scalar\") | key + \"='\" + . + \"'\"),\n\t( select(kind == \"seq\") | key + \"=(\" + (map(\"'\" + . + \"'\") | join(\",\")) + \")\")\n)`\n\nvar nestedBashEnvScript = `.. |(\n\t( select(kind == \"scalar\" and parent | kind != \"seq\") | (path | join(\"_\")) + \"='\" + . + \"'\"),\n\t( select(kind == \"seq\") | (path | join(\"_\")) + \"=(\" + (map(\"'\" + . + \"'\") | join(\",\")) + \")\")\n)`\n\nvar deepPruneExpression = `(\n .. | # recurse through all the nodes\n select(has(\"child1\") or has(\"child2\")) | # match parents that have either child1 or child2\n (.child1, .child2) | # select those children\n select(.) # filter out nulls\n) as $i ireduce({}; # using that set of nodes, create a new result map\n setpath($i | path; $i) # and put in each node, using its original path\n)`\n\nvar recipes = []expressionScenario{\n\t{\n\t\tdescription: \"Find items in an array\",\n\t\tsubdescription: \"We have an array and we want to find the elements with a particular name.\",\n\t\texplanation: []string{\n\t\t\t\"`.[]` splats the array, and puts all the items in the context.\",\n\t\t\t\"These items are then piped (`|`) into `select(.name == \\\"Foo\\\")` which will select all the nodes that have a name property set to 'Foo'.\",\n\t\t\t\"See the [select](https://mikefarah.gitbook.io/yq/operators/select) operator for more information.\",\n\t\t},\n\t\tdocument: `[{name: Foo, numBuckets: 0}, {name: Bar, numBuckets: 0}]`,\n\t\texpression: `.[] | select(.name == \"Foo\")`,\n\t\texpected: []string{\n\t\t\t\"D0, P[0], (!!map)::{name: Foo, numBuckets: 0}\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Find and update items in an array\",\n\t\tsubdescription: \"We have an array and we want to _update_ the elements with a particular name.\",\n\t\tdocument: `[{name: Foo, numBuckets: 0}, {name: Bar, numBuckets: 0}]`,\n\t\texpression: `(.[] | select(.name == \"Foo\") | .numBuckets) |= . + 1`,\n\t\texplanation: []string{\n\t\t\t\"Following from the example above`.[]` splats the array, selects filters the items.\",\n\t\t\t\"We then pipe (`|`) that into `.numBuckets`, which will select that field from all the matching items\",\n\t\t\t\"Splat, select and the field are all in brackets, that whole expression is passed to the `|=` operator as the left hand side expression, with `. + 1` as the right hand side expression.\",\n\t\t\t\"`|=` is the operator that updates fields relative to their own value, which is referenced as dot (`.`).\",\n\t\t\t\"The expression `. + 1` increments the numBuckets counter.\",\n\t\t\t\"See the [assign](https://mikefarah.gitbook.io/yq/operators/assign-update) and [add](https://mikefarah.gitbook.io/yq/operators/add) operators for more information.\",\n\t\t},\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::[{name: Foo, numBuckets: 1}, {name: Bar, numBuckets: 0}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Deeply prune a tree\",\n\t\tsubdescription: \"Say we are only interested in child1 and child2, and want to filter everything else out.\",\n\t\tdocument: `{parentA: [bob],parentB: {child1: i am child1, child3: hiya},parentC: {childX: \"cool\",child2: me child2}}`,\n\t\texpression: deepPruneExpression,\n\t\texplanation: []string{\n\t\t\t\"Find all the matching child1 and child2 nodes\",\n\t\t\t\"Using ireduce, create a new map using just those nodes\",\n\t\t\t\"Set each node into the new map using its original path\",\n\t\t},\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::parentB:\\n child1: i am child1\\nparentC:\\n child2: me child2\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Multiple or complex updates to items in an array\",\n\t\tsubdescription: \"We have an array and we want to _update_ the elements with a particular name in reference to its type.\",\n\t\tdocument: `myArray: [{name: Foo, type: cat}, {name: Bar, type: dog}]`,\n\t\texpression: `with(.myArray[]; .name = .name + \" - \" + .type)`,\n\t\texplanation: []string{\n\t\t\t\"The with operator will effectively loop through each given item in the first given expression, and run the second expression against it.\",\n\t\t\t\"`.myArray[]` splats the array in `myArray`. So `with` will run against each item in that array\",\n\t\t\t\"`.name = .name + \\\" - \\\" + .type` this expression is run against every item, updating the name to be a concatenation of the original name as well as the type.\",\n\t\t\t\"See the [with](https://mikefarah.gitbook.io/yq/operators/with) operator for more information and examples.\",\n\t\t},\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::myArray: [{name: Foo - cat, type: cat}, {name: Bar - dog, type: dog}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Sort an array by a field\",\n\t\tdocument: `myArray: [{name: Foo, numBuckets: 1}, {name: Bar, numBuckets: 0}]`,\n\t\texpression: `.myArray |= sort_by(.numBuckets)`,\n\t\texplanation: []string{\n\t\t\t\"We want to resort `.myArray`.\",\n\t\t\t\"`sort_by` works by piping an array into it, and it pipes out a sorted array.\",\n\t\t\t\"So, we use `|=` to update `.myArray`. This is the same as doing `.myArray = (.myArray | sort_by(.numBuckets))`\",\n\t\t},\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::myArray: [{name: Bar, numBuckets: 0}, {name: Foo, numBuckets: 1}]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Filter, flatten, sort and unique\",\n\t\tsubdescription: \"Lets find the unique set of names from the document.\",\n\t\tdocument: `[{type: foo, names: [Fred, Catherine]}, {type: bar, names: [Zelda]}, {type: foo, names: Fred}, {type: foo, names: Ava}]`,\n\t\texpression: `[.[] | select(.type == \"foo\") | .names] | flatten | sort | unique`,\n\t\texplanation: []string{\n\t\t\t\"`.[] | select(.type == \\\"foo\\\") | .names` will select the array elements of type \\\"foo\\\"\",\n\t\t\t\"Splat `.[]` will unwrap the array and match all the items. We need to do this so we can work on the child items, for instance, filter items out using the `select` operator.\",\n\t\t\t\"But we still want the final results back into an array. So after we're doing working on the children, we wrap everything back into an array using square brackets around the expression. `[.[] | select(.type == \\\"foo\\\") | .names]`\",\n\t\t\t\"Now have have an array of all the 'names' values. Which includes arrays of strings as well as strings on their own.\",\n\t\t\t\"Pipe `|` this array through `flatten`. This will flatten nested arrays. So now we have a flat list of all the name value strings\",\n\t\t\t\"Next we pipe `|` that through `sort` and then `unique` to get a sorted, unique list of the names!\",\n\t\t\t\"See the [flatten](https://mikefarah.gitbook.io/yq/operators/flatten), [sort](https://mikefarah.gitbook.io/yq/operators/sort) and [unique](https://mikefarah.gitbook.io/yq/operators/unique) for more information and examples.\",\n\t\t},\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!seq)::- Ava\\n- Catherine\\n- Fred\\n\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Export as environment variables (script), or any custom format\",\n\t\tsubdescription: \"Given a yaml document, lets output a script that will configure environment variables with that data. This same approach can be used for exporting into custom formats.\",\n\t\tdocument: \"var0: string0\\nvar1: string1\\nfruit: [apple, banana, peach]\\n\",\n\t\texpression: bashEnvScript,\n\t\texpected: []string{\n\t\t\t\"D0, P[var0='string0'], (!!str)::var0='string0'\\n\",\n\t\t\t\"D0, P[var1='string1'], (!!str)::var1='string1'\\n\",\n\t\t\t\"D0, P[fruit=('apple','banana','peach')], (!!str)::fruit=('apple','banana','peach')\\n\",\n\t\t},\n\t\texplanation: []string{\n\t\t\t\"`.[]` matches all top level elements\",\n\t\t\t\"We need a string expression for each of the different types that will produce the bash syntax, we'll use the union operator, to join them together\",\n\t\t\t\"Scalars, we just need the key and quoted value: `( select(kind == \\\"scalar\\\") | key + \\\"='\\\" + . + \\\"'\\\")`\",\n\t\t\t\"Sequences (or arrays) are trickier, we need to quote each value and `join` them with `,`: `map(\\\"'\\\" + . + \\\"'\\\") | join(\\\",\\\")`\",\n\t\t},\n\t},\n\t{\n\t\tdescription: \"Custom format with nested data\",\n\t\tsubdescription: \"Like the previous example, but lets handle nested data structures. In this custom example, we're going to join the property paths with _. The important thing to keep in mind is that our expression is not recursive (despite the data structure being so). Instead we match _all_ elements on the tree and operate on them.\",\n\t\tdocument: \"simple: string0\\nsimpleArray: [apple, banana, peach]\\ndeep:\\n property: value\\n array: [cat]\\n\",\n\t\texpression: nestedBashEnvScript,\n\t\texpected: []string{\n\t\t\t\"D0, P[simple], (!!str)::simple='string0'\\n\",\n\t\t\t\"D0, P[deep property], (!!str)::deep_property='value'\\n\",\n\t\t\t\"D0, P[simpleArray], (!!str)::simpleArray=('apple','banana','peach')\\n\",\n\t\t\t\"D0, P[deep array], (!!str)::deep_array=('cat')\\n\",\n\t\t},\n\t\texplanation: []string{\n\t\t\t\"You'll need to understand how the previous example works to understand this extension.\",\n\t\t\t\"`..` matches _all_ elements, instead of `.[]` from the previous example that just matches top level elements.\",\n\t\t\t\"Like before, we need a string expression for each of the different types that will produce the bash syntax, we'll use the union operator, to join them together\",\n\t\t\t\"This time, however, our expression matches every node in the data structure.\",\n\t\t\t\"We only want to print scalars that are not in arrays (because we handle the separately), so well add `and parent | kind != \\\"seq\\\"` to the select operator expression for scalars\",\n\t\t\t\"We don't just want the key any more, we want the full path. So instead of `key` we have `path | join(\\\"_\\\")`\",\n\t\t\t\"The expression for sequences follows the same logic\",\n\t\t},\n\t},\n}\n\nfunc TestRecipes(t *testing.T) {\n\tfor _, tt := range recipes {\n\t\ttestScenario(t, &tt)\n\t}\n\tgenericScenarios := make([]interface{}, len(recipes))\n\tfor i, s := range recipes {\n\t\tgenericScenarios[i] = s\n\t}\n\tdocumentScenarios(t, \"usage\", \"recipes\", genericScenarios, documentOperatorScenario)\n}\n" }, { "path": "pkg/yqlib/security_prefs.go", "content": "package yqlib\n\ntype SecurityPreferences struct {\n\tDisableEnvOps bool\n\tDisableFileOps bool\n}\n\nvar ConfiguredSecurityPreferences = SecurityPreferences{\n\tDisableEnvOps: false,\n\tDisableFileOps: false,\n}\n" }, { "path": "pkg/yqlib/shellvariables.go", "content": "package yqlib\n\ntype ShellVariablesPreferences struct {\n\tKeySeparator string\n\tUnwrapScalar bool\n}\n\nfunc NewDefaultShellVariablesPreferences() ShellVariablesPreferences {\n\treturn ShellVariablesPreferences{\n\t\tKeySeparator: \"_\",\n\t\tUnwrapScalar: false,\n\t}\n}\n\nvar ConfiguredShellVariablesPreferences = NewDefaultShellVariablesPreferences()\n" }, { "path": "pkg/yqlib/shellvariables_test.go", "content": "package yqlib\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nvar shellVariablesScenarios = []formatScenario{\n\t{\n\t\tdescription: \"Encode shell variables\",\n\t\tsubdescription: \"Note that comments are dropped and values will be enclosed in single quotes as needed.\",\n\t\tinput: \"\" +\n\t\t\t\"# comment\" + \"\\n\" +\n\t\t\t\"name: Mike Wazowski\" + \"\\n\" +\n\t\t\t\"eyes:\" + \"\\n\" +\n\t\t\t\" color: turquoise\" + \"\\n\" +\n\t\t\t\" number: 1\" + \"\\n\" +\n\t\t\t\"friends:\" + \"\\n\" +\n\t\t\t\" - James P. Sullivan\" + \"\\n\" +\n\t\t\t\" - Celia Mae\",\n\t\texpected: \"\" +\n\t\t\t\"name='Mike Wazowski'\" + \"\\n\" +\n\t\t\t\"eyes_color=turquoise\" + \"\\n\" +\n\t\t\t\"eyes_number=1\" + \"\\n\" +\n\t\t\t\"friends_0='James P. Sullivan'\" + \"\\n\" +\n\t\t\t\"friends_1='Celia Mae'\" + \"\\n\",\n\t},\n\t{\n\t\tdescription: \"Encode shell variables: illegal variable names as key.\",\n\t\tsubdescription: \"Keys that would be illegal as variable keys are adapted.\",\n\t\tinput: \"\" +\n\t\t\t\"ascii_=_symbols: replaced with _\" + \"\\n\" +\n\t\t\t\"\\\"ascii_\\t_controls\\\": dropped (this example uses \\\\t)\" + \"\\n\" +\n\t\t\t\"nonascii_\\u05d0_characters: dropped\" + \"\\n\" +\n\t\t\t\"effort_expe\\u00f1ded_t\\u00f2_preserve_accented_latin_letters: moderate (via unicode NFKD)\" + \"\\n\",\n\t\texpected: \"\" +\n\t\t\t\"ascii___symbols='replaced with _'\" + \"\\n\" +\n\t\t\t\"ascii__controls='dropped (this example uses \\\\t)'\" + \"\\n\" +\n\t\t\t\"nonascii__characters=dropped\" + \"\\n\" +\n\t\t\t\"effort_expended_to_preserve_accented_latin_letters='moderate (via unicode NFKD)'\" + \"\\n\",\n\t},\n\t{\n\t\tdescription: \"Encode shell variables: empty values, arrays and maps\",\n\t\tsubdescription: \"Empty values are encoded to empty variables, but empty arrays and maps are skipped.\",\n\t\tinput: \"empty:\\n value:\\n array: []\\n map: {}\",\n\t\texpected: \"empty_value=\" + \"\\n\",\n\t},\n\t{\n\t\tdescription: \"Encode shell variables: single quotes in values\",\n\t\tsubdescription: \"Single quotes in values are encoded as '\\\"'\\\"' (close single quote, double-quoted single quote, open single quote).\",\n\t\tinput: \"name: Miles O'Brien\",\n\t\texpected: `name='Miles O'\"'\"'Brien'` + \"\\n\",\n\t},\n\t{\n\t\tdescription: \"Encode shell variables: custom separator\",\n\t\tsubdescription: \"Use --shell-key-separator to specify a custom separator between keys. This is useful when the original keys contain underscores.\",\n\t\tinput: \"\" +\n\t\t\t\"my_app:\" + \"\\n\" +\n\t\t\t\" db_config:\" + \"\\n\" +\n\t\t\t\" host: localhost\" + \"\\n\" +\n\t\t\t\" port: 5432\",\n\t\texpected: \"\" +\n\t\t\t\"my_app__db_config__host=localhost\" + \"\\n\" +\n\t\t\t\"my_app__db_config__port=5432\" + \"\\n\",\n\t\tscenarioType: \"shell-separator\",\n\t},\n}\n\nfunc TestShellVariableScenarios(t *testing.T) {\n\tfor _, s := range shellVariablesScenarios {\n\t\t//fmt.Printf(\"\\t<%s> <%s>\\n\", s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder()))\n\t\tif s.scenarioType == \"shell-separator\" {\n\t\t\t// Save and restore the original separator\n\t\t\toriginalSeparator := ConfiguredShellVariablesPreferences.KeySeparator\n\t\t\tConfiguredShellVariablesPreferences.KeySeparator = \"__\"\n\t\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder()), s.description)\n\t\t\tConfiguredShellVariablesPreferences.KeySeparator = originalSeparator\n\t\t} else {\n\t\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder()), s.description)\n\t\t}\n\t}\n\tgenericScenarios := make([]interface{}, len(shellVariablesScenarios))\n\tfor i, s := range shellVariablesScenarios {\n\t\tgenericScenarios[i] = s\n\t}\n\tdocumentScenarios(t, \"usage\", \"shellvariables\", genericScenarios, documentShellVariableScenario)\n}\n\nfunc documentShellVariableScenario(_ *testing.T, w *bufio.Writer, i interface{}) {\n\ts := i.(formatScenario)\n\tif s.skipDoc {\n\t\treturn\n\t}\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.yml file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\n\texpression := s.expression\n\n\tif s.scenarioType == \"shell-separator\" {\n\t\twriteOrPanic(w, \"```bash\\nyq -o=shell --shell-key-separator=\\\"__\\\" sample.yml\\n```\\n\")\n\t} else if expression != \"\" {\n\t\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -o=shell '%v' sample.yml\\n```\\n\", expression))\n\t} else {\n\t\twriteOrPanic(w, \"```bash\\nyq -o=shell sample.yml\\n```\\n\")\n\t}\n\twriteOrPanic(w, \"will output\\n\")\n\n\tif s.scenarioType == \"shell-separator\" {\n\t\t// Save and restore the original separator\n\t\toriginalSeparator := ConfiguredShellVariablesPreferences.KeySeparator\n\t\tConfiguredShellVariablesPreferences.KeySeparator = \"__\"\n\t\twriteOrPanic(w, fmt.Sprintf(\"```sh\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder())))\n\t\tConfiguredShellVariablesPreferences.KeySeparator = originalSeparator\n\t} else {\n\t\twriteOrPanic(w, fmt.Sprintf(\"```sh\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder())))\n\t}\n}\n" }, { "path": "pkg/yqlib/stream_evaluator.go", "content": "package yqlib\n\nimport (\n\t\"container/list\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n)\n\n// A yaml expression evaluator that runs the expression multiple times for each given yaml document.\n// Uses less memory than loading all documents and running the expression once, but this cannot process\n// cross document expressions.\ntype StreamEvaluator interface {\n\tEvaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, decoder Decoder) (uint, error)\n\tEvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error\n\tEvaluateNew(expression string, printer Printer) error\n}\n\ntype streamEvaluator struct {\n\ttreeNavigator DataTreeNavigator\n\tfileIndex int\n}\n\nfunc NewStreamEvaluator() StreamEvaluator {\n\treturn &streamEvaluator{treeNavigator: NewDataTreeNavigator()}\n}\n\nfunc (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error {\n\tnode, err := ExpressionParser.ParseExpression(expression)\n\tif err != nil {\n\t\treturn err\n\t}\n\tcandidateNode := createScalarNode(nil, \"\")\n\tinputList := list.New()\n\tinputList.PushBack(candidateNode)\n\n\tresult, errorParsing := s.treeNavigator.GetMatchingNodes(Context{MatchingNodes: inputList}, node)\n\tif errorParsing != nil {\n\t\treturn errorParsing\n\t}\n\treturn printer.PrintResults(result.MatchingNodes)\n}\n\nfunc (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error {\n\tvar totalProcessDocs uint\n\tnode, err := ExpressionParser.ParseExpression(expression)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfor _, filename := range filenames {\n\t\treader, err := readStream(filename)\n\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tprocessedDocs, err := s.Evaluate(filename, reader, node, printer, decoder)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\ttotalProcessDocs = totalProcessDocs + processedDocs\n\n\t\tswitch reader := reader.(type) {\n\t\tcase *os.File:\n\t\t\tsafelyCloseFile(reader)\n\t\t}\n\t}\n\n\tif totalProcessDocs == 0 {\n\t\t// problem is I've already slurped the leading content sadface\n\t\treturn s.EvaluateNew(expression, printer)\n\t}\n\n\treturn nil\n}\n\nfunc (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, decoder Decoder) (uint, error) {\n\n\tvar currentIndex uint\n\terr := decoder.Init(reader)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tfor {\n\t\tcandidateNode, errorReading := decoder.Decode()\n\n\t\tif errors.Is(errorReading, io.EOF) {\n\t\t\ts.fileIndex = s.fileIndex + 1\n\t\t\treturn currentIndex, nil\n\t\t} else if errorReading != nil {\n\t\t\treturn currentIndex, fmt.Errorf(\"bad file '%v': %w\", filename, errorReading)\n\t\t}\n\t\tcandidateNode.document = currentIndex\n\t\tcandidateNode.filename = filename\n\t\tcandidateNode.fileIndex = s.fileIndex\n\n\t\tinputList := list.New()\n\t\tinputList.PushBack(candidateNode)\n\n\t\tresult, errorParsing := s.treeNavigator.GetMatchingNodes(Context{MatchingNodes: inputList}, node)\n\t\tif errorParsing != nil {\n\t\t\treturn currentIndex, errorParsing\n\t\t}\n\t\terr := printer.PrintResults(result.MatchingNodes)\n\n\t\tif err != nil {\n\t\t\treturn currentIndex, err\n\t\t}\n\t\tcurrentIndex = currentIndex + 1\n\t}\n}\n" }, { "path": "pkg/yqlib/string_evaluator.go", "content": "package yqlib\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"container/list\"\n\t\"strings\"\n)\n\ntype StringEvaluator interface {\n\tEvaluate(expression string, input string, encoder Encoder, decoder Decoder) (string, error)\n\tEvaluateAll(expression string, input string, encoder Encoder, decoder Decoder) (string, error)\n}\n\ntype stringEvaluator struct {\n\ttreeNavigator DataTreeNavigator\n}\n\nfunc NewStringEvaluator() StringEvaluator {\n\treturn &stringEvaluator{\n\t\ttreeNavigator: NewDataTreeNavigator(),\n\t}\n}\n\nfunc (s *stringEvaluator) EvaluateAll(expression string, input string, encoder Encoder, decoder Decoder) (string, error) {\n\treader := bufio.NewReader(strings.NewReader(input))\n\tvar documents *list.List\n\tvar results *list.List\n\tvar err error\n\n\tif documents, err = ReadDocuments(reader, decoder); err != nil {\n\t\treturn \"\", err\n\t}\n\n\tevaluator := NewAllAtOnceEvaluator()\n\tif results, err = evaluator.EvaluateCandidateNodes(expression, documents); err != nil {\n\t\treturn \"\", err\n\t}\n\n\tout := new(bytes.Buffer)\n\tprinter := NewPrinter(encoder, NewSinglePrinterWriter(out))\n\tif err := printer.PrintResults(results); err != nil {\n\t\treturn \"\", err\n\t}\n\treturn out.String(), nil\n}\n\nfunc (s *stringEvaluator) Evaluate(expression string, input string, encoder Encoder, decoder Decoder) (string, error) {\n\n\t// Use bytes.Buffer for output of string\n\tout := new(bytes.Buffer)\n\tprinter := NewPrinter(encoder, NewSinglePrinterWriter(out))\n\n\tInitExpressionParser()\n\tnode, err := ExpressionParser.ParseExpression(expression)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\n\treader := bufio.NewReader(strings.NewReader(input))\n\tevaluator := NewStreamEvaluator()\n\tif _, err := evaluator.Evaluate(\"\", reader, node, printer, decoder); err != nil {\n\t\treturn \"\", err\n\t}\n\treturn out.String(), nil\n}\n" }, { "path": "pkg/yqlib/string_evaluator_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nfunc TestStringEvaluator_MultipleDocumentMerge(t *testing.T) {\n\tyamlString := \"a: Hello\\n---\\na: Goodbye\\n\"\n\texpected_output := \"a: Goodbye\\n\"\n\n\tencoder := NewYamlEncoder(ConfiguredYamlPreferences)\n\tdecoder := NewYamlDecoder(ConfiguredYamlPreferences)\n\tresult, err := NewStringEvaluator().EvaluateAll(\"select(di==0) * select(di==1)\", yamlString, encoder, decoder)\n\tif err != nil {\n\t\tt.Error(err)\n\t} else {\n\t\ttest.AssertResult(t, expected_output, result)\n\t}\n}\n\nfunc TestStringEvaluator_Evaluate_Nominal(t *testing.T) {\n\texpected_output := `` +\n\t\t`yq` + \"\\n\" +\n\t\t`---` + \"\\n\" +\n\t\t`jq` + \"\\n\"\n\texpression := \".[].name\"\n\tinput := `` +\n\t\t` - name: yq` + \"\\n\" +\n\t\t` description: yq is a portable command-line YAML, JSON and XML processor` + \"\\n\" +\n\t\t`---` + \"\\n\" +\n\t\t` - name: jq` + \"\\n\" +\n\t\t` description: Command-line JSON processor` + \"\\n\"\n\tencoder := NewYamlEncoder(ConfiguredYamlPreferences)\n\tdecoder := NewYamlDecoder(ConfiguredYamlPreferences)\n\n\tresult, err := NewStringEvaluator().Evaluate(expression, input, encoder, decoder)\n\tif err != nil {\n\t\tt.Error(err)\n\t} else {\n\t\ttest.AssertResult(t, expected_output, result)\n\t}\n}\n" }, { "path": "pkg/yqlib/toml.go", "content": "package yqlib\n\ntype TomlPreferences struct {\n\tColorsEnabled bool\n}\n\nfunc NewDefaultTomlPreferences() TomlPreferences {\n\treturn TomlPreferences{ColorsEnabled: false}\n}\n\nfunc (p *TomlPreferences) Copy() TomlPreferences {\n\treturn TomlPreferences{ColorsEnabled: p.ColorsEnabled}\n}\n\nvar ConfiguredTomlPreferences = NewDefaultTomlPreferences()\n" }, { "path": "pkg/yqlib/toml_test.go", "content": "package yqlib\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"fmt\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/fatih/color\"\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nvar sampleTable = `\nvar = \"x\"\n\n[owner.contact]\nname = \"Tom Preston-Werner\"\nage = 36\n`\n\nvar tableArrayBeforeOwners = `\n[[owner.addresses]]\nstreet = \"first street\"\n\n[owner]\nname = \"Tom Preston-Werner\"\n`\n\nvar expectedTableArrayBeforeOwners = `owner:\n addresses:\n - street: first street\n name: Tom Preston-Werner\n`\n\nvar sampleTableExpected = `var: x\nowner:\n contact:\n name: Tom Preston-Werner\n age: 36\n`\n\nvar doubleArrayTable = `\n[[fruits]]\nname = \"apple\"\n[[fruits.varieties]] # nested array of tables\nname = \"red delicious\"`\n\nvar doubleArrayTableExpected = `fruits:\n - name: apple\n varieties:\n - name: red delicious\n`\n\nvar doubleArrayTableMultipleEntries = `\n[[fruits]]\nname = \"banana\"\n[[fruits]]\nname = \"apple\"\n[[fruits.varieties]] # nested array of tables\nname = \"red delicious\"`\n\nvar doubleArrayTableMultipleEntriesExpected = `fruits:\n - name: banana\n - name: apple\n varieties:\n - name: red delicious\n`\n\nvar doubleArrayTableNothingAbove = `\n[[fruits.varieties]] # nested array of tables\nname = \"red delicious\"`\n\nvar doubleArrayTableNothingAboveExpected = `fruits:\n varieties:\n - name: red delicious\n`\n\nvar doubleArrayTableEmptyAbove = `\n[[fruits]]\n[[fruits.varieties]] # nested array of tables\nname = \"red delicious\"`\n\nvar doubleArrayTableEmptyAboveExpected = `fruits:\n - varieties:\n - name: red delicious\n`\n\nvar emptyArrayTableThenTable = `\n[[fruits]]\n[animals]\n[[fruits.varieties]] # nested array of tables\nname = \"red delicious\"`\n\nvar emptyArrayTableThenTableExpected = `fruits:\n - varieties:\n - name: red delicious\nanimals: {}\n`\n\nvar arrayTableThenArray = `\n[[rootA.kidB]]\ncat = \"meow\"\n\n[rootA.kidB.kidC]\ndog = \"bark\"`\n\nvar arrayTableThenArrayExpected = `rootA:\n kidB:\n - cat: meow\n kidC:\n dog: bark\n`\n\nvar sampleArrayTable = `\n[owner.contact]\nname = \"Tom Preston-Werner\"\nage = 36\n\n[[owner.addresses]]\nstreet = \"first street\"\nsuburb = \"ok\"\n\n[[owner.addresses]]\nstreet = \"second street\"\nsuburb = \"nice\"\n`\n\nvar sampleArrayTableExpected = `owner:\n contact:\n name: Tom Preston-Werner\n age: 36\n addresses:\n - street: first street\n suburb: ok\n - street: second street\n suburb: nice\n`\n\nvar emptyTable = `\n[dependencies]\n`\n\nvar emptyTableExpected = \"dependencies: {}\\n\"\n\nvar multipleEmptyTables = `\n[firstEmptyTable]\n[firstTableWithContent]\nkey = \"value\"\n[secondEmptyTable]\n[thirdEmptyTable]\n[secondTableWithContent]\nkey = \"value\"\n[fourthEmptyTable]\n[fifthEmptyTable]\n`\n\nvar expectedMultipleEmptyTables = `firstEmptyTable: {}\nfirstTableWithContent:\n key: value\nsecondEmptyTable: {}\nthirdEmptyTable: {}\nsecondTableWithContent:\n key: value\nfourthEmptyTable: {}\nfifthEmptyTable: {}\n`\n\nvar sampleWithHeader = `\n[servers]\n\n[servers.alpha]\nip = \"10.0.0.1\"\n`\n\nvar expectedSampleWithHeader = `servers:\n alpha:\n ip: 10.0.0.1\n`\n\n// Roundtrip fixtures\nvar rtInlineTableAttr = `name = { first = \"Tom\", last = \"Preston-Werner\" }\n`\n\nvar rtTableSection = `[owner.contact]\nname = \"Tom\"\nage = 36\n`\n\nvar rtArrayOfTables = `[[fruits]]\nname = \"apple\"\n[[fruits.varieties]]\nname = \"red delicious\"\n`\n\nvar rtArraysAndScalars = `A = [\"hello\", [\"world\", \"again\"]]\nB = 12\n`\n\nvar rtSimple = `A = \"hello\"\nB = 12\n`\n\nvar rtDeepPaths = `[person]\nname = \"hello\"\naddress = \"12 cat st\"\n`\n\nvar rtEmptyArray = `A = []\n`\n\nvar rtSampleTable = `var = \"x\"\n\n[owner.contact]\nname = \"Tom Preston-Werner\"\nage = 36\n`\n\nvar rtEmptyTable = `[dependencies]\n`\n\nvar rtComments = `# This is a comment\nA = \"hello\" # inline comment\nB = 12\n\n# Table comment\n[person]\nname = \"Tom\" # name comment\n`\n\n// Reproduce bug for https://github.com/mikefarah/yq/issues/2588\n// Bug: standalone comments inside a table cause subsequent key-values to be assigned at root.\nvar issue2588RustToolchainWithComments = `[owner]\n# comment\nname = \"Tomer\"\n`\n\nvar tableWithComment = `[owner]\n# comment\n[things]\n`\n\nvar sampleFromWeb = `# This is a TOML document\ntitle = \"TOML Example\"\n\n[owner]\nname = \"Tom Preston-Werner\"\ndob = 1979-05-27T07:32:00-08:00\n\n[database]\nenabled = true\nports = [8000, 8001, 8002]\ndata = [[\"delta\", \"phi\"], [3.14]]\ntemp_targets = { cpu = 79.5, case = 72.0 }\n\n# [servers] yq can't do this one yet\n[servers.alpha]\nip = \"10.0.0.1\"\nrole = \"frontend\"\n\n[servers.beta]\nip = \"10.0.0.2\"\nrole = \"backend\"\n`\n\nvar subArrays = `\n[[array]]\n\n[[array.subarray]]\n\n[[array.subarray.subsubarray]]\n`\n\nvar tomlTableWithComments = `[section]\nthe_array = [\n # comment\n \"value 1\",\n\n # comment\n \"value 2\",\n]\n`\n\nvar expectedSubArrays = `array:\n - subarray:\n - subsubarray:\n - {}\n`\n\nvar tomlScenarios = []formatScenario{\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"blank\",\n\t\tinput: \"\",\n\t\texpected: \"\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"table array before owners\",\n\t\tinput: tableArrayBeforeOwners,\n\t\texpected: expectedTableArrayBeforeOwners,\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"datetime\",\n\t\tinput: \"A = 1979-05-27T07:32:00-08:00\",\n\t\texpected: \"A: 1979-05-27T07:32:00-08:00\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"blank\",\n\t\tinput: `A = \"hello`,\n\t\texpectedError: `bad file 'sample.yml': basic string not terminated by \"`,\n\t\tscenarioType: \"decode-error\",\n\t},\n\t{\n\t\tdescription: \"Parse: Simple\",\n\t\tinput: \"A = \\\"hello\\\"\\nB = 12\\n\",\n\t\texpected: \"A: hello\\nB: 12\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Parse: Deep paths\",\n\t\tinput: \"person.name = \\\"hello\\\"\\nperson.address = \\\"12 cat st\\\"\\n\",\n\t\texpected: \"person:\\n name: hello\\n address: 12 cat st\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Parse: include key information\",\n\t\tinput: \"person.name = \\\"hello\\\"\\nperson.address = \\\"12 cat st\\\"\\n\",\n\t\texpression: \".person.name | key\",\n\t\texpected: \"name\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Parse: include parent information\",\n\t\tinput: \"person.name = \\\"hello\\\"\\nperson.address = \\\"12 cat st\\\"\\n\",\n\t\texpression: \".person.name | parent\",\n\t\texpected: \"name: hello\\naddress: 12 cat st\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Parse: include path information\",\n\t\tinput: \"person.name = \\\"hello\\\"\\nperson.address = \\\"12 cat st\\\"\\n\",\n\t\texpression: \".person.name | path\",\n\t\texpected: \"- person\\n- name\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Encode: Scalar\",\n\t\tinput: \"person.name = \\\"hello\\\"\\nperson.address = \\\"12 cat st\\\"\\n\",\n\t\texpression: \".person.name\",\n\t\texpected: \"hello\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tinput: `A.B = \"hello\"`,\n\t\texpected: \"A:\\n B: hello\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"bool\",\n\t\tinput: `A = true`,\n\t\texpected: \"A: true\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"bool false\",\n\t\tinput: `A = false `,\n\t\texpected: \"A: false\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"number\",\n\t\tinput: `A = 3 `,\n\t\texpected: \"A: 3\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"number\",\n\t\tinput: `A = 0xDEADBEEF`,\n\t\texpression: \" .A += 1\",\n\t\texpected: \"A: 0xDEADBEF0\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"float\",\n\t\tinput: `A = 6.626e-34`,\n\t\texpected: \"A: 6.626e-34\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"empty arraY\",\n\t\tinput: `A = []`,\n\t\texpected: \"A: []\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"array\",\n\t\tinput: `A = [\"hello\", [\"world\", \"again\"]]`,\n\t\texpected: \"A:\\n - hello\\n - - world\\n - again\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Parse: inline table\",\n\t\tinput: `name = { first = \"Tom\", last = \"Preston-Werner\" }`,\n\t\texpected: \"name:\\n first: Tom\\n last: Preston-Werner\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tinput: sampleTable,\n\t\texpected: sampleTableExpected,\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Parse: Array Table\",\n\t\tinput: sampleArrayTable,\n\t\texpected: sampleArrayTableExpected,\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Parse: Array of Array Table\",\n\t\tinput: doubleArrayTable,\n\t\texpected: doubleArrayTableExpected,\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Parse: Array of Array Table; nothing above\",\n\t\tinput: doubleArrayTableNothingAbove,\n\t\texpected: doubleArrayTableNothingAboveExpected,\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Parse: Array of Array Table; empty above\",\n\t\tinput: doubleArrayTableEmptyAbove,\n\t\texpected: doubleArrayTableEmptyAboveExpected,\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Parse: Array of Array Table; multiple entries\",\n\t\tinput: doubleArrayTableMultipleEntries,\n\t\texpected: doubleArrayTableMultipleEntriesExpected,\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Parse: Array of Array Table; then table; then array table\",\n\t\tinput: emptyArrayTableThenTable,\n\t\texpected: emptyArrayTableThenTableExpected,\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Parse: Array of Array Table; then table\",\n\t\tinput: arrayTableThenArray,\n\t\texpected: arrayTableThenArrayExpected,\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Parse: Empty Table\",\n\t\tinput: emptyTable,\n\t\texpected: emptyTableExpected,\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Parse: with header\",\n\t\tskipDoc: true,\n\t\tinput: sampleWithHeader,\n\t\texpected: expectedSampleWithHeader,\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Parse: multiple empty tables\",\n\t\tskipDoc: true,\n\t\tinput: multipleEmptyTables,\n\t\texpected: expectedMultipleEmptyTables,\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"subArrays\",\n\t\tskipDoc: true,\n\t\tinput: subArrays,\n\t\texpected: expectedSubArrays,\n\t\tscenarioType: \"decode\",\n\t},\n\t// Roundtrip scenarios\n\t{\n\t\tdescription: \"Roundtrip: inline table attribute\",\n\t\tinput: rtInlineTableAttr,\n\t\texpression: \".\",\n\t\texpected: rtInlineTableAttr,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip: table section\",\n\t\tinput: rtTableSection,\n\t\texpression: \".\",\n\t\texpected: rtTableSection,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip: array of tables\",\n\t\tinput: rtArrayOfTables,\n\t\texpression: \".\",\n\t\texpected: rtArrayOfTables,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip: arrays and scalars\",\n\t\tinput: rtArraysAndScalars,\n\t\texpression: \".\",\n\t\texpected: rtArraysAndScalars,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip: simple\",\n\t\tinput: rtSimple,\n\t\texpression: \".\",\n\t\texpected: rtSimple,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip: deep paths\",\n\t\tinput: rtDeepPaths,\n\t\texpression: \".\",\n\t\texpected: rtDeepPaths,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip: empty array\",\n\t\tinput: rtEmptyArray,\n\t\texpression: \".\",\n\t\texpected: rtEmptyArray,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip: sample table\",\n\t\tinput: rtSampleTable,\n\t\texpression: \".\",\n\t\texpected: rtSampleTable,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip: empty table\",\n\t\tinput: rtEmptyTable,\n\t\texpression: \".\",\n\t\texpected: rtEmptyTable,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip: comments\",\n\t\tinput: rtComments,\n\t\texpression: \".\",\n\t\texpected: rtComments,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Issue #2588: comments inside table must not flatten (.owner.name)\",\n\t\tinput: issue2588RustToolchainWithComments,\n\t\texpression: \".owner.name\",\n\t\texpected: \"Tomer\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"Issue #2588: comments inside table must not flatten (.name)\",\n\t\tinput: issue2588RustToolchainWithComments,\n\t\texpression: \".name\",\n\t\texpected: \"null\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tinput: issue2588RustToolchainWithComments,\n\t\texpected: issue2588RustToolchainWithComments,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tinput: tableWithComment,\n\t\texpression: \".owner | headComment\",\n\t\texpected: \"comment\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip: sample from web\",\n\t\tinput: sampleFromWeb,\n\t\texpression: \".\",\n\t\texpected: sampleFromWeb,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tinput: tomlTableWithComments,\n\t\texpected: tomlTableWithComments,\n\t\tscenarioType: \"roundtrip\",\n\t},\n}\n\nfunc testTomlScenario(t *testing.T, s formatScenario) {\n\tswitch s.scenarioType {\n\tcase \"\", \"decode\":\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewTomlDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)\n\tcase \"decode-error\":\n\t\tresult, err := processFormatScenario(s, NewTomlDecoder(), NewYamlEncoder(ConfiguredYamlPreferences))\n\t\tif err == nil {\n\t\t\tt.Errorf(\"Expected error '%v' but it worked: %v\", s.expectedError, result)\n\t\t} else {\n\t\t\ttest.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description)\n\t\t}\n\tcase \"roundtrip\":\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewTomlDecoder(), NewTomlEncoder()), s.description)\n\t}\n}\n\nfunc documentTomlDecodeScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.toml file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```toml\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\texpression := s.expression\n\tif expression == \"\" {\n\t\texpression = \".\"\n\t}\n\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -oy '%v' sample.toml\\n```\\n\", expression))\n\twriteOrPanic(w, \"will output\\n\")\n\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewTomlDecoder(), NewYamlEncoder(ConfiguredYamlPreferences))))\n}\n\nfunc documentTomlRoundtripScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.toml file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```toml\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\texpression := s.expression\n\tif expression == \"\" {\n\t\texpression = \".\"\n\t}\n\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq '%v' sample.toml\\n```\\n\", expression))\n\twriteOrPanic(w, \"will output\\n\")\n\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewTomlDecoder(), NewTomlEncoder())))\n}\n\nfunc documentTomlScenario(_ *testing.T, w *bufio.Writer, i interface{}) {\n\ts := i.(formatScenario)\n\n\tif s.skipDoc {\n\t\treturn\n\t}\n\tswitch s.scenarioType {\n\tcase \"\", \"decode\":\n\t\tdocumentTomlDecodeScenario(w, s)\n\tcase \"roundtrip\":\n\t\tdocumentTomlRoundtripScenario(w, s)\n\n\tdefault:\n\t\tpanic(fmt.Sprintf(\"unhandled scenario type %q\", s.scenarioType))\n\t}\n}\n\nfunc TestTomlScenarios(t *testing.T) {\n\tfor _, tt := range tomlScenarios {\n\t\ttestTomlScenario(t, tt)\n\t}\n\tgenericScenarios := make([]interface{}, len(tomlScenarios))\n\tfor i, s := range tomlScenarios {\n\t\tgenericScenarios[i] = s\n\t}\n\tdocumentScenarios(t, \"usage\", \"toml\", genericScenarios, documentTomlScenario)\n}\n\n// TestTomlColourization tests that colourization correctly distinguishes\n// between table section headers and inline arrays\nfunc TestTomlColourization(t *testing.T) {\n\t// Save and restore color state\n\toldNoColor := color.NoColor\n\tcolor.NoColor = false\n\tdefer func() { color.NoColor = oldNoColor }()\n\n\t// Test that inline arrays are not coloured as table sections\n\tencoder := &tomlEncoder{prefs: TomlPreferences{ColorsEnabled: true}}\n\n\t// Create TOML with both table sections and inline arrays\n\tinput := []byte(`[database]\nenabled = true\nports = [8000, 8001, 8002]\n\n[servers]\nalpha = \"test\"\n`)\n\n\tresult := encoder.colorizeToml(input)\n\tresultStr := string(result)\n\n\t// The bug would cause the inline array [8000, 8001, 8002] to be\n\t// coloured with the section colour (Yellow + Bold) instead of being\n\t// left uncoloured or coloured differently.\n\t//\n\t// To test this, we check that the section colour codes appear only\n\t// for actual table sections, not for inline arrays.\n\n\t// Get the ANSI codes for section colour (Yellow + Bold)\n\tsectionColourObj := color.New(color.FgYellow, color.Bold)\n\tsectionColourObj.EnableColor()\n\tsampleSection := sectionColourObj.Sprint(\"[database]\")\n\n\t// Extract just the ANSI codes from the sample\n\t// ANSI codes start with \\x1b[\n\tvar ansiStart string\n\tfor i := 0; i < len(sampleSection); i++ {\n\t\tif sampleSection[i] == '\\x1b' {\n\t\t\t// Find the end of the ANSI sequence (ends with 'm')\n\t\t\tend := i\n\t\t\tfor end < len(sampleSection) && sampleSection[end] != 'm' {\n\t\t\t\tend++\n\t\t\t}\n\t\t\tif end < len(sampleSection) {\n\t\t\t\tansiStart = sampleSection[i : end+1]\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\n\t// Count how many times the section colour appears in the output\n\t// It should appear exactly twice: once for [database] and once for [servers]\n\t// If it appears more times (e.g., for [8000, 8001, 8002]), that's the bug\n\tsectionColourCount := strings.Count(resultStr, ansiStart)\n\n\t// We expect exactly 2 occurrences (for [database] and [servers])\n\t// The bug would cause more occurrences (e.g., also for [8000)\n\tif sectionColourCount != 2 {\n\t\tt.Errorf(\"Expected section colour to appear exactly 2 times (for [database] and [servers]), but it appeared %d times.\\nOutput: %s\", sectionColourCount, resultStr)\n\t}\n}\n\nfunc TestTomlColorisationNumberBug(t *testing.T) {\n\t// Save and restore color state\n\toldNoColor := color.NoColor\n\tcolor.NoColor = false\n\tdefer func() { color.NoColor = oldNoColor }()\n\n\tencoder := NewTomlEncoder()\n\ttomlEncoder := encoder.(*tomlEncoder)\n\n\t// Test case that exposes the bug: \"123-+-45\" should NOT be colourised as a single number\n\tinput := \"A = 123-+-45\\n\"\n\tresult := string(tomlEncoder.colorizeToml([]byte(input)))\n\n\t// The bug causes \"123-+-45\" to be colourised as one token\n\t// It should stop at \"123\" because the next character '-' is not valid in this position\n\tif strings.Contains(result, \"123-+-45\") {\n\t\t// Check if it's colourised as a single token (no color codes in the middle)\n\t\tidx := strings.Index(result, \"123-+-45\")\n\t\t// Look backwards for color code\n\t\tbeforeIdx := idx - 1\n\t\tfor beforeIdx >= 0 && result[beforeIdx] != '\\x1b' {\n\t\t\tbeforeIdx--\n\t\t}\n\t\t// Look forward for reset code\n\t\tafterIdx := idx + 8 // length of \"123-+-45\"\n\t\thasResetAfter := false\n\t\tfor afterIdx < len(result) && afterIdx < idx+20 {\n\t\t\tif result[afterIdx] == '\\x1b' {\n\t\t\t\thasResetAfter = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tafterIdx++\n\t\t}\n\n\t\tif beforeIdx >= 0 && hasResetAfter {\n\t\t\t// The entire \"123-+-45\" is wrapped in color codes - this is the bug!\n\t\t\tt.Errorf(\"BUG DETECTED: '123-+-45' is incorrectly colourised as a single number\")\n\t\t\tt.Errorf(\"Expected only '123' to be colourised as a number, but got the entire '123-+-45'\")\n\t\t\tt.Logf(\"Full output: %q\", result)\n\t\t\tt.Fail()\n\t\t}\n\t}\n\n\t// Additional test cases for the bug\n\tbugTests := []struct {\n\t\tname string\n\t\tinput string\n\t\tinvalidSequence string\n\t\tdescription string\n\t}{\n\t\t{\n\t\t\tname: \"consecutive minuses\",\n\t\t\tinput: \"A = 123--45\\n\",\n\t\t\tinvalidSequence: \"123--45\",\n\t\t\tdescription: \"'123--45' should not be colourised as a single number\",\n\t\t},\n\t\t{\n\t\t\tname: \"plus in middle\",\n\t\t\tinput: \"A = 123+45\\n\",\n\t\t\tinvalidSequence: \"123+45\",\n\t\t\tdescription: \"'123+45' should not be colourised as a single number\",\n\t\t},\n\t}\n\n\tfor _, tt := range bugTests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tresult := string(tomlEncoder.colorizeToml([]byte(tt.input)))\n\t\t\tif strings.Contains(result, tt.invalidSequence) {\n\t\t\t\tidx := strings.Index(result, tt.invalidSequence)\n\t\t\t\tbeforeIdx := idx - 1\n\t\t\t\tfor beforeIdx >= 0 && result[beforeIdx] != '\\x1b' {\n\t\t\t\t\tbeforeIdx--\n\t\t\t\t}\n\t\t\t\tafterIdx := idx + len(tt.invalidSequence)\n\t\t\t\thasResetAfter := false\n\t\t\t\tfor afterIdx < len(result) && afterIdx < idx+20 {\n\t\t\t\t\tif result[afterIdx] == '\\x1b' {\n\t\t\t\t\t\thasResetAfter = true\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t\tafterIdx++\n\t\t\t\t}\n\n\t\t\t\tif beforeIdx >= 0 && hasResetAfter {\n\t\t\t\t\tt.Errorf(\"BUG: %s\", tt.description)\n\t\t\t\t\tt.Logf(\"Full output: %q\", result)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n\n\t// Test that valid scientific notation still works\n\tvalidTests := []struct {\n\t\tname string\n\t\tinput string\n\t}{\n\t\t{\"scientific positive\", \"A = 1.23e+45\\n\"},\n\t\t{\"scientific negative\", \"A = 6.626e-34\\n\"},\n\t\t{\"scientific uppercase\", \"A = 1.23E+10\\n\"},\n\t}\n\n\tfor _, tt := range validTests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tresult := tomlEncoder.colorizeToml([]byte(tt.input))\n\t\t\tif len(result) == 0 {\n\t\t\t\tt.Error(\"Expected non-empty colourised output\")\n\t\t\t}\n\t\t})\n\t}\n}\n\n// Tests that the encoder handles empty path slices gracefully\nfunc TestTomlEmptyPathPanic(t *testing.T) {\n\tencoder := NewTomlEncoder()\n\ttomlEncoder := encoder.(*tomlEncoder)\n\n\tvar buf bytes.Buffer\n\n\t// Create a simple scalar node\n\tscalarNode := &CandidateNode{\n\t\tKind: ScalarNode,\n\t\tTag: \"!!str\",\n\t\tValue: \"test\",\n\t}\n\n\t// Test with empty path - this should not panic\n\terr := tomlEncoder.encodeTopLevelEntry(&buf, []string{}, scalarNode)\n\tif err == nil {\n\t\tt.Error(\"Expected error when encoding with empty path, got nil\")\n\t}\n\n}\n\n// TestTomlStringEscapeColourization tests that string colourization correctly\n// handles escape sequences, particularly escaped quotes at the end of strings\nfunc TestTomlStringEscapeColourization(t *testing.T) {\n\t// Save and restore color state\n\toldNoColor := color.NoColor\n\tcolor.NoColor = false\n\tdefer func() { color.NoColor = oldNoColor }()\n\n\tencoder := NewTomlEncoder()\n\ttomlEncoder := encoder.(*tomlEncoder)\n\n\ttestCases := []struct {\n\t\tname string\n\t\tinput string\n\t\tdescription string\n\t}{\n\t\t{\n\t\t\tname: \"escaped quote at end\",\n\t\t\tinput: `A = \"test\\\"\"` + \"\\n\",\n\t\t\tdescription: \"String ending with escaped quote should be colourised correctly\",\n\t\t},\n\t\t{\n\t\t\tname: \"escaped backslash then quote\",\n\t\t\tinput: `A = \"test\\\\\\\"\"` + \"\\n\",\n\t\t\tdescription: \"String with escaped backslash followed by escaped quote\",\n\t\t},\n\t\t{\n\t\t\tname: \"escaped quote in middle\",\n\t\t\tinput: `A = \"test\\\"middle\"` + \"\\n\",\n\t\t\tdescription: \"String with escaped quote in the middle should be colourised correctly\",\n\t\t},\n\t\t{\n\t\t\tname: \"multiple escaped quotes\",\n\t\t\tinput: `A = \"\\\"test\\\"\"` + \"\\n\",\n\t\t\tdescription: \"String with escaped quotes at start and end\",\n\t\t},\n\t\t{\n\t\t\tname: \"escaped newline\",\n\t\t\tinput: `A = \"test\\n\"` + \"\\n\",\n\t\t\tdescription: \"String with escaped newline should be colourised correctly\",\n\t\t},\n\t\t{\n\t\t\tname: \"single quote with escaped single quote\",\n\t\t\tinput: `A = 'test\\''` + \"\\n\",\n\t\t\tdescription: \"Single-quoted string with escaped single quote\",\n\t\t},\n\t}\n\n\tfor _, tt := range testCases {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\t// The test should not panic and should return some output\n\t\t\tresult := tomlEncoder.colorizeToml([]byte(tt.input))\n\t\t\tif len(result) == 0 {\n\t\t\t\tt.Error(\"Expected non-empty colourised output\")\n\t\t\t}\n\n\t\t\t// Check that the result contains the input string (with color codes)\n\t\t\t// At minimum, it should contain \"A\" and \"=\"\n\t\t\tresultStr := string(result)\n\t\t\tif !strings.Contains(resultStr, \"A\") || !strings.Contains(resultStr, \"=\") {\n\t\t\t\tt.Errorf(\"Expected output to contain 'A' and '=', got: %q\", resultStr)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestTomlEncoderPrintDocumentSeparator(t *testing.T) {\n\tencoder := NewTomlEncoder()\n\tvar buf bytes.Buffer\n\twriter := bufio.NewWriter(&buf)\n\n\terr := encoder.PrintDocumentSeparator(writer)\n\twriter.Flush()\n\n\ttest.AssertResult(t, nil, err)\n\ttest.AssertResult(t, \"\", buf.String())\n}\n\nfunc TestTomlEncoderPrintLeadingContent(t *testing.T) {\n\tencoder := NewTomlEncoder()\n\tvar buf bytes.Buffer\n\twriter := bufio.NewWriter(&buf)\n\n\terr := encoder.PrintLeadingContent(writer, \"some content\")\n\twriter.Flush()\n\n\ttest.AssertResult(t, nil, err)\n\ttest.AssertResult(t, \"\", buf.String())\n}\n\nfunc TestTomlEncoderCanHandleAliases(t *testing.T) {\n\tencoder := NewTomlEncoder()\n\ttest.AssertResult(t, false, encoder.CanHandleAliases())\n}\n" }, { "path": "pkg/yqlib/utils.go", "content": "package yqlib\n\nimport (\n\t\"bufio\"\n\t\"container/list\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n)\n\nfunc readStream(filename string) (io.Reader, error) {\n\tvar reader *bufio.Reader\n\tif filename == \"-\" {\n\t\treader = bufio.NewReader(os.Stdin)\n\t} else {\n\t\t// ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory,\n\t\t// and ensuring that it's not possible to give a path to a file outside that directory.\n\t\tfile, err := os.Open(filename) // #nosec\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treader = bufio.NewReader(file)\n\t}\n\treturn reader, nil\n\n}\n\nfunc writeString(writer io.Writer, txt string) error {\n\t_, errorWriting := writer.Write([]byte(txt))\n\treturn errorWriting\n}\n\nfunc ReadDocuments(reader io.Reader, decoder Decoder) (*list.List, error) {\n\treturn readDocuments(reader, \"\", 0, decoder)\n}\n\nfunc readDocuments(reader io.Reader, filename string, fileIndex int, decoder Decoder) (*list.List, error) {\n\terr := decoder.Init(reader)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tinputList := list.New()\n\tvar currentIndex uint\n\n\tfor {\n\t\tcandidateNode, errorReading := decoder.Decode()\n\n\t\tif errors.Is(errorReading, io.EOF) {\n\t\t\tswitch reader := reader.(type) {\n\t\t\tcase *os.File:\n\t\t\t\tsafelyCloseFile(reader)\n\t\t\t}\n\t\t\treturn inputList, nil\n\t\t} else if errorReading != nil {\n\t\t\treturn nil, fmt.Errorf(\"bad file '%v': %w\", filename, errorReading)\n\t\t}\n\t\tcandidateNode.document = currentIndex\n\t\tcandidateNode.filename = filename\n\t\tcandidateNode.fileIndex = fileIndex\n\t\tcandidateNode.EvaluateTogether = true\n\n\t\tinputList.PushBack(candidateNode)\n\n\t\tcurrentIndex = currentIndex + 1\n\t}\n}\n" }, { "path": "pkg/yqlib/write_in_place_handler.go", "content": "package yqlib\n\nimport (\n\t\"os\"\n)\n\ntype writeInPlaceHandler interface {\n\tCreateTempFile() (*os.File, error)\n\tFinishWriteInPlace(evaluatedSuccessfully bool) error\n}\n\ntype writeInPlaceHandlerImpl struct {\n\tinputFilename string\n\ttempFile *os.File\n}\n\nfunc NewWriteInPlaceHandler(inputFile string) writeInPlaceHandler {\n\n\treturn &writeInPlaceHandlerImpl{inputFile, nil}\n}\n\nfunc (w *writeInPlaceHandlerImpl) CreateTempFile() (*os.File, error) {\n\tfile, err := createTempFile()\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tinfo, err := os.Stat(w.inputFilename)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\terr = os.Chmod(file.Name(), info.Mode())\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif err = changeOwner(info, file); err != nil {\n\t\treturn nil, err\n\t}\n\tlog.Debug(\"WriteInPlaceHandler: writing to tempfile: %v\", file.Name())\n\tw.tempFile = file\n\treturn file, err\n}\n\nfunc (w *writeInPlaceHandlerImpl) FinishWriteInPlace(evaluatedSuccessfully bool) error {\n\tlog.Debug(\"Going to write in place, evaluatedSuccessfully=%v, target=%v\", evaluatedSuccessfully, w.inputFilename)\n\tsafelyCloseFile(w.tempFile)\n\tif evaluatedSuccessfully {\n\t\tlog.Debug(\"Moving temp file to target\")\n\t\treturn tryRenameFile(w.tempFile.Name(), w.inputFilename)\n\t}\n\ttryRemoveTempFile(w.tempFile.Name())\n\n\treturn nil\n}\n" }, { "path": "pkg/yqlib/write_in_place_handler_test.go", "content": "package yqlib\n\nimport (\n\t\"os\"\n\t\"path/filepath\"\n\t\"testing\"\n)\n\nfunc TestWriteInPlaceHandlerImpl_CreateTempFile(t *testing.T) {\n\t// Create a temporary directory and file for testing\n\ttempDir := t.TempDir()\n\tinputFile := filepath.Join(tempDir, \"input.yaml\")\n\n\t// Create input file with some content\n\tcontent := []byte(\"test: value\\n\")\n\terr := os.WriteFile(inputFile, content, 0600)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create input file: %v\", err)\n\t}\n\n\thandler := NewWriteInPlaceHandler(inputFile)\n\ttempFile, err := handler.CreateTempFile()\n\n\tif err != nil {\n\t\tt.Fatalf(\"CreateTempFile failed: %v\", err)\n\t}\n\n\tif tempFile == nil {\n\t\tt.Fatal(\"CreateTempFile returned nil file\")\n\t}\n\n\t// Clean up\n\ttempFile.Close()\n\tos.Remove(tempFile.Name())\n}\n\nfunc TestWriteInPlaceHandlerImpl_CreateTempFile_NonExistentInput(t *testing.T) {\n\t// Test with non-existent input file\n\thandler := NewWriteInPlaceHandler(\"/non/existent/file.yaml\")\n\ttempFile, err := handler.CreateTempFile()\n\n\tif err == nil {\n\t\tt.Error(\"Expected error for non-existent input file, got nil\")\n\t}\n\n\tif tempFile != nil {\n\t\tt.Error(\"Expected nil temp file for non-existent input file\")\n\t\ttempFile.Close()\n\t}\n}\n\nfunc TestWriteInPlaceHandlerImpl_FinishWriteInPlace_Success(t *testing.T) {\n\t// Create a temporary directory and file for testing\n\ttempDir := t.TempDir()\n\tinputFile := filepath.Join(tempDir, \"input.yaml\")\n\n\t// Create input file with some content\n\tcontent := []byte(\"test: value\\n\")\n\terr := os.WriteFile(inputFile, content, 0600)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create input file: %v\", err)\n\t}\n\n\thandler := NewWriteInPlaceHandler(inputFile)\n\ttempFile, err := handler.CreateTempFile()\n\tif err != nil {\n\t\tt.Fatalf(\"CreateTempFile failed: %v\", err)\n\t}\n\tdefer tempFile.Close()\n\n\t// Write some content to temp file\n\ttempContent := []byte(\"updated: content\\n\")\n\t_, err = tempFile.Write(tempContent)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to write to temp file: %v\", err)\n\t}\n\ttempFile.Close()\n\n\t// Test successful finish\n\terr = handler.FinishWriteInPlace(true)\n\tif err != nil {\n\t\tt.Fatalf(\"FinishWriteInPlace failed: %v\", err)\n\t}\n\n\t// Verify the original file was updated\n\tupdatedContent, err := os.ReadFile(inputFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to read updated file: %v\", err)\n\t}\n\n\tif string(updatedContent) != string(tempContent) {\n\t\tt.Errorf(\"File content not updated correctly. Expected %q, got %q\",\n\t\t\tstring(tempContent), string(updatedContent))\n\t}\n}\n\nfunc TestWriteInPlaceHandlerImpl_FinishWriteInPlace_Failure(t *testing.T) {\n\t// Create a temporary directory and file for testing\n\ttempDir := t.TempDir()\n\tinputFile := filepath.Join(tempDir, \"input.yaml\")\n\n\t// Create input file with some content\n\tcontent := []byte(\"test: value\\n\")\n\terr := os.WriteFile(inputFile, content, 0600)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create input file: %v\", err)\n\t}\n\n\thandler := NewWriteInPlaceHandler(inputFile)\n\ttempFile, err := handler.CreateTempFile()\n\tif err != nil {\n\t\tt.Fatalf(\"CreateTempFile failed: %v\", err)\n\t}\n\tdefer tempFile.Close()\n\n\t// Write some content to temp file\n\ttempContent := []byte(\"updated: content\\n\")\n\t_, err = tempFile.Write(tempContent)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to write to temp file: %v\", err)\n\t}\n\ttempFile.Close()\n\n\t// Test failure finish (should not update the original file)\n\terr = handler.FinishWriteInPlace(false)\n\tif err != nil {\n\t\tt.Fatalf(\"FinishWriteInPlace failed: %v\", err)\n\t}\n\n\t// Verify the original file was NOT updated\n\toriginalContent, err := os.ReadFile(inputFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to read original file: %v\", err)\n\t}\n\n\tif string(originalContent) != string(content) {\n\t\tt.Errorf(\"File content should not have been updated. Expected %q, got %q\",\n\t\t\tstring(content), string(originalContent))\n\t}\n}\n\nfunc TestWriteInPlaceHandlerImpl_FinishWriteInPlace_Symlink_Success(t *testing.T) {\n\t// Create a temporary directory and file for testing\n\ttempDir := t.TempDir()\n\tinputFile := filepath.Join(tempDir, \"input.yaml\")\n\tsymlinkFile := filepath.Join(tempDir, \"symlink.yaml\")\n\n\t// Create input file with some content\n\tcontent := []byte(\"test: value\\n\")\n\terr := os.WriteFile(inputFile, content, 0600)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create input file: %v\", err)\n\t}\n\n\terr = os.Symlink(inputFile, symlinkFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to symlink to input file: %v\", err)\n\t}\n\n\thandler := NewWriteInPlaceHandler(symlinkFile)\n\ttempFile, err := handler.CreateTempFile()\n\tif err != nil {\n\t\tt.Fatalf(\"CreateTempFile failed: %v\", err)\n\t}\n\tdefer tempFile.Close()\n\n\t// Write some content to temp file\n\ttempContent := []byte(\"updated: content\\n\")\n\t_, err = tempFile.Write(tempContent)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to write to temp file: %v\", err)\n\t}\n\ttempFile.Close()\n\n\t// Test successful finish\n\terr = handler.FinishWriteInPlace(true)\n\tif err != nil {\n\t\tt.Fatalf(\"FinishWriteInPlace failed: %v\", err)\n\t}\n\n\t// Verify that the symlink is still present\n\tinfo, err := os.Lstat(symlinkFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to lstat input file: %v\", err)\n\t}\n\tif info.Mode()&os.ModeSymlink == 0 {\n\t\tt.Errorf(\"Input file symlink is no longer present\")\n\t}\n\n\t// Verify the original file was updated\n\tupdatedContent, err := os.ReadFile(inputFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to read updated file: %v\", err)\n\t}\n\n\tif string(updatedContent) != string(tempContent) {\n\t\tt.Errorf(\"File content not updated correctly. Expected %q, got %q\",\n\t\t\tstring(tempContent), string(updatedContent))\n\t}\n}\n\nfunc TestWriteInPlaceHandlerImpl_CreateTempFile_Permissions(t *testing.T) {\n\t// Create a temporary directory and file for testing\n\ttempDir := t.TempDir()\n\tinputFile := filepath.Join(tempDir, \"input.yaml\")\n\n\t// Create input file with specific permissions\n\tcontent := []byte(\"test: value\\n\")\n\terr := os.WriteFile(inputFile, content, 0600)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create input file: %v\", err)\n\t}\n\n\thandler := NewWriteInPlaceHandler(inputFile)\n\ttempFile, err := handler.CreateTempFile()\n\tif err != nil {\n\t\tt.Fatalf(\"CreateTempFile failed: %v\", err)\n\t}\n\tdefer tempFile.Close()\n\n\t// Check that temp file has same permissions as input file\n\ttempFileInfo, err := os.Stat(tempFile.Name())\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to stat temp file: %v\", err)\n\t}\n\n\tinputFileInfo, err := os.Stat(inputFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to stat input file: %v\", err)\n\t}\n\n\tif tempFileInfo.Mode() != inputFileInfo.Mode() {\n\t\tt.Errorf(\"Temp file permissions don't match input file. Expected %v, got %v\",\n\t\t\tinputFileInfo.Mode(), tempFileInfo.Mode())\n\t}\n}\n\nfunc TestWriteInPlaceHandlerImpl_Integration(t *testing.T) {\n\t// Create a temporary directory and file for testing\n\ttempDir := t.TempDir()\n\tinputFile := filepath.Join(tempDir, \"integration_test.yaml\")\n\n\t// Create input file with some content\n\toriginalContent := []byte(\"original: content\\n\")\n\terr := os.WriteFile(inputFile, originalContent, 0600)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to create input file: %v\", err)\n\t}\n\n\thandler := NewWriteInPlaceHandler(inputFile)\n\n\t// Create temp file\n\ttempFile, err := handler.CreateTempFile()\n\tif err != nil {\n\t\tt.Fatalf(\"CreateTempFile failed: %v\", err)\n\t}\n\n\t// Write new content to temp file\n\tnewContent := []byte(\"new: content\\n\")\n\t_, err = tempFile.Write(newContent)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to write to temp file: %v\", err)\n\t}\n\ttempFile.Close()\n\n\t// Finish with success\n\terr = handler.FinishWriteInPlace(true)\n\tif err != nil {\n\t\tt.Fatalf(\"FinishWriteInPlace failed: %v\", err)\n\t}\n\n\t// Verify the file was updated\n\tfinalContent, err := os.ReadFile(inputFile)\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to read final file: %v\", err)\n\t}\n\n\tif string(finalContent) != string(newContent) {\n\t\tt.Errorf(\"File not updated correctly. Expected %q, got %q\",\n\t\t\tstring(newContent), string(finalContent))\n\t}\n}\n" }, { "path": "pkg/yqlib/xml.go", "content": "package yqlib\n\ntype XmlPreferences struct {\n\tIndent int\n\tAttributePrefix string\n\tContentName string\n\tStrictMode bool\n\tKeepNamespace bool\n\tUseRawToken bool\n\tProcInstPrefix string\n\tDirectiveName string\n\tSkipProcInst bool\n\tSkipDirectives bool\n}\n\nfunc NewDefaultXmlPreferences() XmlPreferences {\n\treturn XmlPreferences{\n\t\tIndent: 2,\n\t\tAttributePrefix: \"+@\",\n\t\tContentName: \"+content\",\n\t\tStrictMode: false,\n\t\tKeepNamespace: true,\n\t\tUseRawToken: true,\n\t\tProcInstPrefix: \"+p_\",\n\t\tDirectiveName: \"+directive\",\n\t\tSkipProcInst: false,\n\t\tSkipDirectives: false,\n\t}\n}\n\nfunc (p *XmlPreferences) Copy() XmlPreferences {\n\treturn XmlPreferences{\n\t\tIndent: p.Indent,\n\t\tAttributePrefix: p.AttributePrefix,\n\t\tContentName: p.ContentName,\n\t\tStrictMode: p.StrictMode,\n\t\tKeepNamespace: p.KeepNamespace,\n\t\tUseRawToken: p.UseRawToken,\n\t\tProcInstPrefix: p.ProcInstPrefix,\n\t\tDirectiveName: p.DirectiveName,\n\t\tSkipProcInst: p.SkipProcInst,\n\t\tSkipDirectives: p.SkipDirectives,\n\t}\n}\n\nvar ConfiguredXMLPreferences = NewDefaultXmlPreferences()\n" }, { "path": "pkg/yqlib/xml_test.go", "content": "//go:build !yq_noxml\n\npackage yqlib\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nconst yamlInputWithProcInstAndHeadComment = `# cats\n+p_xml: version=\"1.0\"\nthis: is some xml`\n\nconst expectedXmlProcInstAndHeadComment = `\n\nis some xml\n`\n\nconst xmlProcInstAndHeadCommentBlock = `\n\nis some xml\n`\n\nconst expectedYamlProcInstAndHeadCommentBlock = `#\n# cats\n#\n+p_xml: version=\"1.0\"\nthis: is some xml\n`\n\nconst inputXMLWithComments = `\n\n\n\t\n\t3\n\t\n\t\n\t\t\n\t\tz\n\t\t\n\t\t\n\t\n\t\n\n\n`\nconst inputXMLWithCommentsWithSubChild = `\n\n\n\t\n\t3\n\t\n\t\n\t\t\n\t\t\n\t\t\n\t\t\n\t\n\t\n\n\n`\n\nconst expectedDecodeYamlWithSubChild = `# before cat\ncat:\n # in cat before\n x: \"3\" # multi\n # line comment \n # for x\n # before y\n\n y:\n # in y before\n d:\n # in d before\n z:\n +@sweet: cool\n # in d after\n # in y after\n # in_cat_after\n# after cat\n`\n\nconst inputXMLWithCommentsWithArray = `\n\n\n\t\n\t3\n\t\n\t\n\t\t\n\t\t\n \n\t\t\n\t\t\n\t\n\t\n\n\n`\n\nconst expectedDecodeYamlWithArray = `# before cat\ncat:\n # in cat before\n x: \"3\" # multi\n # line comment \n # for x\n # before y\n\n y:\n # in y before\n d:\n - # in d before\n z:\n +@sweet: cool\n # in d after\n - # in d2 before\n z:\n +@sweet: cool2\n # in d2 after\n # in y after\n # in_cat_after\n# after cat\n`\n\nconst expectedDecodeYamlWithComments = `# before cat\ncat:\n # in cat before\n x: \"3\" # multi\n # line comment \n # for x\n # before y\n\n y:\n # in y before\n # in d before\n d: z # in d after\n # in y after\n # in_cat_after\n# after cat\n`\n\nconst expectedRoundtripXMLWithComments = `\n\n 3\n \n z\n \n\n`\n\nconst yamlWithComments = `#\n# header comment\n# above_cat\n#\ncat: # inline_cat\n # above_array\n array: # inline_array\n - val1 # inline_val1\n # above_val2\n - val2 # inline_val2\n# below_cat\n`\n\nconst expectedXMLWithComments = `\n\n\n val1\n val2\n\n`\n\nconst inputXMLWithNamespacedAttr = `\n\n baz\n foobar\n\n`\n\nconst expectedYAMLWithNamespacedAttr = `+p_xml: version=\"1.0\"\nmap:\n +@xmlns: some-namespace\n +@xmlns:xsi: some-instance\n +@xsi:schemaLocation: some-url\n item:\n +content: baz\n +@foo: bar\n xsi:item: foobar\n`\n\nconst expectedYAMLWithRawNamespacedAttr = `+p_xml: version=\"1.0\"\nmap:\n +@xmlns: some-namespace\n +@xmlns:xsi: some-instance\n +@xsi:schemaLocation: some-url\n item:\n +content: baz\n +@foo: bar\n xsi:item: foobar\n`\n\nconst expectedYAMLWithoutRawNamespacedAttr = `+p_xml: version=\"1.0\"\nsome-namespace:map:\n +@xmlns: some-namespace\n +@xmlns:xsi: some-instance\n +@some-instance:schemaLocation: some-url\n some-namespace:item:\n +content: baz\n +@foo: bar\n some-instance:item: foobar\n`\n\nconst xmlWithCustomDtd = `\n\n\n\n]>\n\n &writer;©right;\n`\n\nconst expectedDtd = `\n\n\n]>\n\n &writer;&copyright;\n\n`\n\nconst expectedSkippedDtd = `\n\n &writer;&copyright;\n\n`\n\nconst xmlWithProcInstAndDirectives = `\n\n\n \n \n things\n\n`\n\nconst yamlWithProcInstAndDirectives = `+p_xml: version=\"1.0\"\n+directive: 'DOCTYPE config SYSTEM \"/etc/iwatch/iwatch.dtd\" '\napple:\n +p_coolioo: version=\"1.0\"\n +directive: 'CATYPE meow purr puss '\n b: things\n`\n\nconst expectedXmlWithProcInstAndDirectives = `\n\n\n things\n\n`\n\nvar xmlScenarios = []formatScenario{\n\t{\n\t\tskipDoc: true,\n\t\tdescription: \"bad xml\",\n\t\tinput: ``,\n\t\texpected: \"+p_xml: version=\\\"1.0\\\" encoding=\\\"UTF-8\\\"\\n\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tinput: \" value \",\n\t\texpected: \"root: value # comment\\n\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tinput: \"valuevalue\",\n\t\texpectedError: \"bad file 'sample.yml': invalid XML: Encountered chardata [value] outside of XML node\",\n\t\tscenarioType: \"decode-error\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tinput: \"value\",\n\t\texpected: \"# comment\\nroot: value\\n\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tinput: \" \",\n\t\texpected: \"root: # comment\\n\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tinput: \"valueanotherValue \",\n\t\texpected: \"root:\\n # comment\\n - value\\n - anotherValue\\n\",\n\t},\n\t{\n\t\tskipDoc: true,\n\t\tinput: \"quicksoftsquishy\",\n\t\texpected: \"root:\\n cats:\\n cat:\\n - quick\\n - soft\\n # kitty_comment\\n\\n - squishy\\n\",\n\t},\n\t{\n\t\tdescription: \"ProcInst with head comment\",\n\t\tskipDoc: true,\n\t\tinput: yamlInputWithProcInstAndHeadComment,\n\t\texpected: expectedXmlProcInstAndHeadComment,\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tdescription: \"Scalar roundtrip\",\n\t\tskipDoc: true,\n\t\tinput: \"cat\",\n\t\texpression: \".mike\",\n\t\texpected: \"cat\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"ProcInst with head comment round trip\",\n\t\tskipDoc: true,\n\t\tinput: expectedXmlProcInstAndHeadComment,\n\t\texpected: expectedXmlProcInstAndHeadComment,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"ProcInst with block head comment to yaml\",\n\t\tskipDoc: true,\n\t\tinput: xmlProcInstAndHeadCommentBlock,\n\t\texpected: expectedYamlProcInstAndHeadCommentBlock,\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"ProcInst with block head comment from yaml\",\n\t\tskipDoc: true,\n\t\tinput: expectedYamlProcInstAndHeadCommentBlock,\n\t\texpected: xmlProcInstAndHeadCommentBlock,\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tdescription: \"ProcInst with head comment round trip block\",\n\t\tskipDoc: true,\n\t\tinput: xmlProcInstAndHeadCommentBlock,\n\t\texpected: xmlProcInstAndHeadCommentBlock,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Parse xml: simple\",\n\t\tsubdescription: \"Notice how all the values are strings, see the next example on how you can fix that.\",\n\t\tinput: \"\\n\\n meow\\n 4\\n true\\n\",\n\t\texpected: \"+p_xml: version=\\\"1.0\\\" encoding=\\\"UTF-8\\\"\\ncat:\\n says: meow\\n legs: \\\"4\\\"\\n cute: \\\"true\\\"\\n\",\n\t},\n\t{\n\t\tdescription: \"Parse xml: number\",\n\t\tsubdescription: \"All values are assumed to be strings when parsing XML, but you can use the `from_yaml` operator on all the strings values to autoparse into the correct type.\",\n\t\tinput: \"\\n\\n meow\\n 4\\n true\\n\",\n\t\texpression: \" (.. | select(tag == \\\"!!str\\\")) |= from_yaml\",\n\t\texpected: \"+p_xml: version=\\\"1.0\\\" encoding=\\\"UTF-8\\\"\\ncat:\\n says: meow\\n legs: 4\\n cute: true\\n\",\n\t},\n\t{\n\t\tdescription: \"Parse xml: array\",\n\t\tsubdescription: \"Consecutive nodes with identical xml names are assumed to be arrays.\",\n\t\tinput: \"\\ncat\\ngoat\",\n\t\texpected: \"+p_xml: version=\\\"1.0\\\" encoding=\\\"UTF-8\\\"\\nanimal:\\n - cat\\n - goat\\n\",\n\t},\n\t{\n\t\tdescription: \"Parse xml: force as an array\",\n\t\tsubdescription: \"In XML, if your array has a single item, then yq doesn't know its an array. This is how you can consistently force it to be an array. This handles the 3 scenarios of having nothing in the array, having a single item and having multiple.\",\n\t\tinput: \"cat\",\n\t\texpression: \".zoo.animal |= ([] + .)\",\n\t\texpected: \"zoo:\\n animal:\\n - cat\\n\",\n\t},\n\t{\n\t\tdescription: \"Parse xml: force all as an array\",\n\t\tinput: \"boing\",\n\t\texpression: \".. |= [] + .\",\n\t\texpected: \"- zoo:\\n - thing:\\n - frog:\\n - boing\\n\",\n\t},\n\t{\n\t\tdescription: \"Parse xml: attributes\",\n\t\tsubdescription: \"Attributes are converted to fields, with the default attribute prefix '+'. Use '--xml-attribute-prefix` to set your own.\",\n\t\tinput: \"\\n\\n 7\\n\",\n\t\texpected: \"+p_xml: version=\\\"1.0\\\" encoding=\\\"UTF-8\\\"\\ncat:\\n +@legs: \\\"4\\\"\\n legs: \\\"7\\\"\\n\",\n\t},\n\t{\n\t\tdescription: \"Parse xml: attributes with content\",\n\t\tsubdescription: \"Content is added as a field, using the default content name of `+content`. Use `--xml-content-name` to set your own.\",\n\t\tinput: \"\\nmeow\",\n\t\texpected: \"+p_xml: version=\\\"1.0\\\" encoding=\\\"UTF-8\\\"\\ncat:\\n +content: meow\\n +@legs: \\\"4\\\"\\n\",\n\t},\n\t{\n\t\tdescription: \"Parse xml: content split between comments/children\",\n\t\tsubdescription: \"Multiple content texts are collected into a sequence.\",\n\t\tinput: \" value anotherValue frog cool!\",\n\t\texpected: \"root:\\n +content: # comment\\n - value\\n - anotherValue\\n - cool!\\n a: frog\\n\",\n\t},\n\t{\n\t\tdescription: \"Parse xml: custom dtd\",\n\t\tsubdescription: \"DTD entities are processed as directives.\",\n\t\tinput: xmlWithCustomDtd,\n\t\texpected: expectedDtd,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip with name spaced attributes\",\n\t\tskipDoc: true,\n\t\tinput: inputXMLWithNamespacedAttr,\n\t\texpected: inputXMLWithNamespacedAttr,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Parse xml: skip custom dtd\",\n\t\tsubdescription: \"DTDs are directives, skip over directives to skip DTDs.\",\n\t\tinput: xmlWithCustomDtd,\n\t\texpected: expectedSkippedDtd,\n\t\tscenarioType: \"roundtrip-skip-directives\",\n\t},\n\t{\n\t\tdescription: \"Parse xml: with comments\",\n\t\tsubdescription: \"A best attempt is made to preserve comments.\",\n\t\tinput: inputXMLWithComments,\n\t\texpected: expectedDecodeYamlWithComments,\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Empty doc\",\n\t\tskipDoc: true,\n\t\tinput: \"\",\n\t\texpected: \"\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Empty single node\",\n\t\tskipDoc: true,\n\t\tinput: \"\",\n\t\texpected: \"a:\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Empty close node\",\n\t\tskipDoc: true,\n\t\tinput: \"\",\n\t\texpected: \"a:\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Nested empty\",\n\t\tskipDoc: true,\n\t\tinput: \"\",\n\t\texpected: \"a:\\n b:\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Parse xml: with comments subchild\",\n\t\tskipDoc: true,\n\t\tinput: inputXMLWithCommentsWithSubChild,\n\t\texpected: expectedDecodeYamlWithSubChild,\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Parse xml: with comments array\",\n\t\tskipDoc: true,\n\t\tinput: inputXMLWithCommentsWithArray,\n\t\texpected: expectedDecodeYamlWithArray,\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Parse xml: keep attribute namespace\",\n\t\tsubdescription: fmt.Sprintf(`Defaults to %v`, ConfiguredXMLPreferences.KeepNamespace),\n\t\tskipDoc: false,\n\t\tinput: inputXMLWithNamespacedAttr,\n\t\texpected: expectedYAMLWithNamespacedAttr,\n\t\tscenarioType: \"decode-keep-ns\",\n\t},\n\t{\n\t\tdescription: \"Parse xml: keep raw attribute namespace\",\n\t\tskipDoc: true,\n\t\tinput: inputXMLWithNamespacedAttr,\n\t\texpected: expectedYAMLWithRawNamespacedAttr,\n\t\tscenarioType: \"decode-raw-token\",\n\t},\n\t{\n\t\tdescription: \"Parse xml: keep raw attribute namespace\",\n\t\tsubdescription: fmt.Sprintf(`Defaults to %v`, ConfiguredXMLPreferences.UseRawToken),\n\t\tskipDoc: false,\n\t\tinput: inputXMLWithNamespacedAttr,\n\t\texpected: expectedYAMLWithoutRawNamespacedAttr,\n\t\tscenarioType: \"decode-raw-token-off\",\n\t},\n\t{\n\t\tdescription: \"Encode xml: simple\",\n\t\tinput: \"cat: purrs\",\n\t\texpected: \"purrs\\n\",\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tdescription: \"includes map tags\",\n\t\tskipDoc: true,\n\t\tinput: \"purrs\\n\",\n\t\texpression: `tag`,\n\t\texpected: \"!!map\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"includes array tags\",\n\t\tskipDoc: true,\n\t\tinput: \"purrspurrs\\n\",\n\t\texpression: `.cat | tag`,\n\t\texpected: \"!!seq\\n\",\n\t\tscenarioType: \"decode\",\n\t},\n\t{\n\t\tdescription: \"Encode xml: array\",\n\t\tinput: \"pets:\\n cat:\\n - purrs\\n - meows\",\n\t\texpected: \"\\n purrs\\n meows\\n\\n\",\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tdescription: \"Encode xml: attributes\",\n\t\tsubdescription: \"Fields with the matching xml-attribute-prefix are assumed to be attributes.\",\n\t\tinput: \"cat:\\n +@name: tiger\\n meows: true\\n\",\n\t\texpected: \"\\n true\\n\\n\",\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tdescription: \"double prefix\",\n\t\tskipDoc: true,\n\t\tinput: \"cat:\\n +@+@name: tiger\\n meows: true\\n\",\n\t\texpected: \"\\n true\\n\\n\",\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tdescription: \"arrays cannot be encoded\",\n\t\tskipDoc: true,\n\t\tinput: \"[cat, dog, fish]\",\n\t\texpectedError: \"cannot encode !!seq to XML - only maps can be encoded\",\n\t\tscenarioType: \"encode-error\",\n\t},\n\t{\n\t\tdescription: \"arrays cannot be encoded - 2\",\n\t\tskipDoc: true,\n\t\tinput: \"[cat, dog]\",\n\t\texpectedError: \"cannot encode !!seq to XML - only maps can be encoded\",\n\t\tscenarioType: \"encode-error\",\n\t},\n\t{\n\t\tdescription: \"Encode xml: attributes with content\",\n\t\tsubdescription: \"Fields with the matching xml-content-name is assumed to be content.\",\n\t\tinput: \"cat:\\n +@name: tiger\\n +content: cool\\n\",\n\t\texpected: \"cool\\n\",\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tdescription: \"round trip multiline 1\",\n\t\tskipDoc: true,\n\t\tinput: \"\\n\",\n\t\texpected: \"\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"round trip multiline 2\",\n\t\tskipDoc: true,\n\t\tinput: \"\\n\",\n\t\texpected: \"\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"round trip multiline 3\",\n\t\tskipDoc: true,\n\t\tinput: \"\\n\",\n\t\texpected: \"\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"round trip multiline 4\",\n\t\tskipDoc: true,\n\t\tinput: \"\\n\",\n\t\texpected: \"\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"round trip multiline 5\",\n\t\tskipDoc: true, // pity spaces aren't kept atm.\n\t\tinput: \"\\n\",\n\t\texpected: \"\\n\",\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Encode xml: comments\",\n\t\tsubdescription: \"A best attempt is made to copy comments to xml.\",\n\t\tinput: yamlWithComments,\n\t\texpected: expectedXMLWithComments,\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tdescription: \"Encode: doctype and xml declaration\",\n\t\tsubdescription: \"Use the special xml names to add/modify proc instructions and directives.\",\n\t\tinput: yamlWithProcInstAndDirectives,\n\t\texpected: expectedXmlWithProcInstAndDirectives,\n\t\tscenarioType: \"encode\",\n\t},\n\t{\n\t\tdescription: \"Round trip: with comments\",\n\t\tsubdescription: \"A best effort is made, but comment positions and white space are not preserved perfectly.\",\n\t\tinput: inputXMLWithComments,\n\t\texpected: expectedRoundtripXMLWithComments,\n\t\tscenarioType: \"roundtrip\",\n\t},\n\t{\n\t\tdescription: \"Roundtrip: with doctype and declaration\",\n\t\tsubdescription: \"yq parses XML proc instructions and directives into nodes.\\nUnfortunately the underlying XML parser loses whitespace information.\",\n\t\tinput: xmlWithProcInstAndDirectives,\n\t\texpected: expectedXmlWithProcInstAndDirectives,\n\t\tscenarioType: \"roundtrip\",\n\t},\n}\n\nfunc testXMLScenario(t *testing.T, s formatScenario) {\n\tswitch s.scenarioType {\n\tcase \"\", \"decode\":\n\t\tyamlPrefs := ConfiguredYamlPreferences.Copy()\n\t\tyamlPrefs.Indent = 4\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(yamlPrefs)), s.description)\n\tcase \"encode\":\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(ConfiguredXMLPreferences)), s.description)\n\tcase \"roundtrip\":\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(ConfiguredXMLPreferences)), s.description)\n\tcase \"decode-keep-ns\":\n\t\tprefs := NewDefaultXmlPreferences()\n\t\tprefs.KeepNamespace = true\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)\n\tcase \"decode-raw-token\":\n\t\tprefs := NewDefaultXmlPreferences()\n\t\tprefs.UseRawToken = true\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)\n\tcase \"decode-raw-token-off\":\n\t\tprefs := NewDefaultXmlPreferences()\n\t\tprefs.UseRawToken = false\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)\n\tcase \"roundtrip-skip-directives\":\n\t\tprefs := NewDefaultXmlPreferences()\n\t\tprefs.SkipDirectives = true\n\t\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(prefs)), s.description)\n\tcase \"decode-error\":\n\t\tresult, err := processFormatScenario(s, NewXMLDecoder(NewDefaultXmlPreferences()), NewYamlEncoder(ConfiguredYamlPreferences))\n\t\tif err == nil {\n\t\t\tt.Errorf(\"Expected error '%v' but it worked: %v\", s.expectedError, result)\n\t\t} else {\n\t\t\ttest.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description)\n\t\t}\n\tcase \"encode-error\":\n\t\tresult, err := processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(NewDefaultXmlPreferences()))\n\t\tif err == nil {\n\t\t\tt.Errorf(\"Expected error '%v' but it worked: %v\", s.expectedError, result)\n\t\t} else {\n\t\t\ttest.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description)\n\t\t}\n\n\tdefault:\n\t\tpanic(fmt.Sprintf(\"unhandled scenario type %q\", s.scenarioType))\n\t}\n}\n\nfunc documentXMLScenario(_ *testing.T, w *bufio.Writer, i interface{}) {\n\ts := i.(formatScenario)\n\n\tif s.skipDoc {\n\t\treturn\n\t}\n\tswitch s.scenarioType {\n\tcase \"\", \"decode\":\n\t\tdocumentXMLDecodeScenario(w, s)\n\tcase \"encode\":\n\t\tdocumentXMLEncodeScenario(w, s)\n\tcase \"roundtrip\":\n\t\tdocumentXMLRoundTripScenario(w, s)\n\tcase \"decode-keep-ns\":\n\t\tdocumentXMLDecodeKeepNsScenario(w, s)\n\tcase \"decode-raw-token-off\":\n\t\tdocumentXMLDecodeKeepNsRawTokenScenario(w, s)\n\tcase \"roundtrip-skip-directives\":\n\t\tdocumentXMLSkipDirectivesScenario(w, s)\n\n\tdefault:\n\t\tpanic(fmt.Sprintf(\"unhandled scenario type %q\", s.scenarioType))\n\t}\n}\n\nfunc documentXMLDecodeScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.xml file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```xml\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\texpression := s.expression\n\tif s.expression != \"\" {\n\t\texpression = fmt.Sprintf(\" '%v'\", s.expression)\n\t}\n\twriteOrPanic(w, fmt.Sprintf(\"```bash\\nyq -oy%v sample.xml\\n```\\n\", expression))\n\twriteOrPanic(w, \"will output\\n\")\n\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(ConfiguredYamlPreferences))))\n}\n\nfunc documentXMLDecodeKeepNsScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.xml file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```xml\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\twriteOrPanic(w, \"```bash\\nyq --xml-keep-namespace=false sample.xml\\n```\\n\")\n\twriteOrPanic(w, \"will output\\n\")\n\tprefs := NewDefaultXmlPreferences()\n\tprefs.KeepNamespace = false\n\twriteOrPanic(w, fmt.Sprintf(\"```xml\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(prefs))))\n\n\tprefsWithout := NewDefaultXmlPreferences()\n\tprefs.KeepNamespace = true\n\twriteOrPanic(w, \"instead of\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```xml\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(prefsWithout))))\n}\n\nfunc documentXMLDecodeKeepNsRawTokenScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.xml file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```xml\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\twriteOrPanic(w, \"```bash\\nyq --xml-raw-token=false sample.xml\\n```\\n\")\n\twriteOrPanic(w, \"will output\\n\")\n\n\tprefs := NewDefaultXmlPreferences()\n\tprefs.UseRawToken = false\n\n\twriteOrPanic(w, fmt.Sprintf(\"```xml\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(prefs))))\n\n\tprefsWithout := NewDefaultXmlPreferences()\n\tprefsWithout.UseRawToken = true\n\n\twriteOrPanic(w, \"instead of\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```xml\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(prefsWithout))))\n}\n\nfunc documentXMLEncodeScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.yml file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```yaml\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\twriteOrPanic(w, \"```bash\\nyq -o=xml sample.yml\\n```\\n\")\n\twriteOrPanic(w, \"will output\\n\")\n\n\twriteOrPanic(w, fmt.Sprintf(\"```xml\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(ConfiguredXMLPreferences))))\n}\n\nfunc documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.xml file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```xml\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\twriteOrPanic(w, \"```bash\\nyq sample.xml\\n```\\n\")\n\twriteOrPanic(w, \"will output\\n\")\n\n\twriteOrPanic(w, fmt.Sprintf(\"```xml\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(ConfiguredXMLPreferences))))\n}\n\nfunc documentXMLSkipDirectivesScenario(w *bufio.Writer, s formatScenario) {\n\twriteOrPanic(w, fmt.Sprintf(\"## %v\\n\", s.description))\n\n\tif s.subdescription != \"\" {\n\t\twriteOrPanic(w, s.subdescription)\n\t\twriteOrPanic(w, \"\\n\\n\")\n\t}\n\n\twriteOrPanic(w, \"Given a sample.xml file of:\\n\")\n\twriteOrPanic(w, fmt.Sprintf(\"```xml\\n%v\\n```\\n\", s.input))\n\n\twriteOrPanic(w, \"then\\n\")\n\twriteOrPanic(w, \"```bash\\nyq --xml-skip-directives sample.xml\\n```\\n\")\n\twriteOrPanic(w, \"will output\\n\")\n\tprefs := NewDefaultXmlPreferences()\n\tprefs.SkipDirectives = true\n\n\twriteOrPanic(w, fmt.Sprintf(\"```xml\\n%v```\\n\\n\", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(prefs))))\n}\n\nfunc TestXMLScenarios(t *testing.T) {\n\tfor _, tt := range xmlScenarios {\n\t\ttestXMLScenario(t, tt)\n\t}\n\tgenericScenarios := make([]interface{}, len(xmlScenarios))\n\tfor i, s := range xmlScenarios {\n\t\tgenericScenarios[i] = s\n\t}\n\tdocumentScenarios(t, \"usage\", \"xml\", genericScenarios, documentXMLScenario)\n}\n" }, { "path": "pkg/yqlib/yaml.go", "content": "package yqlib\n\ntype YamlPreferences struct {\n\tIndent int\n\tColorsEnabled bool\n\tLeadingContentPreProcessing bool\n\tPrintDocSeparators bool\n\tUnwrapScalar bool\n\tEvaluateTogether bool\n\tFixMergeAnchorToSpec bool\n\tCompactSequenceIndent bool\n}\n\nfunc NewDefaultYamlPreferences() YamlPreferences {\n\treturn YamlPreferences{\n\t\tIndent: 2,\n\t\tColorsEnabled: false,\n\t\tLeadingContentPreProcessing: true,\n\t\tPrintDocSeparators: true,\n\t\tUnwrapScalar: true,\n\t\tEvaluateTogether: false,\n\t\tFixMergeAnchorToSpec: false,\n\t\tCompactSequenceIndent: false,\n\t}\n}\n\nfunc (p *YamlPreferences) Copy() YamlPreferences {\n\treturn YamlPreferences{\n\t\tIndent: p.Indent,\n\t\tColorsEnabled: p.ColorsEnabled,\n\t\tLeadingContentPreProcessing: p.LeadingContentPreProcessing,\n\t\tPrintDocSeparators: p.PrintDocSeparators,\n\t\tUnwrapScalar: p.UnwrapScalar,\n\t\tEvaluateTogether: p.EvaluateTogether,\n\t\tFixMergeAnchorToSpec: p.FixMergeAnchorToSpec,\n\t\tCompactSequenceIndent: p.CompactSequenceIndent,\n\t}\n}\n\nvar ConfiguredYamlPreferences = NewDefaultYamlPreferences()\n" }, { "path": "pkg/yqlib/yaml_test.go", "content": "package yqlib\n\nimport (\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/test\"\n)\n\nvar yamlFormatScenarios = []formatScenario{\n\t{\n\t\tdescription: \"scalar with doc separator\",\n\t\tskipDoc: true,\n\t\tinput: \"--- cat\",\n\t\texpected: \"---\\ncat\\n\",\n\t},\n\t{\n\t\tdescription: \"CRLF doc separator\",\n\t\tskipDoc: true,\n\t\tinput: \"---\\r\\ncat\\r\\n\",\n\t\texpected: \"---\\r\\ncat\\r\\n\",\n\t},\n\t{\n\t\tdescription: \"yaml directive preserved (LF)\",\n\t\tskipDoc: true,\n\t\tinput: \"%YAML 1.1\\n---\\ncat\\n\",\n\t\texpected: \"%YAML 1.1\\n---\\ncat\\n\",\n\t},\n\t{\n\t\tdescription: \"yaml directive preserved (CRLF)\",\n\t\tskipDoc: true,\n\t\tinput: \"%YAML 1.1\\r\\n---\\r\\ncat\\r\\n\",\n\t\texpected: \"%YAML 1.1\\r\\n---\\r\\ncat\\r\\n\",\n\t},\n\t{\n\t\tdescription: \"comment only no trailing newline\",\n\t\tskipDoc: true,\n\t\tinput: \"# hello\",\n\t\texpected: \"# hello\\n\",\n\t},\n\n\t{\n\t\tdescription: \"scalar with doc separator\",\n\t\tskipDoc: true,\n\t\tinput: \"---cat\",\n\t\texpected: \"---cat\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - null\",\n\t\tskipDoc: true,\n\t\tinput: \"null\",\n\t\texpected: \"null\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - ~\",\n\t\tskipDoc: true,\n\t\tinput: \"~\",\n\t\texpected: \"~\\n\",\n\t},\n\t{\n\t\tdescription: \"octal\",\n\t\tskipDoc: true,\n\t\tinput: \"0o30\",\n\t\texpression: \"tag\",\n\t\texpected: \"!!int\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - [null]\",\n\t\tskipDoc: true,\n\t\tinput: \"[null]\",\n\t\texpected: \"[null]\\n\",\n\t},\n\t{\n\t\tdescription: \"multi document anchor map\",\n\t\tskipDoc: true,\n\t\tinput: \"a: &remember mike\\n---\\nb: *remember\",\n\t\texpression: \"explode(.)\",\n\t\texpected: \"a: mike\\n---\\nb: mike\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - [~]\",\n\t\tskipDoc: true,\n\t\tinput: \"[~]\",\n\t\texpected: \"[~]\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - null map value\",\n\t\tskipDoc: true,\n\t\tinput: \"a: null\",\n\t\texpected: \"a: null\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - number\",\n\t\tskipDoc: true,\n\t\tinput: \"3\",\n\t\texpected: \"3\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - float\",\n\t\tskipDoc: true,\n\t\tinput: \"3.1\",\n\t\texpected: \"3.1\\n\",\n\t},\n\t{\n\t\tdescription: \"basic - float\",\n\t\tskipDoc: true,\n\t\tinput: \"[1, 2]\",\n\t\texpected: \"[1, 2]\\n\",\n\t},\n}\n\nvar yamlParseScenarios = []expressionScenario{\n\t// {\n\t// \tdescription: \"with a unquoted question mark in the string\",\n\t// \tdocument: \"foo: {bar: a?bc}\",\n\t// \texpected: []string{\n\t// \t\t\"D0, P[], (!!map)::a: hello # things\\n\",\n\t// \t},\n\t// },\n\t{\n\t\tdocument: `a: hello # things`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: hello # things\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: \"a: &a apple\\nb: *a\",\n\t\texpression: \".b | explode(.)\",\n\t\texpected: []string{\n\t\t\t\"D0, P[b], (!!str)::apple\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: `a: [1,2]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: [1, 2]\\n\",\n\t\t},\n\t},\n\t{\n\t\tdocument: `a: !horse [a]`,\n\t\texpected: []string{\n\t\t\t\"D0, P[], (!!map)::a: !horse [a]\\n\",\n\t\t},\n\t},\n}\n\nfunc testYamlScenario(t *testing.T, s formatScenario) {\n\ttest.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)\n}\n\nfunc TestYamlParseScenarios(t *testing.T) {\n\tfor _, tt := range yamlParseScenarios {\n\t\ttestScenario(t, &tt)\n\t}\n}\n\nfunc TestYamlFormatScenarios(t *testing.T) {\n\tfor _, tt := range yamlFormatScenarios {\n\t\ttestYamlScenario(t, tt)\n\t}\n}\n" }, { "path": "project-words.txt", "content": "abxbbxdbxebxczzx\nabxbbxdbxebxczzy\naccum\nAccum\nadithyasunil\nAEDT\nágua\nÁGUA\nalecthomas\nappleapple\nAstuff\nautocreating\nautoparse\nAWST\naxbxcxdxe\naxbxcxdxexxx\nbananabanana\nbarp\nnbaz\nbitnami\nblarp\nblddir\nBobo\nBODMAS\nbonapite\nBrien\nBstuff\nBUILDKIT\nbuildpackage\ncatmeow\nCATYPE\nCBVVE\nchardata\nchillum\nchoco\nchomper\ncleanup\ncmlu\ncolorise\ncolors\nColors\ncolourize\ncompinit\ncoolioo\ncoverprofile\ncreatemap\ncsvd\nCSVUTF\ncurrentlabel\ncygpath\nczvf\ndatestring\ndatetime\nDatetime\ndatetimes\nDEBEMAIL\ndebhelper\nDebugf\ndebuild\ndelish\ndelpaths\nDELPATHS\ndevorbitus\ndevscripts\ndimchansky\nDont\ndput\nelliotchance\nendhint\nendofname\nEntriesfrom\nenvsubst\nerrorlevel\nEscandón\nEvalall\nfakefilename\nfakeroot\nFarah\nfatih\nFifi\nfilebytes\nFileish\nfoobar\nfoobaz\nfoof\nfrood\nfullpath\ngitbook\ngithubactions\ngnupg\ngoccy\ngofmt\ngogo\ngolangci\ngoreleaser\nGORELEASER\nGOMODCACHE\nGOPATH\ngosec\ngota\ngoversion\nGOVERSION\nhaha\nhellno\nherbygillot\nhexdump\nHoang\nhostpath\nhotdog\nhowdy\nincase\ninlinetables\ninplace\nints\nireduce\niwatch\njinzhu\njq's\njsond\nkeygrip\nKeygrip\nKEYGRIP\nKEYID\nkeyvalue\nkwak\nlalilu\nldflags\nLDFLAGS\nlexer\nLexer\nlibdistro\nlindex\nlinecomment\nLVAs\nmagiconair\nmapvalues\nMier\nmikefarah\nminideb\nminishift\nmipsle\nmitchellh\nmktemp\nMult\nmultidoc\nmultimaint\nmyenv\nmyenvnonexisting\nmyfile\nmyformat\nndjson\nNDJSON\nNFKD\nnixpkgs\nnojson\nnonascii\nnonempty\nnoninteractive\nNonquoting\nnosec\nnotoml\nnoxml\nnolua\nnullinput\nonea\nOneshot\nopencollect\nopstack\norderedmap\nosarch\noverridign\npacman\nPadder\npandoc\nparsechangelog\npcsv\npelletier\npflag\nprechecking\nPrerelease\nproc\npropsd\nqylib\nreadline\nrealnames\nrealpath\nrepr\nrhash\nrindex\nrisentveber\nrmescandon\nRosey\nroundtrip\nroundtrips\nRoundtrip\nroundtripping\nInterp\ninterp\nrunningvms\nsadface\nselfupdate\nsetpath\nsharedfolder\nSharedfolder\nshellvariables\nshellvars\nshortfunc\nshortpipe\nshunit\nsnapcraft\nsomevalue\nsplt\nsrcdir\nstackoverflow\nstiched\nStrc\nstrenv\nstrload\nstylig\nsubarray\nsubchild\nsubdescription\nsubmatch\nsubmatches\nSUBSTR\ntempfile\ntfstate\nTfstate\nthar\ntimezone\nTimezone\ntimezones\nTimezones\ntojson\nTokenvalue\ntsvd\nTuan\ntzdata\nUhoh\nupdateassign\nurid\nutfbom\nWarningf\nWazowski\nwebi\nWebi\nwherever\nwinget\nwithdots\nwizz\nwoop\nworkdir\nWritable\nxmld\nxyzzy\nyamld\nyqlib\nyuin\nzabbix\ntonumber\nnoyaml\nnolint\nshortfile\nUnmarshalling\nnoini\nnocsv\nnobase64\nnouri\nnoprops\nnosh\nnoshell\ntinygo\nnonexistent\nhclsyntax\nhclwrite\nnohcl\nzclconf\ncty\ngo-cty\nColorisation\ngoimports\nerrorlint\nRDBMS\nexpeñded\nbananabananabananabanana\nedwinjhlee\nflox\nunlabelled\nkyaml\nKYAML\nnokyaml\nbuildvcs\nbehaviour\nGOFLAGS\ngocache\nsubsubarray\nFfile\nFquery\ncoverpkg\ngsub" }, { "path": "release_instructions.txt", "content": "- update release notes\n- make sure local build passes\n- run ./scripts/copy-docs.sh (and commit the changed in the yq-book branch)\n- run ./scripts/bump-version.sh\n- git push\n// to move the v4 branch\n- git push origin -d v4\n- git push --tags\n- use github actions to publish docker and make github release\n- check github updated yq action in marketplace\n\n\n- snapcraft\n - update snapcraft version\n - https://snapcraft.io/yq/builds\n - will auto create a candidate, test it works then promote\n - !! need to update v4/stable as well as latest\n \n sudo snap remove yq\n sudo snap install --edge yq\n \n then use the UI (https://snapcraft.io/yq/release)\n\n- brew\n - brew bump-formula-pr --url=https://github.com/mikefarah/yq/archive/2.2.0.tar.gz yq\n - if that fails with random ruby errors try:\n - clearing out the gems rm -rf .gem/ruby/2.3.0\n - export HOMEBREW_FORCE_VENDOR_RUBY=1\n\n- docker\n - build and push latest and new version tag\n - docker build . -t mikefarah/yq:latest -t mikefarah/yq:3 -t mikefarah/yq:3.X\n\n- debian package (with release script)\n - execute the script `./scripts/release-deb.sh` *on the machine having the private\n gpg key to sign the generated sources* providing the version to release, the\n ppa where deploying the files and the passphrase of the private key if needed:\n ```\n ./scripts/release-deb.sh -o output -p -s --passphrase PASSPHRASE VERSION\n ```\n\n- debian package (manually)\n - ensure you get all vendor dependencies before packaging\n\t```go mod vendor```\n - execute \n ```dch -i```\n - fill debian/changelog with changes from last version\n - build the package sources \n ```debuild -i -I -S -sa```\n (signing with gpg key is required in order to put it to ppa)\n - put to PPA\n ```dput ppa: ../yq__source.changes```\n (current distro repository is ppa:rmescandon/yq. In case that a new version\n is released, please contact rmescandon@gmail.com to bump debian package)\n" }, { "path": "release_notes.txt", "content": "4.52.4:\n - Dropping windows/arm - no longer supported in cross-compile\n\n4.52.3:\n - Fixing comments in TOML arrays (#2592)\n - Bumped dependencies\n\n\n4.52.2:\n - Fixed bad instructions file breaking go-install (#2587) Thanks @theyoprst\n - Fixed TOML table scope after comments (#2588) Thanks @tomers\n - Multiply uses a readonly context (#2558)\n - Fixed merge globbing wildcards in keys (#2564)\n - Fixing TOML subarray parsing issue (#2581)\n\n4.52.1:\n - TOML encoder support - you can now roundtrip! #1364\n - Parent now supports negative indices, and added a 'root' command for referencing the top level document\n - Fixed scalar encoding for HCL\n - Add --yaml-compact-seq-indent / -c flag for compact sequence indentation (#2583) Thanks @jfenal\n - Add symlink check to file rename util (#2576) Thanks @Elias-elastisys\n - Powershell fixed default command used for __completeNoDesc alias (#2568) Thanks @teejaded\n - Unwrap scalars in shell output mode. (#2548) Thanks @flintwinters\n - Added K8S KYAML output format support (#2560) Thanks @robbat2\n\n - Bumped dependencies\n - Special shout out to @ccoVeille for reviewing my PRs!\n\n4.50.1:\n - Added HCL support!\n - Fixing handling of CRLF #2352\n - Bumped dependencies\n\n4.49.2:\n - Fixing escape character bugs :sweat: #2517\n - Fixing snap release pipeline #2518 Thanks @aalexjo\n\n\n4.49.1:\n - Added `--security` flags to disable env and file ops #2515\n - Fixing TOML ArrayTable parsing issues #1758\n - Fixing parsing of escaped characters #2506\n\n4.48.2:\n - Strip whitespace when decoding base64 #2507\n - Upgraded to go-yaml v4! (thanks @ccoVeille, @ingydotnet)\n - Add linux/loong64 to release target (thanks @znley)\n - Added --shell-key-separator flag for customizable shell output format #2497 (thanks @rsleedbx)\n - Bumped dependencies \n\n4.48.1:\n - Added 'parents' operator, to return a list of all the hierarchical parents of a node\n - Added 'first(exp)' operator, to return the first entry matching an expression in an array \n - Fixed xml namespace prefixes #1730 (thanks @baodrate)\n - Fixed out of range panic in yaml decoder #2460 (thanks @n471d)\n - Bumped dependencies \n\n4.47.2:\n - Conversion from TOML to JSON no longer omits empty tables #2459 (thanks @louislouislouislouis)\n - Bumped dependencies\n\n4.47.1:\n - Fixed merge anchor behaviour (`<<`); #2404, #2110, #2386, #2178 Huge thanks to @stevenwdv!\n Note that you will need to set --yaml-fix-merge-anchor-to-spec to see the fixes\n - Fixed panic for syntax error when creating a map #2423\n - Bumped dependencies\n\n4.46.1:\n - Added INI support\n - Fixed 'add' operator when piped in with no data #2378, #2383, #2384\n - Fixed delete after slice problem (bad node path) #2387 Thanks @antoinedeschenes\n - Fixed yq small build Thanks @imzue\n - Switched to YAML org supported go-yaml!\n - Bumped dependencies\n\n\n4.45.4:\n - Fixing wrong map() behaviour on empty map #2359\n - Bumped dependencies\n\n4.45.3:\n - Fixing regression introduced with in 4.45.2 with #2325 fix :sweat: sorry folks!\n - Bumped dependencies\n\n4.45.2:\n - Added windows arm builds (Thanks @albertocavalcante, @ShukantPal)\n - Added s390x platform support (Thanks @ashokpariya0)\n - Additionally push docker images to ghcr.io (Thanks @reegnz)\n - Fixing add when there is no node match #2325\n - sort_by works on maps\n - Bumped dependencies\n\n4.45.1:\n - Create parent directories when --split-exp is used, Thanks @rudo-thomas\n - Bumped dependencies \n\n4.44.6:\n - Fixed deleting items in array bug #2027, #2172; Thanks @jandubois \n - Docker image for armv7 / raspberry pi3, Thanks @brianegge\n - Fixed no-colors regression #2218\n - Fixed various panic scenarios #2211\n - Bumped dependencies \n\n4.44.5:\n - Fixing release pipeline\n\n4.44.4:\n- Format comments with a gray foreground (Thanks @gabe565)\n- Fixed handling of nulls with sort_by expressions #2164\n- Force no color output when NO_COLOR env presents (Thanks @narqo)\n- Fixed array subtraction update bug #2159\n- Fixed index out of range error\n- Can traverse straight from parent operator (parent.blah)\n- Bumped dependencies \n\n4.44.3:\n - Fixed upper-case file extension detection, Thanks @ryenus (#2121)\n - Log printing follow no-colors flag #2082\n - Skip and warn when interpolating strings and theres a unclosed bracket #2083\n - Fixed CSV content starting with # issue #2076\n - Bumped dependencies \n\n4.44.2:\n - Handle numbers with underscores #2039\n - Unique now works on maps and arrays #2068\n - Added support for short hand splat with env[] expression #2071,\n as well as many other operators (split,select,eval,pick..)\n - Bumped dependencies\n \n4.44.1:\n - Added min/max operators (#1992) Thanks @mbenson\n - Added pivot oeprator (#1993) Thanks @mbenson\n - Fix: shell-completion (#2006) Thanks @codekow\n - Handle escaped backslashes (#1997) Thanks @mbenson\n - Fix npe when given filename ending with \".\" (#1994)\n - Fix: linux (w/ selinux) build (#2004) Thanks @codekow\n - Bumped dependencies \n\n4.43.1:\n - Added omit operator #1989 thanks @mbenson!\n - Added tostring #72\n - Added string interpolation #1149\n - Can specify parent(n) levels #1970\n - Can now multiply strings by numbers #1988 thanks @mbenson!\n - Fixed CSV line break issue #1974\n - Adding a EvaluateAll function to StringEvaluator #1966\n - yqlib, default to colors off when using yaml library #1964\n - Removed JSON output warning\n - Bumped dependencies\n\n\n4.42.1:\n - Can execute yq expression files directly with shebang #1851\n - Added --csv-separator flag #1950\n - Added --properties-separator option - thanks @learnitall #1864\n - Added --properties-array-brackets flag for properties encoder #1933\n - Shell completion improvements - thanks @scop #1911\n - Bumped dependencies\n\n\n4.41.1:\n - Can now comment in yq expressions! #1919\n - Fixed Toml decoding when table array defined before parent #1922\n - Added new CSV option to turn off auto-parsing #1947\n - Fixing with_entries context #1925\n - Can now retrieve the alias names of merge anchors #1942\n\n\n4.40.7:\n - Bumped dependencies\n\n4.40.6:\n - Fix: empty TOML table #1924 - Thanks @elibroftw \n - Fixed \"all\" error message #1845\n - Fixed to_entries[]\n - Bumped dependencies\n\n4.40.5:\n - Fixing seg fault on bad XML #1888\n - Fixed handling of --- #1890, #1896\n - Bumped dependencies\n\n4.40.4:\n - Fixed bug with creating maps with values based off keys #1886, #1889\n - Bumped dependencies\n\n4.40.3: \n - Fixed JSON output issue with empty arrays #1880\n\n4.40.2:\n - Do not panic when StdIn is closed (#1867) Thanks @aleskandro!\n - Fixed issue when update against self #1869\n - Fixed multi doc anchor bug #1861\n - Fixes doc line separator issue when reading expression file #1860\n - Bumped dependencies\n\n4.40.1:\n - Added tonumber support (#1664, #71)\n - Added kind operator\n - Lua output fixes (#1811) - Thanks @Zash!\n - Add support for Lua input (#1810) - Thanks @Zash!\n - Rewrote parsing engine - yq now has its own AST!\n - Bumped dependencies\n\n4.35.2:\n - Fix various typos #1798\n - Fixed number parsing as float bug in JSON #1756\n - Fixed string, null concatenation consistency #1712\n - Fixed expression parsing issue #1711\n - Bumped dependencies\n\n4.35.1:\n - Added Lua output support (Thanks @Zash)!\n - Added BSD checksum format (Thanks @viq)!\n - Bumped dependencies\n\n4.34.2:\n - Bumped dependencies\n\n\n4.34.1:\n - Added shell output format thanks @giorgiga\n - Fixed nil pointer dereference (#1649) thanks @ArthurFritz\n - Bumped dependency versions\n\n4.33.3:\n - Fixed bug when splatting empty array #1613\n - Added scalar output for TOML (#1617)\n - Fixed passing of readonly context in pipe (partial fix for #1631)\n - Bumped dependency versions\n\n4.33.2:\n - Add ``--nul-output|-0`` flag to separate element with NUL character (#1550) Thanks @vaab!\n - Add removable-media interface plug declaration to the snap packaging(#1618) Thanks @brlin-tw!\n - Scalar output now handled in csv, tsv and property files\n - Bumped dependency versions\n\n4.33.1:\n - Added read-only TOML support! #1364. Thanks @pelletier for making your API available in your toml lib :)\n - Added warning when auto detect by file type is outputs JSON (#1608)\n\n4.32.2:\n - Fixed behaviour for unknown file types (defaults to yaml) #1609\n\n4.32.1:\n - Added divide and modulo operators (#1593) - thanks @teejaded!\n - Add support for decoding base64 strings without padding (#1555) - thanks @teejaded!\n - Add filter operation (#1588) - thanks @rbren!\n - Detect input format based on file name extension (#1582) - thanks @ryenus!\n - Auto output format when input format is automatically detected\n - Fixed npe in log #1596\n - Improved binary file size! \n - Bumped dependency versions\n\n4.31.2:\n - Fixed variable handling #1458, #1566\n - Fixed merged anchor reference problem #1482\n - Fixed xml encoding of ProcInst #1563, improved XML comment handling\n - Allow build without json and xml support (#1556) Thanks @afbjorklund\n - Bumped dependencies\n\n4.31.1:\n - Added shuffle command #1503\n - Added ability to sort by multiple fields #1541\n - Added @sh encoder #1526\n - Added @uri/@urid encoder/decoder #1529\n - Fixed date comparison with string date #1537\n - Added from_unix/to_unix Operators\n - Bumped dependency versions\n\n4.30.8:\n - Log info message when unable to chown file in linux (e.g. snap confinement) #1521\n\n\n4.30.7:\n - Fixed bug in splice operator #1511\n - Fixed value operator bug #1515\n - Fixed handling of merging null #1501\n - Ownership of file now maintained in linux (thanks @vaguecoder) #1473\n - Bumped dependency versions\n\n4.30.6:\n - Fixed xml comment in array of scalars #1465\n - Include blank newlines in leading header preprocessing #1462\n - Added aarch64 build (#1261)\n - Bumped dependency versions (#1453)\n\n4.30.5:\n - XML Decoder: Comment parsing tweak \n - XML Decoder: Fixed processing comments in empty XML #1446\n - XML Decoder: Checking for invalid content outside of a root node #1448\n - XML Decoder: Fixed issue where content surrounding tags are lost #1447\n - XML Decoder: Fixed xml decode bug when there is content after a comment\n - Fixed loading yaml with header issue #1445\n - guessTagFromCustomType warning log is now a debug.\n - Special thanks to @Kopfbremse for reporting XML issues!\n\n4.30.4:\n - Fixed bug in automated versioning (snap/brew)\n\n4.30.3:\n - Updated release process (automated versioning)\n - Fixed handling of yaml directives (#1424)\n - Fixed parsing of newline character in string expression #1430\n - Fixed length compares to null instead of 0 issue #1427\n\n4.30.2:\n - Actually updated the default xml prefix :facepalm:\n\n4.30.1:\n - XML users note: the default attribute prefix has change to `+@` to avoid naming conflicts!\n - Can use expressions in slice #1419\n - Fixed unhandled exception when decoding CSV thanks @washanhanzi\n - Added array_to_map operator for #1415\n - Fixed sorting by date #1412\n - Added check to ensure only maps can be encoded to XML #1408\n - Check merge alias is a map #1425\n - Explicitly setting unwrap flag works for json output #437, #1409\n - Bumped go version\n\n\n\n4.29.2:\n - Fixed null pointer exception when parsing CSV with empty field #1404\n\n4.29.1:\n - Fixed Square brackets removing update #1342\n - Added slice array operator (.[10:15]) #44\n - XML decoder/encoder now parses directives and proc instructions (#1344). Please use the new skip flags [documented here](https://mikefarah.gitbook.io/yq/usage/xml) to ignore them.\n - XML users note that the default attribute prefix will change to `+@` in the 4.30 release to avoid naming conflicts!\n - Improved comment handling of decoders (breaking change for yqlib users sorry)\n - Fixed load operator bug when loading yaml file with multiple documents\n - Bumped Go compiler version\n - Bumped dependencies\n\n4.28.2:\n - Fixed Github Actions issues (thanks @mattphelps-8451)\n - Fixed bug - can now delete documents #1377\n - Fixed handling of UTF8 encoded CSVs #1373\n - Detect and fail on missing closing brackets #1366\n - yq Github actions now build docker image as part of release\n - Bumped dependencies\n\n4.28.1:\n - Added `setpath` and `delpaths` operators, like jq (#1374)\n - Added `is_key` operator, to check if a match was a key when recursing\n - Added validation when attempting to add sequences to maps (#1341)\n\n4.27.5:\n - Fixed relative merge bug #1333\n\n4.27.4:\n - Fixed bug in alternative (//) operator, RHS being evaluated when it didn't need to be\n - Fixed footer comment issue #1231\n - Github action now runs as root (as recommended by Github Actions doc)\n - Updated dependencies\n\n4.27.3:\n - Added new 'c' merge and assign flag that clobbers custom tags\n - Bumped go dependency to fix CVE (#1316)\n - Updated dependencies\n\n4.27.2:\n - Fixed JSON decoder to maintain object key order.\n\n4.27.1:\n - Added 'json' decoder for support for multiple JSON documents in a single file (e.g. NDJSON)\n - Added 'csv' decoding, array of objects encoding, and round-triping\n - New StringEvaluator when using yq as a lib (thanks @leviliangtw)\n - Fixed XML decoding issue (#1284)\n\n4.26.1:\n - Switched to new expression parser (#1264)\n - Don't clobber anchor when adding nodes (#1269)\n - New error operator for custom validation (#1259)\n - Added support for --wrapScalar=false in properties encoder (#1241) Thanks @dcarbone\n - Fix error on multiple assign (#1257) Thanks @care0717\n - Bumped dependency versions\n\n4.25.4:\n - Fixed panic when using multiply assign on multiple documents #1256 Thanks @care0717\n\n4.25.3:\n - xml decoder now maintains namespaces by default. Use new flags to disable if required. Thanks @rndmit\n - Length and other similar operators no longer return comments (#1231)\n - When split expression includes an extension, dont add .yml automatically (#1165)\n - Map -r to --unwrapScalar to be more a drop in replacement for jq (#1245) Thanks @SuperSandro2000\n - Fixing usage of quoted numeric keys #1247\n - Bumped dependency versions\n\n\n4.25.2:\n - Fixed comments disappearing from end of file (#1217)\n - Fixed empty base64 decoding error (#1209)\n - JSON output now in colors (#1208)\n - Added codeql and fixed minor issues\n - Bumped go-yaml library\n - Bumped go dependency\n\n4.25.1:\n- Can specify a split expression file #1194\n- Fixed append map bug when key matches value in existing map #1200\n- Nicer error message when trying to use merge anchor tags other than maps #1184\n- Fixed Don't automatically read stdin when the null input flag is used\n- Added type as an alias for tag #1195\n- Fixes bug when using write in-place with no expression and multiple files #1193\n\n4.24.5:\n - Fixed scenarios that dropped the first line if it's a comment (#1181)\n - Fixed updating existing empty map resulting in single line styling (#1176)\n - Fixed `with` operation bug (#1174)\n - Bumped go compiler\n\n4.24.4:\n - Fixed docker release build\n\n\n4.24.3:\n - Added from_props\n - Re-releasing, 4.24.2 release failed to publish correctly.\n\n\n4.24.2:\n - Fixing release pipeline for go1.18\n\n4.24.1:\n - Added comparison operators! (#94)\n - Bumped Golang to 1.18 (#1153)\n - XML parser no longer runs in strict mode (added new flag to run in strict mode) (#1155)\n\n4.23.1:\n - Can now supply the envsubst operator with parameters (nounset, noempty, failfast). See [envsubst](https://mikefarah.gitbook.io/yq/operators/env-variable-operators) for details (#1137)\n - Bumped dependencies\n - Fixed '+=' problem with multiple matches #1145\n - Fixed bug with \"and\", \"or\" evaluating the RHS when not needed\n - Fixed potential panic (thanks @mkatychev)\n - Tweaked CLI help (thanks @justin-f-perez)\n\n4.22.1:\n - Added [pick] (https://mikefarah.gitbook.io/yq/operators/pick) operator\n - Can load expression from a file '--from-file' (#1120)\n - Fixed property auto expansion (#1127)\n\n4.21.1:\n - Added [reverse](https://mikefarah.gitbook.io/yq/operators/reverse) operator\n - Added [string case](https://mikefarah.gitbook.io/yq/operators/string-operators) operators\n - Added [base64 support](https://mikefarah.gitbook.io/yq/operators/encode-decode)\n - Added [line](https://mikefarah.gitbook.io/yq/operators/line) and [column](https://mikefarah.gitbook.io/yq/operators/column) operators\n - Bumped dependency versions\n\n4.20.2:\n - Fixed self assignment issue (#1107)\n - Fixed bad capture groups with multiple matches (#1114)\n - No longer auto-read from STDIN if there are files given (#1115)\n - Added missing load_props operator\n\n4.20.1:\n - New [Date Operators](https://mikefarah.gitbook.io/yq/operators/datetime) (now, tz, add and subtract durations from dates) \n - Can now decode property files!\n - New flag to manually set expression if required\n - ZSH completion bug fix (#1108) thanks @whi-tw\n - Fixed SEGV error (#1096)\n - Fixed Github actions issues (it pipes in /dev/null) for XML\n - Fixed bug - handle expressions that match a directory (e.g. \".\")\n\n4.19.1:\n - New [eval](https://mikefarah.gitbook.io/yq/operators/eval) _operator_ that allows dynamic expression evaluation (e.g. from an env variable) (#1087)\n - Adding new elements to array now automatically applies styling of existing elements (#722)\n\n4.18.1:\n - `eval` is now the _default_ command, you can leave it out #113\n - `-` no longer needs to be specified as STDIN, unless you are also working with multiple files. #113\n - Adding to empty maps / arrays now uses idiomatic yaml styling by default\n - Fixed seg fault on bad input #1086\n - New `envsubst` operator! (thanks @sciyoshi)\n - Added support for `*=`, relative multiply/merge\n - Custom tag types now autocast to there actual types #933\n\n4.17.2:\n - Fixed manpath issue (thanks @mr-pmillz)\n \n4.17.1:\n - Added XML support (#491)\n - New merge flag (n) to only merge new fields (#1038)\n - Fixed exit status bug for permission denied error (#1062)\n - Fixed using multiple variables with union (,) operator (#1048)\n - Bumped some versions of dependencies\n\n4.16.2:\n - Bumped go-lang compiler to fix CVE-2021-44717 (#1037)\n - Dependency version bumps via dependabot\n - Added extract-checksum.sh to make it easier to validate checksums (#1011)\n - Report filename on parsing error (#1030)\n\n4.16.1:\n - Added csv, tsv output formats\n - Added map, map_values operators\n - Added sort, sort_by operators (#947, #1024)\n - Fixed bug in collect\n - Fixed permissions issue in Dockerfile (#1014)\n - Fixed assignment operator to no longer overwrite anchor (#1029)\n\n4.15.1:\n - Added 'load/strload' operators for dynamically loading content from files\n - Added 'key' operator\n - Added 'parent' operator\n - Smarter MAN page installation script (thanks @coolaj86)\n - Dockerfile improvements (thanks @actualben)\n - Error handling improvements (thanks @mmorel-35)\n\n4.14.2:\n - Fixed header preprocessing issue (#1000)\n - Bumped version dependencies\n\n4.14.1:\n - Added group_by operator\n - Added encode/decode operators (toyaml, fromjson etc) (#974)\n - Added flatten operator\n - Added --split-exp, for splitting results into multiple files (#966)\n - Fixed json null array bug (#985)\n\n4.13.5:\n - Performance improvement for deepMatch (thanks @pmatseykanets)\n - Added manpage, included in tar.gz downloads as well as a separate tar.gz (#961)\n - Fixed expression parsing bug #970\n - Rebuild fixes CVE (#964)\n - Bumped docker alpine version\n\n\n4.13.4:\n - Fixed select bug (#958)\n - Improved performance of `explode` (this will also speed up json conversion)\n - Improved performance of `merge` (significantly if your merging a small file into a big one)\n\n4.13.3:\n\n- Updated go compiler to 1.17 to fix CVE (#944)\n\n4.13.2:\n\n- Fixing Docker build timeout issues when attempting to release\n\n4.13.1:\n\n- Update to `with` operator, allow for no leading space on the `;`.\n\n4.13.0:\n\nBREAKING CHANGE - the `as` variable operator (e.g. `.a as $x`) now makes a _copy_ of the node(s) at the \npath rather than a reference. This is in order to make it work more like the `jq` equivalent. \n\nThis means any updates made to that variable do not update the original.\n\nThere's a new operator `ref` that will make a reference (and allow multiple updates to the original path by referencing the variable).\nSorry for any inconvenience caused!.\n\n\n- New `with` operator for making multiple changes to a given path\n- New `contains` operator, works like the `jq` equivalent\n- Subtract operator now supports subtracting elements from arrays!\n- Fixed Swapping values using variables #934\n- Github Action now properly supports multiline output #936, thanks @pjxiao\n- Fixed missing closing bracket validation #932\n\n4.12.2:\n- Fix processing of hex numbers #929\n- Fixed alternative and union operator issues #930\n\n4.12.1:\n - Merge comment fix #919\n\n4.12.0:\n- Can now convert yaml to properties properties format (`-o=props`), See [docs](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties) for more info.\n- Fixed document header/footer comment handling when merging (https://github.com/mikefarah/yq/issues/919)\n- pretty print yaml 1.1 compatibility (https://github.com/mikefarah/yq/issues/914)\n" }, { "path": "scripts/acceptance.sh", "content": "#! /bin/bash\nset -e\n\nfor test in acceptance_tests/*.sh; do\n echo \"--------------------------------------------------------------\"\n echo \"$test\"\n echo \"--------------------------------------------------------------\"\n (exec \"$test\");\ndone\n\n" }, { "path": "scripts/build-small-yq.sh", "content": "#!/bin/bash\ngo build -tags \"yq_nolua yq_noini yq_notoml yq_noxml yq_nojson yq_nohcl yq_nokyaml\" -ldflags \"-s -w\" .\n" }, { "path": "scripts/build-tinygo-yq.sh", "content": "#!/bin/bash\n\n# Currently, the `yq_nojson` feature must be enabled when using TinyGo.\ntinygo build -no-debug -tags \"yq_nolua yq_noini yq_notoml yq_noxml yq_nojson yq_nocsv yq_nobase64 yq_nouri yq_noprops yq_nosh yq_noshell yq_nohcl yq_nokyaml\" .\n" }, { "path": "scripts/bump-version.sh", "content": "#!/bin/bash\nset -e\n\nif [ \"$1\" == \"\" ]; then\n echo \"Please specify at a version\"\n exit 1\nfi\n\nversion=$1\n\n# validate version is in the right format\necho $version | sed -r '/v4\\.[0-9][0-9]\\.[0-9][0-9]?$/!{q1}'\n\npreviousVersion=$(cat cmd/version.go| sed -n 's/.*Version = \"\\([^\"]*\\)\"/\\1/p')\n\necho \"Updating from $previousVersion to $version\"\n\nsed -i \"s/\\(.*Version =\\).*/\\1 \\\"$version\\\"/\" cmd/version.go\n\ngo build .\nactualVersion=$(./yq --version)\n\nif [ \"$actualVersion\" != \"yq (https://github.com/mikefarah/yq/) version $version\" ]; then\n echo \"Failed to update version.go\"\n exit 1\nelse\n echo \"version.go updated\"\nfi\n\n version=$version ./yq -i '.version=strenv(version) | .parts.yq.source-tag=strenv(version)' snap/snapcraft.yaml\n\nactualSnapVersion=$(./yq '.version' snap/snapcraft.yaml)\n\nif [ \"$actualSnapVersion\" != \"$version\" ]; then\n echo \"Failed to update snapcraft\"\n exit 1\nelse\n echo \"snapcraft updated\"\nfi\n\nactualSnapVersion=$(./yq '.parts.yq.source-tag' snap/snapcraft.yaml)\n\nif [ \"$actualSnapVersion\" != \"$version\" ]; then\n echo \"Failed to update snapcraft\"\n exit 1\nelse\n echo \"snapcraft updated\"\nfi\n\ngit add cmd/version.go snap/snapcraft.yaml\ngit commit -m 'Bumping version'\ngit tag $version\ngit tag -f v4" }, { "path": "scripts/check.sh", "content": "#!/bin/bash\n\nset -o errexit\nset -o pipefail\n\n# TODO: Check if the found golangci-lint version matches the expected version (e.g., v1.62.0), especially if falling back to PATH version.\n\nGOPATH_LINT=\"$(go env GOPATH)/bin/golangci-lint\"\nBIN_LINT=\"./bin/golangci-lint\"\nLINT_CMD=\"\"\n\nif [ -f \"$GOPATH_LINT\" ]; then\n LINT_CMD=\"$GOPATH_LINT\"\nelif [ -f \"$BIN_LINT\" ]; then\n LINT_CMD=\"$BIN_LINT\"\nelif command -v golangci-lint &> /dev/null; then\n # Using PATH version, ensure compatibility (see TODO)\n LINT_CMD=\"golangci-lint\"\nelse\n echo \"Error: golangci-lint not found in $GOPATH/bin, ./bin, or PATH.\"\n echo \"Please run scripts/devtools.sh or ensure golangci-lint is installed correctly.\"\n exit 1\nfi\n\nGOFLAGS=\"${GOFLAGS}\" \"$LINT_CMD\" run --verbose\n" }, { "path": "scripts/compare-jq.sh", "content": "#!/usr/bin/env bash\nset -e\n\nexp=$1\nfile=$2\n\nif [ \"$2\" == \"\" ]; then\n echo \"yq\"\n ./yq -oj -n \"$1\"\n echo \"jq\"\n jq -n \"$1\"\n \nelse\n\n echo \"yq\"\n ./yq -oj \"$1\" $2\n echo \"jq\"\n ./yq $2 -oj | jq \"$1\"\nfi" }, { "path": "scripts/compare-versions-output.sh", "content": "#!/usr/bin/env bash\n\ntest_data='\n- foo: false\n'\n\nfor version in 4.45.1 4.45.2 4.45.3; do\n for command in '.[] | (select(.foo) | {\"foo\": .foo} // {})' '.[] | (select(.foo) | {.foo} // {})'; do\n echo ${version} \"${command}\"\n echo -------\n echo \"${test_data}\" | podman run -i --rm mikefarah/yq:${version} -o json \"${command}\"\n echo -------\n echo\n done\ndone" }, { "path": "scripts/copy-docs.sh", "content": "#!/bin/bash\n\ncp how-it-works.md ../yq-gitbook/.\ncp pkg/yqlib/doc/operators/*.md ../yq-gitbook/operators/.\ncp pkg/yqlib/doc/usage/*.md ../yq-gitbook/usage/." }, { "path": "scripts/coverage.sh", "content": "#!/bin/bash\n\nset -e\n\necho \"Running tests and generating coverage...\"\npackages=$(go list ./... | grep -v -E 'examples' | grep -v -E 'test' | tr '\\n' ',' | sed 's/,$//')\ntest_packages=$(go list ./... | grep -v -E 'examples' | grep -v -E 'test' | grep -v '^github.com/mikefarah/yq/v4$')\ngo test -coverprofile=coverage.out -coverpkg=\"$packages\" -v $test_packages\n\necho \"Generating HTML coverage report...\"\ngo tool cover -html=coverage.out -o coverage.html\n\necho \"\"\necho \"Generating sorted coverage table...\"\n\n# Create a simple approach using grep and sed to extract file coverage\n# First, get the total coverage\ntotal_coverage=$(go tool cover -func=coverage.out | grep \"^total:\" | sed 's/.*([^)]*)[[:space:]]*\\([0-9.]*\\)%.*/\\1/')\n\n# Extract file-level coverage by finding the last occurrence of each file\ngo tool cover -func=coverage.out | grep -E \"\\.go:[0-9]+:\" | \\\nsed 's/^\\([^:]*\\.go\\):.*[[:space:]]\\([0-9.]*\\)%.*/\\2 \\1/' | \\\nsort -k2 | \\\nawk '{file_coverage[$2] = $1} END {for (file in file_coverage) printf \"%.2f %s\\n\", file_coverage[file], file}' | \\\nsort -nr > coverage_sorted.txt\n\n# Add total coverage to the file\nif [[ -n \"$total_coverage\" && \"$total_coverage\" != \"0\" ]]; then\n echo \"TOTAL: $total_coverage\" >> coverage_sorted.txt\nfi\n\necho \"\"\necho \"Coverage Summary (sorted by percentage - lowest coverage first):\"\necho \"=================================================================\"\nprintf \"%-60s %10s %12s\\n\" \"FILE\" \"COVERAGE\" \"STATUS\"\necho \"=================================================================\"\n\n# Display results with status indicators\ntail -n +1 coverage_sorted.txt | while read percent file; do\n if [[ \"$file\" == \"TOTAL:\" ]]; then\n echo \"\"\n printf \"%-60s %8s%% %12s\\n\" \"OVERALL PROJECT COVERAGE\" \"$percent\" \"📊 TOTAL\"\n echo \"=================================================================\"\n continue\n fi\n \n filename=$(basename \"$file\")\n status=\"\"\n if (( $(echo \"$percent < 50\" | bc -l 2>/dev/null || echo \"0\") )); then\n status=\"🔴 CRITICAL\"\n elif (( $(echo \"$percent < 70\" | bc -l 2>/dev/null || echo \"0\") )); then\n status=\"🟡 LOW\"\n elif (( $(echo \"$percent < 90\" | bc -l 2>/dev/null || echo \"0\") )); then\n status=\"🟢 GOOD\"\n else\n status=\"✅ EXCELLENT\"\n fi\n \n printf \"%-60s %8s%% %12s\\n\" \"$filename\" \"$percent\" \"$status\"\ndone\n\necho \"\"\necho \"Top 10 files by uncovered statements:\"\necho \"=================================================\"\n# Calculate uncovered statements for each file and sort by that\ngo tool cover -func=coverage.out | grep -E \"\\.go:[0-9]+:\" | \\\nawk '{\n # Extract filename and percentage\n split($1, parts, \":\")\n file = parts[1]\n pct = $NF\n gsub(/%/, \"\", pct)\n \n # Track stats per file\n total[file]++\n covered[file] += pct\n}\nEND {\n for (file in total) {\n avg_pct = covered[file] / total[file]\n uncovered = total[file] * (100 - avg_pct) / 100\n covered_count = total[file] - uncovered\n printf \"%.0f %d %.0f %.1f %s\\n\", uncovered, total[file], covered_count, avg_pct, file\n }\n}' | sort -rn | head -10 | while read uncovered total covered pct file; do\n filename=$(basename \"$file\")\n printf \"%-60s %4d uncovered (%4d/%4d, %5.1f%%)\\n\" \"$filename\" \"$uncovered\" \"$covered\" \"$total\" \"$pct\"\ndone\n\necho \"\"\necho \"Coverage reports generated:\"\necho \"- HTML report: coverage.html (detailed line-by-line coverage)\"\necho \"- Sorted table: coverage_sorted.txt\"\necho \"- Use 'go tool cover -func=coverage.out' for function-level details\"\n" }, { "path": "scripts/devtools.sh", "content": "#!/bin/sh\nset -ex\ngo mod download golang.org/x/tools@latest\ncurl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.1.5\ncurl -sSfL https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.22.11" }, { "path": "scripts/extract-checksum.sh", "content": "#!/bin/bash\n\n# This script works with checksums_hashes_order and checksums to extract the relevant\n# sha of the various yq downloads. You can then use your favourite checksum tool to validate.\n# must match an entry in checksums_hashes_order.\n#\n# Usage: ./extract-checksum.sh \n# E.g: ./extract-checksum.sh SHA-256 yq_linux_amd64.tar.gz\n# Outputs: \n# yq_linux_amd64.tar.gz\tacebc9d07aa2d0e482969b2c080ee306e8f58efbd6f2d857eefbce6469da1473\n#\n# Usage with rhash:\n# ./extract-checksum.sh SHA-256 yq_linux_amd64.tar.gz | rhash -c -\n#\n# Tip, if you want the checksum first then the filename (e.g. for the md5sum command)\n# then you can pipe the output of this script into awk to switch the fields around:\n#\n# ./extract-checksum.sh MD5 yq_linux_amd64.tar.gz | awk '{ print $2 \" \" $1}' | md5sum -c -\n#\n#\n\nif [ \"$1\" == \"\" ]; then\n echo \"Please specify at a hash algorithm from the checksum_hashes_order\"\n echo \"Usage: $0 \"\n exit 1\nfi\n\nif [ \"$2\" != \"\" ]; then\n # so we don't match x.tar.gz when 'x' is given\n file=\"$2\\s\"\nelse \n file=\"\"\nfi\n\nif [ ! -f \"checksums_hashes_order\" ]; then\n echo \"This script requires checksums_hashes_order to run\"\n echo \"Download the file from https://github.com/mikefarah/yq/releases/ for the version of yq you are trying to validate\"\n exit 1\nfi\n\nif [ ! -f \"checksums\" ]; then\n echo \"This script requires the checksums file to run\"\n echo \"Download the file from https://github.com/mikefarah/yq/releases/ for the version of yq you are trying to validate\"\n exit 1\nfi\n\n\ngrepMatch=$(grep -m 1 -n \"$1\" checksums_hashes_order)\nif [ \"$grepMatch\" == \"\" ]; then\n echo \"Could not find hash algorithm '$1' in checksums_hashes_order\"\n exit 1\nfi\n\nset -e\n\nlineNumber=$(echo \"$grepMatch\" | cut -f1 -d:)\n\nrealLineNumber=\"$(($lineNumber + 1))\"\n\ngrep \"$file\" checksums | sed 's/ /\\t/g' | cut -f1,$realLineNumber" }, { "path": "scripts/format.sh", "content": "#!/bin/bash\n\ngofmt -w -s .\ngo mod tidy\ngo mod vendor\n" }, { "path": "scripts/generate-man-page-md.sh", "content": "#! /bin/bash\nset -e\n\n# note that this requires pandoc to be installed.\n\ncat ./pkg/yqlib/doc/operators/headers/Main.md > man.md\nprintf \"\\n# HOW IT WORKS\\n\" >> man.md\ntail -n +2 how-it-works.md >> man.md\n\nfor f in ./pkg/yqlib/doc/operators/*.md; do\n cat \"$f\" >> man.md\ndone\n\nfor f in ./pkg/yqlib/doc/usage/*.md; do\n cat \"$f\" >> man.md\ndone\n" }, { "path": "scripts/generate-man-page.sh", "content": "#! /bin/bash\nset -e\n\n# note that this requires pandoc to be installed.\n\npandoc \\\n --variable=title:\"YQ\" \\\n --variable=section:\"1\" \\\n --variable=author:\"Mike Farah\" \\\n --variable=header:\"${MAN_HEADER}\" \\\n --standalone --to man man.md -o yq.1" }, { "path": "scripts/install-man-page.sh", "content": "#!/bin/sh\n\nmy_path=\"$(command -v yq)\"\n\nif [ -z \"$my_path\" ]; then\n echo \"'yq' wasn't found in your PATH, so we don't know where to put the man pages.\"\n echo \"Please update your PATH to include yq, and run this script again.\"\n exit 1\nfi\n\n# ex: ~/.local/bin/yq => ~/.local/\nmy_prefix=\"$(dirname \"$(dirname \"$(command -v yq)\")\")\"\nmkdir -p \"$my_prefix/share/man/man1/\"\ncp yq.1 \"$my_prefix/share/man/man1/\"\n" }, { "path": "scripts/release-deb.sh", "content": "#!/bin/bash -eux\n#\n# Copyright (C) 2021 Roberto Mier Escandón \n#\n# This script creates a .deb package file with yq valid for ubuntu 20.04 by default\n# You can pass \n\nDOCKER_IMAGE_NAME=yq-deb-builder\nDOCKER_IMAGE_TAG=$(git describe --always --tags)\nOUTPUT=\nGOVERSION=\"1.17.4\"\nKEYID=\nMAINTAINER=\nDO_PUBLISH=\nPPA=\"rmescandon/yq\"\nVERSION=\nDISTRIBUTION=\nDO_SIGN=\nPASSPHRASE=\n\nshow_help() {\n echo \" usage: $(basename \"$0\") VERSION [options...]\"\n echo \"\"\n echo \" positional arguments\"\n echo \" VERSION\"\n echo \"\"\n echo \" optional arguments:\"\n echo \" -h, --help\t\t Shows this help\"\n echo \" -d, --distribution DISTRO The distribution to use for the changelog generation. If not provided, last changelog entry\"\n echo \" distribution is considered\"\n echo \" --goversion VERSION The version of Golang to use. Default to $GOVERSION\"\n echo \" -k, --sign-key KEYID Sign the package sources with the provided gpg key id (long format). When not provided this\"\n echo \" parameter, the generated sources are not signed\"\n echo \" -s, --sign Sign the package sources with a gpg key of the maintainer\"\n echo \" -m, --maintainer WHO The maintainer used as author of the changelog. git.name and git.email (see git config) is\"\n echo \" the considered format\"\n echo \" -o DIR, --output DIR The path where leaving the generated debian package. Default to a temporary folder if not set\"\n echo \" -p The resultant file is being published to ppa\"\n echo \" --ppa PPA Push resultant files to indicated ppa. This option should be given along with a signing key.\"\n echo \" Otherwise, the server could reject the package building. Default is set to 'rmescandon/yq'\"\n echo \" --passphrase PASSPHRASE Passphrase to decrypt the signage key\"\n exit 1\n}\n# read input args\nwhile [ $# -ne 0 ]; do\n case $1 in\n -h|--help)\n show_help\n ;;\n -d|--distribution)\n shift\n DISTRIBUTION=\"$1\"\n ;;\n --goversion)\n shift\n GOVERSION=\"$1\"\n ;;\n -k|--sign-key)\n shift\n DO_SIGN='y'\n KEYID=\"$1\"\n ;;\n -s|--sign)\n DO_SIGN='y'\n ;;\n -m|--maintainer)\n shift\n MAINTAINER=\"$1\"\n ;;\n -o|--output)\n shift\n OUTPUT=\"$1\"\n ;;\n -p)\n DO_PUBLISH=\"y\"\n ;;\n --ppa)\n shift\n DO_PUBLISH=\"y\"\n PPA=\"$1\"\n ;;\n --passphrase)\n shift\n PASSPHRASE=\"$1\"\n ;;\n *)\n if [ -z \"$VERSION\" ]; then\n VERSION=\"$1\"\n else\n show_help\n fi\n esac\n shift\ndone\n\n[ -n \"$VERSION\" ] || (echo \"error - you have to provide a version\" && show_help)\n\nif [ -n \"$OUTPUT\" ]; then\n OUTPUT=\"$(realpath \"$OUTPUT\")\"\n mkdir -p \"$OUTPUT\"\nelse\n # Temporary folder where leaving the built deb package in case that output folder is not provided\n OUTPUT=\"$(mktemp -d)\"\nfi \n\n# Define the folders with the source project and the build artifacts and files\nsrcdir=\"$(realpath \"$(dirname \"$0\")\"/..)\"\nblddir=\"$(cd \"${srcdir}\" && mkdir -p build && cd build && echo \"$(pwd)\")\"\n# clean on exit\ncleanup() {\n rm -f \"${blddir}/build.sh\" || true\n rm -f \"${blddir}/Dockerfile\" || true\n rm -f \"${blddir}/dput.cf\" || true\n docker rmi \"${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}\" -f > /dev/null 2>&1 || true\n}\ntrap cleanup EXIT INT\n\n# configure the dput config in case publishing is requested\nlp_id=\"$(echo \"$PPA\" | cut -d'/' -f1)\"\nppa_id=\"$(echo \"$PPA\" | cut -d'/' -f2)\"\ncat << EOF > ${blddir}/dput.cf\n[ppa]\nfqdn = ppa.launchpad.net\nmethod = ftp\nincoming = ~${lp_id}/ubuntu/${ppa_id}\nlogin = anonymous\nEOF\n\n# create the main script\ncat << EOF > ${blddir}/build.sh\n#!/bin/bash \nset -e -o pipefail\n\nPATH=$PATH:/usr/local/go/bin\nexport GPG_TTY=$(tty)\n\ngo mod vendor\n\n### bump debian/changelog\n\n# maintainer\nexport DEBEMAIL=\"$MAINTAINER\"\nif [ -z \"$MAINTAINER\" ]; then\n export DEBEMAIL=\"\\$(dpkg-parsechangelog -S Maintainer)\"\nfi\n\n# prepend a 'v' char to complete the tag name from where calculating the changelog\nSINCE=\"v\\$(dpkg-parsechangelog -S Version)\"\n\n# distribution\nDISTRIBUTION=\"$DISTRIBUTION\"\nif [ -z \"$DISTRIBUTION\" ]; then\n DISTRIBUTION=\"\\$(dpkg-parsechangelog -S Distribution)\"\nfi\n\n# generate changelog\ngbp dch --ignore-branch --no-multimaint -N \"$VERSION\" -s \"\\$SINCE\" -D \"\\$DISTRIBUTION\"\n\n# using -d to prevent failing when searching for Golang dep on control file\nparams=(\"-d\" \"-S\")\n\n# add the -sa option for signing along with the key to use when provided key id\nif [ -n \"$DO_SIGN\" ]; then\n params+=(\"-sa\")\n\n # read from gpg the key id associated with the maintainer if not provided explicitly\n if [ -z \"$KEYID\" ]; then\n KEYID=\"\\$(gpg --list-keys \"\\$(dpkg-parsechangelog -S Maintainer)\" | head -2 | tail -1 | xargs)\"\n else\n KEYID=\"$KEYID\"\n fi\n \n params+=(\"--sign-key=\"\\$KEYID\"\")\n\n if [ -n \"$PASSPHRASE\" ]; then\n gpg-agent --verbose --daemon --options /home/yq/.gnupg/gpg-agent.conf --log-file /tmp/gpg-agent.log --allow-preset-passphrase --default-cache-ttl=31536000\n KEYGRIP=\"\\$(gpg --with-keygrip -k \"\\$KEYID\" | grep 'Keygrip = ' | cut -d'=' -f2 | head -1 | xargs)\"\n /usr/lib/gnupg/gpg-preset-passphrase --preset --passphrase \"$PASSPHRASE\" \"\\$KEYGRIP\"\n fi\n\nelse\n params+=(\"-us\" \"-uc\")\nfi\n\ndebuild \\${params[@]}\nmv ../yq_* /home/yq/output\n\necho \"\"\necho -e \"\\tfind resulting package at: \"$OUTPUT\"\"\n\n# publish to ppa whether given\nif [ -n \"$DO_PUBLISH\" ]; then\n dput -c /etc/dput.cf ppa /home/yq/output/yq_*.changes\nfi\nEOF\nchmod +x \"${blddir}\"/build.sh\n\n# build the docker image with all dependencies\ncat << EOF > ${blddir}/Dockerfile\nFROM bitnami/minideb:bullseye as base\nENV LANG C.UTF-8\nENV LC_ALL C.UTF-8\nENV DEBIAN_FRONTEND noninteractive\nENV GO111MODULE on\nENV GOMODCACHE /home/yq/go\n\nRUN set -e \\\n && sed -i -- 's/# deb-src/deb-src/g' /etc/apt/sources.list \\\n && apt-get -qq update\n\n# install Golang on its $GOVERSION\nFROM base as golang\nRUN apt-get -qq -y --no-install-recommends install \\\n ca-certificates \\\n wget\nRUN wget \"https://golang.org/dl/go${GOVERSION}.linux-amd64.tar.gz\" -4\nRUN tar -C /usr/local -xvf \"go${GOVERSION}.linux-amd64.tar.gz\"\n\nFROM base\nRUN apt-get -qq -y --no-install-recommends install \\\n build-essential \\\n debhelper \\\n devscripts \\\n dput \\\n fakeroot \\\n git-buildpackage \\\n gpg-agent \\\n libdistro-info-perl \\\n pandoc \\\n rsync \\\n sensible-utils && \\\n apt-get clean && \\\n rm -rf /tmp/* /var/tmp/*\n\nCOPY --from=golang /usr/local/go /usr/local/go\n\n# build debian package as yq user\nRUN useradd -ms /bin/bash yq && \\\n mkdir /home/yq/src && chown -R yq: /home/yq/src && \\\n mkdir /home/yq/output && chown -R yq: /home/yq/output\n\nADD ./build/dput.cf /etc/dput.cf\nADD ./build/build.sh /usr/bin/build.sh\nRUN chmod +x /usr/bin/build.sh && chown -R yq: /usr/bin/build.sh\n\nUSER yq\n\nWORKDIR /home/yq/src\nVOLUME [\"/home/yq/src\"]\n\n# dir where output packages are finally left\nVOLUME [\"/home/yq/output\"]\n\nCMD [\"/usr/bin/build.sh\"]\nEOF\n\nDOCKER_BUILDKIT=1 docker build --pull -f \"${blddir}\"/Dockerfile -t \"${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}\" .\n\ndocker run --rm -i \\\n -v \"${srcdir}\":/home/yq/src:delegated \\\n -v \"${OUTPUT}\":/home/yq/output \\\n -v \"${HOME}\"/.gnupg:/home/yq/.gnupg:delegated \\\n -u \"$(id -u)\" \\\n \"${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}\"\n" }, { "path": "scripts/secure.sh", "content": "#!/bin/bash\n\nset -o errexit\nset -o pipefail\n\nOPTS=(\n -exclude-dir=vendor\n -exclude-dir=.gomodcache\n -exclude-dir=.gocache\n)\n\ncommand -v gosec &> /dev/null && BIN=gosec || BIN=./bin/gosec\n\"${BIN}\" \"${OPTS[@]}\" \"${PWD}\" ./...\n" }, { "path": "scripts/setup.sh", "content": "#!/bin/bash\n\nset -eu\n\nfind_mgr() {\n if hash minishift 2>/dev/null; then\n echo \"minishift\"\n else\n if hash docker-machine 2>/dev/null; then\n echo \"docker-machine\"\n fi\n fi\n}\n\nget_vm_name() {\n case \"$1\" in\n minishift)\n echo \"minishift\"\n ;;\n docker-machine)\n echo \"${DOCKER_MACHINE_NAME}\"\n ;;\n *)\n ;;\n esac\n}\n\nis_vm_running() {\n local vm=$1\n declare -a running=($(VBoxManage list runningvms | awk '{ print $1 }'))\n local result='false'\n\n for rvm in \"${running[@]}\"; do\n if [[ \"${rvm}\" == *\"${vm}\"* ]]; then\n result='true'\n fi\n done\n echo \"$result\"\n}\n\nif hash cygpath 2>/dev/null; then\n PROJECT_DIR=$(cygpath -w -a \"$(pwd)\")\nelse\n PROJECT_DIR=$(pwd)\nfi\n\nVM_MGR=$(find_mgr)\nif [[ -z $VM_MGR ]]; then\n echo \"ERROR: No VM Manager found; expected one of ['minishift', 'docker-machine']\"\n exit 1\nfi\n\nVM_NAME=$(get_vm_name \"$VM_MGR\")\nif [[ -z $VM_NAME ]]; then\n echo \"ERROR: No VM found; try running 'eval $(docker-machine env)'\"\n exit 1\nfi\n\nif ! hash VBoxManage 2>/dev/null; then\n echo \"VirtualBox executable 'VBoxManage' not found in path\"\n exit 1\nfi\n\navail=$(is_vm_running \"$VM_NAME\")\nif [[ \"$avail\" == *\"true\"* ]]; then\n res=$(VBoxManage sharedfolder add \"${VM_NAME}\" --name \"${PROJECT}\" --hostpath \"${PROJECT_DIR}\" --transient 2>&1)\n if [[ -z $res || $res == *\"already exists\"* ]]; then\n # no need to show that it already exists\n :\n else\n echo \"$res\"\n exit 1\n fi\n echo \"VM: [${VM_NAME}] -- Added Sharedfolder [${PROJECT}] @Path [${PROJECT_DIR}]\"\nelse\n echo \"$VM_NAME is not currently running; please start your VM and try again.\"\n exit 1\nfi\n\nSSH_CMD=\"sudo mkdir -p /${PROJECT} ; sudo mount -t vboxsf ${PROJECT} /${PROJECT}\"\ncase \"${VM_MGR}\" in\n minishift)\n minishift ssh \"${SSH_CMD}\"\n echo \"VM: [${VM_NAME}] -- Mounted Sharedfolder [${PROJECT}] @VM Path [/${PROJECT}]\"\n ;;\n docker-machine)\n docker-machine ssh \"${VM_NAME}\" \"${SSH_CMD}\"\n echo \"VM: [${VM_NAME}] -- Mounted Sharedfolder [${PROJECT}] @VM Path [/${PROJECT}]\"\n ;;\n *)\n ;;\nesac\n" }, { "path": "scripts/shunit2", "content": "#! /bin/sh\n# vim:et:ft=sh:sts=2:sw=2\n#\n# Copyright 2008-2020 Kate Ward. All Rights Reserved.\n# Released under the Apache 2.0 license.\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# shUnit2 -- Unit testing framework for Unix shell scripts.\n# https://github.com/kward/shunit2\n#\n# Author: kate.ward@forestent.com (Kate Ward)\n#\n# shUnit2 is a xUnit based unit test framework for Bourne shell scripts. It is\n# based on the popular JUnit unit testing framework for Java.\n#\n# $() are not fully portable (POSIX != portable).\n# shellcheck disable=SC2006\n# expr may be antiquated, but it is the only solution in some cases.\n# shellcheck disable=SC2003\n\n# Return if shunit2 already loaded.\ncommand [ -n \"${SHUNIT_VERSION:-}\" ] && exit 0\nSHUNIT_VERSION='2.1.8'\n\n# Return values that scripts can use.\nSHUNIT_TRUE=0\nSHUNIT_FALSE=1\nSHUNIT_ERROR=2\n\n# Logging functions.\n_shunit_warn() {\n ${__SHUNIT_CMD_ECHO_ESC} \\\n \"${__shunit_ansi_yellow}shunit2:WARN${__shunit_ansi_none} $*\" >&2\n}\n_shunit_error() {\n ${__SHUNIT_CMD_ECHO_ESC} \\\n \"${__shunit_ansi_red}shunit2:ERROR${__shunit_ansi_none} $*\" >&2\n}\n_shunit_fatal() {\n ${__SHUNIT_CMD_ECHO_ESC} \\\n \"${__shunit_ansi_red}shunit2:FATAL${__shunit_ansi_none} $*\" >&2\n exit ${SHUNIT_ERROR}\n}\n\n# Determine some reasonable command defaults.\n__SHUNIT_CMD_ECHO_ESC='echo -e'\n# shellcheck disable=SC2039\ncommand [ \"`echo -e test`\" = '-e test' ] && __SHUNIT_CMD_ECHO_ESC='echo'\n\n__SHUNIT_UNAME_S=`uname -s`\ncase \"${__SHUNIT_UNAME_S}\" in\n BSD) __SHUNIT_CMD_EXPR='gexpr' ;;\n *) __SHUNIT_CMD_EXPR='expr' ;;\nesac\n__SHUNIT_CMD_TPUT='tput'\n\n# Commands a user can override if needed.\nSHUNIT_CMD_EXPR=${SHUNIT_CMD_EXPR:-${__SHUNIT_CMD_EXPR}}\nSHUNIT_CMD_TPUT=${SHUNIT_CMD_TPUT:-${__SHUNIT_CMD_TPUT}}\n\n# Enable color output. Options are 'never', 'always', or 'auto'.\nSHUNIT_COLOR=${SHUNIT_COLOR:-auto}\n\n# Specific shell checks.\nif command [ -n \"${ZSH_VERSION:-}\" ]; then\n setopt |grep \"^shwordsplit$\" >/dev/null\n if command [ $? -ne ${SHUNIT_TRUE} ]; then\n _shunit_fatal 'zsh shwordsplit option is required for proper operation'\n fi\n if command [ -z \"${SHUNIT_PARENT:-}\" ]; then\n _shunit_fatal \"zsh does not pass \\$0 through properly. please declare \\\n\\\"SHUNIT_PARENT=\\$0\\\" before calling shUnit2\"\n fi\nfi\n\n#\n# Constants\n#\n\n__SHUNIT_MODE_SOURCED='sourced'\n__SHUNIT_MODE_STANDALONE='standalone'\n__SHUNIT_PARENT=${SHUNIT_PARENT:-$0}\n\n# User provided test prefix to display in front of the name of the test being\n# executed. Define by setting the SHUNIT_TEST_PREFIX variable.\n__SHUNIT_TEST_PREFIX=${SHUNIT_TEST_PREFIX:-}\n\n# ANSI colors.\n__SHUNIT_ANSI_NONE='\\033[0m'\n__SHUNIT_ANSI_RED='\\033[1;31m'\n__SHUNIT_ANSI_GREEN='\\033[1;32m'\n__SHUNIT_ANSI_YELLOW='\\033[1;33m'\n__SHUNIT_ANSI_CYAN='\\033[1;36m'\n\n# Set the constants readonly.\n__shunit_constants=`set |grep '^__SHUNIT_' |cut -d= -f1`\necho \"${__shunit_constants}\" |grep '^Binary file' >/dev/null && \\\n __shunit_constants=`set |grep -a '^__SHUNIT_' |cut -d= -f1`\nfor __shunit_const in ${__shunit_constants}; do\n if command [ -z \"${ZSH_VERSION:-}\" ]; then\n readonly \"${__shunit_const}\"\n else\n case ${ZSH_VERSION} in\n [123].*) readonly \"${__shunit_const}\" ;;\n *) readonly -g \"${__shunit_const}\" # Declare readonly constants globally.\n esac\n fi\ndone\nunset __shunit_const __shunit_constants\n\n#\n# Internal variables.\n#\n\n# Variables.\n__shunit_lineno='' # Line number of executed test.\n__shunit_mode=${__SHUNIT_MODE_SOURCED} # Operating mode.\n__shunit_reportGenerated=${SHUNIT_FALSE} # Is report generated.\n__shunit_script='' # Filename of unittest script (standalone mode).\n__shunit_skip=${SHUNIT_FALSE} # Is skipping enabled.\n__shunit_suite='' # Suite of tests to execute.\n__shunit_clean=${SHUNIT_FALSE} # _shunit_cleanup() was already called.\n\n# ANSI colors (populated by _shunit_configureColor()).\n__shunit_ansi_none=''\n__shunit_ansi_red=''\n__shunit_ansi_green=''\n__shunit_ansi_yellow=''\n__shunit_ansi_cyan=''\n\n# Counts of tests.\n__shunit_testSuccess=${SHUNIT_TRUE}\n__shunit_testsTotal=0\n__shunit_testsPassed=0\n__shunit_testsFailed=0\n\n# Counts of asserts.\n__shunit_assertsTotal=0\n__shunit_assertsPassed=0\n__shunit_assertsFailed=0\n__shunit_assertsSkipped=0\n\n#\n# Macros.\n#\n\n# shellcheck disable=SC2016,SC2089\n_SHUNIT_LINENO_='eval __shunit_lineno=\"\"; if command [ \"${1:-}\" = \"--lineno\" ]; then command [ -n \"$2\" ] && __shunit_lineno=\"[$2] \"; shift 2; fi'\n\n#-----------------------------------------------------------------------------\n# Assertion functions.\n#\n\n# Assert that two values are equal to one another.\n#\n# Args:\n# message: string: failure message [optional]\n# expected: string: expected value\n# actual: string: actual value\n# Returns:\n# integer: success (TRUE/FALSE/ERROR constant)\nassertEquals() {\n # shellcheck disable=SC2090\n ${_SHUNIT_LINENO_}\n if command [ $# -lt 2 -o $# -gt 3 ]; then\n _shunit_error \"assertEquals() requires two or three arguments; $# given\"\n _shunit_assertFail\n return ${SHUNIT_ERROR}\n fi\n _shunit_shouldSkip && return ${SHUNIT_TRUE}\n\n shunit_message_=${__shunit_lineno}\n if command [ $# -eq 3 ]; then\n shunit_message_=\"${shunit_message_}$1\"\n shift\n fi\n shunit_expected_=$1\n shunit_actual_=$2\n\n shunit_return=${SHUNIT_TRUE}\n if command [ \"${shunit_expected_}\" = \"${shunit_actual_}\" ]; then\n _shunit_assertPass\n else\n failNotEquals \"${shunit_message_}\" \"${shunit_expected_}\" \"${shunit_actual_}\"\n shunit_return=${SHUNIT_FALSE}\n fi\n\n unset shunit_message_ shunit_expected_ shunit_actual_\n return ${shunit_return}\n}\n# shellcheck disable=SC2016,SC2034\n_ASSERT_EQUALS_='eval assertEquals --lineno \"${LINENO:-}\"'\n\n# Assert that two values are not equal to one another.\n#\n# Args:\n# message: string: failure message [optional]\n# expected: string: expected value\n# actual: string: actual value\n# Returns:\n# integer: success (TRUE/FALSE/ERROR constant)\nassertNotEquals() {\n # shellcheck disable=SC2090\n ${_SHUNIT_LINENO_}\n if command [ $# -lt 2 -o $# -gt 3 ]; then\n _shunit_error \"assertNotEquals() requires two or three arguments; $# given\"\n _shunit_assertFail\n return ${SHUNIT_ERROR}\n fi\n _shunit_shouldSkip && return ${SHUNIT_TRUE}\n\n shunit_message_=${__shunit_lineno}\n if command [ $# -eq 3 ]; then\n shunit_message_=\"${shunit_message_}$1\"\n shift\n fi\n shunit_expected_=$1\n shunit_actual_=$2\n\n shunit_return=${SHUNIT_TRUE}\n if command [ \"${shunit_expected_}\" != \"${shunit_actual_}\" ]; then\n _shunit_assertPass\n else\n failSame \"${shunit_message_}\" \"${shunit_expected_}\" \"${shunit_actual_}\"\n shunit_return=${SHUNIT_FALSE}\n fi\n\n unset shunit_message_ shunit_expected_ shunit_actual_\n return ${shunit_return}\n}\n# shellcheck disable=SC2016,SC2034\n_ASSERT_NOT_EQUALS_='eval assertNotEquals --lineno \"${LINENO:-}\"'\n\n# Assert that a container contains a content.\n#\n# Args:\n# message: string: failure message [optional]\n# container: string: container to analyze\n# content: string: content to find\n# Returns:\n# integer: success (TRUE/FALSE/ERROR constant)\nassertContains() {\n # shellcheck disable=SC2090\n ${_SHUNIT_LINENO_}\n if command [ $# -lt 2 -o $# -gt 3 ]; then\n _shunit_error \"assertContains() requires two or three arguments; $# given\"\n _shunit_assertFail\n return ${SHUNIT_ERROR}\n fi\n _shunit_shouldSkip && return ${SHUNIT_TRUE}\n\n shunit_message_=${__shunit_lineno}\n if command [ $# -eq 3 ]; then\n shunit_message_=\"${shunit_message_}$1\"\n shift\n fi\n shunit_container_=$1\n shunit_content_=$2\n\n shunit_return=${SHUNIT_TRUE}\n if echo \"$shunit_container_\" | grep -F -- \"$shunit_content_\" > /dev/null; then\n _shunit_assertPass\n else\n failNotFound \"${shunit_message_}\" \"${shunit_content_}\"\n shunit_return=${SHUNIT_FALSE}\n fi\n\n unset shunit_message_ shunit_container_ shunit_content_\n return ${shunit_return}\n}\n# shellcheck disable=SC2016,SC2034\n_ASSERT_CONTAINS_='eval assertContains --lineno \"${LINENO:-}\"'\n\n# Assert that a container does not contain a content.\n#\n# Args:\n# message: string: failure message [optional]\n# container: string: container to analyze\n# content: string: content to look for\n# Returns:\n# integer: success (TRUE/FALSE/ERROR constant)\nassertNotContains() {\n # shellcheck disable=SC2090\n ${_SHUNIT_LINENO_}\n if command [ $# -lt 2 -o $# -gt 3 ]; then\n _shunit_error \"assertNotContains() requires two or three arguments; $# given\"\n _shunit_assertFail\n return ${SHUNIT_ERROR}\n fi\n _shunit_shouldSkip && return ${SHUNIT_TRUE}\n\n shunit_message_=${__shunit_lineno}\n if command [ $# -eq 3 ]; then\n shunit_message_=\"${shunit_message_}$1\"\n shift\n fi\n shunit_container_=$1\n shunit_content_=$2\n\n shunit_return=${SHUNIT_TRUE}\n if echo \"$shunit_container_\" | grep -F -- \"$shunit_content_\" > /dev/null; then\n failFound \"${shunit_message_}\" \"${shunit_content_}\"\n shunit_return=${SHUNIT_FALSE}\n else\n _shunit_assertPass\n fi\n\n unset shunit_message_ shunit_container_ shunit_content_\n return ${shunit_return}\n}\n# shellcheck disable=SC2016,SC2034\n_ASSERT_NOT_CONTAINS_='eval assertNotContains --lineno \"${LINENO:-}\"'\n\n# Assert that a value is null (i.e. an empty string)\n#\n# Args:\n# message: string: failure message [optional]\n# actual: string: actual value\n# Returns:\n# integer: success (TRUE/FALSE/ERROR constant)\nassertNull() {\n # shellcheck disable=SC2090\n ${_SHUNIT_LINENO_}\n if command [ $# -lt 1 -o $# -gt 2 ]; then\n _shunit_error \"assertNull() requires one or two arguments; $# given\"\n _shunit_assertFail\n return ${SHUNIT_ERROR}\n fi\n _shunit_shouldSkip && return ${SHUNIT_TRUE}\n\n shunit_message_=${__shunit_lineno}\n if command [ $# -eq 2 ]; then\n shunit_message_=\"${shunit_message_}$1\"\n shift\n fi\n assertTrue \"${shunit_message_}\" \"[ -z '$1' ]\"\n shunit_return=$?\n\n unset shunit_message_\n return ${shunit_return}\n}\n# shellcheck disable=SC2016,SC2034\n_ASSERT_NULL_='eval assertNull --lineno \"${LINENO:-}\"'\n\n# Assert that a value is not null (i.e. a non-empty string)\n#\n# Args:\n# message: string: failure message [optional]\n# actual: string: actual value\n# Returns:\n# integer: success (TRUE/FALSE/ERROR constant)\nassertNotNull() {\n # shellcheck disable=SC2090\n ${_SHUNIT_LINENO_}\n if command [ $# -gt 2 ]; then # allowing 0 arguments as $1 might actually be null\n _shunit_error \"assertNotNull() requires one or two arguments; $# given\"\n _shunit_assertFail\n return ${SHUNIT_ERROR}\n fi\n _shunit_shouldSkip && return ${SHUNIT_TRUE}\n\n shunit_message_=${__shunit_lineno}\n if command [ $# -eq 2 ]; then\n shunit_message_=\"${shunit_message_}$1\"\n shift\n fi\n shunit_actual_=`_shunit_escapeCharactersInString \"${1:-}\"`\n test -n \"${shunit_actual_}\"\n assertTrue \"${shunit_message_}\" $?\n shunit_return=$?\n\n unset shunit_actual_ shunit_message_\n return ${shunit_return}\n}\n# shellcheck disable=SC2016,SC2034\n_ASSERT_NOT_NULL_='eval assertNotNull --lineno \"${LINENO:-}\"'\n\n# Assert that two values are the same (i.e. equal to one another).\n#\n# Args:\n# message: string: failure message [optional]\n# expected: string: expected value\n# actual: string: actual value\n# Returns:\n# integer: success (TRUE/FALSE/ERROR constant)\nassertSame() {\n # shellcheck disable=SC2090\n ${_SHUNIT_LINENO_}\n if command [ $# -lt 2 -o $# -gt 3 ]; then\n _shunit_error \"assertSame() requires two or three arguments; $# given\"\n _shunit_assertFail\n return ${SHUNIT_ERROR}\n fi\n _shunit_shouldSkip && return ${SHUNIT_TRUE}\n\n shunit_message_=${__shunit_lineno}\n if command [ $# -eq 3 ]; then\n shunit_message_=\"${shunit_message_}$1\"\n shift\n fi\n assertEquals \"${shunit_message_}\" \"$1\" \"$2\"\n shunit_return=$?\n\n unset shunit_message_\n return ${shunit_return}\n}\n# shellcheck disable=SC2016,SC2034\n_ASSERT_SAME_='eval assertSame --lineno \"${LINENO:-}\"'\n\n# Assert that two values are not the same (i.e. not equal to one another).\n#\n# Args:\n# message: string: failure message [optional]\n# expected: string: expected value\n# actual: string: actual value\n# Returns:\n# integer: success (TRUE/FALSE/ERROR constant)\nassertNotSame() {\n # shellcheck disable=SC2090\n ${_SHUNIT_LINENO_}\n if command [ $# -lt 2 -o $# -gt 3 ]; then\n _shunit_error \"assertNotSame() requires two or three arguments; $# given\"\n _shunit_assertFail\n return ${SHUNIT_ERROR}\n fi\n _shunit_shouldSkip && return ${SHUNIT_TRUE}\n\n shunit_message_=${__shunit_lineno}\n if command [ $# -eq 3 ]; then\n shunit_message_=\"${shunit_message_:-}$1\"\n shift\n fi\n assertNotEquals \"${shunit_message_}\" \"$1\" \"$2\"\n shunit_return=$?\n\n unset shunit_message_\n return ${shunit_return}\n}\n# shellcheck disable=SC2016,SC2034\n_ASSERT_NOT_SAME_='eval assertNotSame --lineno \"${LINENO:-}\"'\n\n# Assert that a value or shell test condition is true.\n#\n# In shell, a value of 0 is true and a non-zero value is false. Any integer\n# value passed can thereby be tested.\n#\n# Shell supports much more complicated tests though, and a means to support\n# them was needed. As such, this function tests that conditions are true or\n# false through evaluation rather than just looking for a true or false.\n#\n# The following test will succeed:\n# assertTrue 0\n# assertTrue \"[ 34 -gt 23 ]\"\n# The following test will fail with a message:\n# assertTrue 123\n# assertTrue \"test failed\" \"[ -r '/non/existent/file' ]\"\n#\n# Args:\n# message: string: failure message [optional]\n# condition: string: integer value or shell conditional statement\n# Returns:\n# integer: success (TRUE/FALSE/ERROR constant)\nassertTrue() {\n # shellcheck disable=SC2090\n ${_SHUNIT_LINENO_}\n if command [ $# -lt 1 -o $# -gt 2 ]; then\n _shunit_error \"assertTrue() takes one or two arguments; $# given\"\n _shunit_assertFail\n return ${SHUNIT_ERROR}\n fi\n _shunit_shouldSkip && return ${SHUNIT_TRUE}\n\n shunit_message_=${__shunit_lineno}\n if command [ $# -eq 2 ]; then\n shunit_message_=\"${shunit_message_}$1\"\n shift\n fi\n shunit_condition_=$1\n\n # See if condition is an integer, i.e. a return value.\n shunit_match_=`expr \"${shunit_condition_}\" : '\\([0-9]*\\)'`\n shunit_return=${SHUNIT_TRUE}\n if command [ -z \"${shunit_condition_}\" ]; then\n # Null condition.\n shunit_return=${SHUNIT_FALSE}\n elif command [ -n \"${shunit_match_}\" -a \"${shunit_condition_}\" = \"${shunit_match_}\" ]\n then\n # Possible return value. Treating 0 as true, and non-zero as false.\n command [ \"${shunit_condition_}\" -ne 0 ] && shunit_return=${SHUNIT_FALSE}\n else\n # Hopefully... a condition.\n ( eval \"${shunit_condition_}\" ) >/dev/null 2>&1\n command [ $? -ne 0 ] && shunit_return=${SHUNIT_FALSE}\n fi\n\n # Record the test.\n if command [ ${shunit_return} -eq ${SHUNIT_TRUE} ]; then\n _shunit_assertPass\n else\n _shunit_assertFail \"${shunit_message_}\"\n fi\n\n unset shunit_message_ shunit_condition_ shunit_match_\n return ${shunit_return}\n}\n# shellcheck disable=SC2016,SC2034\n_ASSERT_TRUE_='eval assertTrue --lineno \"${LINENO:-}\"'\n\n# Assert that a value or shell test condition is false.\n#\n# In shell, a value of 0 is true and a non-zero value is false. Any integer\n# value passed can thereby be tested.\n#\n# Shell supports much more complicated tests though, and a means to support\n# them was needed. As such, this function tests that conditions are true or\n# false through evaluation rather than just looking for a true or false.\n#\n# The following test will succeed:\n# assertFalse 1\n# assertFalse \"[ 'apples' = 'oranges' ]\"\n# The following test will fail with a message:\n# assertFalse 0\n# assertFalse \"test failed\" \"[ 1 -eq 1 -a 2 -eq 2 ]\"\n#\n# Args:\n# message: string: failure message [optional]\n# condition: string: integer value or shell conditional statement\n# Returns:\n# integer: success (TRUE/FALSE/ERROR constant)\nassertFalse() {\n # shellcheck disable=SC2090\n ${_SHUNIT_LINENO_}\n if command [ $# -lt 1 -o $# -gt 2 ]; then\n _shunit_error \"assertFalse() requires one or two arguments; $# given\"\n _shunit_assertFail\n return ${SHUNIT_ERROR}\n fi\n _shunit_shouldSkip && return ${SHUNIT_TRUE}\n\n shunit_message_=${__shunit_lineno}\n if command [ $# -eq 2 ]; then\n shunit_message_=\"${shunit_message_}$1\"\n shift\n fi\n shunit_condition_=$1\n\n # See if condition is an integer, i.e. a return value.\n shunit_match_=`expr \"${shunit_condition_}\" : '\\([0-9]*\\)'`\n shunit_return=${SHUNIT_TRUE}\n if command [ -z \"${shunit_condition_}\" ]; then\n # Null condition.\n shunit_return=${SHUNIT_FALSE}\n elif command [ -n \"${shunit_match_}\" -a \"${shunit_condition_}\" = \"${shunit_match_}\" ]\n then\n # Possible return value. Treating 0 as true, and non-zero as false.\n command [ \"${shunit_condition_}\" -eq 0 ] && shunit_return=${SHUNIT_FALSE}\n else\n # Hopefully... a condition.\n ( eval \"${shunit_condition_}\" ) >/dev/null 2>&1\n command [ $? -eq 0 ] && shunit_return=${SHUNIT_FALSE}\n fi\n\n # Record the test.\n if command [ \"${shunit_return}\" -eq \"${SHUNIT_TRUE}\" ]; then\n _shunit_assertPass\n else\n _shunit_assertFail \"${shunit_message_}\"\n fi\n\n unset shunit_message_ shunit_condition_ shunit_match_\n return \"${shunit_return}\"\n}\n# shellcheck disable=SC2016,SC2034\n_ASSERT_FALSE_='eval assertFalse --lineno \"${LINENO:-}\"'\n\n#-----------------------------------------------------------------------------\n# Failure functions.\n#\n\n# Records a test failure.\n#\n# Args:\n# message: string: failure message [optional]\n# Returns:\n# integer: success (TRUE/FALSE/ERROR constant)\nfail() {\n # shellcheck disable=SC2090\n ${_SHUNIT_LINENO_}\n if command [ $# -gt 1 ]; then\n _shunit_error \"fail() requires zero or one arguments; $# given\"\n return ${SHUNIT_ERROR}\n fi\n _shunit_shouldSkip && return ${SHUNIT_TRUE}\n\n shunit_message_=${__shunit_lineno}\n if command [ $# -eq 1 ]; then\n shunit_message_=\"${shunit_message_}$1\"\n shift\n fi\n\n _shunit_assertFail \"${shunit_message_}\"\n\n unset shunit_message_\n return ${SHUNIT_FALSE}\n}\n# shellcheck disable=SC2016,SC2034\n_FAIL_='eval fail --lineno \"${LINENO:-}\"'\n\n# Records a test failure, stating two values were not equal.\n#\n# Args:\n# message: string: failure message [optional]\n# expected: string: expected value\n# actual: string: actual value\n# Returns:\n# integer: success (TRUE/FALSE/ERROR constant)\nfailNotEquals() {\n # shellcheck disable=SC2090\n ${_SHUNIT_LINENO_}\n if command [ $# -lt 2 -o $# -gt 3 ]; then\n _shunit_error \"failNotEquals() requires one or two arguments; $# given\"\n return ${SHUNIT_ERROR}\n fi\n _shunit_shouldSkip && return ${SHUNIT_TRUE}\n\n shunit_message_=${__shunit_lineno}\n if command [ $# -eq 3 ]; then\n shunit_message_=\"${shunit_message_}$1\"\n shift\n fi\n shunit_expected_=$1\n shunit_actual_=$2\n\n shunit_message_=${shunit_message_%% }\n _shunit_assertFail \"${shunit_message_:+${shunit_message_} }expected:<${shunit_expected_}> but was:<${shunit_actual_}>\"\n\n unset shunit_message_ shunit_expected_ shunit_actual_\n return ${SHUNIT_FALSE}\n}\n# shellcheck disable=SC2016,SC2034\n_FAIL_NOT_EQUALS_='eval failNotEquals --lineno \"${LINENO:-}\"'\n\n# Records a test failure, stating a value was found.\n#\n# Args:\n# message: string: failure message [optional]\n# content: string: found value\n# Returns:\n# integer: success (TRUE/FALSE/ERROR constant)\nfailFound() {\n # shellcheck disable=SC2090\n ${_SHUNIT_LINENO_}\n if command [ $# -lt 1 -o $# -gt 2 ]; then\n _shunit_error \"failFound() requires one or two arguments; $# given\"\n return ${SHUNIT_ERROR}\n fi\n _shunit_shouldSkip && return ${SHUNIT_TRUE}\n\n shunit_message_=${__shunit_lineno}\n if command [ $# -eq 2 ]; then\n shunit_message_=\"${shunit_message_}$1\"\n shift\n fi\n\n shunit_message_=${shunit_message_%% }\n _shunit_assertFail \"${shunit_message_:+${shunit_message_} }Found\"\n\n unset shunit_message_\n return ${SHUNIT_FALSE}\n}\n# shellcheck disable=SC2016,SC2034\n_FAIL_FOUND_='eval failFound --lineno \"${LINENO:-}\"'\n\n# Records a test failure, stating a content was not found.\n#\n# Args:\n# message: string: failure message [optional]\n# content: string: content not found\n# Returns:\n# integer: success (TRUE/FALSE/ERROR constant)\nfailNotFound() {\n # shellcheck disable=SC2090\n ${_SHUNIT_LINENO_}\n if command [ $# -lt 1 -o $# -gt 2 ]; then\n _shunit_error \"failNotFound() requires one or two arguments; $# given\"\n return ${SHUNIT_ERROR}\n fi\n _shunit_shouldSkip && return ${SHUNIT_TRUE}\n\n shunit_message_=${__shunit_lineno}\n if command [ $# -eq 2 ]; then\n shunit_message_=\"${shunit_message_}$1\"\n shift\n fi\n shunit_content_=$1\n\n shunit_message_=${shunit_message_%% }\n _shunit_assertFail \"${shunit_message_:+${shunit_message_} }Not found:<${shunit_content_}>\"\n\n unset shunit_message_ shunit_content_\n return ${SHUNIT_FALSE}\n}\n# shellcheck disable=SC2016,SC2034\n_FAIL_NOT_FOUND_='eval failNotFound --lineno \"${LINENO:-}\"'\n\n# Records a test failure, stating two values should have been the same.\n#\n# Args:\n# message: string: failure message [optional]\n# expected: string: expected value\n# actual: string: actual value\n# Returns:\n# integer: success (TRUE/FALSE/ERROR constant)\nfailSame()\n{\n # shellcheck disable=SC2090\n ${_SHUNIT_LINENO_}\n if command [ $# -lt 2 -o $# -gt 3 ]; then\n _shunit_error \"failSame() requires two or three arguments; $# given\"\n return ${SHUNIT_ERROR}\n fi\n _shunit_shouldSkip && return ${SHUNIT_TRUE}\n\n shunit_message_=${__shunit_lineno}\n if command [ $# -eq 3 ]; then\n shunit_message_=\"${shunit_message_}$1\"\n shift\n fi\n\n shunit_message_=${shunit_message_%% }\n _shunit_assertFail \"${shunit_message_:+${shunit_message_} }expected not same\"\n\n unset shunit_message_\n return ${SHUNIT_FALSE}\n}\n# shellcheck disable=SC2016,SC2034\n_FAIL_SAME_='eval failSame --lineno \"${LINENO:-}\"'\n\n# Records a test failure, stating two values were not equal.\n#\n# This is functionally equivalent to calling failNotEquals().\n#\n# Args:\n# message: string: failure message [optional]\n# expected: string: expected value\n# actual: string: actual value\n# Returns:\n# integer: success (TRUE/FALSE/ERROR constant)\nfailNotSame() {\n # shellcheck disable=SC2090\n ${_SHUNIT_LINENO_}\n if command [ $# -lt 2 -o $# -gt 3 ]; then\n _shunit_error \"failNotSame() requires one or two arguments; $# given\"\n return ${SHUNIT_ERROR}\n fi\n _shunit_shouldSkip && return ${SHUNIT_TRUE}\n\n shunit_message_=${__shunit_lineno}\n if command [ $# -eq 3 ]; then\n shunit_message_=\"${shunit_message_}$1\"\n shift\n fi\n failNotEquals \"${shunit_message_}\" \"$1\" \"$2\"\n shunit_return=$?\n\n unset shunit_message_\n return ${shunit_return}\n}\n# shellcheck disable=SC2016,SC2034\n_FAIL_NOT_SAME_='eval failNotSame --lineno \"${LINENO:-}\"'\n\n#-----------------------------------------------------------------------------\n# Skipping functions.\n#\n\n# Force remaining assert and fail functions to be \"skipped\".\n#\n# This function forces the remaining assert and fail functions to be \"skipped\",\n# i.e. they will have no effect. Each function skipped will be recorded so that\n# the total of asserts and fails will not be altered.\n#\n# Args:\n# None\nstartSkipping() { __shunit_skip=${SHUNIT_TRUE}; }\n\n# Resume the normal recording behaviour of assert and fail calls.\n#\n# Args:\n# None\nendSkipping() { __shunit_skip=${SHUNIT_FALSE}; }\n\n# Returns the state of assert and fail call skipping.\n#\n# Args:\n# None\n# Returns:\n# boolean: (TRUE/FALSE constant)\nisSkipping() { return ${__shunit_skip}; }\n\n#-----------------------------------------------------------------------------\n# Suite functions.\n#\n\n# Stub. This function should contains all unit test calls to be made.\n#\n# DEPRECATED (as of 2.1.0)\n#\n# This function can be optionally overridden by the user in their test suite.\n#\n# If this function exists, it will be called when shunit2 is sourced. If it\n# does not exist, shunit2 will search the parent script for all functions\n# beginning with the word 'test', and they will be added dynamically to the\n# test suite.\n#\n# This function should be overridden by the user in their unit test suite.\n# Note: see _shunit_mktempFunc() for actual implementation\n#\n# Args:\n# None\n#suite() { :; } # DO NOT UNCOMMENT THIS FUNCTION\n\n# Adds a function name to the list of tests schedule for execution.\n#\n# This function should only be called from within the suite() function.\n#\n# Args:\n# function: string: name of a function to add to current unit test suite\nsuite_addTest() {\n shunit_func_=${1:-}\n\n __shunit_suite=\"${__shunit_suite:+${__shunit_suite} }${shunit_func_}\"\n __shunit_testsTotal=`expr ${__shunit_testsTotal} + 1`\n\n unset shunit_func_\n}\n\n# Stub. This function will be called once before any tests are run.\n#\n# Common one-time environment preparation tasks shared by all tests can be\n# defined here.\n#\n# This function should be overridden by the user in their unit test suite.\n# Note: see _shunit_mktempFunc() for actual implementation\n#\n# Args:\n# None\n#oneTimeSetUp() { :; } # DO NOT UNCOMMENT THIS FUNCTION\n\n# Stub. This function will be called once after all tests are finished.\n#\n# Common one-time environment cleanup tasks shared by all tests can be defined\n# here.\n#\n# This function should be overridden by the user in their unit test suite.\n# Note: see _shunit_mktempFunc() for actual implementation\n#\n# Args:\n# None\n#oneTimeTearDown() { :; } # DO NOT UNCOMMENT THIS FUNCTION\n\n# Stub. This function will be called before each test is run.\n#\n# Common environment preparation tasks shared by all tests can be defined here.\n#\n# This function should be overridden by the user in their unit test suite.\n# Note: see _shunit_mktempFunc() for actual implementation\n#\n# Args:\n# None\n#setUp() { :; } # DO NOT UNCOMMENT THIS FUNCTION\n\n# Note: see _shunit_mktempFunc() for actual implementation\n# Stub. This function will be called after each test is run.\n#\n# Common environment cleanup tasks shared by all tests can be defined here.\n#\n# This function should be overridden by the user in their unit test suite.\n# Note: see _shunit_mktempFunc() for actual implementation\n#\n# Args:\n# None\n#tearDown() { :; } # DO NOT UNCOMMENT THIS FUNCTION\n\n#------------------------------------------------------------------------------\n# Internal shUnit2 functions.\n#\n\n# Create a temporary directory to store various run-time files in.\n#\n# This function is a cross-platform temporary directory creation tool. Not all\n# OSes have the `mktemp` function, so one is included here.\n#\n# Args:\n# None\n# Outputs:\n# string: the temporary directory that was created\n_shunit_mktempDir() {\n # Try the standard `mktemp` function.\n ( exec mktemp -dqt shunit.XXXXXX 2>/dev/null ) && return\n\n # The standard `mktemp` didn't work. Use our own.\n # shellcheck disable=SC2039\n if command [ -r '/dev/urandom' -a -x '/usr/bin/od' ]; then\n _shunit_random_=`/usr/bin/od -vAn -N4 -tx4 \"${_shunit_file_}\"\n#! /bin/sh\nexit ${SHUNIT_TRUE}\nEOF\n command chmod +x \"${_shunit_file_}\"\n done\n\n unset _shunit_file_\n}\n\n# Final cleanup function to leave things as we found them.\n#\n# Besides removing the temporary directory, this function is in charge of the\n# final exit code of the unit test. The exit code is based on how the script\n# was ended (e.g. normal exit, or via Ctrl-C).\n#\n# Args:\n# name: string: name of the trap called (specified when trap defined)\n_shunit_cleanup() {\n _shunit_name_=$1\n\n case \"${_shunit_name_}\" in\n EXIT) ;;\n INT) _shunit_signal_=130 ;; # 2+128\n TERM) _shunit_signal_=143 ;; # 15+128\n *)\n _shunit_error \"unrecognized trap value (${_shunit_name_})\"\n _shunit_signal_=0\n ;;\n esac\n if command [ \"${_shunit_name_}\" != 'EXIT' ]; then\n _shunit_warn \"trapped and now handling the (${_shunit_name_}) signal\"\n fi\n\n # Do our work.\n if command [ ${__shunit_clean} -eq ${SHUNIT_FALSE} ]; then\n # Ensure tear downs are only called once.\n __shunit_clean=${SHUNIT_TRUE}\n\n tearDown\n command [ $? -eq ${SHUNIT_TRUE} ] \\\n || _shunit_warn \"tearDown() returned non-zero return code.\"\n oneTimeTearDown\n command [ $? -eq ${SHUNIT_TRUE} ] \\\n || _shunit_warn \"oneTimeTearDown() returned non-zero return code.\"\n\n command rm -fr \"${__shunit_tmpDir}\"\n fi\n\n if command [ \"${_shunit_name_}\" != 'EXIT' ]; then\n # Handle all non-EXIT signals.\n trap - 0 # Disable EXIT trap.\n exit ${_shunit_signal_}\n elif command [ ${__shunit_reportGenerated} -eq ${SHUNIT_FALSE} ]; then\n _shunit_assertFail 'unknown failure encountered running a test'\n _shunit_generateReport\n exit ${SHUNIT_ERROR}\n fi\n\n unset _shunit_name_ _shunit_signal_\n}\n\n# configureColor based on user color preference.\n#\n# Args:\n# color: string: color mode (one of `always`, `auto`, or `none`).\n_shunit_configureColor() {\n _shunit_color_=${SHUNIT_FALSE} # By default, no color.\n case $1 in\n 'always') _shunit_color_=${SHUNIT_TRUE} ;;\n 'auto')\n command [ \"`_shunit_colors`\" -ge 8 ] && _shunit_color_=${SHUNIT_TRUE}\n ;;\n 'none') ;;\n *) _shunit_fatal \"unrecognized color option '$1'\" ;;\n esac\n\n case ${_shunit_color_} in\n ${SHUNIT_TRUE})\n __shunit_ansi_none=${__SHUNIT_ANSI_NONE}\n __shunit_ansi_red=${__SHUNIT_ANSI_RED}\n __shunit_ansi_green=${__SHUNIT_ANSI_GREEN}\n __shunit_ansi_yellow=${__SHUNIT_ANSI_YELLOW}\n __shunit_ansi_cyan=${__SHUNIT_ANSI_CYAN}\n ;;\n ${SHUNIT_FALSE})\n __shunit_ansi_none=''\n __shunit_ansi_red=''\n __shunit_ansi_green=''\n __shunit_ansi_yellow=''\n __shunit_ansi_cyan=''\n ;;\n esac\n\n unset _shunit_color_ _shunit_tput_\n}\n\n# colors returns the number of supported colors for the TERM.\n_shunit_colors() {\n _shunit_tput_=`${SHUNIT_CMD_TPUT} colors 2>/dev/null`\n if command [ $? -eq 0 ]; then\n echo \"${_shunit_tput_}\"\n else\n echo 16\n fi\n unset _shunit_tput_\n}\n\n# The actual running of the tests happens here.\n#\n# Args:\n# None\n_shunit_execSuite() {\n for _shunit_test_ in ${__shunit_suite}; do\n __shunit_testSuccess=${SHUNIT_TRUE}\n\n # Disable skipping.\n endSkipping\n\n # Execute the per-test setup function.\n setUp\n command [ $? -eq ${SHUNIT_TRUE} ] \\\n || _shunit_fatal \"setup() returned non-zero return code.\"\n\n # Execute the test.\n echo \"${__SHUNIT_TEST_PREFIX}${_shunit_test_}\"\n eval \"${_shunit_test_}\"\n if command [ $? -ne ${SHUNIT_TRUE} ]; then\n _shunit_error \"${_shunit_test_}() returned non-zero return code.\"\n __shunit_testSuccess=${SHUNIT_ERROR}\n _shunit_incFailedCount\n fi\n\n # Execute the per-test tear-down function.\n tearDown\n command [ $? -eq ${SHUNIT_TRUE} ] \\\n || _shunit_fatal \"tearDown() returned non-zero return code.\"\n\n # Update stats.\n if command [ ${__shunit_testSuccess} -eq ${SHUNIT_TRUE} ]; then\n __shunit_testsPassed=`expr ${__shunit_testsPassed} + 1`\n else\n __shunit_testsFailed=`expr ${__shunit_testsFailed} + 1`\n fi\n done\n\n unset _shunit_test_\n}\n\n# Generates the user friendly report with appropriate OK/FAILED message.\n#\n# Args:\n# None\n# Output:\n# string: the report of successful and failed tests, as well as totals.\n_shunit_generateReport() {\n command [ \"${__shunit_reportGenerated}\" -eq ${SHUNIT_TRUE} ] && return\n\n _shunit_ok_=${SHUNIT_TRUE}\n\n # If no exit code was provided, determine an appropriate one.\n command [ \"${__shunit_testsFailed}\" -gt 0 \\\n -o ${__shunit_testSuccess} -eq ${SHUNIT_FALSE} ] \\\n && _shunit_ok_=${SHUNIT_FALSE}\n\n echo\n _shunit_msg_=\"Ran ${__shunit_ansi_cyan}${__shunit_testsTotal}${__shunit_ansi_none}\"\n if command [ \"${__shunit_testsTotal}\" -eq 1 ]; then\n ${__SHUNIT_CMD_ECHO_ESC} \"${_shunit_msg_} test.\"\n else\n ${__SHUNIT_CMD_ECHO_ESC} \"${_shunit_msg_} tests.\"\n fi\n\n if command [ ${_shunit_ok_} -eq ${SHUNIT_TRUE} ]; then\n _shunit_msg_=\"${__shunit_ansi_green}OK${__shunit_ansi_none}\"\n command [ \"${__shunit_assertsSkipped}\" -gt 0 ] \\\n && _shunit_msg_=\"${_shunit_msg_} (${__shunit_ansi_yellow}skipped=${__shunit_assertsSkipped}${__shunit_ansi_none})\"\n else\n _shunit_msg_=\"${__shunit_ansi_red}FAILED${__shunit_ansi_none}\"\n _shunit_msg_=\"${_shunit_msg_} (${__shunit_ansi_red}failures=${__shunit_assertsFailed}${__shunit_ansi_none}\"\n command [ \"${__shunit_assertsSkipped}\" -gt 0 ] \\\n && _shunit_msg_=\"${_shunit_msg_},${__shunit_ansi_yellow}skipped=${__shunit_assertsSkipped}${__shunit_ansi_none}\"\n _shunit_msg_=\"${_shunit_msg_})\"\n fi\n\n echo\n ${__SHUNIT_CMD_ECHO_ESC} \"${_shunit_msg_}\"\n __shunit_reportGenerated=${SHUNIT_TRUE}\n\n unset _shunit_msg_ _shunit_ok_\n}\n\n# Test for whether a function should be skipped.\n#\n# Args:\n# None\n# Returns:\n# boolean: whether the test should be skipped (TRUE/FALSE constant)\n_shunit_shouldSkip() {\n command [ ${__shunit_skip} -eq ${SHUNIT_FALSE} ] && return ${SHUNIT_FALSE}\n _shunit_assertSkip\n}\n\n# Records a successful test.\n#\n# Args:\n# None\n_shunit_assertPass() {\n __shunit_assertsPassed=`expr ${__shunit_assertsPassed} + 1`\n __shunit_assertsTotal=`expr ${__shunit_assertsTotal} + 1`\n}\n\n# Records a test failure.\n#\n# Args:\n# message: string: failure message to provide user\n_shunit_assertFail() {\n __shunit_testSuccess=${SHUNIT_FALSE}\n _shunit_incFailedCount\n\n \\[ $# -gt 0 ] && ${__SHUNIT_CMD_ECHO_ESC} \\\n \"${__shunit_ansi_red}ASSERT:${__shunit_ansi_none}$*\"\n}\n\n# Increment the count of failed asserts.\n#\n# Args:\n# none\n_shunit_incFailedCount() {\n __shunit_assertsFailed=`expr \"${__shunit_assertsFailed}\" + 1`\n __shunit_assertsTotal=`expr \"${__shunit_assertsTotal}\" + 1`\n}\n\n\n# Records a skipped test.\n#\n# Args:\n# None\n_shunit_assertSkip() {\n __shunit_assertsSkipped=`expr \"${__shunit_assertsSkipped}\" + 1`\n __shunit_assertsTotal=`expr \"${__shunit_assertsTotal}\" + 1`\n}\n\n# Prepare a script filename for sourcing.\n#\n# Args:\n# script: string: path to a script to source\n# Returns:\n# string: filename prefixed with ./ (if necessary)\n_shunit_prepForSourcing() {\n _shunit_script_=$1\n case \"${_shunit_script_}\" in\n /*|./*) echo \"${_shunit_script_}\" ;;\n *) echo \"./${_shunit_script_}\" ;;\n esac\n unset _shunit_script_\n}\n\n# Escape a character in a string.\n#\n# Args:\n# c: string: unescaped character\n# s: string: to escape character in\n# Returns:\n# string: with escaped character(s)\n_shunit_escapeCharInStr() {\n command [ -n \"$2\" ] || return # No point in doing work on an empty string.\n\n # Note: using shorter variable names to prevent conflicts with\n # _shunit_escapeCharactersInString().\n _shunit_c_=$1\n _shunit_s_=$2\n\n # Escape the character.\n # shellcheck disable=SC1003,SC2086\n echo ''${_shunit_s_}'' |command sed 's/\\'${_shunit_c_}'/\\\\\\'${_shunit_c_}'/g'\n\n unset _shunit_c_ _shunit_s_\n}\n\n# Escape a character in a string.\n#\n# Args:\n# str: string: to escape characters in\n# Returns:\n# string: with escaped character(s)\n_shunit_escapeCharactersInString() {\n command [ -n \"$1\" ] || return # No point in doing work on an empty string.\n\n _shunit_str_=$1\n\n # Note: using longer variable names to prevent conflicts with\n # _shunit_escapeCharInStr().\n for _shunit_char_ in '\"' '$' \"'\" '`'; do\n _shunit_str_=`_shunit_escapeCharInStr \"${_shunit_char_}\" \"${_shunit_str_}\"`\n done\n\n echo \"${_shunit_str_}\"\n unset _shunit_char_ _shunit_str_\n}\n\n# Extract list of functions to run tests against.\n#\n# Args:\n# script: string: name of script to extract functions from\n# Returns:\n# string: of function names\n_shunit_extractTestFunctions() {\n _shunit_script_=$1\n\n # Extract the lines with test function names, strip of anything besides the\n # function name, and output everything on a single line.\n _shunit_regex_='^\\s*((function test[A-Za-z0-9_-]*)|(test[A-Za-z0-9_-]* *\\(\\)))'\n # shellcheck disable=SC2196\n egrep \"${_shunit_regex_}\" \"${_shunit_script_}\" \\\n |command sed 's/^[^A-Za-z0-9_-]*//;s/^function //;s/\\([A-Za-z0-9_-]*\\).*/\\1/g' \\\n |xargs\n\n unset _shunit_regex_ _shunit_script_\n}\n\n#------------------------------------------------------------------------------\n# Main.\n#\n\n# Determine the operating mode.\nif command [ $# -eq 0 -o \"${1:-}\" = '--' ]; then\n __shunit_script=${__SHUNIT_PARENT}\n __shunit_mode=${__SHUNIT_MODE_SOURCED}\nelse\n __shunit_script=$1\n command [ -r \"${__shunit_script}\" ] || \\\n _shunit_fatal \"unable to read from ${__shunit_script}\"\n __shunit_mode=${__SHUNIT_MODE_STANDALONE}\nfi\n\n# Create a temporary storage location.\n__shunit_tmpDir=`_shunit_mktempDir`\n\n# Provide a public temporary directory for unit test scripts.\n# TODO(kward): document this.\nSHUNIT_TMPDIR=\"${__shunit_tmpDir}/tmp\"\ncommand mkdir \"${SHUNIT_TMPDIR}\"\n\n# Setup traps to clean up after ourselves.\ntrap '_shunit_cleanup EXIT' 0\ntrap '_shunit_cleanup INT' 2\ntrap '_shunit_cleanup TERM' 15\n\n# Create phantom functions to work around issues with Cygwin.\n_shunit_mktempFunc\nPATH=\"${__shunit_tmpDir}:${PATH}\"\n\n# Make sure phantom functions are executable. This will bite if `/tmp` (or the\n# current `$TMPDIR`) points to a path on a partition that was mounted with the\n# 'noexec' option. The noexec command was created with `_shunit_mktempFunc()`.\nnoexec 2>/dev/null || _shunit_fatal \\\n 'Please declare TMPDIR with path on partition with exec permission.'\n\n# We must manually source the tests in standalone mode.\nif command [ \"${__shunit_mode}\" = \"${__SHUNIT_MODE_STANDALONE}\" ]; then\n # shellcheck disable=SC1090\n command . \"`_shunit_prepForSourcing \\\"${__shunit_script}\\\"`\"\nfi\n\n# Configure default output coloring behaviour.\n_shunit_configureColor \"${SHUNIT_COLOR}\"\n\n# Execute the oneTimeSetUp function (if it exists).\noneTimeSetUp\ncommand [ $? -eq ${SHUNIT_TRUE} ] \\\n || _shunit_fatal \"oneTimeSetUp() returned non-zero return code.\"\n\n# Command line selected tests or suite selected tests\nif command [ \"$#\" -ge 2 ]; then\n # Argument $1 is either the filename of tests or '--'; either way, skip it.\n shift\n # Remaining arguments ($2 .. $#) are assumed to be test function names.\n # Iterate through all remaining args in \"$@\" in a POSIX (likely portable) way.\n # Helpful tip: https://unix.stackexchange.com/questions/314032/how-to-use-arguments-like-1-2-in-a-for-loop\n for _shunit_arg_ do\n suite_addTest \"${_shunit_arg_}\"\n done\n unset _shunit_arg_\nelse\n # Execute the suite function defined in the parent test script.\n # DEPRECATED as of 2.1.0.\n suite\nfi\n\n# If no tests or suite specified, dynamically build a list of functions.\nif command [ -z \"${__shunit_suite}\" ]; then\n shunit_funcs_=`_shunit_extractTestFunctions \"${__shunit_script}\"`\n for shunit_func_ in ${shunit_funcs_}; do\n suite_addTest \"${shunit_func_}\"\n done\nfi\nunset shunit_func_ shunit_funcs_\n\n# Execute the suite of unit tests.\n_shunit_execSuite\n\n# Execute the oneTimeTearDown function (if it exists).\noneTimeTearDown\ncommand [ $? -eq ${SHUNIT_TRUE} ] \\\n || _shunit_fatal \"oneTimeTearDown() returned non-zero return code.\"\n\n# Generate a report summary.\n_shunit_generateReport\n\n# That's it folks.\ncommand [ \"${__shunit_testsFailed}\" -eq 0 ]\nexit $?\n" }, { "path": "scripts/spelling.sh", "content": "#!/bin/bash\n\nnpx cspell --no-progress \"**/*.{sh,go,md}\"" }, { "path": "scripts/test-docker.sh", "content": "#! /bin/bash\nset -e\n\ndocker build . -t temp\ndocker run --rm -it --entrypoint sh temp -c 'touch a'\n" }, { "path": "scripts/test.sh", "content": "#!/bin/bash\n\ngo test $(go list ./... | grep -v -E 'examples' | grep -v -E 'test')\n" }, { "path": "scripts/xcompile.sh", "content": "#!/bin/bash\n\nset -eo pipefail\n\n# You may need to go install github.com/goreleaser/goreleaser/v2@latest first\nGORELEASER=\"goreleaser build --clean\"\nif [ -z \"$CI\" ]; then\n GORELEASER+=\" --snapshot\"\nfi\n\nmkdir -p build\n\n$GORELEASER\n\ncp yq.1 build/yq.1\n\ncd build\n\n# Remove artifacts from goreleaser\nrm artifacts.json config.yaml metadata.json\n\nfind . -executable -type f | xargs -I {} tar czvf {}.tar.gz {} yq.1 -C ../scripts install-man-page.sh\ntar czvf yq_man_page_only.tar.gz yq.1 -C ../scripts install-man-page.sh\n\nrm yq_windows_386.exe.tar.gz\nrm yq_windows_amd64.exe.tar.gz\nrm yq_windows_arm64.exe.tar.gz\n\nzip yq_windows_386.zip yq_windows_386.exe\nzip yq_windows_amd64.zip yq_windows_amd64.exe\nzip yq_windows_arm64.zip yq_windows_arm64.exe\n\nrm yq.1\n\nrhash -r -a . -o checksums\n\nrhash -r -a --bsd . -o checksums-bsd\n\nrhash --list-hashes > checksums_hashes_order\n\ncp ../scripts/extract-checksum.sh .\n" }, { "path": "snap/snapcraft.yaml", "content": "name: yq\nversion: 'v4.52.4'\nsummary: A lightweight and portable command-line data file processor\ndescription: |\n `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml, json, xml, csv, properties and TOML files.\nbase: core24\ngrade: stable # devel|stable. must be 'stable' to release into candidate/stable channels\nconfinement: strict\nplatforms:\n amd64:\n build-on: [amd64]\n build-for: [amd64]\n arm64:\n build-on: [arm64]\n build-for: [arm64]\n armhf:\n build-on: [armhf]\n build-for: [armhf]\n s390x:\n build-on: [s390x]\n build-for: [s390x]\n ppc64el:\n build-on: [ppc64el]\n build-for: [ppc64el]\napps:\n yq:\n command: bin/yq\n plugs: [home, removable-media]\nparts:\n yq:\n plugin: go\n build-environment:\n - CGO_ENABLED: 0\n source: https://github.com/mikefarah/yq.git\n source-tag: v4.52.4\n build-snaps:\n - go/latest/stable\n" }, { "path": "test/format_test.go", "content": "package test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/mikefarah/yq/v4/pkg/yqlib\"\n)\n\n// only test format detection based on filename extension\nfunc TestFormatStringFromFilename(t *testing.T) {\n\tcases := []struct {\n\t\tfilename string\n\t\texpected string\n\t}{\n\t\t// filenames that have extensions\n\t\t{\"file.yaml\", \"yaml\"},\n\t\t{\"FILE.JSON\", \"json\"},\n\t\t{\"file.properties\", \"properties\"},\n\t\t{\"file.xml\", \"xml\"},\n\t\t{\"file.unknown\", \"unknown\"},\n\n\t\t// filenames without extensions\n\t\t{\"file\", \"yaml\"},\n\t\t{\"a.dir/file\", \"yaml\"},\n\t\t{\"file.\", \"yaml\"},\n\t\t{\".\", \"yaml\"},\n\t\t{\"\", \"yaml\"},\n\t}\n\n\tfor _, c := range cases {\n\t\tresult := yqlib.FormatStringFromFilename(c.filename)\n\t\tif result != c.expected {\n\t\t\tt.Errorf(\"FormatStringFromFilename(%q) = %q, wanted: %q\", c.filename, result, c.expected)\n\t\t}\n\t}\n}\n" }, { "path": "test/utils.go", "content": "package test\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"fmt\"\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/pkg/diff\"\n\t\"github.com/pkg/diff/write\"\n)\n\nfunc printDifference(t *testing.T, expectedValue interface{}, actualValue interface{}) {\n\topts := []write.Option{write.TerminalColor()}\n\tvar differenceBuffer bytes.Buffer\n\texpectedString := fmt.Sprintf(\"%v\", expectedValue)\n\tactualString := fmt.Sprintf(\"%v\", actualValue)\n\tif err := diff.Text(\"expected\", \"actual\", expectedString, actualString, bufio.NewWriter(&differenceBuffer), opts...); err != nil {\n\t\tt.Error(err)\n\t} else {\n\t\tt.Error(differenceBuffer.String())\n\t}\n}\n\nfunc AssertResult(t *testing.T, expectedValue interface{}, actualValue interface{}) {\n\tt.Helper()\n\tif expectedValue != actualValue {\n\t\tprintDifference(t, expectedValue, actualValue)\n\t}\n}\n\nfunc AssertResultComplex(t *testing.T, expectedValue interface{}, actualValue interface{}) {\n\tt.Helper()\n\tif !reflect.DeepEqual(expectedValue, actualValue) {\n\t\tprintDifference(t, expectedValue, actualValue)\n\t}\n}\n\nfunc AssertResultComplexWithContext(t *testing.T, expectedValue interface{}, actualValue interface{}, context interface{}) {\n\tt.Helper()\n\tif !reflect.DeepEqual(expectedValue, actualValue) {\n\t\tt.Error(context)\n\t\tprintDifference(t, expectedValue, actualValue)\n\t}\n}\n\nfunc AssertResultWithContext(t *testing.T, expectedValue interface{}, actualValue interface{}, context interface{}) {\n\tt.Helper()\n\topts := []write.Option{write.TerminalColor()}\n\tif expectedValue != actualValue {\n\t\tt.Error(context)\n\t\tvar differenceBuffer bytes.Buffer\n\t\tif err := diff.Text(\"expected\", \"actual\", expectedValue, actualValue, bufio.NewWriter(&differenceBuffer), opts...); err != nil {\n\t\t\tt.Error(err)\n\t\t} else {\n\t\t\tt.Error(differenceBuffer.String())\n\t\t}\n\t}\n}\n" }, { "path": "test.yq", "content": "#!./yq\n.a.b\n" }, { "path": "utf8.csv", "content": "id,first,last\n1,john,smith\n1,jane,smith\n" }, { "path": "yq.go", "content": "package main\n\nimport (\n\t\"os\"\n\n\tcommand \"github.com/mikefarah/yq/v4/cmd\"\n)\n\nfunc main() {\n\tcmd := command.New()\n\n\targs := os.Args[1:]\n\n\t_, _, err := cmd.Find(args)\n\tif err != nil && args[0] != \"__complete\" && args[0] != \"__completeNoDesc\" {\n\t\t// default command when nothing matches...\n\t\tnewArgs := []string{\"eval\"}\n\t\tcmd.SetArgs(append(newArgs, os.Args[1:]...))\n\n\t}\n\n\tif err := cmd.Execute(); err != nil {\n\t\tos.Exit(1)\n\t}\n}\n" }, { "path": "yq_test.go", "content": "package main\n\nimport (\n\t\"testing\"\n\n\tcommand \"github.com/mikefarah/yq/v4/cmd\"\n)\n\nfunc TestMainFunction(t *testing.T) {\n\t// This is a basic smoke test for the main function\n\t// We can't easily test the main function directly since it calls os.Exit\n\t// But we can test the logic that would be executed\n\n\tcmd := command.New()\n\tif cmd == nil {\n\t\tt.Fatal(\"command.New() returned nil\")\n\t}\n\n\tif cmd.Use != \"yq\" {\n\t\tt.Errorf(\"Expected command Use to be 'yq', got %q\", cmd.Use)\n\t}\n}\n\nfunc TestMainFunctionLogic(t *testing.T) {\n\t// Test the logic that would be executed in main()\n\tcmd := command.New()\n\n\targs := []string{}\n\t_, _, err := cmd.Find(args)\n\tif err != nil {\n\t\tt.Errorf(\"Expected no error with empty args, but got: %v\", err)\n\t}\n\n\targs = []string{\"invalid-command\"}\n\t_, _, err = cmd.Find(args)\n\tif err == nil {\n\t\tt.Error(\"Expected error when invalid command found, but got nil\")\n\t}\n\n\targs = []string{\"eval\"}\n\t_, _, err = cmd.Find(args)\n\tif err != nil {\n\t\tt.Errorf(\"Expected no error with valid command 'eval', got: %v\", err)\n\t}\n\n\targs = []string{\"__complete\"}\n\t_, _, err = cmd.Find(args)\n\tif err == nil {\n\t\tt.Error(\"Expected error when no command found for '__complete', but got nil\")\n\t}\n\n\targs = []string{\"__completeNoDesc\"}\n\t_, _, err = cmd.Find(args)\n\tif err == nil {\n\t\tt.Error(\"Expected error when no command found for '__completeNoDesc', but got nil\")\n\t}\n}\n\nfunc TestMainFunctionWithArgs(t *testing.T) {\n\t// Test the argument processing logic\n\tcmd := command.New()\n\n\targs := []string{}\n\t_, _, err := cmd.Find(args)\n\tif err != nil {\n\t\tt.Errorf(\"Expected no error with empty args, but got: %v\", err)\n\t}\n\n\t// When Find fails and args[0] is not \"__complete\", main would set args to [\"eval\"] + original args\n\t// This is the logic: newArgs := []string{\"eval\"}\n\t// cmd.SetArgs(append(newArgs, os.Args[1:]...))\n\n\targs = []string{\"invalid\"}\n\t_, _, err = cmd.Find(args)\n\tif err == nil {\n\t\tt.Error(\"Expected error with invalid command\")\n\t}\n\n\targs = []string{\"__complete\"}\n\t_, _, err = cmd.Find(args)\n\tif err == nil {\n\t\tt.Error(\"Expected error with __complete command\")\n\t}\n\n\targs = []string{\"__completeNoDesc\"}\n\t_, _, err = cmd.Find(args)\n\tif err == nil {\n\t\tt.Error(\"Expected error with __completeNoDesc command\")\n\t}\n}\n\nfunc TestMainFunctionExecution(t *testing.T) {\n\t// Test that the command can be executed without crashing\n\tcmd := command.New()\n\n\tcmd.SetArgs([]string{\"--version\"})\n\n\t// We can't easily test os.Exit(1) behaviour, but we can test that\n\t// the command structure is correct and can be configured\n\tif cmd == nil {\n\t\tt.Fatal(\"Command should not be nil\")\n\t}\n\n\tif cmd.Use != \"yq\" {\n\t\tt.Errorf(\"Expected command Use to be 'yq', got %q\", cmd.Use)\n\t}\n}\n\nfunc TestMainFunctionErrorHandling(t *testing.T) {\n\t// Test the error handling logic that would be in main()\n\tcmd := command.New()\n\n\targs := []string{\"nonexistent-command\"}\n\t_, _, err := cmd.Find(args)\n\tif err == nil {\n\t\tt.Error(\"Expected error with nonexistent command\")\n\t}\n\n\t// The main function logic would be:\n\t// if err != nil && args[0] != \"__complete\" {\n\t// newArgs := []string{\"eval\"}\n\t// cmd.SetArgs(append(newArgs, os.Args[1:]...))\n\t// }\n\n\t// Test that this logic would work\n\tif args[0] != \"__complete\" {\n\t\t// This is what main() would do\n\t\tnewArgs := []string{\"eval\"}\n\t\tcmd.SetArgs(append(newArgs, args...))\n\n\t\t// We can't easily verify the args were set correctly since cmd.Args is a function\n\t\t// But we can test that SetArgs doesn't crash and the command is still valid\n\t\tif cmd == nil {\n\t\t\tt.Error(\"Command should not be nil after SetArgs\")\n\t\t}\n\n\t\t_, _, err := cmd.Find([]string{\"eval\"})\n\t\tif err != nil {\n\t\t\tt.Errorf(\"Should be able to find eval command after SetArgs: %v\", err)\n\t\t}\n\t}\n}\n\nfunc TestMainFunctionWithCompletionCommand(t *testing.T) {\n\t// Test that __complete command doesn't trigger default command logic\n\tcmd := command.New()\n\n\targs := []string{\"__complete\"}\n\t_, _, err := cmd.Find(args)\n\tif err == nil {\n\t\tt.Error(\"Expected error with __complete command\")\n\t}\n\n\t// The main function logic would be:\n\t// if err != nil && args[0] != \"__complete\" {\n\t// // This should NOT execute for __complete\n\t// }\n\n\t// Verify that __complete doesn't trigger the default command logic\n\tif args[0] == \"__complete\" {\n\t\t// This means the default command logic should NOT execute\n\t\tt.Log(\"__complete command correctly identified, default command logic should not execute\")\n\t}\n}\n\nfunc TestMainFunctionWithCompletionNoDescCommand(t *testing.T) {\n\t// Test that __complete command doesn't trigger default command logic\n\tcmd := command.New()\n\n\targs := []string{\"__completeNoDesc\"}\n\t_, _, err := cmd.Find(args)\n\tif err == nil {\n\t\tt.Error(\"Expected error with __completeNoDesc command\")\n\t}\n\n\t// The main function logic would be:\n\t// if err != nil && args[0] != \"__completeNoDesc\" {\n\t// // This should NOT execute for __completeNoDesc\n\t// }\n\n\t// Verify that __completeNoDesc doesn't trigger the default command logic\n\tif args[0] == \"__completeNoDesc\" {\n\t\t// This means the default command logic should NOT execute\n\t\tt.Log(\"__completeNoDesc command correctly identified, default command logic should not execute\")\n\t}\n}\n\nfunc TestMainFunctionIntegration(t *testing.T) {\n\t// Integration test to verify the main function logic works end-to-end\n\n\tcmd := command.New()\n\tcmd.SetArgs([]string{\"eval\", \"--help\"})\n\n\t// This should not crash (we can't test the actual execution due to os.Exit)\n\tif cmd == nil {\n\t\tt.Fatal(\"Command should not be nil\")\n\t}\n\n\tcmd2 := command.New()\n\tcmd2.SetArgs([]string{\"invalid-command\"})\n\n\t// Simulate the main function logic\n\targs := []string{\"invalid-command\"}\n\t_, _, err := cmd2.Find(args)\n\tif err != nil {\n\t\t// This is what main() would do\n\t\tnewArgs := []string{\"eval\"}\n\t\tcmd2.SetArgs(append(newArgs, args...))\n\t}\n\n\t// We can't directly access cmd.Args since it's a function, but we can test\n\t// that SetArgs worked by ensuring the command is still functional\n\tif cmd2 == nil {\n\t\tt.Error(\"Command should not be nil after SetArgs\")\n\t}\n\n\t_, _, err = cmd2.Find([]string{\"eval\"})\n\tif err != nil {\n\t\tt.Errorf(\"Should be able to find eval command after SetArgs: %v\", err)\n\t}\n}\n" } ]