[ { "path": ".codespellrc", "content": "[codespell]\n# ignore test files, go project names, binary files via `skip` and special var/regex via `ignore-words`\nskip = fuzz,*_test.tmpl,testdata,*_test.go,go.mod,go.sum,*.gz\nignore-words = .github/workflows/.ignore_words\ncheck-filenames = true\n" }, { "path": ".github/ISSUE_TEMPLATE/bug_report.md", "content": "---\nname: Bug report\nabout: Create a report to help us improve\ntitle: ''\nlabels: ''\nassignees: ''\n\n---\n\n**Describe the bug**\n\nA clear and concise description of what the bug is.\n\n**To Reproduce**\n\nSteps to reproduce the behavior:\n1. Go to '...'\n2. Click on '....'\n3. Scroll down to '....'\n4. See error\n\n**Expected behavior**\n\nA clear and concise description of what you expected to happen.\n\n**Screenshots**\n\nIf applicable, add screenshots to help explain your problem.\n\n**Sonic version:**\n\nPlease provide the version of Sonic you are using.\n\n**Environment:**\n\nThe output of `go env`.\n\n**Additional context**\n\nAdd any other context about the problem here.\n" }, { "path": ".github/ISSUE_TEMPLATE/feature_request.md", "content": "---\nname: Feature request\nabout: Suggest an idea for this project\ntitle: ''\nlabels: ''\nassignees: ''\n\n---\n\n**Is your feature request related to a problem? Please describe.**\n\nA clear and concise description of what the problem is. Ex. I'm always frustrated when [...]\n\n**Describe the solution you'd like**\n\nA clear and concise description of what you want to happen.\n\n**Describe alternatives you've considered**\n\nA clear and concise description of any alternative solutions or features you've considered.\n\n**Additional context**\n\nAdd any other context or screenshots about the feature request here.\n" }, { "path": ".github/PULL_REQUEST_TEMPLATE.md", "content": "#### What type of PR is this?\n\n\n#### Check the PR title.\n\n- [ ] This PR title match the format: \\(optional scope): \\\n- [ ] The description of this PR title is user-oriented and clear enough for others to understand.\n- [ ] Attach the PR updating the user documentation if the current PR requires user awareness at the usage level. [User docs repo](https://github.com/cloudwego/cloudwego.github.io)\n\n\n#### (Optional) Translate the PR title into Chinese.\n\n\n#### (Optional) More detailed description for this PR(en: English/zh: Chinese).\n\nen: \nzh(optional): \n\n#### (Optional) Which issue(s) this PR fixes:\n\n\n#### (optional) The PR that updates user documentation:\n" }, { "path": ".github/workflows/.ignore_words", "content": "socio-economic\nnd\nregArgs\noders\nure\nalse\n" }, { "path": ".github/workflows/benchmark.yml", "content": "name: Benchmark\n\non: pull_request\n\njobs:\n build:\n strategy:\n matrix:\n os: [ubuntu-latest, ubuntu-24.04-arm]\n runs-on: ${{ matrix.os }}\n steps:\n - name: Clear repository\n run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE\n\n - uses: actions/checkout@v2\n\n - name: Set up Go\n uses: actions/setup-go@v6\n with:\n go-version: 1.22\n\n - uses: actions/cache@v4\n with:\n path: ~/go/pkg/mod\n key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}\n restore-keys: |\n ${{ runner.os }}-go-\n\n - name: Benchmark Target\n continue-on-error: true\n run: |\n export SONIC_NO_ASYNC_GC=1\n go test -run ^$ -count=20 -benchmem -bench 'BenchmarkDecoder_(Generic|Binding)_Sonic' ./decoder >> /var/tmp/sonic_bench_target_${{ github.run_id }}.out\n go test -run ^$ -count=20 -benchmem -bench 'BenchmarkEncoder_(Generic|Binding)_Sonic' ./encoder >> /var/tmp/sonic_bench_target_${{ github.run_id }}.out\n go test -run ^$ -count=20 -benchmem -bench 'Benchmark(Get|Set)One_Sonic|BenchmarkParseSeven_Sonic' ./ast >> /var/tmp/sonic_bench_target_${{ github.run_id }}.out\n\n - name: Clear repository\n run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE\n\n - name: Checkout main\n uses: actions/checkout@v2\n with:\n ref: main\n \n - name: Benchmark main\n continue-on-error: true\n run: |\n export SONIC_NO_ASYNC_GC=1\n go test -run ^$ -count=20 -benchmem -bench 'BenchmarkDecoder_(Generic|Binding)_Sonic' ./decoder >> /var/tmp/sonic_bench_main_${{ github.run_id }}.out\n go test -run ^$ -count=20 -benchmem -bench 'BenchmarkEncoder_(Generic|Binding)_Sonic' ./encoder >> /var/tmp/sonic_bench_main_${{ github.run_id }}.out\n go test -run ^$ -count=20 -benchmem -bench 'Benchmark(Get|Set)One_Sonic|BenchmarkParseSeven_Sonic' ./ast >> /var/tmp/sonic_bench_main_${{ github.run_id }}.out\n\n - name: Diff bench\n continue-on-error: true\n run: |\n ./scripts/bench.py -t 0.20 -d /var/tmp/sonic_bench_target_${{ github.run_id }}.out,/var/tmp/sonic_bench_main_${{ github.run_id }}.out x\n" }, { "path": ".github/workflows/compatibility_test-windows.yml", "content": "name: Compatibility Test Windows-X64\n\non: pull_request\n\njobs:\n build:\n strategy:\n matrix:\n go-version: [1.18.x, 1.21.x, 1.25.x]\n runs-on: windows-latest\n steps:\n - uses: actions/checkout@v2\n\n - name: Set up Go\n uses: actions/setup-go@v6\n with:\n go-version: ${{ matrix.go-version }}\n\n - uses: actions/cache@v4\n with:\n path: ~/go/pkg/mod\n key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}\n restore-keys: |\n ${{ runner.os }}-go-\n\n - name: main\n run: |\n set GOMAXPROCS=4\n go test -race ./\n\n - name: ast\n run: |\n set GOMAXPROCS=4\n go test -race ./ast\n\n - name: external\n run: |\n cd ./external_jsonlib_test\n set GOMAXPROCS=4\n go test -race ./...\n" }, { "path": ".github/workflows/compatibility_test.yml", "content": "name: Compatibility Test Linux-X64|ARM & macOS-ARM\n\non: pull_request\n\njobs:\n build:\n strategy:\n matrix:\n go-version: [1.18.x, 1.19.x, 1.20.x, 1.21.x, 1.22.x, 1.23.x, 1.24.x, 1.25.x]\n os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest]\n runs-on: ${{ matrix.os }}\n steps:\n - name: Clear repository\n run: |\n if [ \"${{ runner.os }}\" = \"Linux\" ]; then\n sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE\n else\n rm -rf $GITHUB_WORKSPACE && mkdir -p $GITHUB_WORKSPACE\n fi\n\n - uses: actions/checkout@v4\n\n - name: Set up Go\n uses: actions/setup-go@v6\n with:\n go-version: ${{ matrix.go-version }}\n\n - uses: actions/cache@v4\n with:\n path: ~/go/pkg/mod\n key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}\n restore-keys: |\n ${{ runner.os }}-go-\n\n - name: main\n run: go test -race -gcflags=\"all=-l\" ./\n\n - name: decoder\n run: go test -race -gcflags=\"all=-l\" ./decoder\n\n - name: encoder\n run: go test -race -gcflags=\"all=-l\" ./encoder\n\n - name: ast\n run: go test -race -gcflags=\"all=-l\" ./ast\n" }, { "path": ".github/workflows/fuzzing.yml", "content": "name: Fuzz Test Linux-X64\n\non: pull_request\n\njobs:\n build:\n strategy:\n max-parallel: 2\n matrix:\n mode: [run, runopt]\n os: [ubuntu-latest, ubuntu-24.04-arm]\n exclude:\n - os: ubuntu-24.04-arm\n mode: runopt\n\n runs-on: ${{ matrix.os }}\n steps:\n - name: Clear repository\n run: sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE\n\n - uses: actions/checkout@v2\n\n - name: Set up Go\n uses: actions/setup-go@v6\n with:\n go-version: 1.20.x\n\n - uses: actions/cache@v4\n with:\n path: ~/go/pkg/mod\n key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}\n restore-keys: |\n ${{ runner.os }}-go-\n\n - name: Fuzz sonic\n run: ./scripts/fuzz.sh ${{ matrix.mode }}\n\n" }, { "path": ".github/workflows/lint.yml", "content": "name: Lint\n\non: pull_request\n\njobs:\n misc:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n - name: spell check\n uses: codespell-project/actions-codespell@v2\n with:\n skip: ./loader/internal/iasm/obj/macho.go,./loader/internal/iasm/x86_64/encodings.go,./loader/internal/iasm/x86_64/program.go,./loader/internal/iasm/expr/ast.go,./loader/internal/iasm/expr/errors.go\n" }, { "path": ".github/workflows/test-arm64.yml", "content": "\nname: Unit Test\n\non: pull_request\n\njobs:\n build:\n permissions:\n contents: read\n id-token: write\n\n strategy:\n matrix:\n go-version: [1.20.x, 1.22.x, 1.25.x]\n runner_arch: [ubuntu-24.04-arm, macos-latest]\n\n runs-on: ${{ matrix.runner_arch }}\n \n steps:\n - name: Clear repository\n run: rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE\n \n - uses: actions/checkout@v4\n\n - name: Set up Go\n uses: actions/setup-go@v6\n with:\n go-version: ${{ matrix.go-version }}\n cache: true\n\n - name: Cache Go modules\n uses: actions/cache@v4\n with:\n path: |\n ~/go/pkg/mod\n ${{ github.workspace }}/go.sum\n key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}\n\n - name: Unit Test\n run: |\n GOMAXPROCS=4 go test -race -covermode=atomic -coverprofile=coverage.txt $(go list ./... | grep -v -E 'loader|jit|avx|x86|sse')\n\n - name: Data Race\n run: |\n ./scripts/test_race.sh\n\n - name: Issue Test\n run: GOMAXPROCS=4 go test -race ./issue_test\n\n - name: Generic Test\n run: GOMAXPROCS=4 go test -race ./generic_test\n\n - name: Codecov\n uses: codecov/codecov-action@v5\n with:\n use_oidc: true\n files: ./coverage.txt\n flags: arm,${{ matrix.runner_arch }}\n fail_ci_if_error: false\n" }, { "path": ".github/workflows/test-x86.yml", "content": "\nname: Unit Test\n\non: pull_request\n\njobs:\n build:\n permissions:\n contents: read\n id-token: write\n\n strategy:\n matrix:\n go-version: [1.18.x, 1.21.x, 1.25.x]\n runner_arch: [ubuntu-latest]\n\n runs-on: ${{ matrix.runner_arch }}\n \n steps:\n - name: Clear repository\n run: rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE\n - uses: actions/checkout@v4\n\n - name: Set up Go\n uses: actions/setup-go@v6\n with:\n go-version: ${{ matrix.go-version }}\n cache: true\n\n - name: Cache Go modules\n uses: actions/cache@v4\n with:\n path: |\n ~/go/pkg/mod\n ${{ github.workspace }}/go.sum\n key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}\n\n - name: Unit Test JIT\n run: |\n GOMAXPROCS=4 go test -race -covermode=atomic -coverprofile=coverage-jit.txt ./...\n\n - name: Unit Test JIT PCSP\n run: |\n GOMAXPROCS=4 go test -v ./internal/decoder/jitdec \n\n - name: Unit Test VM\n run: |\n SONIC_USE_OPTDEC=1 SONIC_USE_FASTMAP=1 SONIC_ENCODER_USE_VM=1 GOMAXPROCS=4 go test -race -covermode=atomic -coverprofile=coverage-vm.txt ./...\n \n - name: Loader Test\n run: |\n cd ./loader\n go test -race ./... \n \n - name: Data Race\n run: |\n ./scripts/test_race.sh\n\n - name: Issue Test\n run: GOMAXPROCS=4 go test -race ./issue_test\n\n - name: PCSP Test\n env:\n GOVERSION: ${{ matrix.go-version }}\n run: python3 ./scripts/test_pcsp.py\n\n - name: Generic Test JIT\n run: GOMAXPROCS=4 go test -race ./generic_test\n\n - name: Generic Test VM\n run: GOMAXPROCS=4 SONIC_USE_OPTDEC=1 SONIC_USE_FASTMAP=1 SONIC_ENCODER_USE_VM=1 go test -v -race ./generic_test\n\n - name: Codecov\n uses: codecov/codecov-action@v5\n with:\n use_oidc: true\n files: ./coverage-jit.txt,./coverage-vm.txt\n flags: x86,${{ matrix.runner_arch }}\n fail_ci_if_error: false\n" }, { "path": ".gitignore", "content": "*.o\n*.swp\n*.swm\n*.swn\n*.a\n*.so\n_obj\n_test\n*.[568vq]\n[568vq].out\n*.cgo1.go\n*.cgo2.c\n_cgo_defun.c\n_cgo_gotypes.go\n_cgo_export.*\n_testmain.go\n*.exe\n*.exe~\n*.test\n*.prof\n*.rar\n*.zip\n*.gz\n*.psd\n*.bmd\n*.cfg\n*.pptx\n*.log\n*nohup.out\n*settings.pyc\n*.sublime-project\n*.sublime-workspace\n.DS_Store\n/.idea/\n/.vscode/\n/output/\n/vendor/\n/Gopkg.lock\n/Gopkg.toml\ncoverage.html\ncoverage.out\ncoverage.xml\njunit.xml\n*.profile\n*.svg\n*.out\nast/test.out\nast/bench.sh\n\n!testdata/**/*.json.gz\nfuzz/testdata\n*__debug_bin*\n*pprof\n*coverage.txt\n.venv/\ntools/venv/*\n" }, { "path": ".gitmodules", "content": "[submodule \"cloudwego\"]\n\tpath = tools/asm2asm\n\turl = https://github.com/cloudwego/asm2asm.git\n[submodule \"tools/simde\"]\n\tpath = tools/simde\n\turl = https://github.com/simd-everywhere/simde.git\n[submodule \"fuzz/go-fuzz-corpus\"]\n\tpath = fuzz/go-fuzz-corpus\n\turl = https://github.com/dvyukov/go-fuzz-corpus.git\n" }, { "path": ".licenserc.yaml", "content": "header:\n license:\n spdx-id: Apache-2.0\n copyright-owner: ByteDance Inc.\n\n paths:\n - '**/*.go'\n - '**/*.s'\n\n paths-ignore:\n - 'ast/asm.s' # empty file\n - 'decoder/asm.s' # empty file\n - 'encoder/asm.s' # empty file\n - 'internal/caching/asm.s' # empty file\n - 'internal/jit/asm.s' # empty file\n - 'internal/native/avx/native_amd64.s' # auto-generated by asm2asm\n - 'internal/native/avx/native_subr_amd64.go' # auto-generated by asm2asm\n - 'internal/native/avx2/native_amd64.s' # auto-generated by asm2asm\n - 'internal/native/avx2/native_subr_amd64.go' # auto-generated by asm2asm\n - 'internal/resolver/asm.s' # empty file\n - 'internal/rt/asm.s' # empty file\n - 'internal/loader/asm.s' # empty file\n\n comment: on-failure" }, { "path": "CODE_OF_CONDUCT.md", "content": "# Contributor Covenant Code of Conduct\n\n## Our Pledge\n\nWe as members, contributors, and leaders pledge to make participation in our\ncommunity a harassment-free experience for everyone, regardless of age, body\nsize, visible or invisible disability, ethnicity, sex characteristics, gender\nidentity and expression, level of experience, education, socio-economic status,\nnationality, personal appearance, race, religion, or sexual identity\nand orientation.\n\nWe pledge to act and interact in ways that contribute to an open, welcoming,\ndiverse, inclusive, and healthy community.\n\n## Our Standards\n\nExamples of behavior that contributes to a positive environment for our\ncommunity include:\n\n* Demonstrating empathy and kindness toward other people\n* Being respectful of differing opinions, viewpoints, and experiences\n* Giving and gracefully accepting constructive feedback\n* Accepting responsibility and apologizing to those affected by our mistakes,\n and learning from the experience\n* Focusing on what is best not just for us as individuals, but for the\n overall community\n\nExamples of unacceptable behavior include:\n\n* The use of sexualized language or imagery, and sexual attention or\n advances of any kind\n* Trolling, insulting or derogatory comments, and personal or political attacks\n* Public or private harassment\n* Publishing others' private information, such as a physical or email\n address, without their explicit permission\n* Other conduct which could reasonably be considered inappropriate in a\n professional setting\n\n## Enforcement Responsibilities\n\nCommunity leaders are responsible for clarifying and enforcing our standards of\nacceptable behavior and will take appropriate and fair corrective action in\nresponse to any behavior that they deem inappropriate, threatening, offensive,\nor harmful.\n\nCommunity leaders have the right and responsibility to remove, edit, or reject\ncomments, commits, code, wiki edits, issues, and other contributions that are\nnot aligned to this Code of Conduct, and will communicate reasons for moderation\ndecisions when appropriate.\n\n## Scope\n\nThis Code of Conduct applies within all community spaces, and also applies when\nan individual is officially representing the community in public spaces.\nExamples of representing our community include using an official e-mail address,\nposting via an official social media account, or acting as an appointed\nrepresentative at an online or offline event.\n\n## Enforcement\n\nInstances of abusive, harassing, or otherwise unacceptable behavior may be\nreported to the community leaders responsible for enforcement at\nwudi.daniel@bytedance.com.\nAll complaints will be reviewed and investigated promptly and fairly.\n\nAll community leaders are obligated to respect the privacy and security of the\nreporter of any incident.\n\n## Enforcement Guidelines\n\nCommunity leaders will follow these Community Impact Guidelines in determining\nthe consequences for any action they deem in violation of this Code of Conduct:\n\n### 1. Correction\n\n**Community Impact**: Use of inappropriate language or other behavior deemed\nunprofessional or unwelcome in the community.\n\n**Consequence**: A private, written warning from community leaders, providing\nclarity around the nature of the violation and an explanation of why the\nbehavior was inappropriate. A public apology may be requested.\n\n### 2. Warning\n\n**Community Impact**: A violation through a single incident or series\nof actions.\n\n**Consequence**: A warning with consequences for continued behavior. No\ninteraction with the people involved, including unsolicited interaction with\nthose enforcing the Code of Conduct, for a specified period of time. This\nincludes avoiding interactions in community spaces as well as external channels\nlike social media. Violating these terms may lead to a temporary or\npermanent ban.\n\n### 3. Temporary Ban\n\n**Community Impact**: A serious violation of community standards, including\nsustained inappropriate behavior.\n\n**Consequence**: A temporary ban from any sort of interaction or public\ncommunication with the community for a specified period of time. No public or\nprivate interaction with the people involved, including unsolicited interaction\nwith those enforcing the Code of Conduct, is allowed during this period.\nViolating these terms may lead to a permanent ban.\n\n### 4. Permanent Ban\n\n**Community Impact**: Demonstrating a pattern of violation of community\nstandards, including sustained inappropriate behavior, harassment of an\nindividual, or aggression toward or disparagement of classes of individuals.\n\n**Consequence**: A permanent ban from any sort of public interaction within\nthe community.\n\n## Attribution\n\nThis Code of Conduct is adapted from the [Contributor Covenant][homepage],\nversion 2.0, available at\nhttps://www.contributor-covenant.org/version/2/0/code_of_conduct.html.\n\nCommunity Impact Guidelines were inspired by [Mozilla's code of conduct\nenforcement ladder](https://github.com/mozilla/diversity).\n\n[homepage]: https://www.contributor-covenant.org\n\nFor answers to common questions about this code of conduct, see the FAQ at\nhttps://www.contributor-covenant.org/faq. Translations are available at\nhttps://www.contributor-covenant.org/translations.\n" }, { "path": "CONTRIBUTING.md", "content": "# How to Contribute\n\n## Your First Pull Request\nWe use GitHub for our codebase. You can start by reading [How To Pull Request](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests).\n\n## Without Semantic Versioning\nWe keep the stable code in branch `main` like `golang.org/x`. Development base on branch `develop`. We promise the **Forward Compatibility** by adding new package directory with suffix `v2/v3` when code has break changes.\n\n## Branch Organization\nWe use [git-flow](https://nvie.com/posts/a-successful-git-branching-model/) as our branch organization, as known as [FDD](https://en.wikipedia.org/wiki/Feature-driven_development)\n\n\n## Bugs\n### 1. How to Find Known Issues\nWe are using [Github Issues](https://github.com/bytedance/sonic/issues) for our public bugs. We keep a close eye on this and try to make it clear when we have an internal fix in progress. Before filing a new task, try to make sure your problem doesn’t already exist.\n\n### 2. Reporting New Issues\nProviding a reduced test code is a recommended way for reporting issues. Then can be placed in:\n- Just in issues\n- [Golang Playground](https://play.golang.org/)\n\n### 3. Security Bugs\nPlease do not report the safe disclosure of bugs to public issues. Contact us by [Support Email](mailto:sonic@bytedance.com)\n\n## How to Get in Touch\n- [Email](mailto:wudi.daniel@bytedance.com)\n\n## Submit a Pull Request\nBefore you submit your Pull Request (PR) consider the following guidelines:\n1. Search [GitHub](https://github.com/bytedance/sonic/pulls) for an open or closed PR that relates to your submission. You don't want to duplicate existing efforts.\n2. Be sure that an issue describes the problem you're fixing, or documents the design for the feature you'd like to add. Discussing the design upfront helps to ensure that we're ready to accept your work.\n3. [Fork](https://docs.github.com/en/github/getting-started-with-github/fork-a-repo) the bytedance/sonic repo.\n4. In your forked repository, make your changes in a new git branch:\n ```\n git checkout -b bugfix/security_bug develop\n ```\n5. Create your patch, including appropriate test cases.\n6. Follow our [Style Guides](#code-style-guides).\n7. Commit your changes using a descriptive commit message that follows [AngularJS Git Commit Message Conventions](https://docs.google.com/document/d/1QrDFcIiPjSLDn3EL15IJygNPiHORgU1_OOAqWjiDU5Y/edit).\n Adherence to these conventions is necessary because release notes will be automatically generated from these messages.\n8. Push your branch to GitHub:\n ```\n git push origin bugfix/security_bug\n ```\n9. In GitHub, send a pull request to `sonic:main`\n\nNote: you must use one of `optimize/feature/bugfix/doc/ci/test/refactor` following a slash(`/`) as the branch prefix.\n\nYour pr title and commit message should follow https://www.conventionalcommits.org/.\n\n## Contribution Prerequisites\n- Our development environment keeps up with [Go Official](https://golang.org/project/).\n- You need fully checking with lint tools before submit your pull request. [gofmt](https://golang.org/pkg/cmd/gofmt/) & [golangci-lint](https://github.com/golangci/golangci-lint)\n- You are familiar with [Github](https://github.com) \n- Maybe you need familiar with [Actions](https://github.com/features/actions)(our default workflow tool).\n\n## Code Style Guides\nSee [Go Code Review Comments](https://github.com/golang/go/wiki/CodeReviewComments).\n\nGood resources:\n- [Effective Go](https://golang.org/doc/effective_go)\n- [Pingcap General advice](https://pingcap.github.io/style-guide/general.html)\n- [Uber Go Style Guide](https://github.com/uber-go/guide/blob/master/style.md)\n" }, { "path": "CREDITS", "content": "" }, { "path": "LICENSE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "README.md", "content": "# Sonic\n\nEnglish | [中文](README_ZH_CN.md)\n\nA blazingly fast JSON serializing & deserializing library, accelerated by JIT (just-in-time compiling) and SIMD (single-instruction-multiple-data).\n\n## Requirement\n\n- Go: 1.18~1.25\n - Notice: Go1.24.0 is not supported due to the [issue](https://github.com/golang/go/issues/71672), please use higher go version or add build tag `--ldflags=\"-checklinkname=0\"` \n- OS: Linux / MacOS / Windows\n- CPU: AMD64 / (ARM64, need go1.20 above)\n\n## Features\n\n- Runtime object binding without code generation\n- Complete APIs for JSON value manipulation\n- Fast, fast, fast!\n\n## APIs\n\nsee [go.dev](https://pkg.go.dev/github.com/bytedance/sonic)\n\n## Benchmarks\n\nFor **all sizes** of json and **all scenarios** of usage, **Sonic performs best**.\n\n- [Medium](https://github.com/bytedance/sonic/blob/main/decoder/testdata_test.go#L19) (13KB, 300+ key, 6 layers)\n\n```powershell\ngoversion: 1.17.1\ngoos: darwin\ngoarch: amd64\ncpu: Intel(R) Core(TM) i9-9880H CPU @ 2.30GHz\nBenchmarkEncoder_Generic_Sonic-16 32393 ns/op 402.40 MB/s 11965 B/op 4 allocs/op\nBenchmarkEncoder_Generic_Sonic_Fast-16 21668 ns/op 601.57 MB/s 10940 B/op 4 allocs/op\nBenchmarkEncoder_Generic_JsonIter-16 42168 ns/op 309.12 MB/s 14345 B/op 115 allocs/op\nBenchmarkEncoder_Generic_GoJson-16 65189 ns/op 199.96 MB/s 23261 B/op 16 allocs/op\nBenchmarkEncoder_Generic_StdLib-16 106322 ns/op 122.60 MB/s 49136 B/op 789 allocs/op\nBenchmarkEncoder_Binding_Sonic-16 6269 ns/op 2079.26 MB/s 14173 B/op 4 allocs/op\nBenchmarkEncoder_Binding_Sonic_Fast-16 5281 ns/op 2468.16 MB/s 12322 B/op 4 allocs/op\nBenchmarkEncoder_Binding_JsonIter-16 20056 ns/op 649.93 MB/s 9488 B/op 2 allocs/op\nBenchmarkEncoder_Binding_GoJson-16 8311 ns/op 1568.32 MB/s 9481 B/op 1 allocs/op\nBenchmarkEncoder_Binding_StdLib-16 16448 ns/op 792.52 MB/s 9479 B/op 1 allocs/op\nBenchmarkEncoder_Parallel_Generic_Sonic-16 6681 ns/op 1950.93 MB/s 12738 B/op 4 allocs/op\nBenchmarkEncoder_Parallel_Generic_Sonic_Fast-16 4179 ns/op 3118.99 MB/s 10757 B/op 4 allocs/op\nBenchmarkEncoder_Parallel_Generic_JsonIter-16 9861 ns/op 1321.84 MB/s 14362 B/op 115 allocs/op\nBenchmarkEncoder_Parallel_Generic_GoJson-16 18850 ns/op 691.52 MB/s 23278 B/op 16 allocs/op\nBenchmarkEncoder_Parallel_Generic_StdLib-16 45902 ns/op 283.97 MB/s 49174 B/op 789 allocs/op\nBenchmarkEncoder_Parallel_Binding_Sonic-16 1480 ns/op 8810.09 MB/s 13049 B/op 4 allocs/op\nBenchmarkEncoder_Parallel_Binding_Sonic_Fast-16 1209 ns/op 10785.23 MB/s 11546 B/op 4 allocs/op\nBenchmarkEncoder_Parallel_Binding_JsonIter-16 6170 ns/op 2112.58 MB/s 9504 B/op 2 allocs/op\nBenchmarkEncoder_Parallel_Binding_GoJson-16 3321 ns/op 3925.52 MB/s 9496 B/op 1 allocs/op\nBenchmarkEncoder_Parallel_Binding_StdLib-16 3739 ns/op 3486.49 MB/s 9480 B/op 1 allocs/op\n\nBenchmarkDecoder_Generic_Sonic-16 66812 ns/op 195.10 MB/s 57602 B/op 723 allocs/op\nBenchmarkDecoder_Generic_Sonic_Fast-16 54523 ns/op 239.07 MB/s 49786 B/op 313 allocs/op\nBenchmarkDecoder_Generic_StdLib-16 124260 ns/op 104.90 MB/s 50869 B/op 772 allocs/op\nBenchmarkDecoder_Generic_JsonIter-16 91274 ns/op 142.81 MB/s 55782 B/op 1068 allocs/op\nBenchmarkDecoder_Generic_GoJson-16 88569 ns/op 147.17 MB/s 66367 B/op 973 allocs/op\nBenchmarkDecoder_Binding_Sonic-16 32557 ns/op 400.38 MB/s 28302 B/op 137 allocs/op\nBenchmarkDecoder_Binding_Sonic_Fast-16 28649 ns/op 455.00 MB/s 24999 B/op 34 allocs/op\nBenchmarkDecoder_Binding_StdLib-16 111437 ns/op 116.97 MB/s 10576 B/op 208 allocs/op\nBenchmarkDecoder_Binding_JsonIter-16 35090 ns/op 371.48 MB/s 14673 B/op 385 allocs/op\nBenchmarkDecoder_Binding_GoJson-16 28738 ns/op 453.59 MB/s 22039 B/op 49 allocs/op\nBenchmarkDecoder_Parallel_Generic_Sonic-16 12321 ns/op 1057.91 MB/s 57233 B/op 723 allocs/op\nBenchmarkDecoder_Parallel_Generic_Sonic_Fast-16 10644 ns/op 1224.64 MB/s 49362 B/op 313 allocs/op\nBenchmarkDecoder_Parallel_Generic_StdLib-16 57587 ns/op 226.35 MB/s 50874 B/op 772 allocs/op\nBenchmarkDecoder_Parallel_Generic_JsonIter-16 38666 ns/op 337.12 MB/s 55789 B/op 1068 allocs/op\nBenchmarkDecoder_Parallel_Generic_GoJson-16 30259 ns/op 430.79 MB/s 66370 B/op 974 allocs/op\nBenchmarkDecoder_Parallel_Binding_Sonic-16 5965 ns/op 2185.28 MB/s 27747 B/op 137 allocs/op\nBenchmarkDecoder_Parallel_Binding_Sonic_Fast-16 5170 ns/op 2521.31 MB/s 24715 B/op 34 allocs/op\nBenchmarkDecoder_Parallel_Binding_StdLib-16 27582 ns/op 472.58 MB/s 10576 B/op 208 allocs/op\nBenchmarkDecoder_Parallel_Binding_JsonIter-16 13571 ns/op 960.51 MB/s 14685 B/op 385 allocs/op\nBenchmarkDecoder_Parallel_Binding_GoJson-16 10031 ns/op 1299.51 MB/s 22111 B/op 49 allocs/op\n\nBenchmarkGetOne_Sonic-16 3276 ns/op 3975.78 MB/s 24 B/op 1 allocs/op\nBenchmarkGetOne_Gjson-16 9431 ns/op 1380.81 MB/s 0 B/op 0 allocs/op\nBenchmarkGetOne_Jsoniter-16 51178 ns/op 254.46 MB/s 27936 B/op 647 allocs/op\nBenchmarkGetOne_Parallel_Sonic-16 216.7 ns/op 60098.95 MB/s 24 B/op 1 allocs/op\nBenchmarkGetOne_Parallel_Gjson-16 1076 ns/op 12098.62 MB/s 0 B/op 0 allocs/op\nBenchmarkGetOne_Parallel_Jsoniter-16 17741 ns/op 734.06 MB/s 27945 B/op 647 allocs/op\nBenchmarkSetOne_Sonic-16 9571 ns/op 1360.61 MB/s 1584 B/op 17 allocs/op\nBenchmarkSetOne_Sjson-16 36456 ns/op 357.22 MB/s 52180 B/op 9 allocs/op\nBenchmarkSetOne_Jsoniter-16 79475 ns/op 163.86 MB/s 45862 B/op 964 allocs/op\nBenchmarkSetOne_Parallel_Sonic-16 850.9 ns/op 15305.31 MB/s 1584 B/op 17 allocs/op\nBenchmarkSetOne_Parallel_Sjson-16 18194 ns/op 715.77 MB/s 52247 B/op 9 allocs/op\nBenchmarkSetOne_Parallel_Jsoniter-16 33560 ns/op 388.05 MB/s 45892 B/op 964 allocs/op\nBenchmarkLoadNode/LoadAll()-16 11384 ns/op 1143.93 MB/s 6307 B/op 25 allocs/op\nBenchmarkLoadNode_Parallel/LoadAll()-16 5493 ns/op 2370.68 MB/s 7145 B/op 25 allocs/op\nBenchmarkLoadNode/Interface()-16 17722 ns/op 734.85 MB/s 13323 B/op 88 allocs/op\nBenchmarkLoadNode_Parallel/Interface()-16 10330 ns/op 1260.70 MB/s 15178 B/op 88 allocs/op\n```\n\n- [Small](https://github.com/bytedance/sonic/blob/main/testdata/small.go) (400B, 11 keys, 3 layers)\n![small benchmarks](./docs/imgs/bench-small.png)\n- [Large](https://github.com/bytedance/sonic/blob/main/testdata/twitter.json) (635KB, 10000+ key, 6 layers)\n![large benchmarks](./docs/imgs/bench-large.png)\n\nSee [bench.sh](https://github.com/bytedance/sonic/blob/main/scripts/bench.sh) for benchmark codes.\n\n## How it works\n\nSee [INTRODUCTION.md](./docs/INTRODUCTION.md).\n\n## Usage\n\n### Marshal/Unmarshal\n\nDefault behaviors are mostly consistent with `encoding/json`, except HTML escaping form (see [Escape HTML](https://github.com/bytedance/sonic/blob/main/README.md#escape-html)) and `SortKeys` feature (optional support see [Sort Keys](https://github.com/bytedance/sonic/blob/main/README.md#sort-keys)) that is **NOT** in conformity to [RFC8259](https://datatracker.ietf.org/doc/html/rfc8259).\n\n ```go\nimport \"github.com/bytedance/sonic\"\n\nvar data YourSchema\n// Marshal\noutput, err := sonic.Marshal(&data)\n// Unmarshal\nerr := sonic.Unmarshal(output, &data)\n ```\n\n### Streaming IO\n\nSonic supports decoding json from `io.Reader` or encoding objects into `io.Writer`, aims at handling multiple values as well as reducing memory consumption.\n\n- encoder\n\n```go\nvar o1 = map[string]interface{}{\n \"a\": \"b\",\n}\nvar o2 = 1\nvar w = bytes.NewBuffer(nil)\nvar enc = sonic.ConfigDefault.NewEncoder(w)\nenc.Encode(o1)\nenc.Encode(o2)\nfmt.Println(w.String())\n// Output:\n// {\"a\":\"b\"}\n// 1\n```\n\n- decoder\n\n```go\nvar o = map[string]interface{}{}\nvar r = strings.NewReader(`{\"a\":\"b\"}{\"1\":\"2\"}`)\nvar dec = sonic.ConfigDefault.NewDecoder(r)\ndec.Decode(&o)\ndec.Decode(&o)\nfmt.Printf(\"%+v\", o)\n// Output:\n// map[1:2 a:b]\n```\n\n### Use Number/Use Int64\n\n ```go\nimport \"github.com/bytedance/sonic/decoder\"\n\nvar input = `1`\nvar data interface{}\n\n// default float64\ndc := decoder.NewDecoder(input)\ndc.Decode(&data) // data == float64(1)\n// use json.Number\ndc = decoder.NewDecoder(input)\ndc.UseNumber()\ndc.Decode(&data) // data == json.Number(\"1\")\n// use int64\ndc = decoder.NewDecoder(input)\ndc.UseInt64()\ndc.Decode(&data) // data == int64(1)\n\nroot, err := sonic.GetFromString(input)\n// Get json.Number\njn := root.Number()\njm := root.InterfaceUseNumber().(json.Number) // jn == jm\n// Get float64\nfn := root.Float64()\nfm := root.Interface().(float64) // jn == jm\n ```\n\n### Sort Keys\n\nOn account of the performance loss from sorting (roughly 10%), sonic doesn't enable this feature by default. If your component depends on it to work (like [zstd](https://github.com/facebook/zstd)), Use it like this:\n\n```go\nimport \"github.com/bytedance/sonic\"\nimport \"github.com/bytedance/sonic/encoder\"\n\n// Binding map only\nm := map[string]interface{}{}\nv, err := encoder.Encode(m, encoder.SortMapKeys)\n\n// Or ast.Node.SortKeys() before marshal\nvar root := sonic.Get(JSON)\nerr := root.SortKeys()\n```\n\n### Escape HTML\n\nOn account of the performance loss (roughly 15%), sonic doesn't enable this feature by default. You can use `encoder.EscapeHTML` option to open this feature (align with `encoding/json.HTMLEscape`).\n\n```go\nimport \"github.com/bytedance/sonic\"\n\nv := map[string]string{\"&&\":\"<>\"}\nret, err := Encode(v, EscapeHTML) // ret == `{\"\\u0026\\u0026\":{\"X\":\"\\u003c\\u003e\"}}`\n```\n\n### Compact Format\n\nSonic encodes primitive objects (struct/map...) as compact-format JSON by default, except marshaling `json.RawMessage` or `json.Marshaler`: sonic ensures validating their output JSON but **DO NOT** compacting them for performance concerns. We provide the option `encoder.CompactMarshaler` to add compacting process.\n\n### Print Error\n\nIf there invalid syntax in input JSON, sonic will return `decoder.SyntaxError`, which supports pretty-printing of error position\n\n```go\nimport \"github.com/bytedance/sonic\"\nimport \"github.com/bytedance/sonic/decoder\"\n\nvar data interface{}\nerr := sonic.UnmarshalString(\"[[[}]]\", &data)\nif err != nil {\n /* One line by default */\n println(e.Error()) // \"Syntax error at index 3: invalid char\\n\\n\\t[[[}]]\\n\\t...^..\\n\"\n /* Pretty print */\n if e, ok := err.(decoder.SyntaxError); ok {\n /*Syntax error at index 3: invalid char\n\n [[[}]]\n ...^..\n */\n print(e.Description())\n } else if me, ok := err.(*decoder.MismatchTypeError); ok {\n // decoder.MismatchTypeError is new to Sonic v1.6.0\n print(me.Description())\n }\n}\n```\n\n#### Mismatched Types [Sonic v1.6.0]\n\nIf there a **mismatch-typed** value for a given key, sonic will report `decoder.MismatchTypeError` (if there are many, report the last one), but still skip wrong the value and keep decoding next JSON.\n\n```go\nimport \"github.com/bytedance/sonic\"\nimport \"github.com/bytedance/sonic/decoder\"\n\nvar data = struct{\n A int\n B int\n}{}\nerr := UnmarshalString(`{\"A\":\"1\",\"B\":1}`, &data)\nprintln(err.Error()) // Mismatch type int with value string \"at index 5: mismatched type with value\\n\\n\\t{\\\"A\\\":\\\"1\\\",\\\"B\\\":1}\\n\\t.....^.........\\n\"\nfmt.Printf(\"%+v\", data) // {A:0 B:1}\n```\n\n### Ast.Node\n\nSonic/ast.Node is a completely self-contained AST for JSON. It implements serialization and deserialization both and provides robust APIs for obtaining and modification of generic data.\n\n#### Get/Index\n\nSearch partial JSON by given paths, which must be non-negative integer or string, or nil\n\n```go\nimport \"github.com/bytedance/sonic\"\n\ninput := []byte(`{\"key1\":[{},{\"key2\":{\"key3\":[1,2,3]}}]}`)\n\n// no path, returns entire json\nroot, err := sonic.Get(input)\nraw := root.Raw() // == string(input)\n\n// multiple paths\nroot, err := sonic.Get(input, \"key1\", 1, \"key2\")\nsub := root.Get(\"key3\").Index(2).Int64() // == 3\n```\n\n**Tip**: since `Index()` uses offset to locate data, which is much faster than scanning like `Get()`, we suggest you use it as much as possible. And sonic also provides another API `IndexOrGet()` to underlying use offset as well as ensure the key is matched.\n\n#### SearchOption\n\n`Searcher` provides some options for user to meet different needs:\n\n```go\nopts := ast.SearchOption{ CopyReturn: true ... }\nval, err := sonic.GetWithOptions(JSON, opts, \"key\")\n```\n\n- CopyReturn\nIndicate the searcher to copy the result JSON string instead of refer from the input. This can help to reduce memory usage if you cache the results\n- ConcurentRead\nSince `ast.Node` use `Lazy-Load` design, it doesn't support Concurrently-Read by default. If you want to read it concurrently, please specify it.\n- ValidateJSON\nIndicate the searcher to validate the entire JSON. This option is enabled by default, which slow down the search speed a little.\n\n#### Set/Unset\n\nModify the json content by Set()/Unset()\n\n```go\nimport \"github.com/bytedance/sonic\"\n\n// Set\nexist, err := root.Set(\"key4\", NewBool(true)) // exist == false\nalias1 := root.Get(\"key4\")\nprintln(alias1.Valid()) // true\nalias2 := root.Index(1)\nprintln(alias1 == alias2) // true\n\n// Unset\nexist, err := root.UnsetByIndex(1) // exist == true\nprintln(root.Get(\"key4\").Check()) // \"value not exist\"\n```\n\n#### Serialize\n\nTo encode `ast.Node` as json, use `MarshalJson()` or `json.Marshal()` (MUST pass the node's pointer)\n\n```go\nimport (\n \"encoding/json\"\n \"github.com/bytedance/sonic\"\n)\n\nbuf, err := root.MarshalJson()\nprintln(string(buf)) // {\"key1\":[{},{\"key2\":{\"key3\":[1,2,3]}}]}\nexp, err := json.Marshal(&root) // WARN: use pointer\nprintln(string(buf) == string(exp)) // true\n```\n\n#### APIs\n\n- validation: `Check()`, `Error()`, `Valid()`, `Exist()`\n- searching: `Index()`, `Get()`, `IndexPair()`, `IndexOrGet()`, `GetByPath()`\n- go-type casting: `Int64()`, `Float64()`, `String()`, `Number()`, `Bool()`, `Map[UseNumber|UseNode]()`, `Array[UseNumber|UseNode]()`, `Interface[UseNumber|UseNode]()`\n- go-type packing: `NewRaw()`, `NewNumber()`, `NewNull()`, `NewBool()`, `NewString()`, `NewObject()`, `NewArray()`\n- iteration: `Values()`, `Properties()`, `ForEach()`, `SortKeys()`\n- modification: `Set()`, `SetByIndex()`, `Add()`\n\n### Ast.Visitor\n\nSonic provides an advanced API for fully parsing JSON into non-standard types (neither `struct` not `map[string]interface{}`) without using any intermediate representation (`ast.Node` or `interface{}`). For example, you might have the following types which are like `interface{}` but actually not `interface{}`:\n\n```go\ntype UserNode interface {}\n\n// the following types implement the UserNode interface.\ntype (\n UserNull struct{}\n UserBool struct{ Value bool }\n UserInt64 struct{ Value int64 }\n UserFloat64 struct{ Value float64 }\n UserString struct{ Value string }\n UserObject struct{ Value map[string]UserNode }\n UserArray struct{ Value []UserNode }\n)\n```\n\nSonic provides the following API to return **the preorder traversal of a JSON AST**. The `ast.Visitor` is a SAX style interface which is used in some C++ JSON library. You should implement `ast.Visitor` by yourself and pass it to `ast.Preorder()` method. In your visitor you can make your custom types to represent JSON values. There may be an O(n) space container (such as stack) in your visitor to record the object / array hierarchy.\n\n```go\nfunc Preorder(str string, visitor Visitor, opts *VisitorOptions) error\n\ntype Visitor interface {\n OnNull() error\n OnBool(v bool) error\n OnString(v string) error\n OnInt64(v int64, n json.Number) error\n OnFloat64(v float64, n json.Number) error\n OnObjectBegin(capacity int) error\n OnObjectKey(key string) error\n OnObjectEnd() error\n OnArrayBegin(capacity int) error\n OnArrayEnd() error\n}\n```\n\nSee [ast/visitor.go](https://github.com/bytedance/sonic/blob/main/ast/visitor.go) for detailed usage. We also implement a demo visitor for `UserNode` in [ast/visitor_test.go](https://github.com/bytedance/sonic/blob/main/ast/visitor_test.go).\n\n## Compatibility\n\nFor developers who want to use sonic to meet different scenarios, we provide some integrated configs as `sonic.API`\n\n- `ConfigDefault`: the sonic's default config (`EscapeHTML=false`,`SortKeys=false`...) to run sonic fast meanwhile ensure security.\n- `ConfigStd`: the std-compatible config (`EscapeHTML=true`,`SortKeys=true`...)\n- `ConfigFastest`: the fastest config (`NoQuoteTextMarshaler=true`) to run on sonic as fast as possible.\nSonic **DOES NOT** ensure to support all environments, due to the difficulty of developing high-performance codes. On non-sonic-supporting environment, the implementation will fall back to `encoding/json`. Thus below configs will all equal to `ConfigStd`.\n\n## Tips\n\n### Pretouch\n\nSince Sonic uses [golang-asm](https://github.com/twitchyliquid64/golang-asm) as a JIT assembler, which is NOT very suitable for runtime compiling, first-hit running of a huge schema may cause request-timeout or even process-OOM. For better stability, we advise **using `Pretouch()` for huge-schema or compact-memory applications** before `Marshal()/Unmarshal()`.\n\n```go\nimport (\n \"reflect\"\n \"github.com/bytedance/sonic\"\n \"github.com/bytedance/sonic/option\"\n)\n\nfunc init() {\n var v HugeStruct\n\n // For most large types (nesting depth <= option.DefaultMaxInlineDepth)\n err := sonic.Pretouch(reflect.TypeOf(v))\n\n // with more CompileOption...\n err := sonic.Pretouch(reflect.TypeOf(v),\n // If the type is too deep nesting (nesting depth > option.DefaultMaxInlineDepth),\n // you can set compile recursive loops in Pretouch for better stability in JIT.\n option.WithCompileRecursiveDepth(loop),\n // For a large nested struct, try to set a smaller depth to reduce compiling time.\n option.WithCompileMaxInlineDepth(depth),\n )\n}\n```\n\n### Copy string\n\nWhen decoding **string values without any escaped characters**, sonic references them from the origin JSON buffer instead of mallocing a new buffer to copy. This helps a lot for CPU performance but may leave the whole JSON buffer in memory as long as the decoded objects are being used. In practice, we found the extra memory introduced by referring JSON buffer is usually 20% ~ 80% of decoded objects. Once an application holds these objects for a long time (for example, cache the decoded objects for reusing), its in-use memory on the server may go up. - `Config.CopyString`/`decoder.CopyString()`: We provide the option for `Decode()` / `Unmarshal()` users to choose not to reference the JSON buffer, which may cause a decline in CPU performance to some degree.\n\n- `GetFromStringNoCopy()`: For memory safety, `sonic.Get()` / `sonic.GetFromString()` now copies return JSON. If users want to get json more quickly and not care about memory usage, you can use `GetFromStringNoCopy()` to return a JSON directly referenced from source.\n\n### Pass string or []byte?\n\nFor alignment to `encoding/json`, we provide API to pass `[]byte` as an argument, but the string-to-bytes copy is conducted at the same time considering safety, which may lose performance when the origin JSON is huge. Therefore, you can use `UnmarshalString()` and `GetFromString()` to pass a string, as long as your origin data is a string or **nocopy-cast** is safe for your []byte. We also provide API `MarshalString()` for convenient **nocopy-cast** of encoded JSON []byte, which is safe since sonic's output bytes is always duplicated and unique.\n\n### Accelerate `encoding.TextMarshaler`\n\nTo ensure data security, sonic.Encoder quotes and escapes string values from `encoding.TextMarshaler` interfaces by default, which may degrade performance much if most of your data is in form of them. We provide `encoder.NoQuoteTextMarshaler` to skip these operations, which means you **MUST** ensure their output string escaped and quoted following [RFC8259](https://datatracker.ietf.org/doc/html/rfc8259).\n\n### Better performance for generic data\n\nIn **fully-parsed** scenario, `Unmarshal()` performs better than `Get()`+`Node.Interface()`. But if you only have a part of the schema for specific json, you can combine `Get()` and `Unmarshal()` together:\n\n```go\nimport \"github.com/bytedance/sonic\"\n\nnode, err := sonic.GetFromString(_TwitterJson, \"statuses\", 3, \"user\")\nvar user User // your partial schema...\nerr = sonic.UnmarshalString(node.Raw(), &user)\n```\n\nEven if you don't have any schema, use `ast.Node` as the container of generic values instead of `map` or `interface`:\n\n```go\nimport \"github.com/bytedance/sonic\"\n\nroot, err := sonic.GetFromString(_TwitterJson)\nuser := root.GetByPath(\"statuses\", 3, \"user\") // === root.Get(\"status\").Index(3).Get(\"user\")\nerr = user.Check()\n\n// err = user.LoadAll() // only call this when you want to use 'user' concurrently...\ngo someFunc(user)\n```\n\nWhy? Because `ast.Node` stores its children using `array`:\n\n- `Array`'s performance is **much better** than `Map` when Inserting (Deserialize) and Scanning (Serialize) data;\n- **Hashing** (`map[x]`) is not as efficient as **Indexing** (`array[x]`), which `ast.Node` can conduct on **both array and object**;\n- Using `Interface()`/`Map()` means Sonic must parse all the underlying values, while `ast.Node` can parse them **on demand**.\n\n**CAUTION:** `ast.Node` **DOESN'T** ensure concurrent security directly, due to its **lazy-load** design. However, you can call `Node.Load()`/`Node.LoadAll()` to achieve that, which may bring performance reduction while it still works faster than converting to `map` or `interface{}`\n\n### Ast.Node or Ast.Visitor?\n\nFor generic data, `ast.Node` should be enough for your needs in most cases.\n\nHowever, `ast.Node` is designed for partially processing JSON string. It has some special designs such as lazy-load which might not be suitable for directly parsing the whole JSON string like `Unmarshal()`. Although `ast.Node` is better then `map` or `interface{}`, it's also a kind of intermediate representation after all if your final types are customized and you have to convert the above types to your custom types after parsing.\n\nFor better performance, in previous case the `ast.Visitor` will be the better choice. It performs JSON decoding like `Unmarshal()` and you can directly use your final types to represents a JSON AST without any intermediate representations.\n\nBut `ast.Visitor` is not a very handy API. You might need to write a lot of code to implement your visitor and carefully maintain the tree hierarchy during decoding. Please read the comments in [ast/visitor.go](https://github.com/bytedance/sonic/blob/main/ast/visitor.go) carefully if you decide to use this API.\n\n### Buffer Size\n\nSonic use memory pool in many places like `encoder.Encode`, `ast.Node.MarshalJSON` to improve performance, which may produce more memory usage (in-use) when server's load is high. See [issue 614](https://github.com/bytedance/sonic/issues/614). Therefore, we introduce some options to let user control the behavior of memory pool. See [option](https://pkg.go.dev/github.com/bytedance/sonic@v1.11.9/option#pkg-variables) package.\n\n### Faster JSON Skip\n\nFor security, sonic use [FSM](native/skip_one.c) algorithm to validate JSON when decoding raw JSON or encoding `json.Marshaler`, which is much slower (1~10x) than [SIMD-searching-pair](native/skip_one_fast.c) algorithm. If user has many redundant JSON value and DO NOT NEED to strictly validate JSON correctness, you can enable below options:\n\n- `Config.NoValidateSkipJSON`: for faster skipping JSON when decoding, such as unknown fields, json.Unmarshaler(json.RawMessage), mismatched values, and redundant array elements\n- `Config.NoValidateJSONMarshaler`: avoid validating JSON when encoding `json.Marshaler`\n- `SearchOption.ValidateJSON`: indicates if validate located JSON value when `Get`\n\n## JSON-Path Support (GJSON)\n\n[tidwall/gjson](https://github.com/tidwall/gjson) has provided a comprehensive and popular JSON-Path API, and\n a lot of older codes heavily relies on it. Therefore, we provides a wrapper library, which combines gjson's API with sonic's SIMD algorithm to boost up the performance. See [cloudwego/gjson](https://github.com/cloudwego/gjson).\n\n## Community\n\nSonic is a subproject of [CloudWeGo](https://www.cloudwego.io/). We are committed to building a cloud native ecosystem.\n" }, { "path": "README_ZH_CN.md", "content": "# Sonic\n\n[English](README.md) | 中文\n\n一个速度奇快的 JSON 序列化/反序列化库，由 JIT （即时编译）和 SIMD （单指令流多数据流）加速。\n\n## 依赖\n\n- Go: 1.18~1.25\n - 注意：Go1.24.0 由于 [issue](https://github.com/golang/go/issues/71672) 不可用，请升级到更高 Go 版本，或添加编译选项 `--ldflags=\"-checklinkname=0\"` \n- OS: Linux / MacOS / Windows\n- CPU: AMD64 / (ARM64, 需要 Go1.20 以上）\n\n## 接口\n\n详见 [go.dev](https://pkg.go.dev/github.com/bytedance/sonic)\n\n## 特色\n\n- 运行时对象绑定，无需代码生成\n- 完备的 JSON 操作 API\n- 快，更快，还要更快！\n\n## 基准测试\n\n对于**所有大小**的 json 和**所有使用场景**， **Sonic 表现均为最佳**。\n\n- [中型](https://github.com/bytedance/sonic/blob/main/decoder/testdata_test.go#L19) (13kB, 300+ 键, 6 层)\n\n```powershell\ngoversion: 1.17.1\ngoos: darwin\ngoarch: amd64\ncpu: Intel(R) Core(TM) i9-9880H CPU @ 2.30GHz\nBenchmarkEncoder_Generic_Sonic-16 32393 ns/op 402.40 MB/s 11965 B/op 4 allocs/op\nBenchmarkEncoder_Generic_Sonic_Fast-16 21668 ns/op 601.57 MB/s 10940 B/op 4 allocs/op\nBenchmarkEncoder_Generic_JsonIter-16 42168 ns/op 309.12 MB/s 14345 B/op 115 allocs/op\nBenchmarkEncoder_Generic_GoJson-16 65189 ns/op 199.96 MB/s 23261 B/op 16 allocs/op\nBenchmarkEncoder_Generic_StdLib-16 106322 ns/op 122.60 MB/s 49136 B/op 789 allocs/op\nBenchmarkEncoder_Binding_Sonic-16 6269 ns/op 2079.26 MB/s 14173 B/op 4 allocs/op\nBenchmarkEncoder_Binding_Sonic_Fast-16 5281 ns/op 2468.16 MB/s 12322 B/op 4 allocs/op\nBenchmarkEncoder_Binding_JsonIter-16 20056 ns/op 649.93 MB/s 9488 B/op 2 allocs/op\nBenchmarkEncoder_Binding_GoJson-16 8311 ns/op 1568.32 MB/s 9481 B/op 1 allocs/op\nBenchmarkEncoder_Binding_StdLib-16 16448 ns/op 792.52 MB/s 9479 B/op 1 allocs/op\nBenchmarkEncoder_Parallel_Generic_Sonic-16 6681 ns/op 1950.93 MB/s 12738 B/op 4 allocs/op\nBenchmarkEncoder_Parallel_Generic_Sonic_Fast-16 4179 ns/op 3118.99 MB/s 10757 B/op 4 allocs/op\nBenchmarkEncoder_Parallel_Generic_JsonIter-16 9861 ns/op 1321.84 MB/s 14362 B/op 115 allocs/op\nBenchmarkEncoder_Parallel_Generic_GoJson-16 18850 ns/op 691.52 MB/s 23278 B/op 16 allocs/op\nBenchmarkEncoder_Parallel_Generic_StdLib-16 45902 ns/op 283.97 MB/s 49174 B/op 789 allocs/op\nBenchmarkEncoder_Parallel_Binding_Sonic-16 1480 ns/op 8810.09 MB/s 13049 B/op 4 allocs/op\nBenchmarkEncoder_Parallel_Binding_Sonic_Fast-16 1209 ns/op 10785.23 MB/s 11546 B/op 4 allocs/op\nBenchmarkEncoder_Parallel_Binding_JsonIter-16 6170 ns/op 2112.58 MB/s 9504 B/op 2 allocs/op\nBenchmarkEncoder_Parallel_Binding_GoJson-16 3321 ns/op 3925.52 MB/s 9496 B/op 1 allocs/op\nBenchmarkEncoder_Parallel_Binding_StdLib-16 3739 ns/op 3486.49 MB/s 9480 B/op 1 allocs/op\n\nBenchmarkDecoder_Generic_Sonic-16 66812 ns/op 195.10 MB/s 57602 B/op 723 allocs/op\nBenchmarkDecoder_Generic_Sonic_Fast-16 54523 ns/op 239.07 MB/s 49786 B/op 313 allocs/op\nBenchmarkDecoder_Generic_StdLib-16 124260 ns/op 104.90 MB/s 50869 B/op 772 allocs/op\nBenchmarkDecoder_Generic_JsonIter-16 91274 ns/op 142.81 MB/s 55782 B/op 1068 allocs/op\nBenchmarkDecoder_Generic_GoJson-16 88569 ns/op 147.17 MB/s 66367 B/op 973 allocs/op\nBenchmarkDecoder_Binding_Sonic-16 32557 ns/op 400.38 MB/s 28302 B/op 137 allocs/op\nBenchmarkDecoder_Binding_Sonic_Fast-16 28649 ns/op 455.00 MB/s 24999 B/op 34 allocs/op\nBenchmarkDecoder_Binding_StdLib-16 111437 ns/op 116.97 MB/s 10576 B/op 208 allocs/op\nBenchmarkDecoder_Binding_JsonIter-16 35090 ns/op 371.48 MB/s 14673 B/op 385 allocs/op\nBenchmarkDecoder_Binding_GoJson-16 28738 ns/op 453.59 MB/s 22039 B/op 49 allocs/op\nBenchmarkDecoder_Parallel_Generic_Sonic-16 12321 ns/op 1057.91 MB/s 57233 B/op 723 allocs/op\nBenchmarkDecoder_Parallel_Generic_Sonic_Fast-16 10644 ns/op 1224.64 MB/s 49362 B/op 313 allocs/op\nBenchmarkDecoder_Parallel_Generic_StdLib-16 57587 ns/op 226.35 MB/s 50874 B/op 772 allocs/op\nBenchmarkDecoder_Parallel_Generic_JsonIter-16 38666 ns/op 337.12 MB/s 55789 B/op 1068 allocs/op\nBenchmarkDecoder_Parallel_Generic_GoJson-16 30259 ns/op 430.79 MB/s 66370 B/op 974 allocs/op\nBenchmarkDecoder_Parallel_Binding_Sonic-16 5965 ns/op 2185.28 MB/s 27747 B/op 137 allocs/op\nBenchmarkDecoder_Parallel_Binding_Sonic_Fast-16 5170 ns/op 2521.31 MB/s 24715 B/op 34 allocs/op\nBenchmarkDecoder_Parallel_Binding_StdLib-16 27582 ns/op 472.58 MB/s 10576 B/op 208 allocs/op\nBenchmarkDecoder_Parallel_Binding_JsonIter-16 13571 ns/op 960.51 MB/s 14685 B/op 385 allocs/op\nBenchmarkDecoder_Parallel_Binding_GoJson-16 10031 ns/op 1299.51 MB/s 22111 B/op 49 allocs/op\n\nBenchmarkGetOne_Sonic-16 3276 ns/op 3975.78 MB/s 24 B/op 1 allocs/op\nBenchmarkGetOne_Gjson-16 9431 ns/op 1380.81 MB/s 0 B/op 0 allocs/op\nBenchmarkGetOne_Jsoniter-16 51178 ns/op 254.46 MB/s 27936 B/op 647 allocs/op\nBenchmarkGetOne_Parallel_Sonic-16 216.7 ns/op 60098.95 MB/s 24 B/op 1 allocs/op\nBenchmarkGetOne_Parallel_Gjson-16 1076 ns/op 12098.62 MB/s 0 B/op 0 allocs/op\nBenchmarkGetOne_Parallel_Jsoniter-16 17741 ns/op 734.06 MB/s 27945 B/op 647 allocs/op\nBenchmarkSetOne_Sonic-16 9571 ns/op 1360.61 MB/s 1584 B/op 17 allocs/op\nBenchmarkSetOne_Sjson-16 36456 ns/op 357.22 MB/s 52180 B/op 9 allocs/op\nBenchmarkSetOne_Jsoniter-16 79475 ns/op 163.86 MB/s 45862 B/op 964 allocs/op\nBenchmarkSetOne_Parallel_Sonic-16 850.9 ns/op 15305.31 MB/s 1584 B/op 17 allocs/op\nBenchmarkSetOne_Parallel_Sjson-16 18194 ns/op 715.77 MB/s 52247 B/op 9 allocs/op\nBenchmarkSetOne_Parallel_Jsoniter-16 33560 ns/op 388.05 MB/s 45892 B/op 964 allocs/op\nBenchmarkLoadNode/LoadAll()-16 11384 ns/op 1143.93 MB/s 6307 B/op 25 allocs/op\nBenchmarkLoadNode_Parallel/LoadAll()-16 5493 ns/op 2370.68 MB/s 7145 B/op 25 allocs/op\nBenchmarkLoadNode/Interface()-16 17722 ns/op 734.85 MB/s 13323 B/op 88 allocs/op\nBenchmarkLoadNode_Parallel/Interface()-16 10330 ns/op 1260.70 MB/s 15178 B/op 88 allocs/op\n```\n\n- [小型](https://github.com/bytedance/sonic/blob/main/testdata/small.go) (400B, 11 个键, 3 层)\n![small benchmarks](./docs/imgs/bench-small.png)\n- [大型](https://github.com/bytedance/sonic/blob/main/testdata/twitter.json) (635kB, 10000+ 个键, 6 层)\n![large benchmarks](./docs/imgs/bench-large.png)\n\n要查看基准测试代码，请参阅 [bench.sh](https://github.com/bytedance/sonic/blob/main/scripts/bench.sh) 。\n\n## 工作原理\n\n请参阅 [INTRODUCTION_ZH_CN.md](./docs/INTRODUCTION_ZH_CN.md).\n\n## 使用方式\n\n### 序列化/反序列化\n\n默认的行为基本上与 `encoding/json` 相一致，除了 HTML 转义形式（参见 [Escape HTML](https://github.com/bytedance/sonic/blob/main/README.md#escape-html)) 和 `SortKeys` 功能（参见 [Sort Keys](https://github.com/bytedance/sonic/blob/main/README.md#sort-keys)）**没有**遵循 [RFC8259](https://datatracker.ietf.org/doc/html/rfc8259) 。\n\n ```go\nimport \"github.com/bytedance/sonic\"\n\nvar data YourSchema\n// Marshal\noutput, err := sonic.Marshal(&data)\n// Unmarshal\nerr := sonic.Unmarshal(output, &data)\n ```\n\n### 流式输入输出\n\nSonic 支持解码 `io.Reader` 中输入的 json，或将对象编码为 json 后输出至 `io.Writer`，以处理多个值并减少内存消耗。\n\n- 编码器\n\n```go\nvar o1 = map[string]interface{}{\n \"a\": \"b\",\n}\nvar o2 = 1\nvar w = bytes.NewBuffer(nil)\nvar enc = sonic.ConfigDefault.NewEncoder(w)\nenc.Encode(o1)\nenc.Encode(o2)\nfmt.Println(w.String())\n// Output:\n// {\"a\":\"b\"}\n// 1\n```\n\n- 解码器\n\n```go\nvar o = map[string]interface{}{}\nvar r = strings.NewReader(`{\"a\":\"b\"}{\"1\":\"2\"}`)\nvar dec = sonic.ConfigDefault.NewDecoder(r)\ndec.Decode(&o)\ndec.Decode(&o)\nfmt.Printf(\"%+v\", o)\n// Output:\n// map[1:2 a:b]\n```\n\n### 使用 `Number` / `int64`\n\n```go\nimport \"github.com/bytedance/sonic/decoder\"\n\nvar input = `1`\nvar data interface{}\n\n// default float64\ndc := decoder.NewDecoder(input)\ndc.Decode(&data) // data == float64(1)\n// use json.Number\ndc = decoder.NewDecoder(input)\ndc.UseNumber()\ndc.Decode(&data) // data == json.Number(\"1\")\n// use int64\ndc = decoder.NewDecoder(input)\ndc.UseInt64()\ndc.Decode(&data) // data == int64(1)\n\nroot, err := sonic.GetFromString(input)\n// Get json.Number\njn := root.Number()\njm := root.InterfaceUseNumber().(json.Number) // jn == jm\n// Get float64\nfn := root.Float64()\nfm := root.Interface().(float64) // jn == jm\n ```\n\n### 对键排序\n\n考虑到排序带来的性能损失（约 10% ）， sonic 默认不会启用这个功能。如果你的组件依赖这个行为（如 [zstd](https://github.com/facebook/zstd)) ，可以仿照下面的例子：\n\n```go\nimport \"github.com/bytedance/sonic\"\nimport \"github.com/bytedance/sonic/encoder\"\n\n// Binding map only\nm := map[string]interface{}{}\nv, err := encoder.Encode(m, encoder.SortMapKeys)\n\n// Or ast.Node.SortKeys() before marshal\nvar root := sonic.Get(JSON)\nerr := root.SortKeys()\n```\n\n### HTML 转义\n\n考虑到性能损失（约15%）， sonic 默认不会启用这个功能。你可以使用 `encoder.EscapeHTML` 选项来开启（与 `encoding/json.HTMLEscape` 行为一致）。\n\n```go\nimport \"github.com/bytedance/sonic\"\n\nv := map[string]string{\"&&\":\"<>\"}\nret, err := Encode(v, EscapeHTML) // ret == `{\"\\u0026\\u0026\":{\"X\":\"\\u003c\\u003e\"}}`\n```\n\n### 紧凑格式\n\nSonic 默认将基本类型（ `struct` ， `map` 等）编码为紧凑格式的 JSON ，除非使用 `json.RawMessage` or `json.Marshaler` 进行编码： sonic 确保输出的 JSON 合法，但出于性能考虑，**不会**加工成紧凑格式。我们提供选项 `encoder.CompactMarshaler` 来添加此过程，\n\n### 打印错误\n\n如果输入的 JSON 存在无效的语法，sonic 将返回 `decoder.SyntaxError`，该错误支持错误位置的美化输出。\n\n```go\nimport \"github.com/bytedance/sonic\"\nimport \"github.com/bytedance/sonic/decoder\"\n\nvar data interface{}\nerr := sonic.UnmarshalString(\"[[[}]]\", &data)\nif err != nil {\n /* One line by default */\n println(e.Error()) // \"Syntax error at index 3: invalid char\\n\\n\\t[[[}]]\\n\\t...^..\\n\"\n /* Pretty print */\n if e, ok := err.(decoder.SyntaxError); ok {\n /*Syntax error at index 3: invalid char\n\n [[[}]]\n ...^..\n */\n print(e.Description())\n } else if me, ok := err.(*decoder.MismatchTypeError); ok {\n // decoder.MismatchTypeError is new to Sonic v1.6.0\n print(me.Description())\n }\n}\n```\n\n#### 类型不匹配 [Sonic v1.6.0]\n\n如果给定键中存在**类型不匹配**的值， sonic 会抛出 `decoder.MismatchTypeError` （如果有多个，只会报告最后一个），但仍会跳过错误的值并解码下一个 JSON 。\n\n```go\nimport \"github.com/bytedance/sonic\"\nimport \"github.com/bytedance/sonic/decoder\"\n\nvar data = struct{\n A int\n B int\n}{}\nerr := UnmarshalString(`{\"A\":\"1\",\"B\":1}`, &data)\nprintln(err.Error()) // Mismatch type int with value string \"at index 5: mismatched type with value\\n\\n\\t{\\\"A\\\":\\\"1\\\",\\\"B\\\":1}\\n\\t.....^.........\\n\"\nfmt.Printf(\"%+v\", data) // {A:0 B:1}\n```\n\n### `Ast.Node`\n\nSonic/ast.Node 是完全独立的 JSON 抽象语法树库。它实现了序列化和反序列化，并提供了获取和修改JSON数据的鲁棒的 API。\n\n#### 查找/索引\n\n通过给定的路径搜索 JSON 片段，路径必须为非负整数，字符串或 `nil` 。\n\n```go\nimport \"github.com/bytedance/sonic\"\n\ninput := []byte(`{\"key1\":[{},{\"key2\":{\"key3\":[1,2,3]}}]}`)\n\n// no path, returns entire json\nroot, err := sonic.Get(input)\nraw := root.Raw() // == string(input)\n\n// multiple paths\nroot, err := sonic.Get(input, \"key1\", 1, \"key2\")\nsub := root.Get(\"key3\").Index(2).Int64() // == 3\n```\n\n**注意**：由于 `Index()` 使用偏移量来定位数据，比使用扫描的 `Get()` 要快的多，建议尽可能的使用 `Index` 。 Sonic 也提供了另一个 API， `IndexOrGet()` ，以偏移量为基础并且也确保键的匹配。\n\n#### 查找选项\n\n`ast.Searcher`提供了一些选项，以满足用户的不同需求:\n\n```go\nopts := ast.SearchOption{CopyReturn: true…}\nval, err := sonic.GetWithOptions(JSON, opts, \"key\")\n```\n\n- CopyReturn\n指示搜索器复制结果JSON字符串，而不是从输入引用。如果用户缓存结果，这有助于减少内存使用\n- ConcurentRead\n因为`ast.Node`使用`Lazy-Load`设计，默认不支持并发读取。如果您想同时读取，请指定它。\n- ValidateJSON\n指示搜索器来验证整个JSON。默认情况下启用该选项, 但是对于查找速度有一定影响。\n\n#### 修改\n\n使用 `Set()` / `Unset()` 修改 json 的内容\n\n```go\nimport \"github.com/bytedance/sonic\"\n\n// Set\nexist, err := root.Set(\"key4\", NewBool(true)) // exist == false\nalias1 := root.Get(\"key4\")\nprintln(alias1.Valid()) // true\nalias2 := root.Index(1)\nprintln(alias1 == alias2) // true\n\n// Unset\nexist, err := root.UnsetByIndex(1) // exist == true\nprintln(root.Get(\"key4\").Check()) // \"value not exist\"\n```\n\n#### 序列化\n\n要将 `ast.Node` 编码为 json ，使用 `MarshalJson()` 或者 `json.Marshal()` （必须传递指向节点的指针）\n\n```go\nimport (\n \"encoding/json\"\n \"github.com/bytedance/sonic\"\n)\n\nbuf, err := root.MarshalJson()\nprintln(string(buf)) // {\"key1\":[{},{\"key2\":{\"key3\":[1,2,3]}}]}\nexp, err := json.Marshal(&root) // WARN: use pointer\nprintln(string(buf) == string(exp)) // true\n```\n\n#### APIs\n\n- 合法性检查： `Check()`, `Error()`, `Valid()`, `Exist()`\n- 索引： `Index()`, `Get()`, `IndexPair()`, `IndexOrGet()`, `GetByPath()`\n- 转换至 go 内置类型： `Int64()`, `Float64()`, `String()`, `Number()`, `Bool()`, `Map[UseNumber|UseNode]()`, `Array[UseNumber|UseNode]()`, `Interface[UseNumber|UseNode]()`\n- go 类型打包： `NewRaw()`, `NewNumber()`, `NewNull()`, `NewBool()`, `NewString()`, `NewObject()`, `NewArray()`\n- 迭代： `Values()`, `Properties()`, `ForEach()`, `SortKeys()`\n- 修改： `Set()`, `SetByIndex()`, `Add()`\n\n### `Ast.Visitor`\n\nSonic 提供了一个高级的 API 用于直接全量解析 JSON 到非标准容器里 (既不是 `struct` 也不是 `map[string]interface{}`) 且不需要借助任何中间表示 (`ast.Node` 或 `interface{}`)。举个例子，你可能定义了下述的类型，它们看起来像 `interface{}`，但实际上并不是：\n\n```go\ntype UserNode interface {}\n\n// the following types implement the UserNode interface.\ntype (\n UserNull struct{}\n UserBool struct{ Value bool }\n UserInt64 struct{ Value int64 }\n UserFloat64 struct{ Value float64 }\n UserString struct{ Value string }\n UserObject struct{ Value map[string]UserNode }\n UserArray struct{ Value []UserNode }\n)\n```\n\nSonic 提供了下述的 API 来返回 **“对 JSON AST 的前序遍历”**。`ast.Visitor` 是一个 SAX 风格的接口，这在某些 C++ 的 JSON 解析库中被使用到。你需要自己实现一个 `ast.Visitor`，将它传递给 `ast.Preorder()` 方法。在你的实现中你可以使用自定义的类型来表示 JSON 的值。在你的 `ast.Visitor` 中，可能需要有一个 O(n) 空间复杂度的容器（比如说栈）来记录 object / array 的层级。\n\n```go\nfunc Preorder(str string, visitor Visitor, opts *VisitorOptions) error\n\ntype Visitor interface {\n OnNull() error\n OnBool(v bool) error\n OnString(v string) error\n OnInt64(v int64, n json.Number) error\n OnFloat64(v float64, n json.Number) error\n OnObjectBegin(capacity int) error\n OnObjectKey(key string) error\n OnObjectEnd() error\n OnArrayBegin(capacity int) error\n OnArrayEnd() error\n}\n```\n\n详细用法参看 [ast/visitor.go](https://github.com/bytedance/sonic/blob/main/ast/visitor.go)，我们还为 `UserNode` 实现了一个示例 `ast.Visitor`，你可以在 [ast/visitor_test.go](https://github.com/bytedance/sonic/blob/main/ast/visitor_test.go) 中找到它。\n\n## 兼容性\n\n对于想要使用sonic来满足不同场景的开发人员，我们提供了一些集成配置:\n\n- `ConfigDefault`: sonic的默认配置 (`EscapeHTML=false`， `SortKeys=false`…) 保证性能同时兼顾安全性。\n- `ConfigStd`: 与 `encoding/json` 保证完全兼容的配置\n- `ConfigFastest`: 最快的配置(`NoQuoteTextMarshaler=true...`) 保证性能最优但是会缺少一些安全性检查（validate UTF8 等）\nSonic **不**确保支持所有环境，由于开发高性能代码的困难。在不支持sonic的环境中，实现将回落到 `encoding/json`。因此上述配置将全部等于`ConfigStd`。\n\n## 注意事项\n\n### 预热\n\n由于 Sonic 使用 [golang-asm](https://github.com/twitchyliquid64/golang-asm) 作为 JIT 汇编器，这个库并不适用于运行时编译，第一次运行一个大型模式可能会导致请求超时甚至进程内存溢出。为了更好地稳定性，我们建议在运行大型模式或在内存有限的应用中，在使用 `Marshal()/Unmarshal()` 前运行 `Pretouch()`。\n\n```go\nimport (\n \"reflect\"\n \"github.com/bytedance/sonic\"\n \"github.com/bytedance/sonic/option\"\n)\n\nfunc init() {\n var v HugeStruct\n\n // For most large types (nesting depth <= option.DefaultMaxInlineDepth)\n err := sonic.Pretouch(reflect.TypeOf(v))\n\n // with more CompileOption...\n err := sonic.Pretouch(reflect.TypeOf(v),\n // If the type is too deep nesting (nesting depth > option.DefaultMaxInlineDepth),\n // you can set compile recursive loops in Pretouch for better stability in JIT.\n option.WithCompileRecursiveDepth(loop),\n // For a large nested struct, try to set a smaller depth to reduce compiling time.\n option.WithCompileMaxInlineDepth(depth),\n )\n}\n```\n\n### 拷贝字符串\n\n当解码 **没有转义字符的字符串**时， sonic 会从原始的 JSON 缓冲区内引用而不是复制到新的一个缓冲区中。这对 CPU 的性能方面很有帮助，但是可能因此在解码后对象仍在使用的时候将整个 JSON 缓冲区保留在内存中。实践中我们发现，通过引用 JSON 缓冲区引入的额外内存通常是解码后对象的 20% 至 80% ，一旦应用长期保留这些对象（如缓存以备重用），服务器所使用的内存可能会增加。我们提供了选项 `decoder.CopyString()` 供用户选择，不引用 JSON 缓冲区。这可能在一定程度上降低 CPU 性能。\n\n### 传递字符串还是字节数组？\n\n为了和 `encoding/json` 保持一致，我们提供了传递 `[]byte` 作为参数的 API ，但考虑到安全性，字符串到字节的复制是同时进行的，这在原始 JSON 非常大时可能会导致性能损失。因此，你可以使用 `UnmarshalString()` 和 `GetFromString()` 来传递字符串，只要你的原始数据是字符串，或**零拷贝类型转换**对于你的字节数组是安全的。我们也提供了 `MarshalString()` 的 API ，以便对编码的 JSON 字节数组进行**零拷贝类型转换**，因为 sonic 输出的字节始终是重复并且唯一的，所以这样是安全的。\n\n### 加速 `encoding.TextMarshaler`\n\n为了保证数据安全性， `sonic.Encoder` 默认会对来自 `encoding.TextMarshaler` 接口的字符串进行引用和转义，如果大部分数据都是这种形式那可能会导致很大的性能损失。我们提供了 `encoder.NoQuoteTextMarshaler` 选项来跳过这些操作，但你**必须**保证他们的输出字符串依照 [RFC8259](https://datatracker.ietf.org/doc/html/rfc8259) 进行了转义和引用。\n\n### 泛型的性能优化\n\n在 **完全解析**的场景下， `Unmarshal()` 表现得比 `Get()`+`Node.Interface()` 更好。但是如果你只有特定 JSON 的部分模式，你可以将 `Get()` 和 `Unmarshal()` 结合使用：\n\n```go\nimport \"github.com/bytedance/sonic\"\n\nnode, err := sonic.GetFromString(_TwitterJson, \"statuses\", 3, \"user\")\nvar user User // your partial schema...\nerr = sonic.UnmarshalString(node.Raw(), &user)\n```\n\n甚至如果你没有任何模式，可以用 `ast.Node` 代替 `map` 或 `interface` 作为泛型的容器：\n\n```go\nimport \"github.com/bytedance/sonic\"\n\nroot, err := sonic.GetFromString(_TwitterJson)\nuser := root.GetByPath(\"statuses\", 3, \"user\") // === root.Get(\"status\").Index(3).Get(\"user\")\nerr = user.Check()\n\n// err = user.LoadAll() // only call this when you want to use 'user' concurrently...\ngo someFunc(user)\n```\n\n为什么？因为 `ast.Node` 使用 `array` 来存储其子节点：\n\n- 在插入（反序列化）和扫描（序列化）数据时，`Array` 的性能比 `Map` **好得多**；\n- **哈希**（`map[x]`）的效率不如**索引**（`array[x]`）高效，而 `ast.Node` 可以在数组和对象上使用索引；\n- 使用 `Interface()` / `Map()` 意味着 sonic 必须解析所有的底层值，而 `ast.Node` 可以**按需解析**它们。\n\n**注意**：由于 `ast.Node` 的惰性加载设计，其**不能**直接保证并发安全性，但你可以调用 `Node.Load()` / `Node.LoadAll()` 来实现并发安全。尽管可能会带来性能损失，但仍比转换成 `map` 或 `interface{}` 更为高效。\n\n### 使用 `ast.Node` 还是 `ast.Visitor`？\n\n对于泛型数据的解析，`ast.Node` 在大多数场景上应该能够满足你的需求。\n\n然而，`ast.Node` 是一种针对部分解析 JSON 而设计的泛型容器，它包含一些特殊设计，比如惰性加载，如果你希望像 `Unmarshal()` 那样直接解析整个 JSON，这些设计可能并不合适。尽管 `ast.Node` 相较于 `map` 或 `interface{}` 来说是更好的一种泛型容器，但它毕竟也是一种中间表示，如果你的最终类型是自定义的，你还得在解析完成后将上述类型转化成你自定义的类型。\n\n在上述场景中，如果想要有更极致的性能，`ast.Visitor` 会是更好的选择。它采用和 `Unmarshal()` 类似的形式解析 JSON，并且你可以直接使用你的最终类型去表示 JSON AST，而不需要经过额外的任何中间表示。\n\n但是，`ast.Visitor` 并不是一个很易用的 API。你可能需要写大量的代码去实现自己的 `ast.Visitor`，并且需要在解析过程中仔细维护树的层级。如果你决定要使用这个 API，请先仔细阅读 [ast/visitor.go](https://github.com/bytedance/sonic/blob/main/ast/visitor.go) 中的注释。\n\n### 缓冲区大小\n\nSonic在许多地方使用内存池，如`encoder.Encode`, `ast.Node.MarshalJSON`等来提高性能，这可能会在服务器负载高时产生更多的内存使用(in-use)。参见[issue 614](https://github.com/bytedance/sonic/issues/614)。因此，我们引入了一些选项来让用户配置内存池的行为。参见[option](https://pkg.go.dev/github.com/bytedance/sonic@v1.11.9/option#pkg-variables)包。\n\n### 更快的 JSON Skip\n\n为了安全起见，在跳过原始JSON 时，sonic decoder 默认使用[FSM](native/skip_one.c)算法扫描来跳过同时校验 JSON。它相比[SIMD-searching-pair](native/skip_one_fast.c)算法跳过要慢得多(1~10倍)。如果用户有很多冗余的JSON值，并且不需要严格验证JSON的正确性，你可以启用以下选项:\n\n- `Config.NoValidateSkipJSON`: 用于在解码时更快地跳过JSON，例如未知字段，`json.RawMessage`，不匹配的值和冗余的数组元素等\n- `Config.NoValidateJSONMarshaler`: 编码JSON时避免验证JSON。封送拆收器\n- `SearchOption.ValidateJSON`: 指示当`Get`时是否验证定位的JSON值\n\n## 社区\n\nSonic 是 [CloudWeGo](https://www.cloudwego.io/) 下的一个子项目。我们致力于构建云原生生态系统。\n" }, { "path": "api.go", "content": "/*\n * Copyright 2021 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage sonic\n\nimport (\n\t\"io\"\n\n\t\"github.com/bytedance/sonic/ast\"\n\t\"github.com/bytedance/sonic/internal/rt\"\n)\n\nconst (\n\t// UseStdJSON indicates you are using fallback implementation (encoding/json)\n\tUseStdJSON = iota\n\t// UseSonicJSON indicates you are using real sonic implementation\n\tUseSonicJSON\n)\n\n// APIKind is the kind of API, 0 is std json, 1 is sonic.\nconst APIKind = apiKind\n\n// Config is a combination of sonic/encoder.Options and sonic/decoder.Options\ntype Config struct {\n\t// EscapeHTML indicates encoder to escape all HTML characters\n\t// after serializing into JSON (see https://pkg.go.dev/encoding/json#HTMLEscape).\n\t// WARNING: This hurts performance A LOT, USE WITH CARE.\n\tEscapeHTML bool\n\n\t// SortMapKeys indicates encoder that the keys of a map needs to be sorted\n\t// before serializing into JSON.\n\t// WARNING: This hurts performance A LOT, USE WITH CARE.\n\tSortMapKeys bool\n\n\t// CompactMarshaler indicates encoder that the output JSON from json.Marshaler\n\t// is always compact and needs no validation\n\tCompactMarshaler bool\n\n\t// NoQuoteTextMarshaler indicates encoder that the output text from encoding.TextMarshaler\n\t// is always escaped string and needs no quoting\n\tNoQuoteTextMarshaler bool\n\n\t// NoNullSliceOrMap indicates encoder that all empty Array or Object are encoded as '[]' or '{}',\n\t// instead of 'null'\n\tNoNullSliceOrMap bool\n\n\t// UseInt64 indicates decoder to unmarshal an integer into an interface{} as an\n\t// int64 instead of as a float64.\n\tUseInt64 bool\n\n\t// UseNumber indicates decoder to unmarshal a number into an interface{} as a\n\t// json.Number instead of as a float64.\n\tUseNumber bool\n\n\t// UseUnicodeErrors indicates decoder to return an error when encounter invalid\n\t// UTF-8 escape sequences.\n\tUseUnicodeErrors bool\n\n\t// DisallowUnknownFields indicates decoder to return an error when the destination\n\t// is a struct and the input contains object keys which do not match any\n\t// non-ignored, exported fields in the destination.\n\tDisallowUnknownFields bool\n\n\t// CopyString indicates decoder to decode string values by copying instead of referring.\n\tCopyString bool\n\n\t// ValidateString indicates decoder and encoder to validate string values: decoder will return errors\n\t// when unescaped control chars(\\u0000-\\u001f) in the string value of JSON.\n\tValidateString bool\n\n\t// NoValidateJSONMarshaler indicates that the encoder should not validate the output string\n\t// after encoding the JSONMarshaler to JSON.\n\tNoValidateJSONMarshaler bool\n\n\t// NoValidateJSONSkip indicates the decoder should not validate the JSON value when skipping it,\n\t// such as unknown-fields, mismatched-type, redundant elements..\n\tNoValidateJSONSkip bool\n\n\t// NoEncoderNewline indicates that the encoder should not add a newline after every message\n\tNoEncoderNewline bool\n\n\t// Encode Infinity or Nan float into `null`, instead of returning an error.\n\tEncodeNullForInfOrNan bool\n\n\t// CaseSensitive indicates that the decoder should not ignore the case of object keys.\n\tCaseSensitive bool\n}\n\nvar (\n\t// ConfigDefault is the default config of APIs, aiming at efficiency and safety.\n\tConfigDefault = Config{}.Froze()\n\n\t// ConfigStd is the standard config of APIs, aiming at being compatible with encoding/json.\n\tConfigStd = Config{\n\t\tEscapeHTML: true,\n\t\tSortMapKeys: true,\n\t\tCompactMarshaler: true,\n\t\tCopyString: true,\n\t\tValidateString: true,\n\t}.Froze()\n\n\t// ConfigFastest is the fastest config of APIs, aiming at speed.\n\tConfigFastest = Config{\n\t\tNoValidateJSONMarshaler: true,\n\t\tNoValidateJSONSkip: true,\n\t}.Froze()\n)\n\n// API is a binding of specific config.\n// This interface is inspired by github.com/json-iterator/go,\n// and has same behaviors under equivalent config.\ntype API interface {\n\t// MarshalToString returns the JSON encoding string of v\n\tMarshalToString(v interface{}) (string, error)\n\t// Marshal returns the JSON encoding bytes of v.\n\tMarshal(v interface{}) ([]byte, error)\n\t// MarshalIndent returns the JSON encoding bytes with indent and prefix.\n\tMarshalIndent(v interface{}, prefix, indent string) ([]byte, error)\n\t// UnmarshalFromString parses the JSON-encoded bytes and stores the result in the value pointed to by v.\n\tUnmarshalFromString(str string, v interface{}) error\n\t// Unmarshal parses the JSON-encoded string and stores the result in the value pointed to by v.\n\tUnmarshal(data []byte, v interface{}) error\n\t// NewEncoder create a Encoder holding writer\n\tNewEncoder(writer io.Writer) Encoder\n\t// NewDecoder create a Decoder holding reader\n\tNewDecoder(reader io.Reader) Decoder\n\t// Valid validates the JSON-encoded bytes and reports if it is valid\n\tValid(data []byte) bool\n}\n\n// Encoder encodes JSON into io.Writer\ntype Encoder interface {\n\t// Encode writes the JSON encoding of v to the stream, followed by a newline character.\n\tEncode(val interface{}) error\n\t// SetEscapeHTML specifies whether problematic HTML characters\n\t// should be escaped inside JSON quoted strings.\n\t// The default behavior NOT ESCAPE\n\tSetEscapeHTML(on bool)\n\t// SetIndent instructs the encoder to format each subsequent encoded value\n\t// as if indented by the package-level function Indent(dst, src, prefix, indent).\n\t// Calling SetIndent(\"\", \"\") disables indentation\n\tSetIndent(prefix, indent string)\n}\n\n// Decoder decodes JSON from io.Read\ntype Decoder interface {\n\t// Decode reads the next JSON-encoded value from its input and stores it in the value pointed to by v.\n\tDecode(val interface{}) error\n\t// Buffered returns a reader of the data remaining in the Decoder's buffer.\n\t// The reader is valid until the next call to Decode.\n\tBuffered() io.Reader\n\t// DisallowUnknownFields causes the Decoder to return an error when the destination is a struct\n\t// and the input contains object keys which do not match any non-ignored, exported fields in the destination.\n\tDisallowUnknownFields()\n\t// More reports whether there is another element in the current array or object being parsed.\n\tMore() bool\n\t// UseNumber causes the Decoder to unmarshal a number into an interface{} as a Number instead of as a float64.\n\tUseNumber()\n}\n\n// Marshal returns the JSON encoding bytes of v.\nfunc Marshal(val interface{}) ([]byte, error) {\n\treturn ConfigDefault.Marshal(val)\n}\n\n// MarshalIndent is like Marshal but applies Indent to format the output.\n// Each JSON element in the output will begin on a new line beginning with prefix\n// followed by one or more copies of indent according to the indentation nesting.\nfunc MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) {\n\treturn ConfigDefault.MarshalIndent(v, prefix, indent)\n}\n\n// MarshalString returns the JSON encoding string of v.\nfunc MarshalString(val interface{}) (string, error) {\n\treturn ConfigDefault.MarshalToString(val)\n}\n\n// Unmarshal parses the JSON-encoded data and stores the result in the value pointed to by v.\n// NOTICE: This API copies given buffer by default,\n// if you want to pass JSON more efficiently, use UnmarshalString instead.\nfunc Unmarshal(buf []byte, val interface{}) error {\n\treturn ConfigDefault.Unmarshal(buf, val)\n}\n\n// UnmarshalString is like Unmarshal, except buf is a string.\nfunc UnmarshalString(buf string, val interface{}) error {\n\treturn ConfigDefault.UnmarshalFromString(buf, val)\n}\n\n// Get searches and locates the given path from src json,\n// and returns a ast.Node representing the partially json.\n//\n// Each path arg must be integer or string:\n// - Integer is target index(>=0), means searching current node as array.\n// - String is target key, means searching current node as object.\n//\n// Notice: It expects the src json is **Well-formed** and **Immutable** when calling,\n// otherwise it may return unexpected result.\n// Considering memory safety, the returned JSON is **Copied** from the input\nfunc Get(src []byte, path ...interface{}) (ast.Node, error) {\n\treturn GetCopyFromString(rt.Mem2Str(src), path...)\n}\n\n// GetWithOptions searches and locates the given path from src json,\n// with specific options of ast.Searcher\nfunc GetWithOptions(src []byte, opts ast.SearchOptions, path ...interface{}) (ast.Node, error) {\n\ts := ast.NewSearcher(rt.Mem2Str(src))\n\ts.SearchOptions = opts\n\treturn s.GetByPath(path...)\n}\n\n// GetFromString is same with Get except src is string.\n//\n// WARNING: The returned JSON is **Referenced** from the input.\n// Caching or long-time holding the returned node may cause OOM.\n// If your src is big, consider use GetFromStringCopy().\nfunc GetFromString(src string, path ...interface{}) (ast.Node, error) {\n\treturn ast.NewSearcher(src).GetByPath(path...)\n}\n\n// GetCopyFromString is same with Get except src is string\nfunc GetCopyFromString(src string, path ...interface{}) (ast.Node, error) {\n\treturn ast.NewSearcher(src).GetByPathCopy(path...)\n}\n\n// Valid reports whether data is a valid JSON encoding.\nfunc Valid(data []byte) bool {\n\treturn ConfigDefault.Valid(data)\n}\n\n// Valid reports whether data is a valid JSON encoding.\nfunc ValidString(data string) bool {\n\treturn ConfigDefault.Valid(rt.Str2Mem(data))\n}\n" }, { "path": "api_test.go", "content": "/*\n * Copyright 2021 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage sonic\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestValid(t *testing.T) {\n\trequire.False(t, Valid(nil))\n\n\ttestCase := []struct {\n\t\tdata string\n\t\texpected bool\n\t}{\n\t\t{``, false},\n\t\t{`s`, false},\n\t\t{`{`, false},\n\t\t{`[`, false},\n\t\t{`[1,2`, false},\n\t\t{`{\"so\":nic\"}`, false},\n\n\t\t{`null`, true},\n\t\t{`\"\"`, true},\n\t\t{`1`, true},\n\t\t{`\"sonic\"`, true},\n\t\t{`{}`, true},\n\t\t{`[]`, true},\n\t\t{`[1,2]`, true},\n\t\t{`{\"so\":\"nic\"}`, true},\n\t}\n\tfor _, tc := range testCase {\n\t\trequire.Equal(t, tc.expected, Valid([]byte(tc.data)), tc.data)\n\t}\n}\n\nfunc TestIdent(t *testing.T) {\n\tfoo := struct {\n\t\tName string\n\t\tAge int\n\t}{\n\t\tName: \"sonic\",\n\t\tAge: 20,\n\t}\n\n\tout, err := MarshalIndent(&foo, \"\", \" \")\n\trequire.Nil(t, err)\n\trequire.Equal(t, `{\n \"Name\": \"sonic\",\n \"Age\": 20\n}`, string(out))\n}\n" }, { "path": "ast/api.go", "content": "//go:build (amd64 && go1.17 && !go1.27) || (arm64 && go1.20 && !go1.27)\n// +build amd64,go1.17,!go1.27 arm64,go1.20,!go1.27\n\n/*\n * Copyright 2022 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage ast\n\nimport (\n\t\"runtime\"\n\t\"unsafe\"\n\n\t\"github.com/bytedance/sonic/encoder\"\n\t\"github.com/bytedance/sonic/internal/encoder/alg\"\n\t\"github.com/bytedance/sonic/internal/native\"\n\t\"github.com/bytedance/sonic/internal/native/types\"\n\t\"github.com/bytedance/sonic/internal/rt\"\n\t\"github.com/bytedance/sonic/utf8\"\n)\n\nvar typeByte = rt.UnpackEface(byte(0)).Type\n\nfunc quote(buf *[]byte, val string) {\n\t*buf = alg.Quote(*buf, val, false)\n}\n\nfunc (self *Parser) decodeValue() (val types.JsonState) {\n\tsv := (*rt.GoString)(unsafe.Pointer(&self.s))\n\tflag := types.F_USE_NUMBER\n\tif self.dbuf != nil {\n\t\tflag = 0\n\t\tval.Dbuf = self.dbuf\n\t\tval.Dcap = types.MaxDigitNums\n\t}\n\tself.p = native.Value(sv.Ptr, sv.Len, self.p, &val, uint64(flag))\n\treturn\n}\n\nfunc (self *Parser) skip() (int, types.ParsingError) {\n\tfsm := types.NewStateMachine()\n\tstart := native.SkipOne(&self.s, &self.p, fsm, 0)\n\ttypes.FreeStateMachine(fsm)\n\n\tif start < 0 {\n\t\treturn self.p, types.ParsingError(-start)\n\t}\n\treturn start, 0\n}\n\nfunc (self *Node) encodeInterface(buf *[]byte) error {\n\t//WARN: NOT compatible with json.Encoder\n\treturn encoder.EncodeInto(buf, self.packAny(), encoder.NoEncoderNewline)\n}\n\nfunc (self *Parser) skipFast() (int, types.ParsingError) {\n\tstart := native.SkipOneFast(&self.s, &self.p)\n\tif start < 0 {\n\t\treturn self.p, types.ParsingError(-start)\n\t}\n\treturn start, 0\n}\n\nfunc (self *Parser) getByPath(validate bool, path ...interface{}) (int, types.ParsingError) {\n\tvar fsm *types.StateMachine\n\tif validate {\n\t\tfsm = types.NewStateMachine()\n\t}\n\tstart := native.GetByPath(&self.s, &self.p, &path, fsm)\n\tif validate {\n\t\ttypes.FreeStateMachine(fsm)\n\t}\n\truntime.KeepAlive(path)\n\tif start < 0 {\n\t\treturn self.p, types.ParsingError(-start)\n\t}\n\treturn start, 0\n}\n\nfunc validate_utf8(str string) bool {\n\treturn utf8.ValidateString(str)\n}\n" }, { "path": "ast/api_compat.go", "content": "//go:build (!amd64 && !arm64) || go1.27 || !go1.17 || (arm64 && !go1.20)\n// +build !amd64,!arm64 go1.27 !go1.17 arm64,!go1.20\n\n/*\n* Copyright 2022 ByteDance Inc.\n*\n* Licensed under the Apache License, Version 2.0 (the \"License\");\n* you may not use this file except in compliance with the License.\n* You may obtain a copy of the License at\n*\n* http://www.apache.org/licenses/LICENSE-2.0\n*\n* Unless required by applicable law or agreed to in writing, software\n* distributed under the License is distributed on an \"AS IS\" BASIS,\n* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n* See the License for the specific language governing permissions and\n* limitations under the License.\n */\n\npackage ast\n\nimport (\n\t\"encoding/json\"\n\t\"unicode/utf8\"\n\n\t\"github.com/bytedance/sonic/internal/compat\"\n\t\"github.com/bytedance/sonic/internal/native/types\"\n)\n\nfunc init() {\n\tcompat.Warn(\"sonic/ast\")\n}\n\nfunc quote(buf *[]byte, val string) {\n\tquoteString(buf, val)\n}\n\nfunc (self *Parser) decodeValue() (val types.JsonState) {\n\te, v := decodeValue(self.s, self.p, self.dbuf == nil)\n\tif e < 0 {\n\t\treturn v\n\t}\n\tself.p = e\n\treturn v\n}\n\nfunc (self *Parser) skip() (int, types.ParsingError) {\n\te, s := skipValue(self.s, self.p)\n\tif e < 0 {\n\t\treturn self.p, types.ParsingError(-e)\n\t}\n\tself.p = e\n\treturn s, 0\n}\n\nfunc (self *Parser) skipFast() (int, types.ParsingError) {\n\te, s := skipValueFast(self.s, self.p)\n\tif e < 0 {\n\t\treturn self.p, types.ParsingError(-e)\n\t}\n\tself.p = e\n\treturn s, 0\n}\n\nfunc (self *Node) encodeInterface(buf *[]byte) error {\n\tout, err := json.Marshal(self.packAny())\n\tif err != nil {\n\t\treturn err\n\t}\n\t*buf = append(*buf, out...)\n\treturn nil\n}\n\nfunc (self *Parser) getByPath(validate bool, path ...interface{}) (int, types.ParsingError) {\n\tfor _, p := range path {\n\t\tif idx, ok := p.(int); ok && idx >= 0 {\n\t\t\tif err := self.searchIndex(idx); err != 0 {\n\t\t\t\treturn self.p, err\n\t\t\t}\n\t\t} else if key, ok := p.(string); ok {\n\t\t\tif err := self.searchKey(key); err != 0 {\n\t\t\t\treturn self.p, err\n\t\t\t}\n\t\t} else {\n\t\t\tpanic(\"path must be either int(>=0) or string\")\n\t\t}\n\t}\n\n\tvar start int\n\tvar e types.ParsingError\n\tif validate {\n\t\tstart, e = self.skip()\n\t} else {\n\t\tstart, e = self.skipFast()\n\t}\n\tif e != 0 {\n\t\treturn self.p, e\n\t}\n\treturn start, 0\n}\n\nfunc validate_utf8(str string) bool {\n\treturn utf8.ValidString(str)\n}\n" }, { "path": "ast/api_native_test.go", "content": "//go:build (amd64 && go1.17 && !go1.27) || (arm64 && go1.20 && !go1.27)\n// +build amd64,go1.17,!go1.27 arm64,go1.20,!go1.27\n\n/*\n * Copyright 2022 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage ast\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"runtime/debug\"\n\t\"testing\"\n\n\t\"github.com/bytedance/sonic/encoder\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestSortNodeTwitter(t *testing.T) {\n\tif encoder.EnableFallback {\n\t\treturn\n\t}\n\troot, err := NewSearcher(_TwitterJson).GetByPath()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tobj, err := root.MapUseNumber()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\texp, err := encoder.Encode(obj, encoder.SortMapKeys|encoder.NoEncoderNewline)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tvar expObj interface{}\n\trequire.NoError(t, json.Unmarshal(exp, &expObj))\n\n\tif err := root.SortKeys(true); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tact, err := root.MarshalJSON()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tvar actObj interface{}\n\trequire.NoError(t, json.Unmarshal(act, &actObj))\n\trequire.Equal(t, expObj, actObj)\n\trequire.Equal(t, len(exp), len(act))\n\trequire.Equal(t, string(exp), string(act))\n}\n\nfunc TestNodeAny(t *testing.T) {\n\tempty := Node{}\n\t_, err := empty.SetAny(\"any\", map[string]interface{}{\"a\": []int{0}})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif m, err := empty.Get(\"any\").Interface(); err != nil {\n\t\tt.Fatal(err)\n\t} else if v, ok := m.(map[string]interface{}); !ok {\n\t\tt.Fatal(v)\n\t}\n\tif buf, err := empty.MarshalJSON(); err != nil {\n\t\tt.Fatal(err)\n\t} else if string(buf) != `{\"any\":{\"a\":[0]}}` {\n\t\tt.Fatal(string(buf))\n\t}\n\tif _, err := empty.Set(\"any2\", Node{}); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif err := empty.Get(\"any2\").AddAny(nil); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif buf, err := empty.MarshalJSON(); err != nil {\n\t\tt.Fatal(err)\n\t} else if string(buf) != `{\"any\":{\"a\":[0]},\"any2\":[null]}` {\n\t\tt.Fatal(string(buf))\n\t}\n\tif _, err := empty.Get(\"any2\").SetAnyByIndex(0, NewNumber(\"-0.0\")); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif buf, err := empty.MarshalJSON(); err != nil {\n\t\tt.Fatal(err)\n\t} else if string(buf) != `{\"any\":{\"a\":[0]},\"any2\":[-0.0]}` {\n\t\tt.Fatal(string(buf))\n\t}\n}\n\nfunc TestTypeCast2(t *testing.T) {\n\ttype tcase struct {\n\t\tmethod string\n\t\tnode Node\n\t\texp interface{}\n\t\terr error\n\t}\n\tvar cases = []tcase{\n\t\t{\"Raw\", NewAny(\"\"), \"\\\"\\\"\", nil},\n\t}\n\n\tfor i, c := range cases {\n\t\tfmt.Println(i, c)\n\t\trt := reflect.ValueOf(&c.node)\n\t\tm := rt.MethodByName(c.method)\n\t\trets := m.Call([]reflect.Value{})\n\t\tif len(rets) != 2 {\n\t\t\tt.Fatal(i, rets)\n\t\t}\n\t\trequire.Equal(t, c.exp, rets[0].Interface())\n\t\tv := rets[1].Interface()\n\t\tif v != c.err {\n\t\t\tt.Fatal(i, v)\n\t\t}\n\t}\n}\n\nfunc TestStackAny(t *testing.T) {\n\tvar obj = stackObj()\n\tany := NewAny(obj)\n\tfmt.Printf(\"any: %#v\\n\", any)\n\truntime.GC()\n\tdebug.FreeOSMemory()\n\tprintln(\"finish GC\")\n\tbuf, err := any.MarshalJSON()\n\tprintln(\"finish marshal\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif string(buf) != \"1\" {\n\t\tt.Fatal(string(buf))\n\t}\n}\n\nfunc Test_Export(t *testing.T) {\n\ttype args struct {\n\t\tsrc string\n\t\tpath []interface{}\n\t}\n\ttests := []struct {\n\t\tname string\n\t\targs args\n\t\twantStart int\n\t\twantEnd int\n\t\twantTyp int\n\t\twantErr bool\n\t\twantValid bool\n\t}{\n\t\t{\"bool\", args{`[true ,2]`, []interface{}{0}}, 1, 5, V_TRUE, false, true},\n\t\t{\"bool\", args{`[t2ue ,2]`, []interface{}{0}}, 1, 5, V_TRUE, false, false},\n\t\t{\"number\", args{`[1 ,2]`, []interface{}{0}}, 1, 2, V_NUMBER, false, true},\n\t\t{\"number\", args{`[1w ,2]`, []interface{}{0}}, 1, 3, V_NUMBER, false, false},\n\t\t{\"string\", args{`[\" \" ,2]`, []interface{}{0}}, 1, 4, V_STRING, false, true},\n\t\t{\"string\", args{`[\" \"] ,2]`, []interface{}{0}}, 1, 4, V_STRING, false, true},\n\t\t{\"object\", args{`[{\"\":\"\"} ,2]`, []interface{}{0}}, 1, 8, V_OBJECT, false, true},\n\t\t{\"object\", args{`[{x} ,2]`, []interface{}{0}}, 1, 4, V_OBJECT, false, false},\n\t\t{\"array\", args{`[[{}] ,2]`, []interface{}{0}}, 1, 5, V_ARRAY, false, true},\n\t\t{\"array\", args{`[[xx] ,2]`, []interface{}{0}}, 1, 5, V_ARRAY, false, false},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tgotStart, gotEnd, gotTyp, err := _GetByPath(tt.args.src, tt.args.path...)\n\t\t\tif (err != nil) != tt.wantErr {\n\t\t\t\tt.Errorf(\"_GetByPath() error = %v, wantErr %v\", err, tt.wantErr)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif gotStart != tt.wantStart {\n\t\t\t\tt.Errorf(\"_GetByPath() gotStart = %v, want %v\", gotStart, tt.wantStart)\n\t\t\t}\n\t\t\tif gotEnd != tt.wantEnd {\n\t\t\t\tt.Errorf(\"_GetByPath() gotEnd = %v, want %v\", gotEnd, tt.wantEnd)\n\t\t\t}\n\t\t\tif gotTyp != tt.wantTyp {\n\t\t\t\tt.Errorf(\"_GetByPath() gotTyp = %v, want %v\", gotTyp, tt.wantTyp)\n\t\t\t}\n\t\t\tgotStart, gotEnd, err = _SkipFast(tt.args.src, tt.wantStart)\n\t\t\tif (err != nil) != tt.wantErr {\n\t\t\t\tt.Errorf(\"_SkipFast() error = %v, wantErr %v\", err, tt.wantErr)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif gotStart != tt.wantStart {\n\t\t\t\tt.Errorf(\"_SkipFast() gotStart = %v, want %v\", gotStart, tt.wantStart)\n\t\t\t}\n\t\t\tif gotEnd != tt.wantEnd {\n\t\t\t\tt.Errorf(\"_SkipFast() gotEnd = %v, want %v\", gotEnd, tt.wantEnd)\n\t\t\t}\n\t\t\tvalid := _ValidSyntax(tt.args.src[tt.wantStart:tt.wantEnd])\n\t\t\tif valid != tt.wantValid {\n\t\t\t\tt.Errorf(\"_ValidSyntax() gotValid = %v, want %v\", valid, tt.wantValid)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "ast/asm.s", "content": "" }, { "path": "ast/buffer.go", "content": "/**\n * Copyright 2023 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage ast\n\nimport (\n\t\"sort\"\n\t\"unsafe\"\n\n\t\"github.com/bytedance/sonic/internal/caching\"\n)\n\ntype nodeChunk [_DEFAULT_NODE_CAP]Node\n\ntype linkedNodes struct {\n\thead nodeChunk\n\ttail []*nodeChunk\n\tsize int\n}\n\nfunc (self *linkedNodes) Cap() int {\n\tif self == nil {\n\t\treturn 0\n\t}\n\treturn (len(self.tail) + 1) * _DEFAULT_NODE_CAP\n}\n\nfunc (self *linkedNodes) Len() int {\n\tif self == nil {\n\t\treturn 0\n\t}\n\treturn self.size\n}\n\nfunc (self *linkedNodes) At(i int) *Node {\n\tif self == nil {\n\t\treturn nil\n\t}\n\tif i >= 0 && i < self.size && i < _DEFAULT_NODE_CAP {\n\t\treturn &self.head[i]\n\t} else if i >= _DEFAULT_NODE_CAP && i < self.size {\n\t\ta, b := i/_DEFAULT_NODE_CAP-1, i%_DEFAULT_NODE_CAP\n\t\tif a < len(self.tail) {\n\t\t\treturn &self.tail[a][b]\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (self *linkedNodes) MoveOne(source int, target int) {\n\tif source == target {\n\t\treturn\n\t}\n\tif source < 0 || source >= self.size || target < 0 || target >= self.size {\n\t\treturn\n\t}\n\t// reserve source\n\tn := *self.At(source)\n\tif source < target {\n\t\t// move every element (source,target] one step back\n\t\tfor i := source; i < target; i++ {\n\t\t\t*self.At(i) = *self.At(i + 1)\n\t\t}\n\t} else {\n\t\t// move every element [target,source) one step forward\n\t\tfor i := source; i > target; i-- {\n\t\t\t*self.At(i) = *self.At(i - 1)\n\t\t}\n\t}\n\t// set target\n\t*self.At(target) = n\n}\n\nfunc (self *linkedNodes) Pop() {\n\tif self == nil || self.size == 0 {\n\t\treturn\n\t}\n\tself.Set(self.size-1, Node{})\n\tself.size--\n}\n\nfunc (self *linkedNodes) Push(v Node) {\n\tself.Set(self.size, v)\n}\n\nfunc (self *linkedNodes) Set(i int, v Node) {\n\tif i < _DEFAULT_NODE_CAP {\n\t\tself.head[i] = v\n\t\tif self.size <= i {\n\t\t\tself.size = i + 1\n\t\t}\n\t\treturn\n\t}\n\ta, b := i/_DEFAULT_NODE_CAP-1, i%_DEFAULT_NODE_CAP\n\tif a < 0 {\n\t\tself.head[b] = v\n\t} else {\n\t\tself.growTailLength(a + 1)\n\t\tvar n = &self.tail[a]\n\t\tif *n == nil {\n\t\t\t*n = new(nodeChunk)\n\t\t}\n\t\t(*n)[b] = v\n\t}\n\tif self.size <= i {\n\t\tself.size = i + 1\n\t}\n}\n\nfunc (self *linkedNodes) growTailLength(l int) {\n\tif l <= len(self.tail) {\n\t\treturn\n\t}\n\tc := cap(self.tail)\n\tfor c < l {\n\t\tc += 1 + c>>_APPEND_GROW_SHIFT\n\t}\n\tif c == cap(self.tail) {\n\t\tself.tail = self.tail[:l]\n\t\treturn\n\t}\n\ttmp := make([]*nodeChunk, l, c)\n\tcopy(tmp, self.tail)\n\tself.tail = tmp\n}\n\nfunc (self *linkedNodes) ToSlice(con []Node) {\n\tif len(con) < self.size {\n\t\treturn\n\t}\n\ti := (self.size - 1)\n\ta, b := i/_DEFAULT_NODE_CAP-1, i%_DEFAULT_NODE_CAP\n\tif a < 0 {\n\t\tcopy(con, self.head[:b+1])\n\t\treturn\n\t} else {\n\t\tcopy(con, self.head[:])\n\t\tcon = con[_DEFAULT_NODE_CAP:]\n\t}\n\n\tfor i := 0; i < a; i++ {\n\t\tcopy(con, self.tail[i][:])\n\t\tcon = con[_DEFAULT_NODE_CAP:]\n\t}\n\tcopy(con, self.tail[a][:b+1])\n}\n\nfunc (self *linkedNodes) FromSlice(con []Node) {\n\tself.size = len(con)\n\ti := self.size - 1\n\ta, b := i/_DEFAULT_NODE_CAP-1, i%_DEFAULT_NODE_CAP\n\tif a < 0 {\n\t\tcopy(self.head[:b+1], con)\n\t\treturn\n\t} else {\n\t\tcopy(self.head[:], con)\n\t\tcon = con[_DEFAULT_NODE_CAP:]\n\t}\n\n\tif cap(self.tail) <= a {\n\t\tc := (a + 1) + (a+1)>>_APPEND_GROW_SHIFT\n\t\tself.tail = make([]*nodeChunk, a+1, c)\n\t}\n\tself.tail = self.tail[:a+1]\n\n\tfor i := 0; i < a; i++ {\n\t\tself.tail[i] = new(nodeChunk)\n\t\tcopy(self.tail[i][:], con)\n\t\tcon = con[_DEFAULT_NODE_CAP:]\n\t}\n\n\tself.tail[a] = new(nodeChunk)\n\tcopy(self.tail[a][:b+1], con)\n}\n\ntype pairChunk [_DEFAULT_NODE_CAP]Pair\n\ntype linkedPairs struct {\n\tindex map[uint64]int\n\thead pairChunk\n\ttail []*pairChunk\n\tsize int\n}\n\nfunc (self *linkedPairs) BuildIndex() {\n\tif self.index == nil {\n\t\tself.index = make(map[uint64]int, self.size)\n\t}\n\tfor i := 0; i < self.size; i++ {\n\t\tp := self.At(i)\n\t\tself.index[p.hash] = i\n\t}\n}\n\nfunc (self *linkedPairs) Cap() int {\n\tif self == nil {\n\t\treturn 0\n\t}\n\treturn (len(self.tail) + 1) * _DEFAULT_NODE_CAP\n}\n\nfunc (self *linkedPairs) Len() int {\n\tif self == nil {\n\t\treturn 0\n\t}\n\treturn self.size\n}\n\nfunc (self *linkedPairs) At(i int) *Pair {\n\tif self == nil {\n\t\treturn nil\n\t}\n\tif i >= 0 && i < _DEFAULT_NODE_CAP && i < self.size {\n\t\treturn &self.head[i]\n\t} else if i >= _DEFAULT_NODE_CAP && i < self.size {\n\t\ta, b := i/_DEFAULT_NODE_CAP-1, i%_DEFAULT_NODE_CAP\n\t\tif a < len(self.tail) {\n\t\t\treturn &self.tail[a][b]\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (self *linkedPairs) Push(v Pair) {\n\tself.Set(self.size, v)\n}\n\nfunc (self *linkedPairs) Pop() {\n\tif self == nil || self.size == 0 {\n\t\treturn\n\t}\n\tself.Unset(self.size - 1)\n\tself.size--\n}\n\nfunc (self *linkedPairs) Unset(i int) {\n\tif self.index != nil {\n\t\tp := self.At(i)\n\t\tdelete(self.index, p.hash)\n\t}\n\tself.set(i, Pair{})\n}\n\nfunc (self *linkedPairs) Set(i int, v Pair) {\n\tif self.index != nil {\n\t\th := v.hash\n\t\tself.index[h] = i\n\t}\n\tself.set(i, v)\n}\n\nfunc (self *linkedPairs) set(i int, v Pair) {\n\tif i < _DEFAULT_NODE_CAP {\n\t\tself.head[i] = v\n\t\tif self.size <= i {\n\t\t\tself.size = i + 1\n\t\t}\n\t\treturn\n\t}\n\ta, b := i/_DEFAULT_NODE_CAP-1, i%_DEFAULT_NODE_CAP\n\tif a < 0 {\n\t\tself.head[b] = v\n\t} else {\n\t\tself.growTailLength(a + 1)\n\t\tvar n = &self.tail[a]\n\t\tif *n == nil {\n\t\t\t*n = new(pairChunk)\n\t\t}\n\t\t(*n)[b] = v\n\t}\n\tif self.size <= i {\n\t\tself.size = i + 1\n\t}\n}\n\nfunc (self *linkedPairs) growTailLength(l int) {\n\tif l <= len(self.tail) {\n\t\treturn\n\t}\n\tc := cap(self.tail)\n\tfor c < l {\n\t\tc += 1 + c>>_APPEND_GROW_SHIFT\n\t}\n\tif c == cap(self.tail) {\n\t\tself.tail = self.tail[:l]\n\t\treturn\n\t}\n\ttmp := make([]*pairChunk, l, c)\n\tcopy(tmp, self.tail)\n\tself.tail = tmp\n}\n\n// linear search\nfunc (self *linkedPairs) Get(key string) (*Pair, int) {\n\tif self.index != nil {\n\t\t// fast-path\n\t\ti, ok := self.index[caching.StrHash(key)]\n\t\tif ok {\n\t\t\tn := self.At(i)\n\t\t\tif n.Key == key {\n\t\t\t\treturn n, i\n\t\t\t}\n\t\t\t// hash conflicts\n\t\t\tgoto linear_search\n\t\t} else {\n\t\t\treturn nil, -1\n\t\t}\n\t}\nlinear_search:\n\tfor i := 0; i < self.size; i++ {\n\t\tif n := self.At(i); n.Key == key {\n\t\t\treturn n, i\n\t\t}\n\t}\n\treturn nil, -1\n}\n\nfunc (self *linkedPairs) ToSlice(con []Pair) {\n\tif len(con) < self.size {\n\t\treturn\n\t}\n\ti := self.size - 1\n\ta, b := i/_DEFAULT_NODE_CAP-1, i%_DEFAULT_NODE_CAP\n\n\tif a < 0 {\n\t\tcopy(con, self.head[:b+1])\n\t\treturn\n\t} else {\n\t\tcopy(con, self.head[:])\n\t\tcon = con[_DEFAULT_NODE_CAP:]\n\t}\n\n\tfor i := 0; i < a; i++ {\n\t\tcopy(con, self.tail[i][:])\n\t\tcon = con[_DEFAULT_NODE_CAP:]\n\t}\n\tcopy(con, self.tail[a][:b+1])\n}\n\nfunc (self *linkedPairs) ToMap(con map[string]Node) {\n\tfor i := 0; i < self.size; i++ {\n\t\tn := self.At(i)\n\t\tcon[n.Key] = n.Value\n\t}\n}\n\nfunc (self *linkedPairs) copyPairs(to []Pair, from []Pair, l int) {\n\tcopy(to, from)\n\tif self.index != nil {\n\t\tfor i := 0; i < l; i++ {\n\t\t\t// NOTICE: in case of user not pass hash, just cal it\n\t\t\th := caching.StrHash(from[i].Key)\n\t\t\tfrom[i].hash = h\n\t\t\tself.index[h] = i\n\t\t}\n\t}\n}\n\nfunc (self *linkedPairs) FromSlice(con []Pair) {\n\tself.size = len(con)\n\ti := self.size - 1\n\ta, b := i/_DEFAULT_NODE_CAP-1, i%_DEFAULT_NODE_CAP\n\tif a < 0 {\n\t\tself.copyPairs(self.head[:b+1], con, b+1)\n\t\treturn\n\t} else {\n\t\tself.copyPairs(self.head[:], con, len(self.head))\n\t\tcon = con[_DEFAULT_NODE_CAP:]\n\t}\n\n\tif cap(self.tail) <= a {\n\t\tc := (a + 1) + (a+1)>>_APPEND_GROW_SHIFT\n\t\tself.tail = make([]*pairChunk, a+1, c)\n\t}\n\tself.tail = self.tail[:a+1]\n\n\tfor i := 0; i < a; i++ {\n\t\tself.tail[i] = new(pairChunk)\n\t\tself.copyPairs(self.tail[i][:], con, len(self.tail[i]))\n\t\tcon = con[_DEFAULT_NODE_CAP:]\n\t}\n\n\tself.tail[a] = new(pairChunk)\n\tself.copyPairs(self.tail[a][:b+1], con, b+1)\n}\n\nfunc (self *linkedPairs) Less(i, j int) bool {\n\treturn lessFrom(self.At(i).Key, self.At(j).Key, 0)\n}\n\nfunc (self *linkedPairs) Swap(i, j int) {\n\ta, b := self.At(i), self.At(j)\n\tif self.index != nil {\n\t\tself.index[a.hash] = j\n\t\tself.index[b.hash] = i\n\t}\n\t*a, *b = *b, *a\n}\n\nfunc (self *linkedPairs) Sort() {\n\tsort.Stable(self)\n}\n\n// Compare two strings from the pos d.\nfunc lessFrom(a, b string, d int) bool {\n\tl := len(a)\n\tif l > len(b) {\n\t\tl = len(b)\n\t}\n\tfor i := d; i < l; i++ {\n\t\tif a[i] == b[i] {\n\t\t\tcontinue\n\t\t}\n\t\treturn a[i] < b[i]\n\t}\n\treturn len(a) < len(b)\n}\n\ntype parseObjectStack struct {\n\tparser Parser\n\tv linkedPairs\n}\n\ntype parseArrayStack struct {\n\tparser Parser\n\tv linkedNodes\n}\n\nfunc newLazyArray(p *Parser) Node {\n\ts := new(parseArrayStack)\n\ts.parser = *p\n\treturn Node{\n\t\tt: _V_ARRAY_LAZY,\n\t\tp: unsafe.Pointer(s),\n\t}\n}\n\nfunc newLazyObject(p *Parser) Node {\n\ts := new(parseObjectStack)\n\ts.parser = *p\n\treturn Node{\n\t\tt: _V_OBJECT_LAZY,\n\t\tp: unsafe.Pointer(s),\n\t}\n}\n\nfunc (self *Node) getParserAndArrayStack() (*Parser, *parseArrayStack) {\n\tstack := (*parseArrayStack)(self.p)\n\treturn &stack.parser, stack\n}\n\nfunc (self *Node) getParserAndObjectStack() (*Parser, *parseObjectStack) {\n\tstack := (*parseObjectStack)(self.p)\n\treturn &stack.parser, stack\n}\n" }, { "path": "ast/buffer_test.go", "content": "/**\n * Copyright 2023 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage ast\n\nimport (\n\t\"strconv\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc makeNodes(l int) []Node {\n\tr := make([]Node, l)\n\tfor i := 0; i < l; i++ {\n\t\tr[i] = NewBool(true)\n\t}\n\treturn r\n}\n\nfunc makePairs(l int) []Pair {\n\tr := make([]Pair, l)\n\tfor i := 0; i < l; i++ {\n\t\tr[i] = NewPair(strconv.Itoa(i), NewBool(true))\n\t}\n\treturn r\n}\n\nfunc Test_linkedPairs_Push(t *testing.T) {\n\ttype args struct {\n\t\tin []Pair\n\t\tv Pair\n\t\texp []Pair\n\t}\n\ttests := []struct {\n\t\tname string\n\t\targs args\n\t}{\n\t\t{\n\t\t\tname: \"add empty\",\n\t\t\targs: args{\n\t\t\t\tin: []Pair{},\n\t\t\t\tv: NewPair(\"a\", NewBool(true)),\n\t\t\t\texp: []Pair{NewPair(\"a\", NewBool(true))},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"add one\",\n\t\t\targs: args{\n\t\t\t\tin: []Pair{NewPair(\"a\", NewBool(false))},\n\t\t\t\tv: NewPair(\"b\", NewBool(true)),\n\t\t\t\texp: []Pair{NewPair(\"a\", NewBool(false)), NewPair(\"b\", NewBool(true))},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"add _DEFAULT_NODE_CAP\",\n\t\t\targs: args{\n\t\t\t\tin: makePairs(_DEFAULT_NODE_CAP),\n\t\t\t\tv: NewPair(strconv.Itoa(_DEFAULT_NODE_CAP), NewBool(true)),\n\t\t\t\texp: makePairs(_DEFAULT_NODE_CAP + 1),\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tself := &linkedPairs{}\n\t\t\tself.FromSlice(tt.args.in)\n\t\t\tself.Push(tt.args.v)\n\t\t\tact := make([]Pair, self.Len())\n\t\t\tself.ToSlice(act)\n\t\t\trequire.Equal(t, tt.args.exp, act)\n\t\t})\n\t}\n}\n\nfunc Test_linkedNodes_Push(t *testing.T) {\n\ttype args struct {\n\t\tin []Node\n\t\tv Node\n\t\texp []Node\n\t}\n\ttests := []struct {\n\t\tname string\n\t\targs args\n\t}{\n\t\t{\n\t\t\tname: \"add empty\",\n\t\t\targs: args{\n\t\t\t\tin: []Node{},\n\t\t\t\tv: NewBool(true),\n\t\t\t\texp: []Node{NewBool(true)},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"add one\",\n\t\t\targs: args{\n\t\t\t\tin: []Node{NewBool(false)},\n\t\t\t\tv: NewBool(true),\n\t\t\t\texp: []Node{NewBool(false), NewBool(true)},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"add _DEFAULT_NODE_CAP\",\n\t\t\targs: args{\n\t\t\t\tin: makeNodes(_DEFAULT_NODE_CAP),\n\t\t\t\tv: NewBool(true),\n\t\t\t\texp: makeNodes(_DEFAULT_NODE_CAP + 1),\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tself := &linkedNodes{}\n\t\t\tself.FromSlice(tt.args.in)\n\t\t\tself.Push(tt.args.v)\n\t\t\tact := make([]Node, self.Len())\n\t\t\tself.ToSlice(act)\n\t\t\trequire.Equal(t, tt.args.exp, act)\n\t\t})\n\t}\n}\n\nfunc Test_linkedNodes_Pop(t *testing.T) {\n\ttype args struct {\n\t\tin []Node\n\t\texp []Node\n\t}\n\ttests := []struct {\n\t\tname string\n\t\targs args\n\t}{\n\t\t{\n\t\t\tname: \"remove empty\",\n\t\t\targs: args{\n\t\t\t\tin: []Node{},\n\t\t\t\texp: []Node{},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"remove one\",\n\t\t\targs: args{\n\t\t\t\tin: []Node{NewBool(false)},\n\t\t\t\texp: []Node{},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"add _DEFAULT_NODE_CAP\",\n\t\t\targs: args{\n\t\t\t\tin: makeNodes(_DEFAULT_NODE_CAP),\n\t\t\t\texp: makeNodes(_DEFAULT_NODE_CAP - 1),\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tself := &linkedNodes{}\n\t\t\tself.FromSlice(tt.args.in)\n\t\t\tself.Pop()\n\t\t\tact := make([]Node, self.Len())\n\t\t\tself.ToSlice(act)\n\t\t\trequire.Equal(t, tt.args.exp, act)\n\t\t})\n\t}\n}\n\nfunc Test_linkedNodes_MoveOne(t *testing.T) {\n\ttype args struct {\n\t\tin []Node\n\t\tsource int\n\t\ttarget int\n\t\texp []Node\n\t}\n\ttests := []struct {\n\t\tname string\n\t\targs args\n\t}{\n\t\t{\n\t\t\tname: \"over index\",\n\t\t\targs: args{\n\t\t\t\tin: []Node{NewBool(true)},\n\t\t\t\tsource: 1,\n\t\t\t\ttarget: 0,\n\t\t\t\texp: []Node{NewBool(true)},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"equal index\",\n\t\t\targs: args{\n\t\t\t\tin: []Node{NewBool(true)},\n\t\t\t\tsource: 0,\n\t\t\t\ttarget: 0,\n\t\t\t\texp: []Node{NewBool(true)},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"forward index\",\n\t\t\targs: args{\n\t\t\t\tin: []Node{NewString(\"a\"), NewString(\"b\"), NewString(\"c\")},\n\t\t\t\tsource: 0,\n\t\t\t\ttarget: 2,\n\t\t\t\texp: []Node{NewString(\"b\"), NewString(\"c\"), NewString(\"a\")},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"backward index\",\n\t\t\targs: args{\n\t\t\t\tin: []Node{NewString(\"a\"), NewString(\"b\"), NewString(\"c\")},\n\t\t\t\tsource: 2,\n\t\t\t\ttarget: 1,\n\t\t\t\texp: []Node{NewString(\"a\"), NewString(\"c\"), NewString(\"b\")},\n\t\t\t},\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tself := &linkedNodes{}\n\t\t\tself.FromSlice(tt.args.in)\n\t\t\tself.MoveOne(tt.args.source, tt.args.target)\n\t\t\tact := make([]Node, self.Len())\n\t\t\tself.ToSlice(act)\n\t\t\trequire.Equal(t, tt.args.exp, act)\n\t\t})\n\t}\n}\n" }, { "path": "ast/decode.go", "content": "/*\n * Copyright 2022 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage ast\n\nimport (\n\t\"encoding/base64\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"unsafe\"\n\n\t\"github.com/bytedance/sonic/internal/native/types\"\n\t\"github.com/bytedance/sonic/internal/rt\"\n\t\"github.com/bytedance/sonic/internal/utils\"\n\t\"github.com/bytedance/sonic/unquote\"\n)\n\nvar bytesNull = []byte(\"null\")\n\nconst (\n\tstrNull = \"null\"\n\tbytesTrue = \"true\"\n\tbytesFalse = \"false\"\n\tbytesObject = \"{}\"\n\tbytesArray = \"[]\"\n)\n\n//go:nocheckptr\nfunc skipBlank(src string, pos int) int {\n\tse := uintptr(rt.IndexChar(src, len(src)))\n\tsp := uintptr(rt.IndexChar(src, pos))\n\n\tfor sp < se {\n\t\tif !utils.IsSpace(*(*byte)(unsafe.Pointer(sp))) {\n\t\t\tbreak\n\t\t}\n\t\tsp += 1\n\t}\n\tif sp >= se {\n\t\treturn -int(types.ERR_EOF)\n\t}\n\truntime.KeepAlive(src)\n\treturn int(sp - uintptr(rt.IndexChar(src, 0)))\n}\n\nfunc decodeNull(src string, pos int) (ret int) {\n\tret = pos + 4\n\tif ret > len(src) {\n\t\treturn -int(types.ERR_EOF)\n\t}\n\tif src[pos:ret] == strNull {\n\t\treturn ret\n\t} else {\n\t\treturn -int(types.ERR_INVALID_CHAR)\n\t}\n}\n\nfunc decodeTrue(src string, pos int) (ret int) {\n\tret = pos + 4\n\tif ret > len(src) {\n\t\treturn -int(types.ERR_EOF)\n\t}\n\tif src[pos:ret] == bytesTrue {\n\t\treturn ret\n\t} else {\n\t\treturn -int(types.ERR_INVALID_CHAR)\n\t}\n\n}\n\nfunc decodeFalse(src string, pos int) (ret int) {\n\tret = pos + 5\n\tif ret > len(src) {\n\t\treturn -int(types.ERR_EOF)\n\t}\n\tif src[pos:ret] == bytesFalse {\n\t\treturn ret\n\t}\n\treturn -int(types.ERR_INVALID_CHAR)\n}\n\n//go:nocheckptr\nfunc decodeString(src string, pos int) (ret int, v string) {\n\tret, ep := skipString(src, pos)\n\tif ep == -1 {\n\t\t(*rt.GoString)(unsafe.Pointer(&v)).Ptr = rt.IndexChar(src, pos+1)\n\t\t(*rt.GoString)(unsafe.Pointer(&v)).Len = ret - pos - 2\n\t\treturn ret, v\n\t}\n\n\tresult, err := unquote.String(src[pos:ret])\n\tif err != 0 {\n\t\treturn -int(types.ERR_INVALID_CHAR), \"\"\n\t}\n\n\truntime.KeepAlive(src)\n\treturn ret, result\n}\n\nfunc decodeBinary(src string, pos int) (ret int, v []byte) {\n\tvar vv string\n\tret, vv = decodeString(src, pos)\n\tif ret < 0 {\n\t\treturn ret, nil\n\t}\n\tvar err error\n\tv, err = base64.StdEncoding.DecodeString(vv)\n\tif err != nil {\n\t\treturn -int(types.ERR_INVALID_CHAR), nil\n\t}\n\treturn ret, v\n}\n\nfunc isDigit(c byte) bool {\n\treturn c >= '0' && c <= '9'\n}\n\n//go:nocheckptr\nfunc decodeInt64(src string, pos int) (ret int, v int64, err error) {\n\tsp := uintptr(rt.IndexChar(src, pos))\n\tss := uintptr(sp)\n\tse := uintptr(rt.IndexChar(src, len(src)))\n\tif uintptr(sp) >= se {\n\t\treturn -int(types.ERR_EOF), 0, nil\n\t}\n\n\tif c := *(*byte)(unsafe.Pointer(sp)); c == '-' {\n\t\tsp += 1\n\t}\n\tif sp == se {\n\t\treturn -int(types.ERR_EOF), 0, nil\n\t}\n\n\tfor ; sp < se; sp += uintptr(1) {\n\t\tif !isDigit(*(*byte)(unsafe.Pointer(sp))) {\n\t\t\tbreak\n\t\t}\n\t}\n\n\tif sp < se {\n\t\tif c := *(*byte)(unsafe.Pointer(sp)); c == '.' || c == 'e' || c == 'E' {\n\t\t\treturn -int(types.ERR_INVALID_NUMBER_FMT), 0, nil\n\t\t}\n\t}\n\n\tvar vv string\n\tret = int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr))\n\t(*rt.GoString)(unsafe.Pointer(&vv)).Ptr = unsafe.Pointer(ss)\n\t(*rt.GoString)(unsafe.Pointer(&vv)).Len = ret - pos\n\n\tv, err = strconv.ParseInt(vv, 10, 64)\n\tif err != nil {\n\t\t//NOTICE: allow overflow here\n\t\tif err.(*strconv.NumError).Err == strconv.ErrRange {\n\t\t\treturn ret, 0, err\n\t\t}\n\t\treturn -int(types.ERR_INVALID_CHAR), 0, err\n\t}\n\n\truntime.KeepAlive(src)\n\treturn ret, v, nil\n}\n\nfunc isNumberChars(c byte) bool {\n\treturn (c >= '0' && c <= '9') || c == '+' || c == '-' || c == 'e' || c == 'E' || c == '.'\n}\n\n//go:nocheckptr\nfunc decodeFloat64(src string, pos int) (ret int, v float64, err error) {\n\tsp := uintptr(rt.IndexChar(src, pos))\n\tss := uintptr(sp)\n\tse := uintptr(rt.IndexChar(src, len(src)))\n\tif uintptr(sp) >= se {\n\t\treturn -int(types.ERR_EOF), 0, nil\n\t}\n\n\tif c := *(*byte)(unsafe.Pointer(sp)); c == '-' {\n\t\tsp += 1\n\t}\n\tif sp == se {\n\t\treturn -int(types.ERR_EOF), 0, nil\n\t}\n\n\tfor ; sp < se; sp += uintptr(1) {\n\t\tif !isNumberChars(*(*byte)(unsafe.Pointer(sp))) {\n\t\t\tbreak\n\t\t}\n\t}\n\n\tvar vv string\n\tret = int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr))\n\t(*rt.GoString)(unsafe.Pointer(&vv)).Ptr = unsafe.Pointer(ss)\n\t(*rt.GoString)(unsafe.Pointer(&vv)).Len = ret - pos\n\n\tv, err = strconv.ParseFloat(vv, 64)\n\tif err != nil {\n\t\t//NOTICE: allow overflow here\n\t\tif err.(*strconv.NumError).Err == strconv.ErrRange {\n\t\t\treturn ret, 0, err\n\t\t}\n\t\treturn -int(types.ERR_INVALID_CHAR), 0, err\n\t}\n\n\truntime.KeepAlive(src)\n\treturn ret, v, nil\n}\n\nfunc decodeValue(src string, pos int, skipnum bool) (ret int, v types.JsonState) {\n\tpos = skipBlank(src, pos)\n\tif pos < 0 {\n\t\treturn pos, types.JsonState{Vt: types.ValueType(pos)}\n\t}\n\tswitch c := src[pos]; c {\n\tcase 'n':\n\t\tret = decodeNull(src, pos)\n\t\tif ret < 0 {\n\t\t\treturn ret, types.JsonState{Vt: types.ValueType(ret)}\n\t\t}\n\t\treturn ret, types.JsonState{Vt: types.V_NULL}\n\tcase '\"':\n\t\tvar ep int\n\t\tret, ep = skipString(src, pos)\n\t\tif ret < 0 {\n\t\t\treturn ret, types.JsonState{Vt: types.ValueType(ret)}\n\t\t}\n\t\treturn ret, types.JsonState{Vt: types.V_STRING, Iv: int64(pos + 1), Ep: ep}\n\tcase '{':\n\t\treturn pos + 1, types.JsonState{Vt: types.V_OBJECT}\n\tcase '[':\n\t\treturn pos + 1, types.JsonState{Vt: types.V_ARRAY}\n\tcase 't':\n\t\tret = decodeTrue(src, pos)\n\t\tif ret < 0 {\n\t\t\treturn ret, types.JsonState{Vt: types.ValueType(ret)}\n\t\t}\n\t\treturn ret, types.JsonState{Vt: types.V_TRUE}\n\tcase 'f':\n\t\tret = decodeFalse(src, pos)\n\t\tif ret < 0 {\n\t\t\treturn ret, types.JsonState{Vt: types.ValueType(ret)}\n\t\t}\n\t\treturn ret, types.JsonState{Vt: types.V_FALSE}\n\tcase '-', '+', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':\n\t\tif skipnum {\n\t\t\tret = skipNumber(src, pos)\n\t\t\tif ret >= 0 {\n\t\t\t\treturn ret, types.JsonState{Vt: types.V_DOUBLE, Iv: 0, Ep: pos}\n\t\t\t} else {\n\t\t\t\treturn ret, types.JsonState{Vt: types.ValueType(ret)}\n\t\t\t}\n\t\t} else {\n\t\t\tvar iv int64\n\t\t\tret, iv, _ = decodeInt64(src, pos)\n\t\t\tif ret >= 0 {\n\t\t\t\treturn ret, types.JsonState{Vt: types.V_INTEGER, Iv: iv, Ep: pos}\n\t\t\t} else if ret != -int(types.ERR_INVALID_NUMBER_FMT) {\n\t\t\t\treturn ret, types.JsonState{Vt: types.ValueType(ret)}\n\t\t\t}\n\t\t\tvar fv float64\n\t\t\tret, fv, _ = decodeFloat64(src, pos)\n\t\t\tif ret >= 0 {\n\t\t\t\treturn ret, types.JsonState{Vt: types.V_DOUBLE, Dv: fv, Ep: pos}\n\t\t\t} else {\n\t\t\t\treturn ret, types.JsonState{Vt: types.ValueType(ret)}\n\t\t\t}\n\t\t}\n\n\tdefault:\n\t\treturn -int(types.ERR_INVALID_CHAR), types.JsonState{Vt: -types.ValueType(types.ERR_INVALID_CHAR)}\n\t}\n}\n\n//go:nocheckptr\nfunc skipNumber(src string, pos int) (ret int) {\n\treturn utils.SkipNumber(src, pos)\n}\n\n//go:nocheckptr\nfunc skipString(src string, pos int) (ret int, ep int) {\n\tif pos+1 >= len(src) {\n\t\treturn -int(types.ERR_EOF), -1\n\t}\n\n\tsp := uintptr(rt.IndexChar(src, pos))\n\tse := uintptr(rt.IndexChar(src, len(src)))\n\n\t// not start with quote\n\tif *(*byte)(unsafe.Pointer(sp)) != '\"' {\n\t\treturn -int(types.ERR_INVALID_CHAR), -1\n\t}\n\tsp += 1\n\n\tep = -1\n\tfor sp < se {\n\t\tc := *(*byte)(unsafe.Pointer(sp))\n\t\tif c == '\\\\' {\n\t\t\tif ep == -1 {\n\t\t\t\tep = int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr))\n\t\t\t}\n\t\t\tsp += 2\n\t\t\tcontinue\n\t\t}\n\t\tsp += 1\n\t\tif c == '\"' {\n\t\t\treturn int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr)), ep\n\t\t}\n\t}\n\n\truntime.KeepAlive(src)\n\t// not found the closed quote until EOF\n\treturn -int(types.ERR_EOF), -1\n}\n\n//go:nocheckptr\nfunc skipPair(src string, pos int, lchar byte, rchar byte) (ret int) {\n\tif pos+1 >= len(src) {\n\t\treturn -int(types.ERR_EOF)\n\t}\n\n\tsp := uintptr(rt.IndexChar(src, pos))\n\tse := uintptr(rt.IndexChar(src, len(src)))\n\n\tif *(*byte)(unsafe.Pointer(sp)) != lchar {\n\t\treturn -int(types.ERR_INVALID_CHAR)\n\t}\n\n\tsp += 1\n\tnbrace := 1\n\tinquote := false\n\n\tfor sp < se {\n\t\tc := *(*byte)(unsafe.Pointer(sp))\n\t\tif c == '\\\\' {\n\t\t\tsp += 2\n\t\t\tcontinue\n\t\t} else if c == '\"' {\n\t\t\tinquote = !inquote\n\t\t} else if c == lchar {\n\t\t\tif !inquote {\n\t\t\t\tnbrace += 1\n\t\t\t}\n\t\t} else if c == rchar {\n\t\t\tif !inquote {\n\t\t\t\tnbrace -= 1\n\t\t\t\tif nbrace == 0 {\n\t\t\t\t\tsp += 1\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tsp += 1\n\t}\n\n\tif nbrace != 0 {\n\t\treturn -int(types.ERR_INVALID_CHAR)\n\t}\n\n\truntime.KeepAlive(src)\n\treturn int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr))\n}\n\nfunc skipValueFast(src string, pos int) (ret int, start int) {\n\tpos = skipBlank(src, pos)\n\tif pos < 0 {\n\t\treturn pos, -1\n\t}\n\tswitch c := src[pos]; c {\n\tcase 'n':\n\t\tret = decodeNull(src, pos)\n\tcase '\"':\n\t\tret, _ = skipString(src, pos)\n\tcase '{':\n\t\tret = skipPair(src, pos, '{', '}')\n\tcase '[':\n\t\tret = skipPair(src, pos, '[', ']')\n\tcase 't':\n\t\tret = decodeTrue(src, pos)\n\tcase 'f':\n\t\tret = decodeFalse(src, pos)\n\tcase '-', '+', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':\n\t\tret = skipNumber(src, pos)\n\tdefault:\n\t\tret = -int(types.ERR_INVALID_CHAR)\n\t}\n\treturn ret, pos\n}\n\nfunc skipValue(src string, pos int) (ret int, start int) {\n\tpos = skipBlank(src, pos)\n\tif pos < 0 {\n\t\treturn pos, -1\n\t}\n\tswitch c := src[pos]; c {\n\tcase 'n':\n\t\tret = decodeNull(src, pos)\n\tcase '\"':\n\t\tret, _ = skipString(src, pos)\n\tcase '{':\n\t\tret, _ = skipObject(src, pos)\n\tcase '[':\n\t\tret, _ = skipArray(src, pos)\n\tcase 't':\n\t\tret = decodeTrue(src, pos)\n\tcase 'f':\n\t\tret = decodeFalse(src, pos)\n\tcase '-', '+', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':\n\t\tret = skipNumber(src, pos)\n\tdefault:\n\t\tret = -int(types.ERR_INVALID_CHAR)\n\t}\n\treturn ret, pos\n}\n\nfunc skipObject(src string, pos int) (ret int, start int) {\n\tstart = skipBlank(src, pos)\n\tif start < 0 {\n\t\treturn start, -1\n\t}\n\n\tif src[start] != '{' {\n\t\treturn -int(types.ERR_INVALID_CHAR), -1\n\t}\n\n\tpos = start + 1\n\tpos = skipBlank(src, pos)\n\tif pos < 0 {\n\t\treturn pos, -1\n\t}\n\tif src[pos] == '}' {\n\t\treturn pos + 1, start\n\t}\n\n\tfor {\n\t\tpos, _ = skipString(src, pos)\n\t\tif pos < 0 {\n\t\t\treturn pos, -1\n\t\t}\n\n\t\tpos = skipBlank(src, pos)\n\t\tif pos < 0 {\n\t\t\treturn pos, -1\n\t\t}\n\t\tif src[pos] != ':' {\n\t\t\treturn -int(types.ERR_INVALID_CHAR), -1\n\t\t}\n\n\t\tpos++\n\t\tpos, _ = skipValue(src, pos)\n\t\tif pos < 0 {\n\t\t\treturn pos, -1\n\t\t}\n\n\t\tpos = skipBlank(src, pos)\n\t\tif pos < 0 {\n\t\t\treturn pos, -1\n\t\t}\n\t\tif src[pos] == '}' {\n\t\t\treturn pos + 1, start\n\t\t}\n\t\tif src[pos] != ',' {\n\t\t\treturn -int(types.ERR_INVALID_CHAR), -1\n\t\t}\n\n\t\tpos++\n\t\tpos = skipBlank(src, pos)\n\t\tif pos < 0 {\n\t\t\treturn pos, -1\n\t\t}\n\n\t}\n}\n\nfunc skipArray(src string, pos int) (ret int, start int) {\n\tstart = skipBlank(src, pos)\n\tif start < 0 {\n\t\treturn start, -1\n\t}\n\n\tif src[start] != '[' {\n\t\treturn -int(types.ERR_INVALID_CHAR), -1\n\t}\n\n\tpos = start + 1\n\tpos = skipBlank(src, pos)\n\tif pos < 0 {\n\t\treturn pos, -1\n\t}\n\tif src[pos] == ']' {\n\t\treturn pos + 1, start\n\t}\n\n\tfor {\n\t\tpos, _ = skipValue(src, pos)\n\t\tif pos < 0 {\n\t\t\treturn pos, -1\n\t\t}\n\n\t\tpos = skipBlank(src, pos)\n\t\tif pos < 0 {\n\t\t\treturn pos, -1\n\t\t}\n\t\tif src[pos] == ']' {\n\t\t\treturn pos + 1, start\n\t\t}\n\t\tif src[pos] != ',' {\n\t\t\treturn -int(types.ERR_INVALID_CHAR), -1\n\t\t}\n\t\tpos++\n\t}\n}\n\n// DecodeString decodes a JSON string from pos and return golang string.\n// - needEsc indicates if to unescaped escaping chars\n// - hasEsc tells if the returned string has escaping chars\n// - validStr enables validating UTF8 charset\nfunc _DecodeString(src string, pos int, needEsc bool, validStr bool) (v string, ret int, hasEsc bool) {\n\tp := NewParserObj(src)\n\tp.p = pos\n\tswitch val := p.decodeValue(); val.Vt {\n\tcase types.V_STRING:\n\t\tstr := p.s[val.Iv : p.p-1]\n\t\tif validStr && !validate_utf8(str) {\n\t\t\treturn \"\", -int(types.ERR_INVALID_UTF8), false\n\t\t}\n\t\t/* fast path: no escape sequence */\n\t\tif val.Ep == -1 {\n\t\t\treturn str, p.p, false\n\t\t} else if !needEsc {\n\t\t\treturn str, p.p, true\n\t\t}\n\t\t/* unquote the string */\n\t\tout, err := unquote.String(str)\n\t\t/* check for errors */\n\t\tif err != 0 {\n\t\t\treturn \"\", -int(err), true\n\t\t} else {\n\t\t\treturn out, p.p, true\n\t\t}\n\tdefault:\n\t\treturn \"\", -int(_ERR_UNSUPPORT_TYPE), false\n\t}\n}\n" }, { "path": "ast/decode_test.go", "content": "/*\n * Copyright 2022 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage ast\n\nimport (\n\t\"testing\"\n\t\"unicode/utf8\"\n\n\t\"github.com/bytedance/sonic/internal/rt\"\n)\n\nfunc Test_DecodeString(t *testing.T) {\n\ttype args struct {\n\t\tsrc string\n\t\tpos int\n\t\tneedEsc bool\n\t\tvalidStr bool\n\t}\n\tinvalidstr := rt.Mem2Str([]byte{'\"', 193, 255, '\"'})\n\tprintln(utf8.ValidString(invalidstr))\n\n\ttests := []struct {\n\t\tname string\n\t\targs args\n\t\twantV string\n\t\twantRet int\n\t\twantHasEsc bool\n\t}{\n\t\t{\"empty\", args{`\"\"`, 0, false, false}, \"\", 2, false},\n\t\t{\"one\", args{`\"1\"`, 0, false, false}, \"1\", 3, false},\n\t\t{\"escape\", args{`\"\\\\\"`, 0, false, false}, `\\\\`, 4, true},\n\t\t{\"escape\", args{`\"\\\\\"`, 0, true, true}, `\\`, 4, true},\n\t\t{\"uft8\", args{`\"\\u263a\"`, 0, false, false}, `\\u263a`, 8, true},\n\t\t{\"uft8\", args{`\"\\u263a\"`, 0, true, true}, `☺`, 8, true},\n\t\t{\"invalid uft8\", args{`\"\\xx\"`, 0, false, false}, `\\xx`, 5, true},\n\t\t{\"invalid escape\", args{`\"\\xx\"`, 0, false, true}, `\\xx`, 5, true},\n\t\t{\"invalid escape\", args{`\"\\xx\"`, 0, true, true}, ``, -3, true},\n\t\t{\"invalid string\", args{invalidstr, 0, false, false}, invalidstr[1:3], 4, false},\n\t\t{\"invalid string\", args{invalidstr, 0, true, true}, \"\", -10, false},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tgotV, gotRet, gotHasEsc := _DecodeString(tt.args.src, tt.args.pos, tt.args.needEsc, tt.args.validStr)\n\t\t\tif gotV != tt.wantV {\n\t\t\t\tt.Errorf(\"_DecodeString() gotV = %v, want %v\", gotV, tt.wantV)\n\t\t\t}\n\t\t\tif gotRet != tt.wantRet {\n\t\t\t\tt.Errorf(\"_DecodeString() gotRet = %v, want %v\", gotRet, tt.wantRet)\n\t\t\t}\n\t\t\tif gotHasEsc != tt.wantHasEsc {\n\t\t\t\tt.Errorf(\"_DecodeString() gotHasEsc = %v, want %v\", gotHasEsc, tt.wantHasEsc)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "ast/encode.go", "content": "/*\n * Copyright 2021 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage ast\n\nimport (\n\t\"sync\"\n\t\"unicode/utf8\"\n\n\t\"github.com/bytedance/gopkg/lang/dirtmake\"\n\t\"github.com/bytedance/sonic/internal/rt\"\n\t\"github.com/bytedance/sonic/option\"\n)\n\nfunc quoteString(e *[]byte, s string) {\n\t*e = append(*e, '\"')\n\tstart := 0\n\tfor i := 0; i < len(s); {\n\t\tif b := s[i]; b < utf8.RuneSelf {\n\t\t\tif rt.SafeSet[b] {\n\t\t\t\ti++\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif start < i {\n\t\t\t\t*e = append(*e, s[start:i]...)\n\t\t\t}\n\t\t\t*e = append(*e, '\\\\')\n\t\t\tswitch b {\n\t\t\tcase '\\\\', '\"':\n\t\t\t\t*e = append(*e, b)\n\t\t\tcase '\\n':\n\t\t\t\t*e = append(*e, 'n')\n\t\t\tcase '\\r':\n\t\t\t\t*e = append(*e, 'r')\n\t\t\tcase '\\t':\n\t\t\t\t*e = append(*e, 't')\n\t\t\tdefault:\n\t\t\t\t// This encodes bytes < 0x20 except for \\t, \\n and \\r.\n\t\t\t\t// If escapeHTML is set, it also escapes <, >, and &\n\t\t\t\t// because they can lead to security holes when\n\t\t\t\t// user-controlled strings are rendered into JSON\n\t\t\t\t// and served to some browsers.\n\t\t\t\t*e = append(*e, `u00`...)\n\t\t\t\t*e = append(*e, rt.Hex[b>>4])\n\t\t\t\t*e = append(*e, rt.Hex[b&0xF])\n\t\t\t}\n\t\t\ti++\n\t\t\tstart = i\n\t\t\tcontinue\n\t\t}\n\t\tc, size := utf8.DecodeRuneInString(s[i:])\n\t\t// if c == utf8.RuneError && size == 1 {\n\t\t// if start < i {\n\t\t// e.Write(s[start:i])\n\t\t// }\n\t\t// e.WriteString(`\\ufffd`)\n\t\t// i += size\n\t\t// start = i\n\t\t// continue\n\t\t// }\n\t\tif c == '\\u2028' || c == '\\u2029' {\n\t\t\tif start < i {\n\t\t\t\t*e = append(*e, s[start:i]...)\n\t\t\t}\n\t\t\t*e = append(*e, `\\u202`...)\n\t\t\t*e = append(*e, rt.Hex[c&0xF])\n\t\t\ti += size\n\t\t\tstart = i\n\t\t\tcontinue\n\t\t}\n\t\ti += size\n\t}\n\tif start < len(s) {\n\t\t*e = append(*e, s[start:]...)\n\t}\n\t*e = append(*e, '\"')\n}\n\nvar bytesPool = sync.Pool{}\n\nfunc (self *Node) MarshalJSON() ([]byte, error) {\n\tif self == nil {\n\t\treturn bytesNull, nil\n\t}\n\n\t// fast path for raw node\n\tif self.isRaw() {\n\t\treturn rt.Str2Mem(self.toString()), nil\n\t}\n\n\tbuf := newBuffer()\n\terr := self.encode(buf)\n\tif err != nil {\n\t\tfreeBuffer(buf)\n\t\treturn nil, err\n\t}\n\tvar ret []byte\n\tif !rt.CanSizeResue(cap(*buf)) {\n\t\tret = *buf\n\t} else {\n\t\tret = dirtmake.Bytes(len(*buf), len(*buf))\n\t\tcopy(ret, *buf)\n\t\tfreeBuffer(buf)\n\t}\n\treturn ret, err\n}\n\nfunc newBuffer() *[]byte {\n\tif ret := bytesPool.Get(); ret != nil {\n\t\treturn ret.(*[]byte)\n\t} else {\n\t\tbuf := make([]byte, 0, option.DefaultAstBufferSize)\n\t\treturn &buf\n\t}\n}\n\nfunc freeBuffer(buf *[]byte) {\n\tif !rt.CanSizeResue(cap(*buf)) {\n\t\treturn\n\t}\n\t*buf = (*buf)[:0]\n\tbytesPool.Put(buf)\n}\n\nfunc (self *Node) encode(buf *[]byte) error {\n\tif self.isRaw() {\n\t\treturn self.encodeRaw(buf)\n\t}\n\tswitch int(self.itype()) {\n\tcase V_NONE:\n\t\treturn ErrNotExist\n\tcase V_ERROR:\n\t\treturn self.Check()\n\tcase V_NULL:\n\t\treturn self.encodeNull(buf)\n\tcase V_TRUE:\n\t\treturn self.encodeTrue(buf)\n\tcase V_FALSE:\n\t\treturn self.encodeFalse(buf)\n\tcase V_ARRAY:\n\t\treturn self.encodeArray(buf)\n\tcase V_OBJECT:\n\t\treturn self.encodeObject(buf)\n\tcase V_STRING:\n\t\treturn self.encodeString(buf)\n\tcase V_NUMBER:\n\t\treturn self.encodeNumber(buf)\n\tcase V_ANY:\n\t\treturn self.encodeInterface(buf)\n\tdefault:\n\t\treturn ErrUnsupportType\n\t}\n}\n\nfunc (self *Node) encodeRaw(buf *[]byte) error {\n\tlock := self.rlock()\n\tif !self.isRaw() {\n\t\tself.runlock()\n\t\treturn self.encode(buf)\n\t}\n\traw := self.toString()\n\tif lock {\n\t\tself.runlock()\n\t}\n\t*buf = append(*buf, raw...)\n\treturn nil\n}\n\nfunc (self *Node) encodeNull(buf *[]byte) error {\n\t*buf = append(*buf, strNull...)\n\treturn nil\n}\n\nfunc (self *Node) encodeTrue(buf *[]byte) error {\n\t*buf = append(*buf, bytesTrue...)\n\treturn nil\n}\n\nfunc (self *Node) encodeFalse(buf *[]byte) error {\n\t*buf = append(*buf, bytesFalse...)\n\treturn nil\n}\n\nfunc (self *Node) encodeNumber(buf *[]byte) error {\n\tstr := self.toString()\n\t*buf = append(*buf, str...)\n\treturn nil\n}\n\nfunc (self *Node) encodeString(buf *[]byte) error {\n\tif self.l == 0 {\n\t\t*buf = append(*buf, '\"', '\"')\n\t\treturn nil\n\t}\n\n\tquote(buf, self.toString())\n\treturn nil\n}\n\nfunc (self *Node) encodeArray(buf *[]byte) error {\n\tif self.isLazy() {\n\t\tif err := self.skipAllIndex(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tnb := self.len()\n\tif nb == 0 {\n\t\t*buf = append(*buf, bytesArray...)\n\t\treturn nil\n\t}\n\n\t*buf = append(*buf, '[')\n\n\tvar started bool\n\tfor i := 0; i < nb; i++ {\n\t\tn := self.nodeAt(i)\n\t\tif !n.Exists() {\n\t\t\tcontinue\n\t\t}\n\t\tif started {\n\t\t\t*buf = append(*buf, ',')\n\t\t}\n\t\tstarted = true\n\t\tif err := n.encode(buf); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\t*buf = append(*buf, ']')\n\treturn nil\n}\n\nfunc (self *Pair) encode(buf *[]byte) error {\n\tif len(*buf) == 0 {\n\t\t*buf = append(*buf, '\"', '\"', ':')\n\t\treturn self.Value.encode(buf)\n\t}\n\n\tquote(buf, self.Key)\n\t*buf = append(*buf, ':')\n\n\treturn self.Value.encode(buf)\n}\n\nfunc (self *Node) encodeObject(buf *[]byte) error {\n\tif self.isLazy() {\n\t\tif err := self.skipAllKey(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tnb := self.len()\n\tif nb == 0 {\n\t\t*buf = append(*buf, bytesObject...)\n\t\treturn nil\n\t}\n\n\t*buf = append(*buf, '{')\n\n\tvar started bool\n\tfor i := 0; i < nb; i++ {\n\t\tn := self.pairAt(i)\n\t\tif n == nil || !n.Value.Exists() {\n\t\t\tcontinue\n\t\t}\n\t\tif started {\n\t\t\t*buf = append(*buf, ',')\n\t\t}\n\t\tstarted = true\n\t\tif err := n.encode(buf); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\t*buf = append(*buf, '}')\n\treturn nil\n}\n" }, { "path": "ast/encode_test.go", "content": "/*\n * Copyright 2021 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage ast\n\nimport (\n\t\"encoding/json\"\n\t\"runtime\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\n\t\"github.com/bytedance/sonic/internal/native/types\"\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestGC_Encode(t *testing.T) {\n\tif debugSyncGC {\n\t\treturn\n\t}\n\troot, err := NewSearcher(_TwitterJson).GetByPath()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\troot.LoadAll()\n\t_, err = root.MarshalJSON()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\twg := &sync.WaitGroup{}\n\tN := 10000\n\tfor i := 0; i < N; i++ {\n\t\twg.Add(1)\n\t\tgo func(wg *sync.WaitGroup) {\n\t\t\tdefer wg.Done()\n\t\t\troot, err := NewSearcher(_TwitterJson).GetByPath()\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\troot.Load()\n\t\t\t_, err = root.MarshalJSON()\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\truntime.GC()\n\t\t}(wg)\n\t}\n\twg.Wait()\n}\n\nfunc TestEncodeValue(t *testing.T) {\n\tobj := new(_TwitterStruct)\n\tif err := json.Unmarshal([]byte(_TwitterJson), obj); err != nil {\n\t\tt.Fatal(err)\n\t}\n\t// buf, err := encoder.Encode(obj, encoder.EscapeHTML|encoder.SortMapKeys)\n\tbuf, err := json.Marshal(obj)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tquote, err := json.Marshal(_TwitterJson)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\ttype Case struct {\n\t\tnode Node\n\t\texp string\n\t\terr bool\n\t}\n\tinput := []Case{\n\t\t{NewNull(), \"null\", false},\n\t\t{NewBool(true), \"true\", false},\n\t\t{NewBool(false), \"false\", false},\n\t\t{NewNumber(\"0.0\"), \"0.0\", false},\n\t\t{NewString(\"\"), `\"\"`, false},\n\t\t{NewString(`\\\"\\\"`), `\"\\\\\\\"\\\\\\\"\"`, false},\n\t\t{NewString(_TwitterJson), string(quote), false},\n\t\t{NewArray([]Node{}), \"[]\", false},\n\t\t{NewArray([]Node{NewString(\"\"), NewNull()}), `[\"\",null]`, false},\n\t\t{NewArray([]Node{NewBool(true), NewString(\"true\"), NewString(\"\\t\")}), `[true,\"true\",\"\\t\"]`, false},\n\t\t{NewObject([]Pair{NewPair(\"a\", NewNull()), NewPair(\"b\", NewNumber(\"0\"))}), `{\"a\":null,\"b\":0}`, false},\n\t\t{NewObject([]Pair{NewPair(\"\\ta\", NewString(\"\\t\")), NewPair(\"\\bb\", NewString(\"\\b\")), NewPair(\"\\nb\", NewString(\"\\n\")), NewPair(\"\\ra\", NewString(\"\\r\"))}), `{\"\\ta\":\"\\t\",\"\\u0008b\":\"\\u0008\",\"\\nb\":\"\\n\",\"\\ra\":\"\\r\"}`, false},\n\t\t{NewObject([]Pair{}), `{}`, false},\n\t\t{NewObject([]Pair{Pair{Key: \"\", Value: NewNull()}}), `{\"\":null}`, false},\n\t\t{NewBytes([]byte(\"hello, world\")), `\"aGVsbG8sIHdvcmxk\"`, false},\n\t\t{NewAny(obj), string(buf), false},\n\t\t{NewRaw(`[{ }]`), \"[{}]\", false},\n\t\t{Node{}, \"\", true},\n\t\t{Node{t: types.ValueType(1)}, \"\", true},\n\t}\n\tfor i, c := range input {\n\t\tt.Log(i)\n\t\tbuf, err := json.Marshal(&c.node)\n\t\tif c.err {\n\t\t\tif err == nil {\n\t\t\t\tt.Fatal(i)\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Fatal(i, err)\n\t\t}\n\t\tassert.Equal(t, c.exp, string(buf))\n\t}\n}\n\nfunc BenchmarkNil(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\tnull := (*Node)(nil)\n\t\t_, _ = null.MarshalJSON()\n\t}\n}\n\nfunc TestEncodeNode(t *testing.T) {\n\tnull := (*Node)(nil)\n\tjs, err := null.MarshalJSON()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif string(js) != \"null\" {\n\t\tt.Fatal(string(js))\n\t}\n\n\tdata := `{\"a\":[{},[],-0.1,true,false,null,\"\"],\"b\":0,\"c\":true,\"d\":false,\"e\":null,\"g\":\"\"}`\n\troot, e := NewSearcher(data).GetByPath()\n\tif e != nil {\n\t\tt.Fatal(root)\n\t}\n\tret, err := root.MarshalJSON()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif string(ret) != data {\n\t\tt.Fatal(string(ret))\n\t}\n\troot.skipAllKey()\n\tret, err = root.MarshalJSON()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif string(ret) != data {\n\t\tt.Fatal(string(ret))\n\t}\n\troot.Load()\n\tret, err = root.MarshalJSON()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif string(ret) != data {\n\t\tt.Fatal(string(ret))\n\t}\n}\n\ntype SortableNode struct {\n\tsorted bool\n\t*Node\n}\n\nfunc (j *SortableNode) UnmarshalJSON(data []byte) error {\n\tj.Node = new(Node)\n\treturn j.Node.UnmarshalJSON(data)\n}\n\nfunc (j *SortableNode) MarshalJSON() ([]byte, error) {\n\tif !j.sorted {\n\t\tj.Node.SortKeys(true)\n\t\tj.sorted = true\n\t}\n\treturn j.Node.MarshalJSON()\n}\n\nfunc TestMarshalSort(t *testing.T) {\n\tvar data = `{\"d\":3,\"a\":{\"c\":1,\"b\":2},\"e\":null}`\n\tvar obj map[string]*SortableNode\n\trequire.NoError(t, json.Unmarshal([]byte(data), &obj))\n\tout, err := json.Marshal(obj)\n\trequire.NoError(t, err)\n\trequire.Equal(t, `{\"a\":{\"b\":2,\"c\":1},\"d\":3,\"e\":null}`, string(out))\n\tout, err = json.Marshal(obj)\n\trequire.NoError(t, err)\n\trequire.Equal(t, `{\"a\":{\"b\":2,\"c\":1},\"d\":3,\"e\":null}`, string(out))\n}\n\nfunc BenchmarkEncodeRaw_Sonic(b *testing.B) {\n\tdata := _TwitterJson\n\troot, e := NewSearcher(data).GetByPath()\n\tif e != nil {\n\t\tb.Fatal(root)\n\t}\n\t_, err := root.MarshalJSON()\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\tb.SetBytes(int64(len(data)))\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\t_, err := root.MarshalJSON()\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkEncodeSkip_Sonic(b *testing.B) {\n\tdata := _TwitterJson\n\troot, e := NewParser(data).Parse()\n\tif e != 0 {\n\t\tb.Fatal(root)\n\t}\n\troot.skipAllKey()\n\t_, err := root.MarshalJSON()\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\tb.SetBytes(int64(len(data)))\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\t_, err := root.MarshalJSON()\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkEncodeLoad_Sonic(b *testing.B) {\n\tdata := _TwitterJson\n\troot, e := NewParser(data).Parse()\n\tif e != 0 {\n\t\tb.Fatal(root)\n\t}\n\troot.Load()\n\t_, err := root.MarshalJSON()\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\tb.SetBytes(int64(len(data)))\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\t_, err := root.MarshalJSON()\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n\nfunc TestEncodeNone(t *testing.T) {\n\tn := NewObject([]Pair{{Key: \"a\", Value: Node{}}})\n\tout, err := n.MarshalJSON()\n\trequire.NoError(t, err)\n\trequire.Equal(t, \"{}\", string(out))\n\tn = NewObject([]Pair{{Key: \"a\", Value: NewNull()}, {Key: \"b\", Value: Node{}}})\n\tout, err = n.MarshalJSON()\n\trequire.NoError(t, err)\n\trequire.Equal(t, `{\"a\":null}`, string(out))\n\n\tn = NewArray([]Node{Node{}})\n\tout, err = n.MarshalJSON()\n\trequire.NoError(t, err)\n\trequire.Equal(t, \"[]\", string(out))\n\tn = NewArray([]Node{NewNull(), Node{}})\n\tout, err = n.MarshalJSON()\n\trequire.NoError(t, err)\n\trequire.Equal(t, `[null]`, string(out))\n}\n\ntype Path = []interface{}\n\ntype testGetApi struct {\n\tjson string\n\tpath Path\n}\n\ntype checkError func(error) bool\n\nfunc isSyntaxError(err error) bool {\n\tif err == nil {\n\t\treturn false\n\t}\n\treturn strings.HasPrefix(err.Error(), `\"Syntax error at index`)\n}\n\nfunc isEmptySource(err error) bool {\n\tif err == nil {\n\t\treturn false\n\t}\n\treturn strings.Contains(err.Error(), \"no sources available\")\n}\n\nfunc isErrNotExist(err error) bool {\n\treturn err == ErrNotExist\n}\n\nfunc isErrUnsupportType(err error) bool {\n\treturn err == ErrUnsupportType\n}\n\nfunc testSyntaxJson(t *testing.T, json string, path ...interface{}) {\n\tsearch := NewSearcher(json)\n\t_, err := search.GetByPath(path...)\n\tassert.True(t, isSyntaxError(err))\n}\n\nfunc TestGetFromEmptyJson(t *testing.T) {\n\ttests := []testGetApi{\n\t\t{\"\", nil},\n\t\t{\"\", Path{}},\n\t\t{\"\", Path{\"\"}},\n\t\t{\"\", Path{0}},\n\t\t{\"\", Path{\"\", \"\"}},\n\t}\n\tfor _, test := range tests {\n\t\tf := func(t *testing.T) {\n\t\t\tsearch := NewSearcher(test.json)\n\t\t\t_, err := search.GetByPath(test.path...)\n\t\t\tassert.True(t, isEmptySource(err))\n\t\t}\n\t\tt.Run(test.json, f)\n\t}\n}\n\nfunc TestGetFromSyntaxError(t *testing.T) {\n\ttests := []testGetApi{\n\t\t{\" \\r\\n\\f\\t\", Path{}},\n\t\t{\"123.\", Path{}},\n\t\t{\"+124\", Path{}},\n\t\t{\"-\", Path{}},\n\t\t{\"-e123\", Path{}},\n\t\t{\"-1.e123\", Path{}},\n\t\t{\"-12e456.1\", Path{}},\n\t\t{\"-12e.1\", Path{}},\n\t\t{\"[\", Path{}},\n\t\t{\"{\", Path{}},\n\t\t{\"[}\", Path{}},\n\t\t{\"{]\", Path{}},\n\t\t{\"{,}\", Path{}},\n\t\t{\"[,]\", Path{}},\n\t\t{\"tru\", Path{}},\n\t\t{\"fals\", Path{}},\n\t\t{\"nul\", Path{}},\n\t\t{`{\"a\":\"`, Path{\"a\"}},\n\t\t{`{\"`, Path{}},\n\t\t{`\"`, Path{}},\n\t\t{`\"\\\"`, Path{}},\n\t\t{`\"\\\\\\\"`, Path{}},\n\t\t{`\"hello`, Path{}},\n\t\t{`{{}}`, Path{}},\n\t\t{`{[]}`, Path{}},\n\t\t{`{:,}`, Path{}},\n\t\t{`{test:error}`, Path{}},\n\t\t{`{\":true}`, Path{}},\n\t\t{`{\"\" false}`, Path{}},\n\t\t{`{ \"\" : \"false }`, Path{}},\n\t\t{`{\"\":\"\",}`, Path{}},\n\t\t{`{ \" test : true}`, Path{}},\n\t\t{`{ \"test\" : tru }`, Path{}},\n\t\t{`{ \"test\" : true , }`, Path{}},\n\t\t{`{ {\"test\" : true , } }`, Path{}},\n\t\t{`{\"test\":1. }`, Path{}},\n\t\t{`{\"\\\\\\\"\"`, Path{}},\n\t\t{`{\"\\\\\\\"\":`, Path{}},\n\t\t{`{\"\\\\\\\":\",\"\"}`, Path{}},\n\t\t{`[{]`, Path{}},\n\t\t{`[tru]`, Path{}},\n\t\t{`[-1.]`, Path{}},\n\t\t{`[[]`, Path{}},\n\t\t{`[[],`, Path{}},\n\t\t{`[ true , false , [ ]`, Path{}},\n\t\t{`[true, false, [],`, Path{}},\n\t\t{`[true, false, [],]`, Path{}},\n\t\t{`{\"key\": [true, false, []], \"key2\": {{}}`, Path{}},\n\t}\n\n\tfor _, test := range tests {\n\t\tf := func(t *testing.T) {\n\t\t\ttestSyntaxJson(t, test.json, test.path...)\n\t\t\tpath := append(Path{\"key\"}, test.path...)\n\t\t\ttestSyntaxJson(t, `{\"key\":`+test.json, path...)\n\t\t\tpath = append(Path{\"\"}, test.path...)\n\t\t\ttestSyntaxJson(t, `{\"\":`+test.json, path...)\n\t\t\tpath = append(Path{1}, test.path...)\n\t\t\ttestSyntaxJson(t, `[\"\",`+test.json, path...)\n\t\t}\n\t\tt.Run(test.json, f)\n\t}\n}\n\n// NOTE: GetByPath API not validate the undemanded fields for performance.\nfunc TestGetWithInvalidUndemandedField(t *testing.T) {\n\ttype Any = interface{}\n\ttests := []struct {\n\t\tjson string\n\t\tpath Path\n\t\texp Any\n\t}{\n\t\t{\"-0xyz\", Path{}, Any(float64(-0))},\n\t\t{\"-12e4xyz\", Path{}, Any(float64(-12e4))},\n\t\t{\"truex\", Path{}, Any(true)},\n\t\t{\"false,\", Path{}, Any(false)},\n\t\t{`{\"a\":{,xxx},\"b\":true}`, Path{\"b\"}, Any(true)},\n\t\t{`{\"a\":[,xxx],\"b\":true}`, Path{\"b\"}, Any(true)},\n\t}\n\n\tfor _, test := range tests {\n\t\tf := func(t *testing.T) {\n\t\t\tsearch := NewSearcher(test.json)\n\t\t\tnode, err := search.GetByPath(test.path...)\n\t\t\tassert.NoError(t, err)\n\t\t\tv, err := node.Interface()\n\t\t\tassert.NoError(t, err)\n\t\t\tassert.Equal(t, v, test.exp)\n\t\t}\n\t\tt.Run(test.json, f)\n\t}\n}\n\nfunc TestGet_InvalidPathType(t *testing.T) {\n\tassert.Panics(t, assert.PanicTestFunc(func() {\n\t\tdata := `{\"a\":[{\"b\":true}]}`\n\t\ts := NewSearcher(data)\n\t\ts.GetByPath(\"a\", true)\n\n\t\ts = NewSearcher(data)\n\t\ts.GetByPath(\"a\", nil)\n\n\t\ts = NewSearcher(data)\n\t\ts.GetByPath(\"a\", -1)\n\t}))\n}\n" }, { "path": "ast/error.go", "content": "package ast\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n\t\"unsafe\"\n\n\t\"github.com/bytedance/sonic/internal/native/types\"\n)\n\nfunc newError(err types.ParsingError, msg string) *Node {\n\treturn &Node{\n\t\tt: V_ERROR,\n\t\tl: uint(err),\n\t\tp: unsafe.Pointer(&msg),\n\t}\n}\n\nfunc newErrorPair(err SyntaxError) *Pair {\n\treturn &Pair{0, \"\", *newSyntaxError(err)}\n}\n\n// Error returns error message if the node is invalid\nfunc (self Node) Error() string {\n\tif self.t != V_ERROR {\n\t\treturn \"\"\n\t} else {\n\t\treturn *(*string)(self.p)\n\t}\n}\n\nfunc newSyntaxError(err SyntaxError) *Node {\n\tmsg := err.Description()\n\treturn &Node{\n\t\tt: V_ERROR,\n\t\tl: uint(err.Code),\n\t\tp: unsafe.Pointer(&msg),\n\t}\n}\n\nfunc (self *Parser) syntaxError(err types.ParsingError) SyntaxError {\n\treturn SyntaxError{\n\t\tPos: self.p,\n\t\tSrc: self.s,\n\t\tCode: err,\n\t}\n}\n\nfunc unwrapError(err error) *Node {\n\tif se, ok := err.(*Node); ok {\n\t\treturn se\n\t} else if sse, ok := err.(Node); ok {\n\t\treturn &sse\n\t} else {\n\t\tmsg := err.Error()\n\t\treturn &Node{\n\t\t\tt: V_ERROR,\n\t\t\tp: unsafe.Pointer(&msg),\n\t\t}\n\t}\n}\n\ntype SyntaxError struct {\n\tPos int\n\tSrc string\n\tCode types.ParsingError\n\tMsg string\n}\n\nfunc (self SyntaxError) Error() string {\n\treturn fmt.Sprintf(\"%q\", self.Description())\n}\n\nfunc (self SyntaxError) Description() string {\n\treturn \"Syntax error \" + self.description()\n}\n\nfunc (self SyntaxError) description() string {\n\ti := 16\n\tp := self.Pos - i\n\tq := self.Pos + i\n\n\t/* check for empty source */\n\tif self.Src == \"\" {\n\t\treturn fmt.Sprintf(\"no sources available, the input json is empty: %#v\", self)\n\t}\n\n\t/* prevent slicing before the beginning */\n\tif p < 0 {\n\t\tp, q, i = 0, q-p, i+p\n\t}\n\n\t/* prevent slicing beyond the end */\n\tif n := len(self.Src); q > n {\n\t\tn = q - n\n\t\tq = len(self.Src)\n\n\t\t/* move the left bound if possible */\n\t\tif p > n {\n\t\t\ti += n\n\t\t\tp -= n\n\t\t}\n\t}\n\n\t/* left and right length */\n\tx := clamp_zero(i)\n\ty := clamp_zero(q - p - i - 1)\n\n\t/* compose the error description */\n\treturn fmt.Sprintf(\n\t\t\"at index %d: %s\\n\\n\\t%s\\n\\t%s^%s\\n\",\n\t\tself.Pos,\n\t\tself.Message(),\n\t\tself.Src[p:q],\n\t\tstrings.Repeat(\".\", x),\n\t\tstrings.Repeat(\".\", y),\n\t)\n}\n\nfunc (self SyntaxError) Message() string {\n\tif self.Msg == \"\" {\n\t\treturn self.Code.Message()\n\t}\n\treturn self.Msg\n}\n\nfunc clamp_zero(v int) int {\n\tif v < 0 {\n\t\treturn 0\n\t} else {\n\t\treturn v\n\t}\n}\n" }, { "path": "ast/iterator.go", "content": "/*\n * Copyright 2021 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage ast\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/bytedance/sonic/internal/caching\"\n\t\"github.com/bytedance/sonic/internal/native/types\"\n)\n\ntype Pair struct {\n\thash uint64\n\tKey string\n\tValue Node\n}\n\nfunc NewPair(key string, val Node) Pair {\n\treturn Pair{\n\t\thash: caching.StrHash(key),\n\t\tKey: key,\n\t\tValue: val,\n\t}\n}\n\n// Values returns iterator for array's children traversal\nfunc (self *Node) Values() (ListIterator, error) {\n\tif err := self.should(types.V_ARRAY); err != nil {\n\t\treturn ListIterator{}, err\n\t}\n\treturn self.values(), nil\n}\n\nfunc (self *Node) values() ListIterator {\n\treturn ListIterator{Iterator{p: self}}\n}\n\n// Properties returns iterator for object's children traversal\nfunc (self *Node) Properties() (ObjectIterator, error) {\n\tif err := self.should(types.V_OBJECT); err != nil {\n\t\treturn ObjectIterator{}, err\n\t}\n\treturn self.properties(), nil\n}\n\nfunc (self *Node) properties() ObjectIterator {\n\treturn ObjectIterator{Iterator{p: self}}\n}\n\ntype Iterator struct {\n\ti int\n\tp *Node\n}\n\nfunc (self *Iterator) Pos() int {\n\treturn self.i\n}\n\nfunc (self *Iterator) Len() int {\n\treturn self.p.len()\n}\n\n// HasNext reports if it is the end of iteration or has error.\nfunc (self *Iterator) HasNext() bool {\n\tif !self.p.isLazy() {\n\t\treturn self.p.Valid() && self.i < self.p.len()\n\t} else if self.p.t == _V_ARRAY_LAZY {\n\t\treturn self.p.skipNextNode().Valid()\n\t} else if self.p.t == _V_OBJECT_LAZY {\n\t\tpair := self.p.skipNextPair()\n\t\tif pair == nil {\n\t\t\treturn false\n\t\t}\n\t\treturn pair.Value.Valid()\n\t}\n\treturn false\n}\n\n// ListIterator is specialized iterator for V_ARRAY\ntype ListIterator struct {\n\tIterator\n}\n\n// ObjectIterator is specialized iterator for V_ARRAY\ntype ObjectIterator struct {\n\tIterator\n}\n\nfunc (self *ListIterator) next() *Node {\nnext_start:\n\tif !self.HasNext() {\n\t\treturn nil\n\t} else {\n\t\tn := self.p.nodeAt(self.i)\n\t\tself.i++\n\t\tif !n.Exists() {\n\t\t\tgoto next_start\n\t\t}\n\t\treturn n\n\t}\n}\n\n// Next scans through children of underlying V_ARRAY,\n// copies each child to v, and returns .HasNext().\nfunc (self *ListIterator) Next(v *Node) bool {\n\tn := self.next()\n\tif n == nil {\n\t\treturn false\n\t}\n\t*v = *n\n\treturn true\n}\n\nfunc (self *ObjectIterator) next() *Pair {\nnext_start:\n\tif !self.HasNext() {\n\t\treturn nil\n\t} else {\n\t\tn := self.p.pairAt(self.i)\n\t\tself.i++\n\t\tif n == nil || !n.Value.Exists() {\n\t\t\tgoto next_start\n\t\t}\n\t\treturn n\n\t}\n}\n\n// Next scans through children of underlying V_OBJECT,\n// copies each child to v, and returns .HasNext().\nfunc (self *ObjectIterator) Next(p *Pair) bool {\n\tn := self.next()\n\tif n == nil {\n\t\treturn false\n\t}\n\t*p = *n\n\treturn true\n}\n\n// Sequence represents scanning path of single-layer nodes.\n// Index indicates the value's order in both V_ARRAY and V_OBJECT json.\n// Key is the value's key (for V_OBJECT json only, otherwise it will be nil).\ntype Sequence struct {\n\tIndex int\n\tKey *string\n\t// Level int\n}\n\n// String is string representation of one Sequence\nfunc (s Sequence) String() string {\n\tk := \"\"\n\tif s.Key != nil {\n\t\tk = *s.Key\n\t}\n\treturn fmt.Sprintf(\"Sequence(%d, %q)\", s.Index, k)\n}\n\ntype Scanner func(path Sequence, node *Node) bool\n\n// ForEach scans one V_OBJECT node's children from JSON head to tail,\n// and pass the Sequence and Node of corresponding JSON value.\n//\n// Especially, if the node is not V_ARRAY or V_OBJECT,\n// the node itself will be returned and Sequence.Index == -1.\n//\n// NOTICE: An unset node WON'T trigger sc, but its index still counts into Path.Index\nfunc (self *Node) ForEach(sc Scanner) error {\n\tif err := self.checkRaw(); err != nil {\n\t\treturn err\n\t}\n\tswitch self.itype() {\n\tcase types.V_ARRAY:\n\t\titer, err := self.Values()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tv := iter.next()\n\t\tfor v != nil {\n\t\t\tif !sc(Sequence{iter.i - 1, nil}, v) {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tv = iter.next()\n\t\t}\n\tcase types.V_OBJECT:\n\t\titer, err := self.Properties()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tv := iter.next()\n\t\tfor v != nil {\n\t\t\tif !sc(Sequence{iter.i - 1, &v.Key}, &v.Value) {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\tv = iter.next()\n\t\t}\n\tdefault:\n\t\tif self.Check() != nil {\n\t\t\treturn self\n\t\t}\n\t\tsc(Sequence{-1, nil}, self)\n\t}\n\treturn nil\n}\n" }, { "path": "ast/iterator_test.go", "content": "/*\n * Copyright 2021 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage ast\n\nimport (\n\t\"fmt\"\n\t\"strconv\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc getTestIteratorSample(loop int) (string, int) {\n\tvar data []int\n\tvar v1 = \"\"\n\tvar v2 = \"\"\n\tfor i := 0; i < loop; i++ {\n\t\tdata = append(data, i*i)\n\t\tv1 += strconv.Itoa(i)\n\t\tv2 += `\"k` + strconv.Itoa(i) + `\":` + strconv.Itoa(i)\n\t\tif i != loop-1 {\n\t\t\tv1 += `,`\n\t\t\tv2 += `,`\n\t\t}\n\t}\n\treturn `{\"array\":[` + v1 + `], \"object\":{` + v2 + `}}`, loop\n}\n\nfunc TestForEach(t *testing.T) {\n\tpathes := []Sequence{}\n\tvalues := []*Node{}\n\tsc := func(path Sequence, node *Node) bool {\n\t\tpathes = append(pathes, path)\n\t\tvalues = append(values, node)\n\t\tif path.Key != nil && *path.Key == \"array\" {\n\t\t\tnode.ForEach(func(path Sequence, node *Node) bool {\n\t\t\t\tpathes = append(pathes, path)\n\t\t\t\tvalues = append(values, node)\n\t\t\t\treturn true\n\t\t\t})\n\t\t}\n\t\treturn true\n\t}\n\n\tstr, _ := getTestIteratorSample(3)\n\tfmt.Println(str)\n\troot, err := NewSearcher(str).GetByPath()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = root.ForEach(sc)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\teObjKey := \"object\"\n\teArrKey := \"array\"\n\texpPath := []Sequence{\n\t\t{0, &eArrKey},\n\t\t{0, nil},\n\t\t{1, nil},\n\t\t{2, nil},\n\t\t{1, &eObjKey},\n\t}\n\texpValue := []*Node{\n\t\troot.Get(\"array\"),\n\t\troot.GetByPath(\"array\", 0),\n\t\troot.GetByPath(\"array\", 1),\n\t\troot.GetByPath(\"array\", 2),\n\t\troot.Get(\"object\"),\n\t}\n\t// fmt.Printf(\"pathes:%+v\\n\", pathes)\n\t// fmt.Printf(\"values:%+v\\n\", values)\n\tassert.Equal(t, expPath, pathes)\n\tassert.Equal(t, expValue, values)\n\n}\n\nfunc TestRawIterator(t *testing.T) {\n\tstr, loop := getTestIteratorSample(_DEFAULT_NODE_CAP)\n\tfmt.Println(str)\n\n\troot, err := NewSearcher(str).GetByPath(\"array\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tai, _ := root.Values()\n\ti := int64(0)\n\tfor ai.HasNext() {\n\t\tv := &Node{}\n\t\tif !ai.Next(v) {\n\t\t\tt.Fatalf(\"no next\")\n\t\t}\n\t\tx, _ := v.Int64()\n\t\tif i < int64(loop) && x != i {\n\t\t\tt.Fatalf(\"exp:%v, got:%v\", i, v)\n\t\t}\n\t\tif i != int64(ai.Pos())-1 || i >= int64(ai.Len()) {\n\t\t\tt.Fatal(i)\n\t\t}\n\t\ti++\n\t}\n\tif i != int64(loop) {\n\t\tt.Fatal(i)\n\t}\n\n\troot, err = NewSearcher(str).GetByPath(\"object\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tmi, _ := root.Properties()\n\ti = int64(0)\n\tfor mi.HasNext() {\n\t\tv := &Pair{}\n\t\tif !mi.Next(v) {\n\t\t\tt.Fatalf(\"no next\")\n\t\t}\n\t\tx, _ := v.Value.Int64()\n\t\tif i < int64(loop) && (x != i || v.Key != fmt.Sprintf(\"k%d\", i)) {\n\t\t\tvv, _ := v.Value.Interface()\n\t\t\tt.Fatalf(\"exp:%v, got:%v\", i, vv)\n\t\t}\n\t\tif i != int64(mi.Pos())-1 || i >= int64(mi.Len()) {\n\t\t\tt.Fatal(i)\n\t\t}\n\t\ti++\n\t}\n\tif i != int64(loop) {\n\t\tt.Fatal(i)\n\t}\n}\n\nfunc TestIterator(t *testing.T) {\n\tstr, loop := getTestIteratorSample(_DEFAULT_NODE_CAP)\n\tfmt.Println(str)\n\n\troot, err := NewParser(str).Parse()\n\tif err != 0 {\n\t\tt.Fatal(err)\n\t}\n\tai, _ := root.Get(\"array\").Values()\n\ti := int64(0)\n\tfor ai.HasNext() {\n\t\tv := &Node{}\n\t\tif !ai.Next(v) {\n\t\t\tt.Fatalf(\"no next\")\n\t\t}\n\t\tx, _ := v.Int64()\n\t\tif i < int64(loop) && x != i {\n\t\t\tt.Fatalf(\"exp:%v, got:%v\", i, v)\n\t\t}\n\t\tif i != int64(ai.Pos())-1 || i >= int64(ai.Len()) {\n\t\t\tt.Fatal(i)\n\t\t}\n\t\ti++\n\t}\n\tif i != int64(loop) {\n\t\tt.Fatal(i)\n\t}\n\n\troot, err = NewParser(str).Parse()\n\tif err != 0 {\n\t\tt.Fatal(err)\n\t}\n\tmi, _ := root.Get(\"object\").Properties()\n\ti = int64(0)\n\tfor mi.HasNext() {\n\t\tv := &Pair{}\n\t\tif !mi.Next(v) {\n\t\t\tt.Fatalf(\"no next\")\n\t\t}\n\t\tx, _ := v.Value.Int64()\n\t\tif i < int64(loop) && (x != i || v.Key != fmt.Sprintf(\"k%d\", i)) {\n\t\t\tvv, _ := v.Value.Interface()\n\t\t\tt.Fatalf(\"exp:%v, got:%v\", i, vv)\n\t\t}\n\t\tif i != int64(mi.Pos())-1 || i >= int64(mi.Len()) {\n\t\t\tt.Fatal(i)\n\t\t}\n\t\ti++\n\t}\n\tif i != int64(loop) {\n\t\tt.Fatal(i)\n\t}\n\n\tstr, _ = getTestIteratorSample(0)\n\troot, err = NewParser(str).Parse()\n\tif err != 0 {\n\t\tt.Fatal(err)\n\t}\n\tmi, _ = root.Get(\"object\").Properties()\n\tif mi.HasNext() {\n\t\tt.Fatalf(\"should not have next\")\n\t}\n}\n\nfunc TestExist(t *testing.T) {\n\tn := NewRaw(`null`)\n\tif !n.Exists() {\n\t\tt.Fatal()\n\t}\n\tnn := n.Get(\"xx\")\n\tif nn.Exists() {\n\t\tt.Fatal()\n\t}\n\n\troot := NewRaw(`{\"a\":1, \"b\":[1,2], \"c\":{\"1\":1, \"2\":2}}`)\n\tif !root.Exists() {\n\t\tt.Fatal()\n\t}\n\texi, err := root.Unset(\"a\")\n\tif !exi || err != nil {\n\t\tt.Fatal(exi, err)\n\t}\n\n\troot.ForEach(func(path Sequence, node *Node) bool {\n\t\tif path.Key != nil && *path.Key == \"a\" {\n\t\t\tt.Fatal()\n\t\t}\n\t\tif path.Index == 0 {\n\t\t\tif *path.Key != \"b\" {\n\t\t\t\tt.Fatal()\n\t\t\t}\n\t\t\texi, err := node.UnsetByIndex(1)\n\t\t\tif !exi || err != nil {\n\t\t\t\tt.Fatal(exi, err)\n\t\t\t}\n\t\t\tnode.ForEach(func(path Sequence, node *Node) bool {\n\t\t\t\tif path.Index == 1 {\n\t\t\t\t\tt.Fatal()\n\t\t\t\t}\n\t\t\t\treturn true\n\t\t\t})\n\t\t}\n\n\t\tif path.Index == 1 {\n\t\t\tif *path.Key != \"c\" {\n\t\t\t\tt.Fatal()\n\t\t\t}\n\t\t\texi, err := node.UnsetByIndex(1)\n\t\t\tif !exi || err != nil {\n\t\t\t\tt.Fatal(exi, err)\n\t\t\t}\n\t\t\tnode.ForEach(func(path Sequence, node *Node) bool {\n\t\t\t\tif path.Index == 1 {\n\t\t\t\t\tt.Fatal()\n\t\t\t\t}\n\t\t\t\treturn true\n\t\t\t})\n\t\t}\n\t\treturn true\n\t})\n\n\tout, err := root.Raw()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\trequire.Equal(t, `{\"b\":[1],\"c\":{\"1\":1}}`, out)\n\n}\n\nfunc BenchmarkArrays(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\troot, err := NewSearcher(_TwitterJson).GetByPath(\"statuses\", 1, \"entities\", \"hashtags\")\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t\ta, _ := root.Array()\n\t\tfor _, v := range a {\n\t\t\t_ = v\n\t\t}\n\t}\n}\n\nfunc BenchmarkListIterator(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\troot, err := NewSearcher(_TwitterJson).GetByPath(\"statuses\", 1, \"entities\", \"hashtags\")\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t\tit, _ := root.Values()\n\t\tfor it.HasNext() {\n\t\t\tv := &Node{}\n\t\t\tif !it.Next(v) {\n\t\t\t\tb.Fatalf(\"no value\")\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc BenchmarkMap(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\troot, err := NewSearcher(_TwitterJson).GetByPath(\"statuses\", 1, \"user\")\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t\tm, _ := root.Map()\n\t\tfor k, v := range m {\n\t\t\t_ = v\n\t\t\t_ = k\n\t\t}\n\t}\n}\n\nfunc BenchmarkObjectIterator(b *testing.B) {\n\tfor i := 0; i < b.N; i++ {\n\t\troot, err := NewSearcher(_TwitterJson).GetByPath(\"statuses\", 1, \"user\")\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t\tit, _ := root.Properties()\n\t\tfor it.HasNext() {\n\t\t\tv := &Pair{}\n\t\t\tif !it.Next(v) {\n\t\t\t\tb.Fatalf(\"no value\")\n\t\t\t}\n\t\t}\n\t}\n}\n" }, { "path": "ast/node.go", "content": "/*\n * Copyright 2021 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage ast\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"strconv\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"unsafe\"\n\n\t\"github.com/bytedance/sonic/internal/native/types\"\n\t\"github.com/bytedance/sonic/internal/rt\"\n)\n\nconst (\n\t_V_NONE types.ValueType = 0\n\t_V_NODE_BASE types.ValueType = 1 << 5\n\t_V_LAZY types.ValueType = 1 << 7\n\t_V_RAW types.ValueType = 1 << 8\n\t_V_NUMBER = _V_NODE_BASE + 1\n\t_V_ANY = _V_NODE_BASE + 2\n\t_V_ARRAY_LAZY = _V_LAZY | types.V_ARRAY\n\t_V_OBJECT_LAZY = _V_LAZY | types.V_OBJECT\n\t_MASK_LAZY = _V_LAZY - 1\n\t_MASK_RAW = _V_RAW - 1\n)\n\nconst (\n\tV_NONE = 0\n\tV_ERROR = 1\n\tV_NULL = int(types.V_NULL)\n\tV_TRUE = int(types.V_TRUE)\n\tV_FALSE = int(types.V_FALSE)\n\tV_ARRAY = int(types.V_ARRAY)\n\tV_OBJECT = int(types.V_OBJECT)\n\tV_STRING = int(types.V_STRING)\n\tV_NUMBER = int(_V_NUMBER)\n\tV_ANY = int(_V_ANY)\n)\n\ntype Node struct {\n\tt types.ValueType\n\tl uint\n\tp unsafe.Pointer\n\tm *sync.RWMutex\n}\n\n// UnmarshalJSON is just an adapter to json.Unmarshaler.\n// If you want better performance, use Searcher.GetByPath() directly\nfunc (self *Node) UnmarshalJSON(data []byte) (err error) {\n\t*self = newRawNode(rt.Mem2Str(data), switchRawType(data[0]), false)\n\treturn nil\n}\n\n/** Node Type Accessor **/\n\n// Type returns json type represented by the node\n// It will be one of bellows:\n//\n//\tV_NONE = 0 (empty node, key not exists)\n//\tV_ERROR = 1 (error node)\n//\tV_NULL = 2 (json value `null`, key exists)\n//\tV_TRUE = 3 (json value `true`)\n//\tV_FALSE = 4 (json value `false`)\n//\tV_ARRAY = 5 (json value array)\n//\tV_OBJECT = 6 (json value object)\n//\tV_STRING = 7 (json value string)\n//\tV_NUMBER = 33 (json value number )\n//\tV_ANY = 34 (golang interface{})\n//\n// Deprecated: not concurrent safe. Use TypeSafe instead\nfunc (self Node) Type() int {\n\treturn int(self.t & _MASK_LAZY & _MASK_RAW)\n}\n\n// Type concurrently-safe returns json type represented by the node\n// It will be one of bellows:\n//\n//\tV_NONE = 0 (empty node, key not exists)\n//\tV_ERROR = 1 (error node)\n//\tV_NULL = 2 (json value `null`, key exists)\n//\tV_TRUE = 3 (json value `true`)\n//\tV_FALSE = 4 (json value `false`)\n//\tV_ARRAY = 5 (json value array)\n//\tV_OBJECT = 6 (json value object)\n//\tV_STRING = 7 (json value string)\n//\tV_NUMBER = 33 (json value number )\n//\tV_ANY = 34 (golang interface{})\nfunc (self *Node) TypeSafe() int {\n\treturn int(self.loadt() & _MASK_LAZY & _MASK_RAW)\n}\n\nfunc (self *Node) itype() types.ValueType {\n\treturn self.t & _MASK_LAZY & _MASK_RAW\n}\n\n// Exists returns false only if the self is nil or empty node V_NONE\nfunc (self *Node) Exists() bool {\n\tif self == nil {\n\t\treturn false\n\t}\n\tt := self.loadt()\n\treturn t != V_ERROR && t != _V_NONE\n}\n\n// Valid reports if self is NOT V_ERROR or nil\nfunc (self *Node) Valid() bool {\n\tif self == nil {\n\t\treturn false\n\t}\n\treturn self.loadt() != V_ERROR\n}\n\n// Check checks if the node itself is valid, and return:\n// - ErrNotExist If the node is nil\n// - Its underlying error If the node is V_ERROR\nfunc (self *Node) Check() error {\n\tif self == nil {\n\t\treturn ErrNotExist\n\t} else if self.loadt() != V_ERROR {\n\t\treturn nil\n\t} else {\n\t\treturn self\n\t}\n}\n\nfunc (self *Node) checkFast() error {\n\tif self == nil {\n\t\treturn ErrNotExist\n\t} else if self.t != V_ERROR {\n\t\treturn nil\n\t} else {\n\t\treturn self\n\t}\n}\n\n// isRaw returns true if node's underlying value is raw json\n//\n// Deprecated: not concurrent safe\nfunc (self Node) IsRaw() bool {\n\treturn self.t&_V_RAW != 0\n}\n\n// IsRaw returns true if node's underlying value is raw json\nfunc (self *Node) isRaw() bool {\n\treturn self.loadt()&_V_RAW != 0\n}\n\nfunc (self *Node) isLazy() bool {\n\treturn self != nil && self.t&_V_LAZY != 0\n}\n\nfunc (self *Node) isAny() bool {\n\treturn self != nil && self.loadt() == _V_ANY\n}\n\n/** Simple Value Methods **/\n\n// Raw returns json representation of the node,\nfunc (self *Node) Raw() (string, error) {\n\tif self == nil {\n\t\treturn \"\", ErrNotExist\n\t}\n\tlock := self.rlock()\n\tif !self.isRaw() {\n\t\tif lock {\n\t\t\tself.runlock()\n\t\t}\n\t\tbuf, err := self.MarshalJSON()\n\t\treturn rt.Mem2Str(buf), err\n\t}\n\tret := self.toString()\n\tif lock {\n\t\tself.runlock()\n\t}\n\treturn ret, nil\n}\n\nfunc (self *Node) checkRaw() error {\n\tif self == nil {\n\t\treturn ErrNotExist\n\t}\n\n\tt := self.loadt()\n\tif t == V_ERROR {\n\t\treturn self\n\t}\n\n\tif t&_V_RAW != 0 {\n\t\tself.parseRaw(false)\n\t}\n\n\treturn self.checkFast()\n}\n\n// Bool returns bool value represented by this node,\n// including types.V_TRUE|V_FALSE|V_NUMBER|V_STRING|V_ANY|V_NULL,\n// V_NONE will return error\nfunc (self *Node) Bool() (bool, error) {\n\tif err := self.checkRaw(); err != nil {\n\t\treturn false, err\n\t}\n\tswitch self.t {\n\tcase types.V_TRUE:\n\t\treturn true, nil\n\tcase types.V_FALSE:\n\t\treturn false, nil\n\tcase types.V_NULL:\n\t\treturn false, nil\n\tcase _V_NUMBER:\n\t\tif i, err := self.toInt64(); err == nil {\n\t\t\treturn i != 0, nil\n\t\t} else if f, err := self.toFloat64(); err == nil {\n\t\t\treturn f != 0, nil\n\t\t} else {\n\t\t\treturn false, err\n\t\t}\n\tcase types.V_STRING:\n\t\treturn strconv.ParseBool(self.toString())\n\tcase _V_ANY:\n\t\tany := self.packAny()\n\t\tswitch v := any.(type) {\n\t\tcase bool:\n\t\t\treturn v, nil\n\t\tcase int:\n\t\t\treturn v != 0, nil\n\t\tcase int8:\n\t\t\treturn v != 0, nil\n\t\tcase int16:\n\t\t\treturn v != 0, nil\n\t\tcase int32:\n\t\t\treturn v != 0, nil\n\t\tcase int64:\n\t\t\treturn v != 0, nil\n\t\tcase uint:\n\t\t\treturn v != 0, nil\n\t\tcase uint8:\n\t\t\treturn v != 0, nil\n\t\tcase uint16:\n\t\t\treturn v != 0, nil\n\t\tcase uint32:\n\t\t\treturn v != 0, nil\n\t\tcase uint64:\n\t\t\treturn v != 0, nil\n\t\tcase float32:\n\t\t\treturn v != 0, nil\n\t\tcase float64:\n\t\t\treturn v != 0, nil\n\t\tcase string:\n\t\t\treturn strconv.ParseBool(v)\n\t\tcase json.Number:\n\t\t\tif i, err := v.Int64(); err == nil {\n\t\t\t\treturn i != 0, nil\n\t\t\t} else if f, err := v.Float64(); err == nil {\n\t\t\t\treturn f != 0, nil\n\t\t\t} else {\n\t\t\t\treturn false, err\n\t\t\t}\n\t\tdefault:\n\t\t\treturn false, ErrUnsupportType\n\t\t}\n\tdefault:\n\t\treturn false, ErrUnsupportType\n\t}\n}\n\n// Int64 casts the node to int64 value,\n// including V_NUMBER|V_TRUE|V_FALSE|V_ANY|V_STRING\n// V_NONE it will return error\nfunc (self *Node) Int64() (int64, error) {\n\tif err := self.checkRaw(); err != nil {\n\t\treturn 0, err\n\t}\n\tswitch self.t {\n\tcase _V_NUMBER, types.V_STRING:\n\t\tif i, err := self.toInt64(); err == nil {\n\t\t\treturn i, nil\n\t\t} else if f, err := self.toFloat64(); err == nil {\n\t\t\treturn int64(f), nil\n\t\t} else {\n\t\t\treturn 0, err\n\t\t}\n\tcase types.V_TRUE:\n\t\treturn 1, nil\n\tcase types.V_FALSE:\n\t\treturn 0, nil\n\tcase types.V_NULL:\n\t\treturn 0, nil\n\tcase _V_ANY:\n\t\tany := self.packAny()\n\t\tswitch v := any.(type) {\n\t\tcase bool:\n\t\t\tif v {\n\t\t\t\treturn 1, nil\n\t\t\t} else {\n\t\t\t\treturn 0, nil\n\t\t\t}\n\t\tcase int:\n\t\t\treturn int64(v), nil\n\t\tcase int8:\n\t\t\treturn int64(v), nil\n\t\tcase int16:\n\t\t\treturn int64(v), nil\n\t\tcase int32:\n\t\t\treturn int64(v), nil\n\t\tcase int64:\n\t\t\treturn int64(v), nil\n\t\tcase uint:\n\t\t\treturn int64(v), nil\n\t\tcase uint8:\n\t\t\treturn int64(v), nil\n\t\tcase uint16:\n\t\t\treturn int64(v), nil\n\t\tcase uint32:\n\t\t\treturn int64(v), nil\n\t\tcase uint64:\n\t\t\treturn int64(v), nil\n\t\tcase float32:\n\t\t\treturn int64(v), nil\n\t\tcase float64:\n\t\t\treturn int64(v), nil\n\t\tcase string:\n\t\t\tif i, err := strconv.ParseInt(v, 10, 64); err == nil {\n\t\t\t\treturn i, nil\n\t\t\t} else if f, err := strconv.ParseFloat(v, 64); err == nil {\n\t\t\t\treturn int64(f), nil\n\t\t\t} else {\n\t\t\t\treturn 0, err\n\t\t\t}\n\t\tcase json.Number:\n\t\t\tif i, err := v.Int64(); err == nil {\n\t\t\t\treturn i, nil\n\t\t\t} else if f, err := v.Float64(); err == nil {\n\t\t\t\treturn int64(f), nil\n\t\t\t} else {\n\t\t\t\treturn 0, err\n\t\t\t}\n\t\tdefault:\n\t\t\treturn 0, ErrUnsupportType\n\t\t}\n\tdefault:\n\t\treturn 0, ErrUnsupportType\n\t}\n}\n\n// StrictInt64 exports underlying int64 value, including V_NUMBER, V_ANY\nfunc (self *Node) StrictInt64() (int64, error) {\n\tif err := self.checkRaw(); err != nil {\n\t\treturn 0, err\n\t}\n\tswitch self.t {\n\tcase _V_NUMBER:\n\t\treturn self.toInt64()\n\tcase _V_ANY:\n\t\tany := self.packAny()\n\t\tswitch v := any.(type) {\n\t\tcase int:\n\t\t\treturn int64(v), nil\n\t\tcase int8:\n\t\t\treturn int64(v), nil\n\t\tcase int16:\n\t\t\treturn int64(v), nil\n\t\tcase int32:\n\t\t\treturn int64(v), nil\n\t\tcase int64:\n\t\t\treturn int64(v), nil\n\t\tcase uint:\n\t\t\treturn int64(v), nil\n\t\tcase uint8:\n\t\t\treturn int64(v), nil\n\t\tcase uint16:\n\t\t\treturn int64(v), nil\n\t\tcase uint32:\n\t\t\treturn int64(v), nil\n\t\tcase uint64:\n\t\t\treturn int64(v), nil\n\t\tcase json.Number:\n\t\t\tif i, err := v.Int64(); err == nil {\n\t\t\t\treturn i, nil\n\t\t\t} else {\n\t\t\t\treturn 0, err\n\t\t\t}\n\t\tdefault:\n\t\t\treturn 0, ErrUnsupportType\n\t\t}\n\tdefault:\n\t\treturn 0, ErrUnsupportType\n\t}\n}\n\nfunc castNumber(v bool) json.Number {\n\tif v {\n\t\treturn json.Number(\"1\")\n\t} else {\n\t\treturn json.Number(\"0\")\n\t}\n}\n\n// Number casts node to float64,\n// including V_NUMBER|V_TRUE|V_FALSE|V_ANY|V_STRING|V_NULL,\n// V_NONE it will return error\nfunc (self *Node) Number() (json.Number, error) {\n\tif err := self.checkRaw(); err != nil {\n\t\treturn json.Number(\"\"), err\n\t}\n\tswitch self.t {\n\tcase _V_NUMBER:\n\t\treturn self.toNumber(), nil\n\tcase types.V_STRING:\n\t\tif _, err := self.toInt64(); err == nil {\n\t\t\treturn self.toNumber(), nil\n\t\t} else if _, err := self.toFloat64(); err == nil {\n\t\t\treturn self.toNumber(), nil\n\t\t} else {\n\t\t\treturn json.Number(\"\"), err\n\t\t}\n\tcase types.V_TRUE:\n\t\treturn json.Number(\"1\"), nil\n\tcase types.V_FALSE:\n\t\treturn json.Number(\"0\"), nil\n\tcase types.V_NULL:\n\t\treturn json.Number(\"0\"), nil\n\tcase _V_ANY:\n\t\tany := self.packAny()\n\t\tswitch v := any.(type) {\n\t\tcase bool:\n\t\t\treturn castNumber(v), nil\n\t\tcase int:\n\t\t\treturn castNumber(v != 0), nil\n\t\tcase int8:\n\t\t\treturn castNumber(v != 0), nil\n\t\tcase int16:\n\t\t\treturn castNumber(v != 0), nil\n\t\tcase int32:\n\t\t\treturn castNumber(v != 0), nil\n\t\tcase int64:\n\t\t\treturn castNumber(v != 0), nil\n\t\tcase uint:\n\t\t\treturn castNumber(v != 0), nil\n\t\tcase uint8:\n\t\t\treturn castNumber(v != 0), nil\n\t\tcase uint16:\n\t\t\treturn castNumber(v != 0), nil\n\t\tcase uint32:\n\t\t\treturn castNumber(v != 0), nil\n\t\tcase uint64:\n\t\t\treturn castNumber(v != 0), nil\n\t\tcase float32:\n\t\t\treturn castNumber(v != 0), nil\n\t\tcase float64:\n\t\t\treturn castNumber(v != 0), nil\n\t\tcase string:\n\t\t\tif _, err := strconv.ParseFloat(v, 64); err == nil {\n\t\t\t\treturn json.Number(v), nil\n\t\t\t} else {\n\t\t\t\treturn json.Number(\"\"), err\n\t\t\t}\n\t\tcase json.Number:\n\t\t\treturn v, nil\n\t\tdefault:\n\t\t\treturn json.Number(\"\"), ErrUnsupportType\n\t\t}\n\tdefault:\n\t\treturn json.Number(\"\"), ErrUnsupportType\n\t}\n}\n\n// Number exports underlying float64 value, including V_NUMBER, V_ANY of json.Number\nfunc (self *Node) StrictNumber() (json.Number, error) {\n\tif err := self.checkRaw(); err != nil {\n\t\treturn json.Number(\"\"), err\n\t}\n\tswitch self.t {\n\tcase _V_NUMBER:\n\t\treturn self.toNumber(), nil\n\tcase _V_ANY:\n\t\tif v, ok := self.packAny().(json.Number); ok {\n\t\t\treturn v, nil\n\t\t} else {\n\t\t\treturn json.Number(\"\"), ErrUnsupportType\n\t\t}\n\tdefault:\n\t\treturn json.Number(\"\"), ErrUnsupportType\n\t}\n}\n\n// String cast node to string,\n// including V_NUMBER|V_TRUE|V_FALSE|V_ANY|V_STRING|V_NULL,\n// V_NONE it will return error\nfunc (self *Node) String() (string, error) {\n\tif err := self.checkRaw(); err != nil {\n\t\treturn \"\", err\n\t}\n\tswitch self.t {\n\tcase types.V_NULL:\n\t\treturn \"\", nil\n\tcase types.V_TRUE:\n\t\treturn \"true\", nil\n\tcase types.V_FALSE:\n\t\treturn \"false\", nil\n\tcase types.V_STRING, _V_NUMBER:\n\t\treturn self.toString(), nil\n\tcase _V_ANY:\n\t\tany := self.packAny()\n\t\tswitch v := any.(type) {\n\t\tcase bool:\n\t\t\treturn strconv.FormatBool(v), nil\n\t\tcase int:\n\t\t\treturn strconv.Itoa(v), nil\n\t\tcase int8:\n\t\t\treturn strconv.Itoa(int(v)), nil\n\t\tcase int16:\n\t\t\treturn strconv.Itoa(int(v)), nil\n\t\tcase int32:\n\t\t\treturn strconv.Itoa(int(v)), nil\n\t\tcase int64:\n\t\t\treturn strconv.Itoa(int(v)), nil\n\t\tcase uint:\n\t\t\treturn strconv.Itoa(int(v)), nil\n\t\tcase uint8:\n\t\t\treturn strconv.Itoa(int(v)), nil\n\t\tcase uint16:\n\t\t\treturn strconv.Itoa(int(v)), nil\n\t\tcase uint32:\n\t\t\treturn strconv.Itoa(int(v)), nil\n\t\tcase uint64:\n\t\t\treturn strconv.Itoa(int(v)), nil\n\t\tcase float32:\n\t\t\treturn strconv.FormatFloat(float64(v), 'g', -1, 64), nil\n\t\tcase float64:\n\t\t\treturn strconv.FormatFloat(float64(v), 'g', -1, 64), nil\n\t\tcase string:\n\t\t\treturn v, nil\n\t\tcase json.Number:\n\t\t\treturn v.String(), nil\n\t\tdefault:\n\t\t\treturn \"\", ErrUnsupportType\n\t\t}\n\tdefault:\n\t\treturn \"\", ErrUnsupportType\n\t}\n}\n\n// StrictString returns string value (unescaped), including V_STRING, V_ANY of string.\n// In other cases, it will return empty string.\nfunc (self *Node) StrictString() (string, error) {\n\tif err := self.checkRaw(); err != nil {\n\t\treturn \"\", err\n\t}\n\tswitch self.t {\n\tcase types.V_STRING:\n\t\treturn self.toString(), nil\n\tcase _V_ANY:\n\t\tif v, ok := self.packAny().(string); ok {\n\t\t\treturn v, nil\n\t\t} else {\n\t\t\treturn \"\", ErrUnsupportType\n\t\t}\n\tdefault:\n\t\treturn \"\", ErrUnsupportType\n\t}\n}\n\n// Float64 cast node to float64,\n// including V_NUMBER|V_TRUE|V_FALSE|V_ANY|V_STRING|V_NULL,\n// V_NONE it will return error\nfunc (self *Node) Float64() (float64, error) {\n\tif err := self.checkRaw(); err != nil {\n\t\treturn 0.0, err\n\t}\n\tswitch self.t {\n\tcase _V_NUMBER, types.V_STRING:\n\t\treturn self.toFloat64()\n\tcase types.V_TRUE:\n\t\treturn 1.0, nil\n\tcase types.V_FALSE:\n\t\treturn 0.0, nil\n\tcase types.V_NULL:\n\t\treturn 0.0, nil\n\tcase _V_ANY:\n\t\tany := self.packAny()\n\t\tswitch v := any.(type) {\n\t\tcase bool:\n\t\t\tif v {\n\t\t\t\treturn 1.0, nil\n\t\t\t} else {\n\t\t\t\treturn 0.0, nil\n\t\t\t}\n\t\tcase int:\n\t\t\treturn float64(v), nil\n\t\tcase int8:\n\t\t\treturn float64(v), nil\n\t\tcase int16:\n\t\t\treturn float64(v), nil\n\t\tcase int32:\n\t\t\treturn float64(v), nil\n\t\tcase int64:\n\t\t\treturn float64(v), nil\n\t\tcase uint:\n\t\t\treturn float64(v), nil\n\t\tcase uint8:\n\t\t\treturn float64(v), nil\n\t\tcase uint16:\n\t\t\treturn float64(v), nil\n\t\tcase uint32:\n\t\t\treturn float64(v), nil\n\t\tcase uint64:\n\t\t\treturn float64(v), nil\n\t\tcase float32:\n\t\t\treturn float64(v), nil\n\t\tcase float64:\n\t\t\treturn float64(v), nil\n\t\tcase string:\n\t\t\tif f, err := strconv.ParseFloat(v, 64); err == nil {\n\t\t\t\treturn float64(f), nil\n\t\t\t} else {\n\t\t\t\treturn 0, err\n\t\t\t}\n\t\tcase json.Number:\n\t\t\tif f, err := v.Float64(); err == nil {\n\t\t\t\treturn float64(f), nil\n\t\t\t} else {\n\t\t\t\treturn 0, err\n\t\t\t}\n\t\tdefault:\n\t\t\treturn 0, ErrUnsupportType\n\t\t}\n\tdefault:\n\t\treturn 0.0, ErrUnsupportType\n\t}\n}\n\nfunc (self *Node) StrictBool() (bool, error) {\n\tif err := self.checkRaw(); err != nil {\n\t\treturn false, err\n\t}\n\tswitch self.t {\n\tcase types.V_TRUE:\n\t\treturn true, nil\n\tcase types.V_FALSE:\n\t\treturn false, nil\n\tcase _V_ANY:\n\t\tany := self.packAny()\n\t\tswitch v := any.(type) {\n\t\tcase bool:\n\t\t\treturn v, nil\n\t\tdefault:\n\t\t\treturn false, ErrUnsupportType\n\t\t}\n\tdefault:\n\t\treturn false, ErrUnsupportType\n\t}\n}\n\n// Float64 exports underlying float64 value, including V_NUMBER, V_ANY\nfunc (self *Node) StrictFloat64() (float64, error) {\n\tif err := self.checkRaw(); err != nil {\n\t\treturn 0.0, err\n\t}\n\tswitch self.t {\n\tcase _V_NUMBER:\n\t\treturn self.toFloat64()\n\tcase _V_ANY:\n\t\tany := self.packAny()\n\t\tswitch v := any.(type) {\n\t\tcase float32:\n\t\t\treturn float64(v), nil\n\t\tcase float64:\n\t\t\treturn float64(v), nil\n\t\tdefault:\n\t\t\treturn 0, ErrUnsupportType\n\t\t}\n\tdefault:\n\t\treturn 0.0, ErrUnsupportType\n\t}\n}\n\n/** Sequential Value Methods **/\n\n// Len returns children count of a array|object|string node\n// WARN: For partially loaded node, it also works but only counts the parsed children\nfunc (self *Node) Len() (int, error) {\n\tif err := self.checkRaw(); err != nil {\n\t\treturn 0, err\n\t}\n\tif self.t == types.V_ARRAY || self.t == types.V_OBJECT || self.t == _V_ARRAY_LAZY || self.t == _V_OBJECT_LAZY || self.t == types.V_STRING {\n\t\treturn int(self.l), nil\n\t} else if self.t == _V_NONE || self.t == types.V_NULL {\n\t\treturn 0, nil\n\t} else {\n\t\treturn 0, ErrUnsupportType\n\t}\n}\n\nfunc (self *Node) len() int {\n\treturn int(self.l)\n}\n\n// Cap returns malloc capacity of a array|object node for children\nfunc (self *Node) Cap() (int, error) {\n\tif err := self.checkRaw(); err != nil {\n\t\treturn 0, err\n\t}\n\tswitch self.t {\n\tcase types.V_ARRAY:\n\t\treturn (*linkedNodes)(self.p).Cap(), nil\n\tcase types.V_OBJECT:\n\t\treturn (*linkedPairs)(self.p).Cap(), nil\n\tcase _V_ARRAY_LAZY:\n\t\treturn (*parseArrayStack)(self.p).v.Cap(), nil\n\tcase _V_OBJECT_LAZY:\n\t\treturn (*parseObjectStack)(self.p).v.Cap(), nil\n\tcase _V_NONE, types.V_NULL:\n\t\treturn 0, nil\n\tdefault:\n\t\treturn 0, ErrUnsupportType\n\t}\n}\n\n// Set sets the node of given key under self, and reports if the key has existed.\n//\n// If self is V_NONE or V_NULL, it becomes V_OBJECT and sets the node at the key.\nfunc (self *Node) Set(key string, node Node) (bool, error) {\n\tif err := self.checkRaw(); err != nil {\n\t\treturn false, err\n\t}\n\n\t// check the node, not use Check() to avoid unescape the node parameter\n\tif node.t == V_ERROR {\n\t\treturn false, node\n\t}\n\n\tif self.t == _V_NONE || self.t == types.V_NULL {\n\t\t*self = NewObject([]Pair{NewPair(key, node)})\n\t\treturn false, nil\n\t} else if self.itype() != types.V_OBJECT {\n\t\treturn false, ErrUnsupportType\n\t}\n\n\tp := self.Get(key)\n\n\tif !p.Exists() {\n\t\t// self must be fully-loaded here\n\t\tif self.len() == 0 {\n\t\t\t*self = newObject(new(linkedPairs))\n\t\t}\n\t\ts := (*linkedPairs)(self.p)\n\t\ts.Push(NewPair(key, node))\n\t\tself.l++\n\t\treturn false, nil\n\n\t} else if err := p.Check(); err != nil {\n\t\treturn false, err\n\t}\n\n\t*p = node\n\treturn true, nil\n}\n\n// SetAny wraps val with V_ANY node, and Set() the node.\nfunc (self *Node) SetAny(key string, val interface{}) (bool, error) {\n\treturn self.Set(key, NewAny(val))\n}\n\n// Unset REMOVE (soft) the node of given key under object parent, and reports if the key has existed.\nfunc (self *Node) Unset(key string) (bool, error) {\n\tif err := self.should(types.V_OBJECT); err != nil {\n\t\treturn false, err\n\t}\n\t// NOTICE: must get accurate length before deduct\n\tif err := self.skipAllKey(); err != nil {\n\t\treturn false, err\n\t}\n\tp, i := self.skipKey(key)\n\tif !p.Exists() {\n\t\treturn false, nil\n\t} else if err := p.Check(); err != nil {\n\t\treturn false, err\n\t}\n\tself.removePairAt(i)\n\treturn true, nil\n}\n\n// SetByIndex sets the node of given index, and reports if the key has existed.\n//\n// The index must be within self's children.\nfunc (self *Node) SetByIndex(index int, node Node) (bool, error) {\n\tif err := self.checkRaw(); err != nil {\n\t\treturn false, err\n\t}\n\tif err := node.Check(); err != nil {\n\t\treturn false, err\n\t}\n\n\tif index == 0 && (self.t == _V_NONE || self.t == types.V_NULL) {\n\t\t*self = NewArray([]Node{node})\n\t\treturn false, nil\n\t}\n\n\tp := self.Index(index)\n\tif !p.Exists() {\n\t\treturn false, ErrNotExist\n\t} else if err := p.Check(); err != nil {\n\t\treturn false, err\n\t}\n\n\t*p = node\n\treturn true, nil\n}\n\n// SetAny wraps val with V_ANY node, and SetByIndex() the node.\nfunc (self *Node) SetAnyByIndex(index int, val interface{}) (bool, error) {\n\treturn self.SetByIndex(index, NewAny(val))\n}\n\n// UnsetByIndex REMOVE (softly) the node of given index.\n//\n// WARN: this will change address of elements, which is a dangerous action.\n// Use Unset() for object or Pop() for array instead.\nfunc (self *Node) UnsetByIndex(index int) (bool, error) {\n\tif err := self.checkRaw(); err != nil {\n\t\treturn false, err\n\t}\n\n\tvar p *Node\n\tit := self.itype()\n\n\tif it == types.V_ARRAY {\n\t\tif err := self.skipAllIndex(); err != nil {\n\t\t\treturn false, err\n\t\t}\n\t\tp = self.nodeAt(index)\n\t} else if it == types.V_OBJECT {\n\t\tif err := self.skipAllKey(); err != nil {\n\t\t\treturn false, err\n\t\t}\n\t\tpr := self.pairAt(index)\n\t\tif pr == nil {\n\t\t\treturn false, ErrNotExist\n\t\t}\n\t\tp = &pr.Value\n\t} else {\n\t\treturn false, ErrUnsupportType\n\t}\n\n\tif !p.Exists() {\n\t\treturn false, ErrNotExist\n\t}\n\n\t// last elem\n\tif index == self.len()-1 {\n\t\treturn true, self.Pop()\n\t}\n\n\t// not last elem, self.len() change but linked-chunk not change\n\tif it == types.V_ARRAY {\n\t\tself.removeNode(index)\n\t} else if it == types.V_OBJECT {\n\t\tself.removePair(index)\n\t}\n\treturn true, nil\n}\n\n// Add appends the given node under self.\n//\n// If self is V_NONE or V_NULL, it becomes V_ARRAY and sets the node at index 0.\nfunc (self *Node) Add(node Node) error {\n\tif err := self.checkRaw(); err != nil {\n\t\treturn err\n\t}\n\n\tif self != nil && (self.t == _V_NONE || self.t == types.V_NULL) {\n\t\t*self = NewArray([]Node{node})\n\t\treturn nil\n\t}\n\tif err := self.should(types.V_ARRAY); err != nil {\n\t\treturn err\n\t}\n\n\ts, err := self.unsafeArray()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Notice: array won't have unset node in tail\n\ts.Push(node)\n\tself.l++\n\treturn nil\n}\n\n// Pop remove the last child of the V_Array or V_Object node.\nfunc (self *Node) Pop() error {\n\tif err := self.checkRaw(); err != nil {\n\t\treturn err\n\t}\n\n\tif it := self.itype(); it == types.V_ARRAY {\n\t\ts, err := self.unsafeArray()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// remove tail unset nodes\n\t\tfor i := s.Len() - 1; i >= 0; i-- {\n\t\t\tif s.At(i).Exists() {\n\t\t\t\ts.Pop()\n\t\t\t\tself.l--\n\t\t\t\tbreak\n\t\t\t}\n\t\t\ts.Pop()\n\t\t}\n\n\t} else if it == types.V_OBJECT {\n\t\ts, err := self.unsafeMap()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// remove tail unset nodes\n\t\tfor i := s.Len() - 1; i >= 0; i-- {\n\t\t\tif p := s.At(i); p != nil && p.Value.Exists() {\n\t\t\t\ts.Pop()\n\t\t\t\tself.l--\n\t\t\t\tbreak\n\t\t\t}\n\t\t\ts.Pop()\n\t\t}\n\n\t} else {\n\t\treturn ErrUnsupportType\n\t}\n\n\treturn nil\n}\n\n// Move moves the child at src index to dst index,\n// meanwhile slides siblings from src+1 to dst.\n//\n// WARN: this will change address of elements, which is a dangerous action.\nfunc (self *Node) Move(dst, src int) error {\n\tif err := self.should(types.V_ARRAY); err != nil {\n\t\treturn err\n\t}\n\n\ts, err := self.unsafeArray()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// check if any unset node exists\n\tif l := s.Len(); self.len() != l {\n\t\tdi, si := dst, src\n\t\t// find real pos of src and dst\n\t\tfor i := 0; i < l; i++ {\n\t\t\tif s.At(i).Exists() {\n\t\t\t\tdi--\n\t\t\t\tsi--\n\t\t\t}\n\t\t\tif di == -1 {\n\t\t\t\tdst = i\n\t\t\t\tdi--\n\t\t\t}\n\t\t\tif si == -1 {\n\t\t\t\tsrc = i\n\t\t\t\tsi--\n\t\t\t}\n\t\t\tif di == -2 && si == -2 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\n\ts.MoveOne(src, dst)\n\treturn nil\n}\n\n// AddAny wraps val with V_ANY node, and Add() the node.\nfunc (self *Node) AddAny(val interface{}) error {\n\treturn self.Add(NewAny(val))\n}\n\n// GetByPath load given path on demands,\n// which only ensure nodes before this path got parsed.\n//\n// Note, the api expects the json is well-formed at least,\n// otherwise it may return unexpected result.\nfunc (self *Node) GetByPath(path ...interface{}) *Node {\n\tif !self.Valid() {\n\t\treturn self\n\t}\n\tvar s = self\n\tfor _, p := range path {\n\t\tswitch p := p.(type) {\n\t\tcase int:\n\t\t\ts = s.Index(p)\n\t\t\tif !s.Valid() {\n\t\t\t\treturn s\n\t\t\t}\n\t\tcase string:\n\t\t\ts = s.Get(p)\n\t\t\tif !s.Valid() {\n\t\t\t\treturn s\n\t\t\t}\n\t\tdefault:\n\t\t\tpanic(\"path must be either int or string\")\n\t\t}\n\t}\n\treturn s\n}\n\n// Get loads given key of an object node on demands\nfunc (self *Node) Get(key string) *Node {\n\tif err := self.should(types.V_OBJECT); err != nil {\n\t\treturn unwrapError(err)\n\t}\n\tn, _ := self.skipKey(key)\n\treturn n\n}\n\n// Index indexies node at given idx,\n// node type CAN be either V_OBJECT or V_ARRAY\nfunc (self *Node) Index(idx int) *Node {\n\tif err := self.checkRaw(); err != nil {\n\t\treturn unwrapError(err)\n\t}\n\n\tit := self.itype()\n\tif it == types.V_ARRAY {\n\t\treturn self.skipIndex(idx)\n\n\t} else if it == types.V_OBJECT {\n\t\tpr := self.skipIndexPair(idx)\n\t\tif pr == nil {\n\t\t\treturn newError(_ERR_NOT_FOUND, \"value not exists\")\n\t\t}\n\t\treturn &pr.Value\n\n\t} else {\n\t\treturn newError(_ERR_UNSUPPORT_TYPE, fmt.Sprintf(\"unsupported type: %v\", self.itype()))\n\t}\n}\n\n// IndexPair indexies pair at given idx,\n// node type MUST be either V_OBJECT\nfunc (self *Node) IndexPair(idx int) *Pair {\n\tif err := self.should(types.V_OBJECT); err != nil {\n\t\treturn nil\n\t}\n\treturn self.skipIndexPair(idx)\n}\n\nfunc (self *Node) indexOrGet(idx int, key string) (*Node, int) {\n\tif err := self.should(types.V_OBJECT); err != nil {\n\t\treturn unwrapError(err), idx\n\t}\n\n\tpr := self.skipIndexPair(idx)\n\tif pr != nil && pr.Key == key {\n\t\treturn &pr.Value, idx\n\t}\n\n\treturn self.skipKey(key)\n}\n\n// IndexOrGet firstly use idx to index a value and check if its key matches\n// If not, then use the key to search value\nfunc (self *Node) IndexOrGet(idx int, key string) *Node {\n\tnode, _ := self.indexOrGet(idx, key)\n\treturn node\n}\n\n// IndexOrGetWithIdx attempts to retrieve a node by index and key, returning the node and its correct index.\n// If the key does not match at the given index, it searches by key and returns the node with its updated index.\nfunc (self *Node) IndexOrGetWithIdx(idx int, key string) (*Node, int) {\n\treturn self.indexOrGet(idx, key)\n}\n\n/** Generic Value Converters **/\n\n// Map loads all keys of an object node\nfunc (self *Node) Map() (map[string]interface{}, error) {\n\tif self.isAny() {\n\t\tany := self.packAny()\n\t\tif v, ok := any.(map[string]interface{}); ok {\n\t\t\treturn v, nil\n\t\t} else {\n\t\t\treturn nil, ErrUnsupportType\n\t\t}\n\t}\n\tif err := self.should(types.V_OBJECT); err != nil {\n\t\treturn nil, err\n\t}\n\tif err := self.loadAllKey(false); err != nil {\n\t\treturn nil, err\n\t}\n\treturn self.toGenericObject()\n}\n\n// MapUseNumber loads all keys of an object node, with numeric nodes cast to json.Number\nfunc (self *Node) MapUseNumber() (map[string]interface{}, error) {\n\tif self.isAny() {\n\t\tany := self.packAny()\n\t\tif v, ok := any.(map[string]interface{}); ok {\n\t\t\treturn v, nil\n\t\t} else {\n\t\t\treturn nil, ErrUnsupportType\n\t\t}\n\t}\n\tif err := self.should(types.V_OBJECT); err != nil {\n\t\treturn nil, err\n\t}\n\tif err := self.loadAllKey(false); err != nil {\n\t\treturn nil, err\n\t}\n\treturn self.toGenericObjectUseNumber()\n}\n\n// MapUseNode scans both parsed and non-parsed children nodes,\n// and map them by their keys\nfunc (self *Node) MapUseNode() (map[string]Node, error) {\n\tif self.isAny() {\n\t\tany := self.packAny()\n\t\tif v, ok := any.(map[string]Node); ok {\n\t\t\treturn v, nil\n\t\t} else {\n\t\t\treturn nil, ErrUnsupportType\n\t\t}\n\t}\n\tif err := self.should(types.V_OBJECT); err != nil {\n\t\treturn nil, err\n\t}\n\tif err := self.skipAllKey(); err != nil {\n\t\treturn nil, err\n\t}\n\treturn self.toGenericObjectUseNode()\n}\n\n// MapUnsafe exports the underlying pointer to its children map\n// WARN: don't use it unless you know what you are doing\n//\n// Deprecated: this API now returns copied nodes instead of directly reference,\n// func (self *Node) UnsafeMap() ([]Pair, error) {\n// if err := self.should(types.V_OBJECT, \"an object\"); err != nil {\n// return nil, err\n// }\n// if err := self.skipAllKey(); err != nil {\n// return nil, err\n// }\n// return self.toGenericObjectUsePair()\n// }\n\n//go:nocheckptr\nfunc (self *Node) unsafeMap() (*linkedPairs, error) {\n\tif err := self.skipAllKey(); err != nil {\n\t\treturn nil, err\n\t}\n\tif self.p == nil {\n\t\t*self = newObject(new(linkedPairs))\n\t}\n\treturn (*linkedPairs)(self.p), nil\n}\n\n// SortKeys sorts children of a V_OBJECT node in ascending key-order.\n// If recurse is true, it recursively sorts children's children as long as a V_OBJECT node is found.\nfunc (self *Node) SortKeys(recurse bool) error {\n\t// check raw node first\n\tif err := self.checkRaw(); err != nil {\n\t\treturn err\n\t}\n\tif self.itype() == types.V_OBJECT {\n\t\treturn self.sortKeys(recurse)\n\t} else if self.itype() == types.V_ARRAY {\n\t\tvar err error\n\t\terr2 := self.ForEach(func(path Sequence, node *Node) bool {\n\t\t\tit := node.itype()\n\t\t\tif it == types.V_ARRAY || it == types.V_OBJECT {\n\t\t\t\terr = node.SortKeys(recurse)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\treturn err2\n\t} else {\n\t\treturn nil\n\t}\n}\n\nfunc (self *Node) sortKeys(recurse bool) (err error) {\n\t// check raw node first\n\tif err := self.checkRaw(); err != nil {\n\t\treturn err\n\t}\n\tps, err := self.unsafeMap()\n\tif err != nil {\n\t\treturn err\n\t}\n\tps.Sort()\n\tif recurse {\n\t\tvar sc Scanner\n\t\tsc = func(path Sequence, node *Node) bool {\n\t\t\tif node.itype() == types.V_OBJECT {\n\t\t\t\tif err := node.sortKeys(recurse); err != nil {\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t}\n\t\t\tif node.itype() == types.V_ARRAY {\n\t\t\t\tif err := node.ForEach(sc); err != nil {\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn true\n\t\t}\n\t\tif err := self.ForEach(sc); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// Array loads all indexes of an array node\nfunc (self *Node) Array() ([]interface{}, error) {\n\tif self.isAny() {\n\t\tany := self.packAny()\n\t\tif v, ok := any.([]interface{}); ok {\n\t\t\treturn v, nil\n\t\t} else {\n\t\t\treturn nil, ErrUnsupportType\n\t\t}\n\t}\n\tif err := self.should(types.V_ARRAY); err != nil {\n\t\treturn nil, err\n\t}\n\tif err := self.loadAllIndex(false); err != nil {\n\t\treturn nil, err\n\t}\n\treturn self.toGenericArray()\n}\n\n// ArrayUseNumber loads all indexes of an array node, with numeric nodes cast to json.Number\nfunc (self *Node) ArrayUseNumber() ([]interface{}, error) {\n\tif self.isAny() {\n\t\tany := self.packAny()\n\t\tif v, ok := any.([]interface{}); ok {\n\t\t\treturn v, nil\n\t\t} else {\n\t\t\treturn nil, ErrUnsupportType\n\t\t}\n\t}\n\tif err := self.should(types.V_ARRAY); err != nil {\n\t\treturn nil, err\n\t}\n\tif err := self.loadAllIndex(false); err != nil {\n\t\treturn nil, err\n\t}\n\treturn self.toGenericArrayUseNumber()\n}\n\n// ArrayUseNode copies both parsed and non-parsed children nodes,\n// and indexes them by original order\nfunc (self *Node) ArrayUseNode() ([]Node, error) {\n\tif self.isAny() {\n\t\tany := self.packAny()\n\t\tif v, ok := any.([]Node); ok {\n\t\t\treturn v, nil\n\t\t} else {\n\t\t\treturn nil, ErrUnsupportType\n\t\t}\n\t}\n\tif err := self.should(types.V_ARRAY); err != nil {\n\t\treturn nil, err\n\t}\n\tif err := self.skipAllIndex(); err != nil {\n\t\treturn nil, err\n\t}\n\treturn self.toGenericArrayUseNode()\n}\n\n// ArrayUnsafe exports the underlying pointer to its children array\n// WARN: don't use it unless you know what you are doing\n//\n// Deprecated: this API now returns copied nodes instead of directly reference,\n// which has no difference with ArrayUseNode\n// func (self *Node) UnsafeArray() ([]Node, error) {\n// if err := self.should(types.V_ARRAY, \"an array\"); err != nil {\n// return nil, err\n// }\n// if err := self.skipAllIndex(); err != nil {\n// return nil, err\n// }\n// return self.toGenericArrayUseNode()\n// }\n\nfunc (self *Node) unsafeArray() (*linkedNodes, error) {\n\tif err := self.skipAllIndex(); err != nil {\n\t\treturn nil, err\n\t}\n\tif self.p == nil {\n\t\t*self = newArray(new(linkedNodes))\n\t}\n\treturn (*linkedNodes)(self.p), nil\n}\n\n// Interface loads all children under all paths from this node,\n// and converts itself as generic type.\n// WARN: all numeric nodes are cast to float64\nfunc (self *Node) Interface() (interface{}, error) {\n\tif err := self.checkRaw(); err != nil {\n\t\treturn nil, err\n\t}\n\tswitch self.t {\n\tcase V_ERROR:\n\t\treturn nil, self.Check()\n\tcase types.V_NULL:\n\t\treturn nil, nil\n\tcase types.V_TRUE:\n\t\treturn true, nil\n\tcase types.V_FALSE:\n\t\treturn false, nil\n\tcase types.V_ARRAY:\n\t\treturn self.toGenericArray()\n\tcase types.V_OBJECT:\n\t\treturn self.toGenericObject()\n\tcase types.V_STRING:\n\t\treturn self.toString(), nil\n\tcase _V_NUMBER:\n\t\tv, err := self.toFloat64()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn v, nil\n\tcase _V_ARRAY_LAZY:\n\t\tif err := self.loadAllIndex(false); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn self.toGenericArray()\n\tcase _V_OBJECT_LAZY:\n\t\tif err := self.loadAllKey(false); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn self.toGenericObject()\n\tcase _V_ANY:\n\t\tswitch v := self.packAny().(type) {\n\t\tcase Node:\n\t\t\treturn v.Interface()\n\t\tcase *Node:\n\t\t\treturn v.Interface()\n\t\tdefault:\n\t\t\treturn v, nil\n\t\t}\n\tdefault:\n\t\treturn nil, ErrUnsupportType\n\t}\n}\n\nfunc (self *Node) packAny() interface{} {\n\treturn *(*interface{})(self.p)\n}\n\n// InterfaceUseNumber works same with Interface()\n// except numeric nodes are cast to json.Number\nfunc (self *Node) InterfaceUseNumber() (interface{}, error) {\n\tif err := self.checkRaw(); err != nil {\n\t\treturn nil, err\n\t}\n\tswitch self.t {\n\tcase V_ERROR:\n\t\treturn nil, self.Check()\n\tcase types.V_NULL:\n\t\treturn nil, nil\n\tcase types.V_TRUE:\n\t\treturn true, nil\n\tcase types.V_FALSE:\n\t\treturn false, nil\n\tcase types.V_ARRAY:\n\t\treturn self.toGenericArrayUseNumber()\n\tcase types.V_OBJECT:\n\t\treturn self.toGenericObjectUseNumber()\n\tcase types.V_STRING:\n\t\treturn self.toString(), nil\n\tcase _V_NUMBER:\n\t\treturn self.toNumber(), nil\n\tcase _V_ARRAY_LAZY:\n\t\tif err := self.loadAllIndex(false); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn self.toGenericArrayUseNumber()\n\tcase _V_OBJECT_LAZY:\n\t\tif err := self.loadAllKey(false); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn self.toGenericObjectUseNumber()\n\tcase _V_ANY:\n\t\treturn self.packAny(), nil\n\tdefault:\n\t\treturn nil, ErrUnsupportType\n\t}\n}\n\n// InterfaceUseNode clone itself as a new node,\n// or its children as map[string]Node (or []Node)\nfunc (self *Node) InterfaceUseNode() (interface{}, error) {\n\tif err := self.checkRaw(); err != nil {\n\t\treturn nil, err\n\t}\n\tswitch self.t {\n\tcase types.V_ARRAY:\n\t\treturn self.toGenericArrayUseNode()\n\tcase types.V_OBJECT:\n\t\treturn self.toGenericObjectUseNode()\n\tcase _V_ARRAY_LAZY:\n\t\tif err := self.skipAllIndex(); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn self.toGenericArrayUseNode()\n\tcase _V_OBJECT_LAZY:\n\t\tif err := self.skipAllKey(); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn self.toGenericObjectUseNode()\n\tdefault:\n\t\treturn *self, self.Check()\n\t}\n}\n\n// LoadAll loads the node's children\n// and ensure all its children can be READ concurrently (include its children's children)\nfunc (self *Node) LoadAll() error {\n\treturn self.Load()\n}\n\n// Load loads the node's children as parsed.\n// and ensure all its children can be READ concurrently (include its children's children)\nfunc (self *Node) Load() error {\n\tswitch self.t {\n\tcase _V_ARRAY_LAZY:\n\t\tself.loadAllIndex(true)\n\tcase _V_OBJECT_LAZY:\n\t\tself.loadAllKey(true)\n\tcase V_ERROR:\n\t\treturn self\n\tcase V_NONE:\n\t\treturn nil\n\t}\n\tif self.m == nil {\n\t\tself.m = new(sync.RWMutex)\n\t}\n\treturn self.checkRaw()\n}\n\n/**---------------------------------- Internal Helper Methods ----------------------------------**/\n\nfunc (self *Node) should(t types.ValueType) error {\n\tif err := self.checkRaw(); err != nil {\n\t\treturn err\n\t}\n\tif self.itype() != t {\n\t\treturn ErrUnsupportType\n\t}\n\treturn nil\n}\n\nfunc (self *Node) nodeAt(i int) *Node {\n\tvar p *linkedNodes\n\tif self.isLazy() {\n\t\t_, stack := self.getParserAndArrayStack()\n\t\tp = &stack.v\n\t} else {\n\t\tp = (*linkedNodes)(self.p)\n\t\tif l := p.Len(); l != self.len() {\n\t\t\t// some nodes got unset, iterate to skip them\n\t\t\tfor j := 0; j < l; j++ {\n\t\t\t\tv := p.At(j)\n\t\t\t\tif v.Exists() {\n\t\t\t\t\ti--\n\t\t\t\t}\n\t\t\t\tif i < 0 {\n\t\t\t\t\treturn v\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\t}\n\treturn p.At(i)\n}\n\nfunc (self *Node) pairAt(i int) *Pair {\n\tvar p *linkedPairs\n\tif self.isLazy() {\n\t\t_, stack := self.getParserAndObjectStack()\n\t\tp = &stack.v\n\t} else {\n\t\tp = (*linkedPairs)(self.p)\n\t\tif l := p.Len(); l != self.len() {\n\t\t\t// some nodes got unset, iterate to skip them\n\t\t\tfor j := 0; j < l; j++ {\n\t\t\t\tv := p.At(j)\n\t\t\t\tif v != nil && v.Value.Exists() {\n\t\t\t\t\ti--\n\t\t\t\t}\n\t\t\t\tif i < 0 {\n\t\t\t\t\treturn v\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t}\n\t}\n\treturn p.At(i)\n}\n\nfunc (self *Node) skipAllIndex() error {\n\tif !self.isLazy() {\n\t\treturn nil\n\t}\n\tvar err types.ParsingError\n\tparser, stack := self.getParserAndArrayStack()\n\tparser.skipValue = true\n\tparser.noLazy = true\n\t*self, err = parser.decodeArray(&stack.v)\n\tif err != 0 {\n\t\treturn parser.ExportError(err)\n\t}\n\treturn nil\n}\n\nfunc (self *Node) skipAllKey() error {\n\tif !self.isLazy() {\n\t\treturn nil\n\t}\n\tvar err types.ParsingError\n\tparser, stack := self.getParserAndObjectStack()\n\tparser.skipValue = true\n\tparser.noLazy = true\n\t*self, err = parser.decodeObject(&stack.v)\n\tif err != 0 {\n\t\treturn parser.ExportError(err)\n\t}\n\treturn nil\n}\n\nfunc (self *Node) skipKey(key string) (*Node, int) {\n\tnb := self.len()\n\tlazy := self.isLazy()\n\n\tif nb > 0 {\n\t\t/* linear search */\n\t\tvar p *Pair\n\t\tvar i int\n\t\tif lazy {\n\t\t\ts := (*parseObjectStack)(self.p)\n\t\t\tp, i = s.v.Get(key)\n\t\t} else {\n\t\t\tp, i = (*linkedPairs)(self.p).Get(key)\n\t\t}\n\n\t\tif p != nil {\n\t\t\treturn &p.Value, i\n\t\t}\n\t}\n\n\t/* not found */\n\tif !lazy {\n\t\treturn nil, -1\n\t}\n\n\t// lazy load\n\tfor last, i := self.skipNextPair(), nb; last != nil; last, i = self.skipNextPair(), i+1 {\n\t\tif last.Value.Check() != nil {\n\t\t\treturn &last.Value, -1\n\t\t}\n\t\tif last.Key == key {\n\t\t\treturn &last.Value, i\n\t\t}\n\t}\n\n\treturn nil, -1\n}\n\nfunc (self *Node) skipIndex(index int) *Node {\n\tnb := self.len()\n\tif nb > index {\n\t\tv := self.nodeAt(index)\n\t\treturn v\n\t}\n\tif !self.isLazy() {\n\t\treturn nil\n\t}\n\n\t// lazy load\n\tfor last := self.skipNextNode(); last != nil; last = self.skipNextNode() {\n\t\tif last.Check() != nil {\n\t\t\treturn last\n\t\t}\n\t\tif self.len() > index {\n\t\t\treturn last\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (self *Node) skipIndexPair(index int) *Pair {\n\tnb := self.len()\n\tif nb > index {\n\t\treturn self.pairAt(index)\n\t}\n\tif !self.isLazy() {\n\t\treturn nil\n\t}\n\n\t// lazy load\n\tfor last := self.skipNextPair(); last != nil; last = self.skipNextPair() {\n\t\tif last.Value.Check() != nil {\n\t\t\treturn last\n\t\t}\n\t\tif self.len() > index {\n\t\t\treturn last\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (self *Node) loadAllIndex(loadOnce bool) error {\n\tif !self.isLazy() {\n\t\treturn nil\n\t}\n\tvar err types.ParsingError\n\tparser, stack := self.getParserAndArrayStack()\n\tif !loadOnce {\n\t\tparser.noLazy = true\n\t} else {\n\t\tparser.loadOnce = true\n\t}\n\t*self, err = parser.decodeArray(&stack.v)\n\tif err != 0 {\n\t\treturn parser.ExportError(err)\n\t}\n\treturn nil\n}\n\nfunc (self *Node) loadAllKey(loadOnce bool) error {\n\tif !self.isLazy() {\n\t\treturn nil\n\t}\n\tvar err types.ParsingError\n\tparser, stack := self.getParserAndObjectStack()\n\tif !loadOnce {\n\t\tparser.noLazy = true\n\t\t*self, err = parser.decodeObject(&stack.v)\n\t} else {\n\t\tparser.loadOnce = true\n\t\t*self, err = parser.decodeObject(&stack.v)\n\t}\n\tif err != 0 {\n\t\treturn parser.ExportError(err)\n\t}\n\treturn nil\n}\n\nfunc (self *Node) removeNode(i int) {\n\tnode := self.nodeAt(i)\n\tif node == nil {\n\t\treturn\n\t}\n\t*node = Node{}\n\t// NOTICE: not be consistent with linkedNode.Len()\n\tself.l--\n}\n\nfunc (self *Node) removePair(i int) {\n\tlast := self.pairAt(i)\n\tif last == nil {\n\t\treturn\n\t}\n\t*last = Pair{}\n\t// NOTICE: should be consistent with linkedPair.Len()\n\tself.l--\n}\n\nfunc (self *Node) removePairAt(i int) {\n\tp := (*linkedPairs)(self.p).At(i)\n\tif p == nil {\n\t\treturn\n\t}\n\t*p = Pair{}\n\t// NOTICE: should be consistent with linkedPair.Len()\n\tself.l--\n}\n\nfunc (self *Node) toGenericArray() ([]interface{}, error) {\n\tnb := self.len()\n\tif nb == 0 {\n\t\treturn []interface{}{}, nil\n\t}\n\tret := make([]interface{}, 0, nb)\n\n\t/* convert each item */\n\tit := self.values()\n\tfor v := it.next(); v != nil; v = it.next() {\n\t\tvv, err := v.Interface()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tret = append(ret, vv)\n\t}\n\n\t/* all done */\n\treturn ret, nil\n}\n\nfunc (self *Node) toGenericArrayUseNumber() ([]interface{}, error) {\n\tnb := self.len()\n\tif nb == 0 {\n\t\treturn []interface{}{}, nil\n\t}\n\tret := make([]interface{}, 0, nb)\n\n\t/* convert each item */\n\tit := self.values()\n\tfor v := it.next(); v != nil; v = it.next() {\n\t\tvv, err := v.InterfaceUseNumber()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tret = append(ret, vv)\n\t}\n\n\t/* all done */\n\treturn ret, nil\n}\n\nfunc (self *Node) toGenericArrayUseNode() ([]Node, error) {\n\tvar nb = self.len()\n\tif nb == 0 {\n\t\treturn []Node{}, nil\n\t}\n\n\tvar s = (*linkedNodes)(self.p)\n\tvar out = make([]Node, nb)\n\ts.ToSlice(out)\n\n\treturn out, nil\n}\n\nfunc (self *Node) toGenericObject() (map[string]interface{}, error) {\n\tnb := self.len()\n\tif nb == 0 {\n\t\treturn map[string]interface{}{}, nil\n\t}\n\tret := make(map[string]interface{}, nb)\n\n\t/* convert each item */\n\tit := self.properties()\n\tfor v := it.next(); v != nil; v = it.next() {\n\t\tvv, err := v.Value.Interface()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tret[v.Key] = vv\n\t}\n\n\t/* all done */\n\treturn ret, nil\n}\n\nfunc (self *Node) toGenericObjectUseNumber() (map[string]interface{}, error) {\n\tnb := self.len()\n\tif nb == 0 {\n\t\treturn map[string]interface{}{}, nil\n\t}\n\tret := make(map[string]interface{}, nb)\n\n\t/* convert each item */\n\tit := self.properties()\n\tfor v := it.next(); v != nil; v = it.next() {\n\t\tvv, err := v.Value.InterfaceUseNumber()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tret[v.Key] = vv\n\t}\n\n\t/* all done */\n\treturn ret, nil\n}\n\nfunc (self *Node) toGenericObjectUseNode() (map[string]Node, error) {\n\tvar nb = self.len()\n\tif nb == 0 {\n\t\treturn map[string]Node{}, nil\n\t}\n\n\tvar s = (*linkedPairs)(self.p)\n\tvar out = make(map[string]Node, nb)\n\ts.ToMap(out)\n\n\t/* all done */\n\treturn out, nil\n}\n\n/**------------------------------------ Factory Methods ------------------------------------**/\n\nvar (\n\tnullNode = Node{t: types.V_NULL}\n\ttrueNode = Node{t: types.V_TRUE}\n\tfalseNode = Node{t: types.V_FALSE}\n)\n\n// NewRaw creates a node of raw json.\n// If the input json is invalid, NewRaw returns a error Node.\nfunc NewRaw(json string) Node {\n\tparser := NewParserObj(json)\n\tstart, err := parser.skip()\n\tif err != 0 {\n\t\treturn *newError(err, err.Message())\n\t}\n\tit := switchRawType(parser.s[start])\n\tif it == _V_NONE {\n\t\treturn Node{}\n\t}\n\treturn newRawNode(parser.s[start:parser.p], it, false)\n}\n\n// NewRawConcurrentRead creates a node of raw json, which can be READ\n// (GetByPath/Get/Index/GetOrIndex/Int64/Bool/Float64/String/Number/Interface/Array/Map/Raw/MarshalJSON) concurrently.\n// If the input json is invalid, NewRaw returns a error Node.\nfunc NewRawConcurrentRead(json string) Node {\n\tparser := NewParserObj(json)\n\tstart, err := parser.skip()\n\tif err != 0 {\n\t\treturn *newError(err, err.Message())\n\t}\n\tit := switchRawType(parser.s[start])\n\tif it == _V_NONE {\n\t\treturn Node{}\n\t}\n\treturn newRawNode(parser.s[start:parser.p], it, true)\n}\n\n// NewAny creates a node of type V_ANY if any's type isn't Node or *Node,\n// which stores interface{} and can be only used for `.Interface()`\\`.MarshalJSON()`.\nfunc NewAny(any interface{}) Node {\n\tswitch n := any.(type) {\n\tcase Node:\n\t\treturn n\n\tcase *Node:\n\t\treturn *n\n\tdefault:\n\t\treturn Node{\n\t\t\tt: _V_ANY,\n\t\t\tp: unsafe.Pointer(&any),\n\t\t}\n\t}\n}\n\n// NewBytes encodes given src with Base64 (RFC 4648), and creates a node of type V_STRING.\nfunc NewBytes(src []byte) Node {\n\tif len(src) == 0 {\n\t\tpanic(\"empty src bytes\")\n\t}\n\tout := rt.EncodeBase64ToString(src)\n\treturn NewString(out)\n}\n\n// NewNull creates a node of type V_NULL\nfunc NewNull() Node {\n\treturn Node{\n\t\tp: nil,\n\t\tt: types.V_NULL,\n\t}\n}\n\n// NewBool creates a node of type bool:\n//\n//\tIf v is true, returns V_TRUE node\n//\tIf v is false, returns V_FALSE node\nfunc NewBool(v bool) Node {\n\tvar t = types.V_FALSE\n\tif v {\n\t\tt = types.V_TRUE\n\t}\n\treturn Node{\n\t\tp: nil,\n\t\tt: t,\n\t}\n}\n\n// NewNumber creates a json.Number node\n// v must be a decimal string complying with RFC8259\nfunc NewNumber(v string) Node {\n\treturn Node{\n\t\tl: uint(len(v)),\n\t\tp: rt.StrPtr(v),\n\t\tt: _V_NUMBER,\n\t}\n}\n\nfunc (node *Node) toNumber() json.Number {\n\treturn json.Number(rt.StrFrom(node.p, int64(node.l)))\n}\n\nfunc (self *Node) toString() string {\n\treturn rt.StrFrom(self.p, int64(self.l))\n}\n\nfunc (node *Node) toFloat64() (float64, error) {\n\tret, err := node.toNumber().Float64()\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\treturn ret, nil\n}\n\nfunc (node *Node) toInt64() (int64, error) {\n\tret, err := node.toNumber().Int64()\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\treturn ret, nil\n}\n\nfunc newBytes(v []byte) Node {\n\treturn Node{\n\t\tt: types.V_STRING,\n\t\tp: mem2ptr(v),\n\t\tl: uint(len(v)),\n\t}\n}\n\n// NewString creates a node of type V_STRING.\n// v is considered to be a valid UTF-8 string,\n// which means it won't be validated and unescaped.\n// when the node is encoded to json, v will be escaped.\nfunc NewString(v string) Node {\n\treturn Node{\n\t\tt: types.V_STRING,\n\t\tp: rt.StrPtr(v),\n\t\tl: uint(len(v)),\n\t}\n}\n\n// NewArray creates a node of type V_ARRAY,\n// using v as its underlying children\nfunc NewArray(v []Node) Node {\n\ts := new(linkedNodes)\n\ts.FromSlice(v)\n\treturn newArray(s)\n}\n\nconst _Threshold_Index = 16\n\nfunc newArray(v *linkedNodes) Node {\n\treturn Node{\n\t\tt: types.V_ARRAY,\n\t\tl: uint(v.Len()),\n\t\tp: unsafe.Pointer(v),\n\t}\n}\n\nfunc (self *Node) setArray(v *linkedNodes) {\n\tself.t = types.V_ARRAY\n\tself.l = uint(v.Len())\n\tself.p = unsafe.Pointer(v)\n}\n\n// NewObject creates a node of type V_OBJECT,\n// using v as its underlying children\nfunc NewObject(v []Pair) Node {\n\ts := new(linkedPairs)\n\ts.FromSlice(v)\n\treturn newObject(s)\n}\n\nfunc newObject(v *linkedPairs) Node {\n\tif v.size > _Threshold_Index {\n\t\tv.BuildIndex()\n\t}\n\treturn Node{\n\t\tt: types.V_OBJECT,\n\t\tl: uint(v.Len()),\n\t\tp: unsafe.Pointer(v),\n\t}\n}\n\nfunc (self *Node) setObject(v *linkedPairs) {\n\tif v.size > _Threshold_Index {\n\t\tv.BuildIndex()\n\t}\n\tself.t = types.V_OBJECT\n\tself.l = uint(v.Len())\n\tself.p = unsafe.Pointer(v)\n}\n\nfunc (self *Node) parseRaw(full bool) {\n\tlock := self.lock()\n\tdefer self.unlock()\n\tif !self.isRaw() {\n\t\treturn\n\t}\n\traw := self.toString()\n\tparser := NewParserObj(raw)\n\tvar e types.ParsingError\n\tif full {\n\t\tparser.noLazy = true\n\t\t*self, e = parser.Parse()\n\t} else if lock {\n\t\tvar n Node\n\t\tparser.noLazy = true\n\t\tparser.loadOnce = true\n\t\tn, e = parser.Parse()\n\t\tself.assign(n)\n\t} else {\n\t\t*self, e = parser.Parse()\n\t}\n\tif e != 0 {\n\t\t*self = *newSyntaxError(parser.syntaxError(e))\n\t}\n}\n\nfunc (self *Node) assign(n Node) {\n\tself.l = n.l\n\tself.p = n.p\n\tatomic.StoreInt64(&self.t, n.t)\n}\n" }, { "path": "ast/node_test.go", "content": "/*\n * Copyright 2021 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage ast\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/bytedance/sonic/internal/native/types\"\n\t\"github.com/bytedance/sonic/internal/rt\"\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestNodeSortKeys(t *testing.T) {\n\tvar src = `{\"b\":1,\"a\":2,\"c\":3}`\n\troot, err := NewSearcher(src).GetByPath()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tobj, err := root.MapUseNumber()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\texp, err := json.Marshal(obj)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif err := root.SortKeys(true); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tact, err := root.MarshalJSON()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tassert.Equal(t, len(exp), len(act))\n\tassert.Equal(t, string(exp), string(act))\n\n\tsrc = `[[1], {\"b\":1,\"a\":2,\"c\":3}, [], {}, [{\"b\":1,\"a\":2,\"c\":3,\"d\":[],\"e\":{}}]]`\n\troot, err = NewSearcher(src).GetByPath()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tvv, err := root.Interface()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\texp, err = json.Marshal(vv)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif err := root.SortKeys(true); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tact, err = root.MarshalJSON()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tassert.Equal(t, string(exp), string(act))\n\n}\n\nfunc BenchmarkNodeSortKeys(b *testing.B) {\n\troot, err := NewSearcher(_TwitterJson).GetByPath()\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\t// if err := root.LoadAll(); err != nil {\n\t// b.Fatal(err)\n\t// }\n\n\tb.Run(\"single\", func(b *testing.B) {\n\t\tr := root.Get(\"statuses\")\n\t\tif r.Check() != nil {\n\t\t\tb.Fatal(r.Error())\n\t\t}\n\t\tb.SetBytes(int64(len(_TwitterJson)))\n\t\tb.ResetTimer()\n\t\tfor i := 0; i < b.N; i++ {\n\t\t\t_ = root.SortKeys(false)\n\t\t}\n\t})\n\tb.Run(\"recurse\", func(b *testing.B) {\n\t\tb.SetBytes(int64(len(_TwitterJson)))\n\t\tb.ResetTimer()\n\t\tfor i := 0; i < b.N; i++ {\n\t\t\t_ = root.SortKeys(true)\n\t\t}\n\t})\n}\n\nfunc TestNodeSortKeys2(t *testing.T) {\n\troot, err := NewSearcher(_TwitterJson).GetByPath()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\t// if err := root.LoadAll(); err != nil {\n\t// b.Fatal(err)\n\t// }\n\tt.Run(\"single\", func(t *testing.T) {\n\t\tr := root.Get(\"statuses\")\n\t\tif r.Check() != nil {\n\t\t\tt.Fatal(r.Error())\n\t\t}\n\t\trequire.NoError(t, root.SortKeys(false))\n\t})\n\tt.Run(\"recurse\", func(t *testing.T) {\n\t\trequire.NoError(t, root.SortKeys(true))\n\t})\n}\n\n//go:noinline\nfunc stackObj() interface{} {\n\tvar a int = 1\n\treturn rt.UnpackEface(a).Pack()\n}\n\nfunc TestLoadAll(t *testing.T) {\n\te := Node{}\n\terr := e.Load()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\terr = e.LoadAll()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\troot, err := NewSearcher(`{\"a\":{\"1\":[1],\"2\":2},\"b\":[{\"1\":1},2],\"c\":[1,2]}`).GetByPath()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif err = root.Load(); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif l, _ := root.Len(); l != 3 {\n\t\tt.Fatal(root.Len())\n\t}\n\n\tc := root.Get(\"c\")\n\tif !c.isRaw() {\n\t\tt.Fatal(err)\n\t}\n\terr = c.LoadAll()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif l, _ := c.Len(); l != 2 {\n\t\tt.Fatal(c.Len())\n\t}\n\tc1 := c.nodeAt(0)\n\tif n, err := c1.Int64(); err != nil || n != 1 {\n\t\tt.Fatal(n, err)\n\t}\n\n\ta := root.pairAt(0)\n\tif a.Key != \"a\" {\n\t\tt.Fatal(a.Key)\n\t} else if !a.Value.isRaw() {\n\t\tt.Fatal(a.Value.itype())\n\t} else if n, err := a.Value.Len(); n != 2 || err != nil {\n\t\tt.Fatal(n, err)\n\t}\n\tif err := a.Value.Load(); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif l, _ := a.Value.Len(); l != 2 {\n\t\tt.Fatal(a.Value.Len())\n\t}\n\ta1 := a.Value.Get(\"1\")\n\tif !a1.isRaw() {\n\t\tt.Fatal(a1)\n\t}\n\ta.Value.LoadAll()\n\tif l, _ := a1.Len(); a1.t != types.V_ARRAY || l != 1 {\n\t\tt.Fatal(a1.t)\n\t}\n\n\tb := root.pairAt(1)\n\tif b.Key != \"b\" {\n\t\tt.Fatal(b.Key)\n\t} else if !b.Value.isRaw() {\n\t\tt.Fatal(b.Value.itype())\n\t} else if n, err := b.Value.Len(); n != 2 || err != nil {\n\t\tt.Fatal(n, err)\n\t}\n\tif err := b.Value.Load(); err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif l, _ := b.Value.Len(); l != 2 {\n\t\tt.Fatal(b.Value.Len())\n\t}\n\tb1 := b.Value.Index(0)\n\tif !b1.isRaw() {\n\t\tt.Fatal(b1)\n\t}\n\tb.Value.LoadAll()\n\tif l, _ := b1.Len(); b1.t != types.V_OBJECT || l != 1 {\n\t\tt.Fatal(a1.Len())\n\t}\n}\n\nfunc TestIndexPair(t *testing.T) {\n\troot, _ := NewParser(`{\"a\":1,\"b\":2}`).Parse()\n\ta := root.IndexPair(0)\n\tif a == nil || a.Key != \"a\" {\n\t\tt.Fatal(a)\n\t}\n\tb := root.IndexPair(1)\n\tif b == nil || b.Key != \"b\" {\n\t\tt.Fatal(b)\n\t}\n\tc := root.IndexPair(2)\n\tif c != nil {\n\t\tt.Fatal(c)\n\t}\n}\n\nfunc TestIndexOrGet(t *testing.T) {\n\troot, _ := NewParser(`{\"a\":1,\"b\":2}`).Parse()\n\ta := root.IndexOrGet(0, \"a\")\n\tif v, err := a.Int64(); err != nil || v != int64(1) {\n\t\tt.Fatal(a)\n\t}\n\ta = root.IndexOrGet(0, \"b\")\n\tif v, err := a.Int64(); err != nil || v != int64(2) {\n\t\tt.Fatal(a)\n\t}\n\ta = root.IndexOrGet(0, \"c\")\n\tif a.Valid() {\n\t\tt.Fatal(a)\n\t}\n}\n\nfunc TestIndexOrGetWithIdx(t *testing.T) {\n\troot, _ := NewParser(`{\"a\":1,\"b\":2}`).Parse()\n\tb, idx := root.IndexOrGetWithIdx(0, \"b\")\n\tif v, err := b.Int64(); err != nil || v != int64(2) {\n\t\tt.Fatal(b, idx)\n\t}\n\tif idx != 1 {\n\t\tt.Fatal(b, idx)\n\t}\n}\n\nfunc TestTypeCast(t *testing.T) {\n\ttype tcase struct {\n\t\tmethod string\n\t\tnode Node\n\t\texp interface{}\n\t\terr error\n\t}\n\tvar nonEmptyErr error = errors.New(\"\")\n\ta1 := NewAny(1)\n\tlazyArray, _ := NewParser(\"[\" + strings.Repeat(\"1,\", _DEFAULT_NODE_CAP) + \"1]\").Parse()\n\tlazyObject, _ := NewParser(`{\"0\":0,\"1\":1,\"2\":2,\"3\":3,\"4\":4,\"5\":5,\"6\":6,\"7\":7,\"8\":8,\"9\":9,\"10\":10,\"11\":11,\"12\":12,\"13\":13,\"14\":14,\"15\":15,\"16\":16}`).Parse()\n\tvar cases = []tcase{\n\t\t{\"Interface\", Node{}, interface{}(nil), ErrUnsupportType},\n\t\t{\"Interface\", NewAny(NewNumber(\"1\")), float64(1), nil},\n\t\t{\"Interface\", NewAny(int64(1)), int64(1), nil},\n\t\t{\"Interface\", NewNumber(\"1\"), float64(1), nil},\n\t\t{\"InterfaceUseNode\", Node{}, Node{}, nil},\n\t\t{\"InterfaceUseNode\", a1, a1, nil},\n\t\t{\"InterfaceUseNode\", NewNumber(\"1\"), NewNumber(\"1\"), nil},\n\t\t{\"InterfaceUseNumber\", Node{}, interface{}(nil), ErrUnsupportType},\n\t\t{\"InterfaceUseNumber\", NewAny(1), 1, nil},\n\t\t{\"InterfaceUseNumber\", NewNumber(\"1\"), json.Number(\"1\"), nil},\n\t\t{\"Map\", Node{}, map[string]interface{}(nil), ErrUnsupportType},\n\t\t{\"Map\", NewAny(map[string]Node{\"a\": NewNumber(\"1\")}), map[string]interface{}(nil), ErrUnsupportType},\n\t\t{\"Map\", NewAny(map[string]interface{}{\"a\": 1}), map[string]interface{}{\"a\": 1}, nil},\n\t\t{\"Map\", NewObject([]Pair{NewPair(\"a\", NewNumber(\"1\"))}), map[string]interface{}{\"a\": float64(1.0)}, nil},\n\t\t{\"MapUseNode\", Node{}, map[string]Node(nil), ErrUnsupportType},\n\t\t{\"MapUseNode\", NewAny(map[string]interface{}{\"a\": 1}), map[string]Node(nil), ErrUnsupportType},\n\t\t{\"MapUseNode\", NewAny(map[string]Node{\"a\": NewNumber(\"1\")}), map[string]Node{\"a\": NewNumber(\"1\")}, nil},\n\t\t{\"MapUseNode\", NewObject([]Pair{NewPair(\"a\", NewNumber(\"1\"))}), map[string]Node{\"a\": NewNumber(\"1\")}, nil},\n\t\t{\"MapUseNumber\", Node{}, map[string]interface{}(nil), ErrUnsupportType},\n\t\t{\"MapUseNumber\", NewAny(map[string]interface{}{\"a\": 1}), map[string]interface{}{\"a\": 1}, nil},\n\t\t{\"MapUseNumber\", NewObject([]Pair{NewPair(\"a\", NewNumber(\"1\"))}), map[string]interface{}{\"a\": json.Number(\"1\")}, nil},\n\t\t{\"Array\", Node{}, []interface{}(nil), ErrUnsupportType},\n\t\t{\"Array\", NewAny([]interface{}{1}), []interface{}{1}, nil},\n\t\t{\"Array\", NewArray([]Node{NewNumber(\"1\")}), []interface{}{float64(1.0)}, nil},\n\t\t{\"ArrayUseNode\", Node{}, []Node(nil), ErrUnsupportType},\n\t\t{\"ArrayUseNode\", NewAny([]interface{}{1}), []Node(nil), ErrUnsupportType},\n\t\t{\"ArrayUseNode\", NewAny([]Node{NewNumber(\"1\")}), []Node{NewNumber(\"1\")}, nil},\n\t\t{\"ArrayUseNode\", NewArray([]Node{NewNumber(\"1\")}), []Node{NewNumber(\"1\")}, nil},\n\t\t{\"ArrayUseNumber\", Node{}, []interface{}(nil), ErrUnsupportType},\n\t\t{\"ArrayUseNumber\", NewAny([]interface{}{1}), []interface{}{1}, nil},\n\t\t{\"ArrayUseNumber\", NewAny([]Node{NewNumber(\"1\")}), []interface{}(nil), ErrUnsupportType},\n\t\t{\"ArrayUseNumber\", NewArray([]Node{NewNumber(\"1\")}), []interface{}{json.Number(\"1\")}, nil},\n\t\t{\"Raw\", Node{}, \"\", ErrNotExist},\n\t\t{\"Raw\", NewRaw(\" \"), \"\", nonEmptyErr},\n\t\t{\"Raw\", NewRaw(\" [ ] \"), \"[ ]\", nil},\n\t\t{\"Raw\", NewRaw(\"[ ]\"), \"[ ]\", nil},\n\t\t{\"Raw\", NewRaw(` { \"a\" : [ true, -1.2e34 ] } `), `{ \"a\" : [ true, -1.2e34 ] }`, nil},\n\t\t{\"Raw\", NewRaw(` { \"a\" : [ true, -1.2e34 ] `), \"\", nonEmptyErr},\n\t\t{\"Raw\", NewRaw(` { \"a\" : [ true, -1.2e34 }`), \"\", nonEmptyErr},\n\t\t{\"Raw\", NewBool(true), \"true\", nil},\n\t\t{\"Raw\", NewNumber(\"-0.0\"), \"-0.0\", nil},\n\t\t{\"Raw\", NewString(\"\"), `\"\"`, nil},\n\t\t{\"Raw\", NewBytes([]byte(\"hello, world\")), `\"aGVsbG8sIHdvcmxk\"`, nil},\n\t\t{\"Bool\", Node{}, false, ErrUnsupportType},\n\t\t{\"Bool\", NewAny(true), true, nil},\n\t\t{\"Bool\", NewAny(false), false, nil},\n\t\t{\"Bool\", NewAny(int(0)), false, nil},\n\t\t{\"Bool\", NewAny(int8(1)), true, nil},\n\t\t{\"Bool\", NewAny(int16(1)), true, nil},\n\t\t{\"Bool\", NewAny(int32(1)), true, nil},\n\t\t{\"Bool\", NewAny(int64(1)), true, nil},\n\t\t{\"Bool\", NewAny(uint(1)), true, nil},\n\t\t{\"Bool\", NewAny(uint16(1)), true, nil},\n\t\t{\"Bool\", NewAny(uint32(1)), true, nil},\n\t\t{\"Bool\", NewAny(uint64(1)), true, nil},\n\t\t{\"Bool\", NewAny(float64(0)), false, nil},\n\t\t{\"Bool\", NewAny(float32(1)), true, nil},\n\t\t{\"Bool\", NewAny(float64(1)), true, nil},\n\t\t{\"Bool\", NewAny(json.Number(\"0\")), false, nil},\n\t\t{\"Bool\", NewAny(json.Number(\"1\")), true, nil},\n\t\t{\"Bool\", NewAny(json.Number(\"1.1\")), true, nil},\n\t\t{\"Bool\", NewAny(json.Number(\"+x1.1\")), false, nonEmptyErr},\n\t\t{\"Bool\", NewAny(string(\"0\")), false, nil},\n\t\t{\"Bool\", NewAny(string(\"t\")), true, nil},\n\t\t{\"Bool\", NewAny([]byte{0}), false, nonEmptyErr},\n\t\t{\"Bool\", NewRaw(\"true\"), true, nil},\n\t\t{\"Bool\", NewRaw(\"false\"), false, nil},\n\t\t{\"Bool\", NewRaw(\"null\"), false, nil},\n\t\t{\"Bool\", NewString(`true`), true, nil},\n\t\t{\"Bool\", NewString(`false`), false, nil},\n\t\t{\"Bool\", NewString(``), false, nonEmptyErr},\n\t\t{\"Bool\", NewNumber(\"2\"), true, nil},\n\t\t{\"Bool\", NewNumber(\"-2.1\"), true, nil},\n\t\t{\"Bool\", NewNumber(\"-x-2.1\"), false, nonEmptyErr},\n\t\t{\"StrictBool\", NewBool(false), false, nil},\n\t\t{\"StrictBool\", NewBool(true), true, nil},\n\t\t{\"StrictBool\", NewRaw(`null`), false, ErrUnsupportType},\n\t\t{\"StrictBool\", NewAny(true), true, nil},\n\t\t{\"StrictBool\", NewAny(false), false, nil},\n\t\t{\"StrictBool\", NewAny(int(1)), false, ErrUnsupportType},\n\t\t{\"Int64\", NewRaw(\"true\"), int64(1), nil},\n\t\t{\"Int64\", NewRaw(\"false\"), int64(0), nil},\n\t\t{\"Int64\", NewRaw(\"\\\"1\\\"\"), int64(1), nil},\n\t\t{\"Int64\", NewRaw(\"\\\"1.1\\\"\"), int64(1), nil},\n\t\t{\"Int64\", NewRaw(\"\\\"1.0\\\"\"), int64(1), nil},\n\t\t{\"Int64\", NewNumber(\"+x.0\"), int64(0), nonEmptyErr},\n\t\t{\"Int64\", NewAny(false), int64(0), nil},\n\t\t{\"Int64\", NewAny(true), int64(1), nil},\n\t\t{\"Int64\", NewAny(int(1)), int64(1), nil},\n\t\t{\"Int64\", NewAny(int8(1)), int64(1), nil},\n\t\t{\"Int64\", NewAny(int16(1)), int64(1), nil},\n\t\t{\"Int64\", NewAny(int32(1)), int64(1), nil},\n\t\t{\"Int64\", NewAny(int64(1)), int64(1), nil},\n\t\t{\"Int64\", NewAny(uint(1)), int64(1), nil},\n\t\t{\"Int64\", NewAny(uint8(1)), int64(1), nil},\n\t\t{\"Int64\", NewAny(uint32(1)), int64(1), nil},\n\t\t{\"Int64\", NewAny(uint64(1)), int64(1), nil},\n\t\t{\"Int64\", NewAny(float32(1)), int64(1), nil},\n\t\t{\"Int64\", NewAny(float64(1)), int64(1), nil},\n\t\t{\"Int64\", NewAny(\"1\"), int64(1), nil},\n\t\t{\"Int64\", NewAny(\"1.1\"), int64(1), nil},\n\t\t{\"Int64\", NewAny(\"+1x.1\"), int64(0), nonEmptyErr},\n\t\t{\"Int64\", NewAny(json.Number(\"1\")), int64(1), nil},\n\t\t{\"Int64\", NewAny(json.Number(\"1.1\")), int64(1), nil},\n\t\t{\"Int64\", NewAny(json.Number(\"+1x.1\")), int64(0), nonEmptyErr},\n\t\t{\"Int64\", NewAny([]byte{0}), int64(0), ErrUnsupportType},\n\t\t{\"Int64\", Node{}, int64(0), ErrUnsupportType},\n\t\t{\"Int64\", NewRaw(\"0\"), int64(0), nil},\n\t\t{\"Int64\", NewRaw(\"null\"), int64(0), nil},\n\t\t{\"StrictInt64\", NewRaw(\"true\"), int64(0), ErrUnsupportType},\n\t\t{\"StrictInt64\", NewRaw(\"false\"), int64(0), ErrUnsupportType},\n\t\t{\"StrictInt64\", NewAny(int(0)), int64(0), nil},\n\t\t{\"StrictInt64\", NewAny(int8(0)), int64(0), nil},\n\t\t{\"StrictInt64\", NewAny(int16(0)), int64(0), nil},\n\t\t{\"StrictInt64\", NewAny(int32(0)), int64(0), nil},\n\t\t{\"StrictInt64\", NewAny(int64(0)), int64(0), nil},\n\t\t{\"StrictInt64\", NewAny(uint(0)), int64(0), nil},\n\t\t{\"StrictInt64\", NewAny(uint8(0)), int64(0), nil},\n\t\t{\"StrictInt64\", NewAny(uint32(0)), int64(0), nil},\n\t\t{\"StrictInt64\", NewAny(uint64(0)), int64(0), nil},\n\t\t{\"StrictInt64\", Node{}, int64(0), ErrUnsupportType},\n\t\t{\"StrictInt64\", NewRaw(\"0\"), int64(0), nil},\n\t\t{\"StrictInt64\", NewRaw(\"null\"), int64(0), ErrUnsupportType},\n\t\t{\"Float64\", NewRaw(\"true\"), float64(1), nil},\n\t\t{\"Float64\", NewRaw(\"false\"), float64(0), nil},\n\t\t{\"Float64\", NewRaw(\"\\\"1.0\\\"\"), float64(1.0), nil},\n\t\t{\"Float64\", NewRaw(\"\\\"xx\\\"\"), float64(0), nonEmptyErr},\n\t\t{\"Float64\", Node{}, float64(0), ErrUnsupportType},\n\t\t{\"Float64\", NewAny(false), float64(0), nil},\n\t\t{\"Float64\", NewAny(true), float64(1), nil},\n\t\t{\"Float64\", NewAny(int(1)), float64(1), nil},\n\t\t{\"Float64\", NewAny(int8(1)), float64(1), nil},\n\t\t{\"Float64\", NewAny(int16(1)), float64(1), nil},\n\t\t{\"Float64\", NewAny(int32(1)), float64(1), nil},\n\t\t{\"Float64\", NewAny(int64(1)), float64(1), nil},\n\t\t{\"Float64\", NewAny(uint(1)), float64(1), nil},\n\t\t{\"Float64\", NewAny(uint8(1)), float64(1), nil},\n\t\t{\"Float64\", NewAny(uint32(1)), float64(1), nil},\n\t\t{\"Float64\", NewAny(uint64(1)), float64(1), nil},\n\t\t{\"Float64\", NewAny(float32(1)), float64(1), nil},\n\t\t{\"Float64\", NewAny(float64(1)), float64(1), nil},\n\t\t{\"Float64\", NewAny(\"1.1\"), float64(1.1), nil},\n\t\t{\"Float64\", NewAny(\"+1x.1\"), float64(0), nonEmptyErr},\n\t\t{\"Float64\", NewAny(json.Number(\"0\")), float64(0), nil},\n\t\t{\"Float64\", NewAny(json.Number(\"x\")), float64(0), nonEmptyErr},\n\t\t{\"Float64\", NewAny([]byte{0}), float64(0), ErrUnsupportType},\n\t\t{\"Float64\", NewRaw(\"0.0\"), float64(0.0), nil},\n\t\t{\"Float64\", NewRaw(\"1\"), float64(1.0), nil},\n\t\t{\"Float64\", NewRaw(\"null\"), float64(0.0), nil},\n\t\t{\"StrictFloat64\", NewRaw(\"true\"), float64(0), ErrUnsupportType},\n\t\t{\"StrictFloat64\", NewRaw(\"false\"), float64(0), ErrUnsupportType},\n\t\t{\"StrictFloat64\", Node{}, float64(0), ErrUnsupportType},\n\t\t{\"StrictFloat64\", NewAny(float32(0)), float64(0), nil},\n\t\t{\"StrictFloat64\", NewAny(float64(0)), float64(0), nil},\n\t\t{\"StrictFloat64\", NewRaw(\"0.0\"), float64(0.0), nil},\n\t\t{\"StrictFloat64\", NewRaw(\"null\"), float64(0.0), ErrUnsupportType},\n\t\t{\"Number\", Node{}, json.Number(\"\"), ErrUnsupportType},\n\t\t{\"Number\", NewAny(false), json.Number(\"0\"), nil},\n\t\t{\"Number\", NewAny(true), json.Number(\"1\"), nil},\n\t\t{\"Number\", NewAny(int(1)), json.Number(\"1\"), nil},\n\t\t{\"Number\", NewAny(int8(1)), json.Number(\"1\"), nil},\n\t\t{\"Number\", NewAny(int16(1)), json.Number(\"1\"), nil},\n\t\t{\"Number\", NewAny(int32(1)), json.Number(\"1\"), nil},\n\t\t{\"Number\", NewAny(int64(1)), json.Number(\"1\"), nil},\n\t\t{\"Number\", NewAny(uint(1)), json.Number(\"1\"), nil},\n\t\t{\"Number\", NewAny(uint8(1)), json.Number(\"1\"), nil},\n\t\t{\"Number\", NewAny(uint32(1)), json.Number(\"1\"), nil},\n\t\t{\"Number\", NewAny(uint64(1)), json.Number(\"1\"), nil},\n\t\t{\"Number\", NewAny(float32(1)), json.Number(\"1\"), nil},\n\t\t{\"Number\", NewAny(float64(1)), json.Number(\"1\"), nil},\n\t\t{\"Number\", NewAny(\"1.1\"), json.Number(\"1.1\"), nil},\n\t\t{\"Number\", NewAny(\"+1x.1\"), json.Number(\"\"), nonEmptyErr},\n\t\t{\"Number\", NewAny(json.Number(\"0\")), json.Number(\"0\"), nil},\n\t\t{\"Number\", NewAny(json.Number(\"x\")), json.Number(\"x\"), nil},\n\t\t{\"Number\", NewAny(json.Number(\"+1x.1\")), json.Number(\"+1x.1\"), nil},\n\t\t{\"Number\", NewAny([]byte{0}), json.Number(\"\"), ErrUnsupportType},\n\t\t{\"Number\", NewRaw(\"x\"), json.Number(\"\"), nonEmptyErr},\n\t\t{\"Number\", NewRaw(\"0.0\"), json.Number(\"0.0\"), nil},\n\t\t{\"Number\", NewRaw(\"\\\"1\\\"\"), json.Number(\"1\"), nil},\n\t\t{\"Number\", NewRaw(\"\\\"1.1\\\"\"), json.Number(\"1.1\"), nil},\n\t\t{\"Number\", NewRaw(\"\\\"0.x0\\\"\"), json.Number(\"\"), nonEmptyErr},\n\t\t{\"Number\", NewRaw(\"{]\"), json.Number(\"\"), nonEmptyErr},\n\t\t{\"Number\", NewRaw(\"true\"), json.Number(\"1\"), nil},\n\t\t{\"Number\", NewRaw(\"false\"), json.Number(\"0\"), nil},\n\t\t{\"Number\", NewRaw(\"null\"), json.Number(\"0\"), nil},\n\t\t{\"StrictNumber\", NewRaw(\"true\"), json.Number(\"\"), ErrUnsupportType},\n\t\t{\"StrictNumber\", NewRaw(\"false\"), json.Number(\"\"), ErrUnsupportType},\n\t\t{\"StrictNumber\", Node{}, json.Number(\"\"), ErrUnsupportType},\n\t\t{\"StrictNumber\", NewAny(json.Number(\"0\")), json.Number(\"0\"), nil},\n\t\t{\"StrictNumber\", NewRaw(\"0.0\"), json.Number(\"0.0\"), nil},\n\t\t{\"StrictNumber\", NewRaw(\"null\"), json.Number(\"\"), ErrUnsupportType},\n\t\t{\"String\", Node{}, \"\", ErrUnsupportType},\n\t\t{\"String\", NewAny(`\\u263a`), `\\u263a`, nil},\n\t\t{\"String\", NewRaw(`\"\\u263a\"`), `☺`, nil},\n\t\t{\"String\", NewString(`\\u263a`), `\\u263a`, nil},\n\t\t{\"String\", NewRaw(`0.0`), \"0.0\", nil},\n\t\t{\"String\", NewRaw(`true`), \"true\", nil},\n\t\t{\"String\", NewRaw(`false`), \"false\", nil},\n\t\t{\"String\", NewRaw(`null`), \"\", nil},\n\t\t{\"String\", NewAny(false), \"false\", nil},\n\t\t{\"String\", NewAny(true), \"true\", nil},\n\t\t{\"String\", NewAny(int(1)), \"1\", nil},\n\t\t{\"String\", NewAny(int8(1)), \"1\", nil},\n\t\t{\"String\", NewAny(int16(1)), \"1\", nil},\n\t\t{\"String\", NewAny(int32(1)), \"1\", nil},\n\t\t{\"String\", NewAny(int64(1)), \"1\", nil},\n\t\t{\"String\", NewAny(uint(1)), \"1\", nil},\n\t\t{\"String\", NewAny(uint8(1)), \"1\", nil},\n\t\t{\"String\", NewAny(uint32(1)), \"1\", nil},\n\t\t{\"String\", NewAny(uint64(1)), \"1\", nil},\n\t\t{\"String\", NewAny(float32(1)), \"1\", nil},\n\t\t{\"String\", NewAny(float64(1)), \"1\", nil},\n\t\t{\"String\", NewAny(\"1.1\"), \"1.1\", nil},\n\t\t{\"String\", NewAny(\"+1x.1\"), \"+1x.1\", nil},\n\t\t{\"String\", NewAny(json.Number(\"0\")), (\"0\"), nil},\n\t\t{\"String\", NewAny(json.Number(\"x\")), (\"x\"), nil},\n\t\t{\"String\", NewAny([]byte{0}), (\"\"), ErrUnsupportType},\n\t\t{\"StrictString\", Node{}, \"\", ErrUnsupportType},\n\t\t{\"StrictString\", NewAny(`\\u263a`), `\\u263a`, nil},\n\t\t{\"StrictString\", NewRaw(`\"\\u263a\"`), `☺`, nil},\n\t\t{\"StrictString\", NewString(`\\u263a`), `\\u263a`, nil},\n\t\t{\"StrictString\", NewRaw(`0.0`), \"\", ErrUnsupportType},\n\t\t{\"StrictString\", NewRaw(`true`), \"\", ErrUnsupportType},\n\t\t{\"StrictString\", NewRaw(`false`), \"\", ErrUnsupportType},\n\t\t{\"StrictString\", NewRaw(`null`), \"\", ErrUnsupportType},\n\t\t{\"Len\", Node{}, 0, nil},\n\t\t{\"Len\", NewAny(0), 0, ErrUnsupportType},\n\t\t{\"Len\", NewNull(), 0, nil},\n\t\t{\"Len\", NewRaw(`\"1\"`), 1, nil},\n\t\t{\"Len\", NewRaw(`[1]`), 0, nil},\n\t\t{\"Len\", NewArray([]Node{NewNull()}), 1, nil},\n\t\t{\"Len\", lazyArray, 0, nil},\n\t\t{\"Len\", NewRaw(`{\"a\":1}`), 0, nil},\n\t\t{\"Len\", lazyObject, 0, nil},\n\t\t{\"Cap\", Node{}, 0, nil},\n\t\t{\"Cap\", NewAny(0), 0, ErrUnsupportType},\n\t\t{\"Cap\", NewNull(), 0, nil},\n\t\t{\"Cap\", NewRaw(`[1]`), _DEFAULT_NODE_CAP, nil},\n\t\t{\"Cap\", NewObject([]Pair{NewPair(\"\", NewNull())}), _DEFAULT_NODE_CAP, nil},\n\t\t{\"Cap\", NewRaw(`{\"a\":1}`), _DEFAULT_NODE_CAP, nil},\n\t}\n\tlazyArray.skipAllIndex()\n\tlazyObject.skipAllKey()\n\tcases = append(cases,\n\t\ttcase{\"Len\", lazyArray, _DEFAULT_NODE_CAP + 1, nil},\n\t\ttcase{\"Len\", lazyObject, 17, nil},\n\t\ttcase{\"Cap\", lazyArray, _DEFAULT_NODE_CAP * 2, nil},\n\t\ttcase{\"Cap\", lazyObject, _DEFAULT_NODE_CAP * 2, nil},\n\t)\n\n\tfor i, c := range cases {\n\t\tfmt.Println(i, c)\n\t\trt := reflect.ValueOf(&c.node)\n\t\tm := rt.MethodByName(c.method)\n\t\trets := m.Call([]reflect.Value{})\n\t\tif len(rets) != 2 {\n\t\t\tt.Error(i, rets)\n\t\t}\n\t\tif !reflect.DeepEqual(rets[0].Interface(), c.exp) {\n\t\t\tt.Error(i, rets[0].Interface(), c.exp)\n\t\t}\n\t\tv := rets[1].Interface()\n\t\tif c.err == nonEmptyErr {\n\t\t\tif reflect.ValueOf(v).IsNil() {\n\t\t\t\tt.Error(i, v)\n\t\t\t}\n\t\t} else if v != c.err {\n\t\t\tt.Error(i, v)\n\t\t}\n\t}\n}\n\nfunc TestCheckError_Nil(t *testing.T) {\n\tnill := (*Node)(nil)\n\tif nill.Valid() || nill.Check() == nil {\n\t\tt.Fail()\n\t}\n\tif nill.Get(\"\").Check() == nil {\n\t\tt.Fatal()\n\t}\n\tif nill.GetByPath(\"\").Check() == nil {\n\t\tt.Fatal()\n\t}\n\tif nill.Index(1).Check() == nil {\n\t\tt.Fatal()\n\t}\n\tif nill.IndexOrGet(1, \"a\").Check() == nil {\n\t\tt.Fatal()\n\t}\n\tif nill.IndexPair(1) != nil {\n\t\tt.Fatal()\n\t}\n\tif _, err := nill.Set(\"\", Node{}); err == nil {\n\t\tt.Fatal()\n\t}\n\tif _, err := nill.SetByIndex(1, Node{}); err == nil {\n\t\tt.Fatal()\n\t}\n\tif _, err := nill.SetAny(\"\", 1); err == nil {\n\t\tt.Fatal()\n\t}\n\tif _, err := nill.SetAnyByIndex(1, 1); err == nil {\n\t\tt.Fatal()\n\t}\n\tif _, err := nill.Unset(\"\"); err == nil {\n\t\tt.Fatal()\n\t}\n\tif _, err := nill.UnsetByIndex(1); err == nil {\n\t\tt.Fatal()\n\t}\n\tif err := nill.Add(Node{}); err == nil {\n\t\tt.Fatal()\n\t}\n\tif err := nill.AddAny(1); err == nil {\n\t\tt.Fatal()\n\t}\n}\n\nfunc TestCheckError_None(t *testing.T) {\n\tnill := Node{}\n\tif !nill.Valid() || nill.Check() != nil {\n\t\tt.Fail()\n\t}\n\tif nill.Get(\"\").Check() == nil {\n\t\tt.Fatal()\n\t}\n\tif nill.GetByPath(\"\").Check() == nil {\n\t\tt.Fatal()\n\t}\n\tif nill.Index(1).Check() == nil {\n\t\tt.Fatal()\n\t}\n\tif nill.IndexOrGet(1, \"a\").Check() == nil {\n\t\tt.Fatal()\n\t}\n\tif nill.IndexPair(1) != nil {\n\t\tt.Fatal()\n\t}\n\tnill = Node{}\n\tif _, err := nill.Set(\"a\", Node{}); err != nil {\n\t\tt.Fatal()\n\t}\n\tnill = Node{}\n\tif _, err := nill.SetByIndex(0, Node{}); err != nil {\n\t\tt.Fatal()\n\t}\n\tnill = Node{}\n\tif _, err := nill.SetByIndex(1, Node{}); err == nil {\n\t\tt.Fatal()\n\t}\n\tnill = Node{}\n\tif _, err := nill.SetAny(\"a\", 1); err != nil {\n\t\tt.Fatal()\n\t}\n\tnill = Node{}\n\tif _, err := nill.SetAnyByIndex(0, 1); err != nil {\n\t\tt.Fatal()\n\t}\n\tnill = Node{}\n\tif _, err := nill.SetAnyByIndex(1, 1); err == nil {\n\t\tt.Fatal()\n\t}\n\tnill = Node{}\n\tif _, err := nill.Unset(\"\"); err == nil {\n\t\tt.Fatal()\n\t}\n\tnill = Node{}\n\tif _, err := nill.UnsetByIndex(1); err == nil {\n\t\tt.Fatal()\n\t}\n\tnill = Node{}\n\tif err := nill.Add(Node{}); err != nil {\n\t\tt.Fatal()\n\t}\n\tnill = Node{}\n\tif err := nill.AddAny(1); err != nil {\n\t\tt.Fatal()\n\t}\n}\n\nfunc TestCheckError_Error(t *testing.T) {\n\tnill := newError(types.ERR_EOF, \"\")\n\tif nill.Valid() || nill.Check() == nil {\n\t\tt.Fail()\n\t}\n\tif nill.Get(\"\").Check() == nil {\n\t\tt.Fatal()\n\t}\n\tif nill.GetByPath(\"\").Check() == nil {\n\t\tt.Fatal()\n\t}\n\tif nill.Index(1).Check() == nil {\n\t\tt.Fatal()\n\t}\n\tif nill.IndexOrGet(1, \"a\").Check() == nil {\n\t\tt.Fatal()\n\t}\n\tif nill.IndexPair(1) != nil {\n\t\tt.Fatal()\n\t}\n\tif _, err := nill.Set(\"\", Node{}); err == nil {\n\t\tt.Fatal()\n\t}\n\tif _, err := nill.SetByIndex(1, Node{}); err == nil {\n\t\tt.Fatal()\n\t}\n\tif _, err := nill.SetAny(\"\", 1); err == nil {\n\t\tt.Fatal()\n\t}\n\tif _, err := nill.SetAnyByIndex(1, 1); err == nil {\n\t\tt.Fatal()\n\t}\n\tif _, err := nill.Unset(\"\"); err == nil {\n\t\tt.Fatal()\n\t}\n\tif _, err := nill.UnsetByIndex(1); err == nil {\n\t\tt.Fatal()\n\t}\n\tif err := nill.Add(Node{}); err == nil {\n\t\tt.Fatal()\n\t}\n\tif err := nill.AddAny(1); err == nil {\n\t\tt.Fatal()\n\t}\n}\n\nfunc TestCheckError_Empty(t *testing.T) {\n\tempty := Node{}\n\tif !empty.Valid() || empty.Check() != nil || empty.Error() != \"\" {\n\t\tt.Fatal()\n\t}\n\n\tn := newRawNode(\"[hello]\", types.V_ARRAY, false)\n\tn.parseRaw(false)\n\tif n.Check() != nil {\n\t\tt.Fatal(n.Check())\n\t}\n\tn = newRawNode(\"[hello]\", types.V_ARRAY, false)\n\tn.parseRaw(true)\n\tp := NewParser(\"[hello]\")\n\tp.noLazy = true\n\tp.skipValue = false\n\t_, x := p.Parse()\n\tif n.Error() != newSyntaxError(p.syntaxError(x)).Error() {\n\t\tt.Fatal(n.Check())\n\t}\n\n\ts, err := NewParser(`{\"a\":{}, \"b\":talse, \"c\":{}}`).Parse()\n\tif err != 0 {\n\t\tt.Fatal(err)\n\t}\n\n\troot := s.GetByPath()\n\t// fmt.Println(root.Check())\n\ta := root.Get(\"a\")\n\tif a.Check() != nil {\n\t\tt.Fatal(a.Check())\n\t}\n\tc := root.Get(\"c\")\n\tif c.Check() == nil {\n\t\tt.Fatal()\n\t}\n\tfmt.Println(c.Check())\n\n\t_, e := a.Properties()\n\tif e != nil {\n\t\tt.Fatal(e)\n\t}\n\texist, e := a.Set(\"d\", newRawNode(\"x\", types.V_OBJECT, false))\n\tif exist || e != nil {\n\t\tt.Fatal(err)\n\t}\n\tif a.len() != 1 {\n\t\tt.Fail()\n\t}\n\td := a.Get(\"d\").Get(\"\")\n\tif d.Check() == nil {\n\t\tt.Fatal(d)\n\t}\n\texist, e = a.Set(\"e\", newRawNode(\"[}\", types.V_ARRAY, false))\n\tif e != nil {\n\t\tt.Fatal(e)\n\t}\n\tif a.len() != 2 {\n\t\tt.Fail()\n\t}\n\td = a.Index(1).Index(0)\n\tif d.Check() == nil {\n\t\tt.Fatal(d)\n\t}\n\n\tit, e := root.Interface()\n\tif e == nil {\n\t\tt.Fatal(it)\n\t}\n\tfmt.Println(e)\n}\n\nfunc TestIndex(t *testing.T) {\n\troot, derr := NewParser(_TwitterJson).Parse()\n\tif derr != 0 {\n\t\tt.Fatalf(\"decode failed: %v\", derr.Error())\n\t}\n\tstatus := root.GetByPath(\"statuses\", 0)\n\tx, _ := status.Index(4).String()\n\ty, _ := status.Get(\"id_str\").String()\n\tif x != y {\n\t\tt.Fail()\n\t}\n}\n\nfunc TestUnset(t *testing.T) {\n\troot, derr := NewParser(_TwitterJson).Parse()\n\tif derr != 0 {\n\t\tt.Fatalf(\"decode failed: %v\", derr.Error())\n\t}\n\tentities := root.GetByPath(\"statuses\", 0, \"entities\")\n\tif !entities.Exists() || entities.Check() != nil {\n\t\tt.Fatal(entities.Check().Error())\n\t}\n\texist, err := entities.Unset(\"urls\")\n\tif !exist || err != nil {\n\t\tt.Fatal()\n\t}\n\te := entities.Get(\"urls\")\n\tif e.Exists() {\n\t\tt.Fatal()\n\t}\n\tif entities.len() != 2 {\n\t\tt.Fatal(entities.len())\n\t}\n\n\tes, err := entities.Interface()\n\trequire.NoError(t, err)\n\trequire.Equal(t, map[string]interface{}{\n\t\t\"hashtags\": []interface{}{\n\t\t\tmap[string]interface{}{\n\t\t\t\t\"text\": \"freebandnames\",\n\t\t\t\t\"indices\": []interface{}{\n\t\t\t\t\tfloat64(20), float64(34),\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t\"user_mentions\": []interface{}{},\n\t}, es)\n\n\tout, err := entities.MarshalJSON()\n\trequire.NoError(t, err)\n\tprintln(string(out))\n\tbuf := bytes.NewBuffer(nil)\n\trequire.NoError(t, json.Compact(buf, out))\n\trequire.Equal(t,\n\t\t`{\"hashtags\":[{\"text\":\"freebandnames\",\"indices\":[20,34]}],\"user_mentions\":[]}`, buf.String())\n\n\tentities.Set(\"urls\", NewString(\"a\"))\n\te = entities.Get(\"urls\")\n\tx, _ := e.String()\n\tif !e.Exists() || x != \"a\" {\n\t\tt.Fatal()\n\t}\n\n\tout, err = entities.MarshalJSON()\n\trequire.NoError(t, err)\n\tbuf = bytes.NewBuffer(nil)\n\tjson.Compact(buf, out)\n\trequire.Equal(t,\n\t\t`{\"hashtags\":[{\"text\":\"freebandnames\",\"indices\":[20,34]}],\"user_mentions\":[],\"urls\":\"a\"}`, buf.String())\n\n\t// reload entities\n\t*entities = NewRaw(string(out))\n\n\thashtags := entities.Get(\"hashtags\").Index(0)\n\thashtags.Set(\"text2\", NewRaw(`{}`))\n\texist, err = hashtags.Unset(\"indices\") // NOTICE: Unset() won't change node.Len() here\n\tif !exist || err != nil || hashtags.len() != 2 {\n\t\tt.Fatal(hashtags.len())\n\t}\n\ty, _ := hashtags.Get(\"text\").String()\n\tif y != \"freebandnames\" {\n\t\tt.Fatal()\n\t}\n\tif hashtags.Get(\"text2\").Type() != V_OBJECT {\n\t\tt.Fatal()\n\t}\n\n\tentities.Load()\n\texist, err = entities.UnsetByIndex(entities.len() - 1)\n\tif !exist || err != nil {\n\t\tt.Fatal()\n\t}\n\tif entities.len() != 2 {\n\t\tt.Fatal(entities.len())\n\t}\n\te = entities.Index(entities.len())\n\tif e.Exists() {\n\t\tt.Fatal()\n\t}\n\n\tums := entities.Get(\"user_mentions\")\n\tums.Add(NewNull())\n\tums.Add(NewBool(false))\n\tums.Add(NewBool(true))\n\tif ums.len() != 3 {\n\t\tt.Fatal()\n\t}\n\texist, err = ums.UnsetByIndex(1)\n\tif !exist || err != nil {\n\t\tt.Fatal()\n\t}\n\trequire.Equal(t, 2, ums.len())\n\tumses, err := ums.Interface()\n\trequire.NoError(t, err)\n\trequire.Equal(t, []interface{}{interface{}(nil), true}, umses)\n\n\tv1, _ := ums.Index(0).Interface()\n\tv2, _ := ums.Index(1).Interface() // NOTICE: unset index 1 still can be find here\n\tv3, _ := ums.Index(2).Interface()\n\tif v1 != nil {\n\t\tt.Fatal()\n\t}\n\tif v2 != true {\n\t\tt.Fatal()\n\t}\n\tif v3 != nil {\n\t\tt.Fatal()\n\t}\n\tout, err = entities.MarshalJSON()\n\trequire.NoError(t, err)\n\trequire.Equal(t,\n\t\t`{\"hashtags\":[{\"text\":\"freebandnames\",\"text2\":{}}],\"user_mentions\":[null,true]}`, string(out))\n\n}\n\n// func TestUnsafeNode(t *testing.T) {\n// str, loop := getTestIteratorSample(_DEFAULT_NODE_CAP)\n\n// root, err := NewSearcher(str).GetByPath(\"array\")\n// if err != nil {\n// t.Fatal(err)\n// }\n// a, _ := root.UnsafeArray()\n// if len(a) != loop {\n// t.Fatalf(\"exp:%v, got:%v\", loop, len(a))\n// }\n// for i := int64(0); i= 0; i-- {\n\t\troot.GetByPath(\"statuses\", 3).Set(\"id_str\"+strconv.Itoa(i), app)\n\t}\n\tval, _ = root.GetByPath(\"statuses\", 3, \"id_str0\").Int64()\n\tif val != 111 {\n\t\tt.Fatalf(\"exp: %+v, got: %+v\", 111, val)\n\t}\n\n\tnroot, derr := NewParser(`{\"a\":[0.1,true,0,\"name\",{\"b\":\"c\"}]}`).Parse()\n\tif derr != 0 {\n\t\tt.Fatalf(\"decode failed: %v\", derr.Error())\n\t}\n\troot.GetByPath(\"statuses\", 3).Set(\"id_str2\", nroot)\n\tval2, _ := root.GetByPath(\"statuses\", 3, \"id_str2\", \"a\", 4, \"b\").String()\n\tif val2 != \"c\" {\n\t\tt.Fatalf(\"exp:%+v, got:%+v\", \"c\", val2)\n\t}\n}\n\nfunc TestNodeSetByIndex(t *testing.T) {\n\troot, derr := NewParser(_TwitterJson).Parse()\n\tif derr != 0 {\n\t\tt.Fatalf(\"decode failed: %v\", derr.Error())\n\t}\n\tapp, _ := NewParser(\"111\").Parse()\n\tst := root.GetByPath(\"statuses\")\n\tst.SetByIndex(0, app)\n\tst = root.GetByPath(\"statuses\")\n\tval := st.Index(0)\n\tx, _ := val.Int64()\n\tif x != 111 {\n\t\tt.Fatalf(\"exp: %+v, got: %+v\", 111, val)\n\t}\n\n\tnroot, derr := NewParser(`{\"a\":[0.1,true,0,\"name\",{\"b\":\"c\"}]}`).Parse()\n\tif derr != 0 {\n\t\tt.Fatalf(\"decode failed: %v\", derr.Error())\n\t}\n\troot.GetByPath(\"statuses\").SetByIndex(0, nroot)\n\tval2, _ := root.GetByPath(\"statuses\", 0, \"a\", 4, \"b\").String()\n\tif val2 != \"c\" {\n\t\tt.Fatalf(\"exp:%+v, got:%+v\", \"c\", val2)\n\t}\n}\n\nfunc TestNodeAdd(t *testing.T) {\n\troot, derr := NewParser(_TwitterJson).Parse()\n\tif derr != 0 {\n\t\tt.Fatalf(\"decode failed: %v\", derr.Error())\n\t}\n\tapp, _ := NewParser(\"111\").Parse()\n\n\tfor i, _ := root.GetByPath(\"statuses\").Cap(); i >= 0; i-- {\n\t\troot.GetByPath(\"statuses\").Add(app)\n\t}\n\tval, _ := root.GetByPath(\"statuses\", 4).Int64()\n\tif val != 111 {\n\t\tt.Fatalf(\"exp: %+v, got: %+v\", 111, val)\n\t}\n\tval, _ = root.GetByPath(\"statuses\", root.GetByPath(\"statuses\").len()-1).Int64()\n\tif val != 111 {\n\t\tt.Fatalf(\"exp: %+v, got: %+v\", 111, val)\n\t}\n\n\tnroot, derr := NewParser(`{\"a\":[0.1,true,0,\"name\",{\"b\":\"c\"}]}`).Parse()\n\tif derr != 0 {\n\t\tt.Fatalf(\"decode failed: %v\", derr.Error())\n\t}\n\troot.GetByPath(\"statuses\").Add(nroot)\n\tval2, _ := root.GetByPath(\"statuses\", root.GetByPath(\"statuses\").len()-1, \"a\", 4, \"b\").String()\n\tif val2 != \"c\" {\n\t\tt.Fatalf(\"exp:%+v, got:%+v\", \"c\", val2)\n\t}\n}\n\nfunc BenchmarkLoadNode(b *testing.B) {\n\tb.Run(\"Interface()\", func(b *testing.B) {\n\t\tb.SetBytes(int64(len(_TwitterJson)))\n\t\tb.ResetTimer()\n\t\tfor i := 0; i < b.N; i++ {\n\t\t\troot, err := NewSearcher(_TwitterJson).GetByPath(\"statuses\", 0)\n\t\t\tif err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\t\t\t_, _ = root.Interface()\n\t\t}\n\t})\n\n\tb.Run(\"LoadAll()\", func(b *testing.B) {\n\t\tb.SetBytes(int64(len(_TwitterJson)))\n\t\tb.ResetTimer()\n\t\tfor i := 0; i < b.N; i++ {\n\t\t\troot, err := NewSearcher(_TwitterJson).GetByPath(\"statuses\", 0)\n\t\t\tif err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\t\t\t_ = root.LoadAll()\n\t\t}\n\t})\n\n\tb.Run(\"InterfaceUseNode()\", func(b *testing.B) {\n\t\tb.SetBytes(int64(len(_TwitterJson)))\n\t\tb.ResetTimer()\n\t\tfor i := 0; i < b.N; i++ {\n\t\t\troot, err := NewSearcher(_TwitterJson).GetByPath(\"statuses\", 0)\n\t\t\tif err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\t\t\t_, _ = root.InterfaceUseNode()\n\t\t}\n\t})\n\n\tb.Run(\"Load()\", func(b *testing.B) {\n\t\tb.SetBytes(int64(len(_TwitterJson)))\n\t\tb.ResetTimer()\n\t\tfor i := 0; i < b.N; i++ {\n\t\t\troot, err := NewSearcher(_TwitterJson).GetByPath(\"statuses\", 0)\n\t\t\tif err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\t\t\t_ = root.Load()\n\t\t}\n\t})\n}\n\nfunc BenchmarkLoadNode_Parallel(b *testing.B) {\n\tb.Run(\"Interface()\", func(b *testing.B) {\n\t\tb.SetBytes(int64(len(_TwitterJson)))\n\t\tb.ResetTimer()\n\t\tb.RunParallel(func(pb *testing.PB) {\n\t\t\tfor pb.Next() {\n\t\t\t\troot, err := NewSearcher(_TwitterJson).GetByPath(\"statuses\", 0)\n\t\t\t\tif err != nil {\n\t\t\t\t\tb.Fatal(err)\n\t\t\t\t}\n\t\t\t\t_, _ = root.Interface()\n\t\t\t}\n\t\t})\n\t})\n\n\tb.Run(\"LoadAll()\", func(b *testing.B) {\n\t\tb.SetBytes(int64(len(_TwitterJson)))\n\t\tb.ResetTimer()\n\t\tb.RunParallel(func(pb *testing.PB) {\n\t\t\tfor pb.Next() {\n\t\t\t\troot, err := NewSearcher(_TwitterJson).GetByPath(\"statuses\", 0)\n\t\t\t\tif err != nil {\n\t\t\t\t\tb.Fatal(err)\n\t\t\t\t}\n\t\t\t\t_ = root.LoadAll()\n\t\t\t}\n\t\t})\n\t})\n\n\tb.Run(\"InterfaceUseNode()\", func(b *testing.B) {\n\t\tb.SetBytes(int64(len(_TwitterJson)))\n\t\tb.ResetTimer()\n\t\tb.RunParallel(func(pb *testing.PB) {\n\t\t\tfor pb.Next() {\n\t\t\t\troot, err := NewSearcher(_TwitterJson).GetByPath(\"statuses\", 0)\n\t\t\t\tif err != nil {\n\t\t\t\t\tb.Fatal(err)\n\t\t\t\t}\n\t\t\t\t_, _ = root.InterfaceUseNode()\n\t\t\t}\n\t\t})\n\t})\n\n\tb.Run(\"Load()\", func(b *testing.B) {\n\t\tb.SetBytes(int64(len(_TwitterJson)))\n\t\tb.ResetTimer()\n\t\tb.RunParallel(func(pb *testing.PB) {\n\t\t\tfor pb.Next() {\n\t\t\t\troot, err := NewSearcher(_TwitterJson).GetByPath(\"statuses\", 0)\n\t\t\t\tif err != nil {\n\t\t\t\t\tb.Fatal(err)\n\t\t\t\t}\n\t\t\t\t_ = root.Load()\n\t\t\t}\n\t\t})\n\t})\n}\n\nfunc BenchmarkNodeGetByPath(b *testing.B) {\n\troot, derr := NewParser(_TwitterJson).Parse()\n\tif derr != 0 {\n\t\tb.Fatalf(\"decode failed: %v\", derr.Error())\n\t}\n\t_, _ = root.GetByPath(\"statuses\", 3, \"entities\", \"hashtags\", 0, \"text\").String()\n\tb.ResetTimer()\n\n\tb.RunParallel(func(pb *testing.PB) {\n\t\tfor pb.Next() {\n\t\t\t_, _ = root.GetByPath(\"statuses\", 3, \"entities\", \"hashtags\", 0, \"text\").String()\n\t\t}\n\t})\n}\n\nfunc BenchmarkStructGetByPath(b *testing.B) {\n\tvar root = _TwitterStruct{}\n\terr := json.Unmarshal([]byte(_TwitterJson), &root)\n\tif err != nil {\n\t\tb.Fatalf(\"unmarshal failed: %v\", err)\n\t}\n\n\tb.ResetTimer()\n\n\tb.RunParallel(func(pb *testing.PB) {\n\t\tfor pb.Next() {\n\t\t\t_ = root.Statuses[3].Entities.Hashtags[0].Text\n\t\t}\n\t})\n}\n\nfunc BenchmarkNodeIndex(b *testing.B) {\n\troot, derr := NewParser(_TwitterJson).Parse()\n\tif derr != 0 {\n\t\tb.Fatalf(\"decode failed: %v\", derr.Error())\n\t}\n\tnode := root.Get(\"statuses\").Index(3).Get(\"entities\").Get(\"hashtags\").Index(0)\n\tnode.Set(\"test1\", NewNumber(\"1\"))\n\tnode.Set(\"test2\", NewNumber(\"2\"))\n\tnode.Set(\"test3\", NewNumber(\"3\"))\n\tnode.Set(\"test4\", NewNumber(\"4\"))\n\tnode.Set(\"test5\", NewNumber(\"5\"))\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tnode.Index(2)\n\t}\n}\n\nfunc BenchmarkStructIndex(b *testing.B) {\n\ttype T struct {\n\t\tA Node\n\t\tB Node\n\t\tC Node\n\t\tD Node\n\t\tE Node\n\t}\n\tvar obj = new(T)\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\t_ = obj.C\n\t}\n}\n\nfunc BenchmarkSliceIndex(b *testing.B) {\n\tvar obj = []Node{Node{}, Node{}, Node{}, Node{}, Node{}}\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\t_ = obj[2]\n\t}\n}\n\nfunc BenchmarkMapIndex(b *testing.B) {\n\tvar obj = map[string]interface{}{\"test1\": Node{}, \"test2\": Node{}, \"test3\": Node{}, \"test4\": Node{}, \"test5\": Node{}}\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tfor k := range obj {\n\t\t\tif k == \"test3\" {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc BenchmarkNodeGet(b *testing.B) {\n\tvar N = 5\n\tvar half = \"test\" + strconv.Itoa(N/2+1)\n\troot, derr := NewParser(_TwitterJson).Parse()\n\tif derr != 0 {\n\t\tb.Fatalf(\"decode failed: %v\", derr.Error())\n\t}\n\tnode := root.Get(\"statuses\").Index(3).Get(\"entities\").Get(\"hashtags\").Index(0)\n\tfor i := 0; i < N; i++ {\n\t\tnode.Set(\"test\"+strconv.Itoa(i), NewNumber(strconv.Itoa(i)))\n\t}\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\t_ = node.Get(half)\n\t}\n}\n\nfunc BenchmarkSliceGet(b *testing.B) {\n\tvar obj = []string{\"test1\", \"test2\", \"test3\", \"test4\", \"test5\"}\n\tstr := \"test3\"\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tfor _, k := range obj {\n\t\t\tif k == str {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc BenchmarkMapGet(b *testing.B) {\n\troot, derr := NewParser(_TwitterJson).Parse()\n\tif derr != 0 {\n\t\tb.Fatalf(\"decode failed: %v\", derr.Error())\n\t}\n\tnode := root.Get(\"statuses\").Index(3).Get(\"entities\").Get(\"hashtags\").Index(0)\n\tnode.Set(\"test1\", NewNumber(\"1\"))\n\tnode.Set(\"test2\", NewNumber(\"2\"))\n\tnode.Set(\"test3\", NewNumber(\"3\"))\n\tnode.Set(\"test4\", NewNumber(\"4\"))\n\tnode.Set(\"test5\", NewNumber(\"5\"))\n\tm, _ := node.Map()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\t_ = m[\"test3\"]\n\t}\n}\n\nfunc BenchmarkNodeSet(b *testing.B) {\n\troot, derr := NewParser(_TwitterJson).Parse()\n\tif derr != 0 {\n\t\tb.Fatalf(\"decode failed: %v\", derr.Error())\n\t}\n\tnode := root.Get(\"statuses\").Index(3).Get(\"entities\").Get(\"hashtags\").Index(0)\n\tnode.Set(\"test1\", NewNumber(\"1\"))\n\tnode.Set(\"test2\", NewNumber(\"2\"))\n\tnode.Set(\"test3\", NewNumber(\"3\"))\n\tnode.Set(\"test4\", NewNumber(\"4\"))\n\tnode.Set(\"test5\", NewNumber(\"5\"))\n\tn := NewNull()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tnode.Set(\"test3\", n)\n\t}\n}\n\nfunc BenchmarkMapSet(b *testing.B) {\n\troot, derr := NewParser(_TwitterJson).Parse()\n\tif derr != 0 {\n\t\tb.Fatalf(\"decode failed: %v\", derr.Error())\n\t}\n\tnode := root.Get(\"statuses\").Index(3).Get(\"entities\").Get(\"hashtags\").Index(0)\n\tnode.Set(\"test1\", NewNumber(\"1\"))\n\tnode.Set(\"test2\", NewNumber(\"2\"))\n\tnode.Set(\"test3\", NewNumber(\"3\"))\n\tnode.Set(\"test4\", NewNumber(\"4\"))\n\tnode.Set(\"test5\", NewNumber(\"5\"))\n\tm, _ := node.Map()\n\tn := NewNull()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tm[\"test3\"] = n\n\t}\n}\n\nfunc BenchmarkNodeSetByIndex(b *testing.B) {\n\troot, derr := NewParser(_TwitterJson).Parse()\n\tif derr != 0 {\n\t\tb.Fatalf(\"decode failed: %v\", derr.Error())\n\t}\n\tnode := root.Get(\"statuses\").Index(3).Get(\"entities\").Get(\"hashtags\")\n\tnode.Add(NewNumber(\"1\"))\n\tnode.Add(NewNumber(\"2\"))\n\tnode.Add(NewNumber(\"3\"))\n\tnode.Add(NewNumber(\"4\"))\n\tnode.Add(NewNumber(\"5\"))\n\tn := NewNull()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tnode.SetByIndex(2, n)\n\t}\n}\n\nfunc BenchmarkSliceSetByIndex(b *testing.B) {\n\troot, derr := NewParser(_TwitterJson).Parse()\n\tif derr != 0 {\n\t\tb.Fatalf(\"decode failed: %v\", derr.Error())\n\t}\n\tnode := root.Get(\"statuses\").Index(3).Get(\"entities\").Get(\"hashtags\")\n\tnode.Add(NewNumber(\"1\"))\n\tnode.Add(NewNumber(\"2\"))\n\tnode.Add(NewNumber(\"3\"))\n\tnode.Add(NewNumber(\"4\"))\n\tnode.Add(NewNumber(\"5\"))\n\tm, _ := node.Array()\n\tn := NewNull()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tm[2] = n\n\t}\n}\n\nfunc BenchmarkStructSetByIndex(b *testing.B) {\n\ttype T struct {\n\t\tA Node\n\t\tB Node\n\t\tC Node\n\t\tD Node\n\t\tE Node\n\t}\n\tvar obj = new(T)\n\tn := NewNull()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tobj.C = n\n\t}\n}\n\nfunc BenchmarkNodeUnset(b *testing.B) {\n\troot, derr := NewParser(_TwitterJson).Parse()\n\tif derr != 0 {\n\t\tb.Fatalf(\"decode failed: %v\", derr.Error())\n\t}\n\tnode := root.Get(\"statuses\").Index(3).Get(\"entities\").Get(\"hashtags\").Index(0)\n\tnode.Set(\"test1\", NewNumber(\"1\"))\n\tnode.Set(\"test2\", NewNumber(\"2\"))\n\tnode.Set(\"test3\", NewNumber(\"3\"))\n\tnode.Set(\"test4\", NewNumber(\"4\"))\n\tnode.Set(\"test5\", NewNumber(\"5\"))\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tnode.Unset(\"test3\")\n\t}\n}\n\nfunc BenchmarkMapUnset(b *testing.B) {\n\troot, derr := NewParser(_TwitterJson).Parse()\n\tif derr != 0 {\n\t\tb.Fatalf(\"decode failed: %v\", derr.Error())\n\t}\n\tnode := root.Get(\"statuses\").Index(3).Get(\"entities\").Get(\"hashtags\").Index(0)\n\tnode.Set(\"test1\", NewNumber(\"1\"))\n\tnode.Set(\"test2\", NewNumber(\"2\"))\n\tnode.Set(\"test3\", NewNumber(\"3\"))\n\tnode.Set(\"test4\", NewNumber(\"4\"))\n\tnode.Set(\"test5\", NewNumber(\"5\"))\n\tm, _ := node.Map()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tdelete(m, \"test3\")\n\t}\n}\n\nfunc BenchmarkNodUnsetByIndex(b *testing.B) {\n\troot, derr := NewParser(_TwitterJson).Parse()\n\tif derr != 0 {\n\t\tb.Fatalf(\"decode failed: %v\", derr.Error())\n\t}\n\tnode := root.Get(\"statuses\").Index(3).Get(\"entities\").Get(\"hashtags\")\n\tnode.Add(NewNumber(\"1\"))\n\tnode.Add(NewNumber(\"2\"))\n\tnode.Add(NewNumber(\"3\"))\n\tnode.Add(NewNumber(\"4\"))\n\tnode.Add(NewNumber(\"5\"))\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tnode.UnsetByIndex(2)\n\t}\n}\n\nfunc BenchmarkSliceUnsetByIndex(b *testing.B) {\n\troot, derr := NewParser(_TwitterJson).Parse()\n\tif derr != 0 {\n\t\tb.Fatalf(\"decode failed: %v\", derr.Error())\n\t}\n\tnode := root.Get(\"statuses\").Index(3).Get(\"entities\").Get(\"hashtags\")\n\tnode.Add(NewNumber(\"1\"))\n\tnode.Add(NewNumber(\"2\"))\n\tnode.Add(NewNumber(\"3\"))\n\tnode.Add(NewNumber(\"4\"))\n\tnode.Add(NewNumber(\"5\"))\n\tm, _ := node.Array()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tfor i := 3; i < 5; i++ {\n\t\t\tm[i-1] = m[i]\n\t\t}\n\t}\n}\n\nfunc BenchmarkNodeAdd(b *testing.B) {\n\tn := NewObject([]Pair{NewPair(\"test\", NewNumber(\"1\"))})\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tnode := NewArray([]Node{})\n\t\tnode.Add(n)\n\t}\n}\n\nfunc BenchmarkSliceAdd(b *testing.B) {\n\tn := NewObject([]Pair{NewPair(\"test\", NewNumber(\"1\"))})\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tnode := []Node{}\n\t\tnode = append(node, n)\n\t}\n}\n\nfunc BenchmarkMapAdd(b *testing.B) {\n\tn := NewObject([]Pair{NewPair(\"test\", NewNumber(\"1\"))})\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tnode := map[string]Node{}\n\t\tnode[\"test3\"] = n\n\t}\n}\n\nfunc TestNode_Move(t *testing.T) {\n\tvar us = NewRaw(`[\"a\",\"1\",\"b\",\"c\"]`)\n\tif ex, e := us.UnsetByIndex(1); !ex || e != nil {\n\t\tt.Fail()\n\t}\n\tvar us2 = NewRaw(`[\"a\",\"b\",\"c\",\"1\"]`)\n\tif ex, e := us2.UnsetByIndex(3); !ex || e != nil {\n\t\tt.Fail()\n\t}\n\ttests := []struct {\n\t\tname string\n\t\tin Node\n\t\tsrc int\n\t\tdst int\n\t\tout Node\n\t\twantErr bool\n\t}{\n\t\t{\n\t\t\tname: \"over index\",\n\t\t\tin: NewArray([]Node{}),\n\t\t\tsrc: 0,\n\t\t\tdst: 1,\n\t\t\tout: NewArray([]Node{}),\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"equal index\",\n\t\t\tin: NewArray([]Node{NewBool(true)}),\n\t\t\tsrc: 0,\n\t\t\tdst: 0,\n\t\t\tout: NewArray([]Node{NewBool(true)}),\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"forward\",\n\t\t\tin: NewArray([]Node{NewString(\"a\"), NewString(\"b\"), NewString(\"c\")}),\n\t\t\tsrc: 0,\n\t\t\tdst: 2,\n\t\t\tout: NewArray([]Node{NewString(\"b\"), NewString(\"c\"), NewString(\"a\")}),\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"backward\",\n\t\t\tin: NewArray([]Node{NewString(\"a\"), NewString(\"b\"), NewString(\"c\")}),\n\t\t\tsrc: 2,\n\t\t\tdst: 0,\n\t\t\tout: NewArray([]Node{NewString(\"c\"), NewString(\"a\"), NewString(\"b\")}),\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"lazy\",\n\t\t\tin: NewRaw(`[\"a\",\"b\",\"c\"]`),\n\t\t\tsrc: 2,\n\t\t\tdst: 0,\n\t\t\tout: NewArray([]Node{NewString(\"c\"), NewString(\"a\"), NewString(\"b\")}),\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"unset back\",\n\t\t\tin: us,\n\t\t\tsrc: 2,\n\t\t\tdst: 0,\n\t\t\tout: NewArray([]Node{NewString(\"c\"), NewString(\"a\"), NewString(\"b\")}),\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"unset forward\",\n\t\t\tin: us2,\n\t\t\tsrc: 0,\n\t\t\tdst: 2,\n\t\t\tout: NewArray([]Node{NewString(\"b\"), NewString(\"c\"), NewString(\"a\")}),\n\t\t\twantErr: false,\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\terr := tt.in.Move(tt.dst, tt.src)\n\t\t\trequire.NoError(t, err)\n\t\t\tej, _ := tt.out.MarshalJSON()\n\t\t\taj, _ := tt.in.MarshalJSON()\n\t\t\trequire.Equal(t, string(ej), string(aj))\n\t\t})\n\t}\n\n}\n\nfunc TestNode_Pop(t *testing.T) {\n\tvar us = NewRaw(`[1,2,3]`)\n\tif ex, e := us.UnsetByIndex(0); !ex || e != nil {\n\t\tt.Fail()\n\t}\n\tvar us2 = NewRaw(`[1,2,3]`)\n\tif ex, e := us2.UnsetByIndex(2); !ex || e != nil {\n\t\tt.Fail()\n\t}\n\ttests := []struct {\n\t\tname string\n\t\tin Node\n\t\tout Node\n\t\twantErr bool\n\t}{\n\t\t{\n\t\t\tname: \"empty\",\n\t\t\tin: NewArray([]Node{}),\n\t\t\tout: NewArray([]Node{}),\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"one\",\n\t\t\tin: NewArray([]Node{NewString(\"a\")}),\n\t\t\tout: NewArray([]Node{}),\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"raw\",\n\t\t\tin: NewRaw(`[1]`),\n\t\t\tout: NewArray([]Node{}),\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"unset head\",\n\t\t\tin: us,\n\t\t\tout: NewRaw(`[2]`),\n\t\t\twantErr: false,\n\t\t},\n\t\t{\n\t\t\tname: \"unset tail\",\n\t\t\tin: us2,\n\t\t\tout: NewRaw(`[1]`),\n\t\t\twantErr: false,\n\t\t},\n\t}\n\tfor _, tt := range tests {\n\t\tt.Run(tt.name, func(t *testing.T) {\n\t\t\tif err := tt.in.Pop(); (err != nil) != tt.wantErr {\n\t\t\t\tt.Errorf(\"Node.Pop() error = %v, wantErr %v\", err, tt.wantErr)\n\t\t\t}\n\t\t\tej, _ := tt.out.MarshalJSON()\n\t\t\taj, _ := tt.in.MarshalJSON()\n\t\t\trequire.Equal(t, string(ej), string(aj))\n\t\t})\n\t}\n}\n" }, { "path": "ast/parser.go", "content": "/*\n * Copyright 2021 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage ast\n\nimport (\n\t\"fmt\"\n\t\"sync\"\n\t\"sync/atomic\"\n\n\t\"github.com/bytedance/sonic/internal/native/types\"\n\t\"github.com/bytedance/sonic/internal/rt\"\n\t\"github.com/bytedance/sonic/internal/utils\"\n\t\"github.com/bytedance/sonic/unquote\"\n)\n\nconst (\n\t_DEFAULT_NODE_CAP int = 16\n\t_APPEND_GROW_SHIFT = 1\n)\n\nconst (\n\t_ERR_NOT_FOUND types.ParsingError = 33\n\t_ERR_UNSUPPORT_TYPE types.ParsingError = 34\n)\n\nvar (\n\t// ErrNotExist means both key and value doesn't exist\n\tErrNotExist error = newError(_ERR_NOT_FOUND, \"value not exists\")\n\n\t// ErrUnsupportType means API on the node is unsupported\n\tErrUnsupportType error = newError(_ERR_UNSUPPORT_TYPE, \"unsupported type\")\n)\n\ntype Parser struct {\n\tp int\n\ts string\n\tnoLazy bool\n\tloadOnce bool\n\tskipValue bool\n\tdbuf *byte\n}\n\n/** Parser Private Methods **/\n\nfunc (self *Parser) delim() types.ParsingError {\n\tn := len(self.s)\n\tp := self.lspace(self.p)\n\n\t/* check for EOF */\n\tif p >= n {\n\t\treturn types.ERR_EOF\n\t}\n\n\t/* check for the delimiter */\n\tif self.s[p] != ':' {\n\t\treturn types.ERR_INVALID_CHAR\n\t}\n\n\t/* update the read pointer */\n\tself.p = p + 1\n\treturn 0\n}\n\nfunc (self *Parser) object() types.ParsingError {\n\tn := len(self.s)\n\tp := self.lspace(self.p)\n\n\t/* check for EOF */\n\tif p >= n {\n\t\treturn types.ERR_EOF\n\t}\n\n\t/* check for the delimiter */\n\tif self.s[p] != '{' {\n\t\treturn types.ERR_INVALID_CHAR\n\t}\n\n\t/* update the read pointer */\n\tself.p = p + 1\n\treturn 0\n}\n\nfunc (self *Parser) array() types.ParsingError {\n\tn := len(self.s)\n\tp := self.lspace(self.p)\n\n\t/* check for EOF */\n\tif p >= n {\n\t\treturn types.ERR_EOF\n\t}\n\n\t/* check for the delimiter */\n\tif self.s[p] != '[' {\n\t\treturn types.ERR_INVALID_CHAR\n\t}\n\n\t/* update the read pointer */\n\tself.p = p + 1\n\treturn 0\n}\n\nfunc (self *Parser) lspace(sp int) int {\n\tns := len(self.s)\n\tfor ; sp < ns && utils.IsSpace(self.s[sp]); sp += 1 {\n\t}\n\n\treturn sp\n}\n\nfunc (self *Parser) backward() {\n\tfor ; self.p >= 0 && utils.IsSpace(self.s[self.p]); self.p -= 1 {\n\t}\n}\n\nfunc (self *Parser) decodeArray(ret *linkedNodes) (Node, types.ParsingError) {\n\tsp := self.p\n\tns := len(self.s)\n\n\t/* check for EOF */\n\tif self.p = self.lspace(sp); self.p >= ns {\n\t\treturn Node{}, types.ERR_EOF\n\t}\n\n\t/* check for empty array */\n\tif self.s[self.p] == ']' {\n\t\tself.p++\n\t\treturn Node{t: types.V_ARRAY}, 0\n\t}\n\n\t/* allocate array space and parse every element */\n\tfor {\n\t\tvar val Node\n\t\tvar err types.ParsingError\n\n\t\tif self.skipValue {\n\t\t\t/* skip the value */\n\t\t\tvar start int\n\t\t\tif start, err = self.skipFast(); err != 0 {\n\t\t\t\treturn Node{}, err\n\t\t\t}\n\t\t\tif self.p > ns {\n\t\t\t\treturn Node{}, types.ERR_EOF\n\t\t\t}\n\t\t\tt := switchRawType(self.s[start])\n\t\t\tif t == _V_NONE {\n\t\t\t\treturn Node{}, types.ERR_INVALID_CHAR\n\t\t\t}\n\t\t\tval = newRawNode(self.s[start:self.p], t, false)\n\t\t} else {\n\t\t\t/* decode the value */\n\t\t\tif val, err = self.Parse(); err != 0 {\n\t\t\t\treturn Node{}, err\n\t\t\t}\n\t\t}\n\n\t\t/* add the value to result */\n\t\tret.Push(val)\n\t\tself.p = self.lspace(self.p)\n\n\t\t/* check for EOF */\n\t\tif self.p >= ns {\n\t\t\treturn Node{}, types.ERR_EOF\n\t\t}\n\n\t\t/* check for the next character */\n\t\tswitch self.s[self.p] {\n\t\tcase ',':\n\t\t\tself.p++\n\t\tcase ']':\n\t\t\tself.p++\n\t\t\treturn newArray(ret), 0\n\t\tdefault:\n\t\t\t// if val.isLazy() {\n\t\t\t// return newLazyArray(self, ret), 0\n\t\t\t// }\n\t\t\treturn Node{}, types.ERR_INVALID_CHAR\n\t\t}\n\t}\n}\n\nfunc (self *Parser) decodeObject(ret *linkedPairs) (Node, types.ParsingError) {\n\tsp := self.p\n\tns := len(self.s)\n\n\t/* check for EOF */\n\tif self.p = self.lspace(sp); self.p >= ns {\n\t\treturn Node{}, types.ERR_EOF\n\t}\n\n\t/* check for empty object */\n\tif self.s[self.p] == '}' {\n\t\tself.p++\n\t\treturn Node{t: types.V_OBJECT}, 0\n\t}\n\n\t/* decode each pair */\n\tfor {\n\t\tvar val Node\n\t\tvar njs types.JsonState\n\t\tvar err types.ParsingError\n\n\t\t/* decode the key */\n\t\tif njs = self.decodeValue(); njs.Vt != types.V_STRING {\n\t\t\treturn Node{}, types.ERR_INVALID_CHAR\n\t\t}\n\n\t\t/* extract the key */\n\t\tidx := self.p - 1\n\t\tkey := self.s[njs.Iv:idx]\n\n\t\t/* check for escape sequence */\n\t\tif njs.Ep != -1 {\n\t\t\tif key, err = unquote.String(key); err != 0 {\n\t\t\t\treturn Node{}, err\n\t\t\t}\n\t\t}\n\n\t\t/* expect a ':' delimiter */\n\t\tif err = self.delim(); err != 0 {\n\t\t\treturn Node{}, err\n\t\t}\n\n\t\tif self.skipValue {\n\t\t\t/* skip the value */\n\t\t\tvar start int\n\t\t\tif start, err = self.skipFast(); err != 0 {\n\t\t\t\treturn Node{}, err\n\t\t\t}\n\t\t\tif self.p > ns {\n\t\t\t\treturn Node{}, types.ERR_EOF\n\t\t\t}\n\t\t\tt := switchRawType(self.s[start])\n\t\t\tif t == _V_NONE {\n\t\t\t\treturn Node{}, types.ERR_INVALID_CHAR\n\t\t\t}\n\t\t\tval = newRawNode(self.s[start:self.p], t, false)\n\t\t} else {\n\t\t\t/* decode the value */\n\t\t\tif val, err = self.Parse(); err != 0 {\n\t\t\t\treturn Node{}, err\n\t\t\t}\n\t\t}\n\n\t\t/* add the value to result */\n\t\t// FIXME: ret's address may change here, thus previous referred node in ret may be invalid !!\n\t\tret.Push(NewPair(key, val))\n\t\tself.p = self.lspace(self.p)\n\n\t\t/* check for EOF */\n\t\tif self.p >= ns {\n\t\t\treturn Node{}, types.ERR_EOF\n\t\t}\n\n\t\t/* check for the next character */\n\t\tswitch self.s[self.p] {\n\t\tcase ',':\n\t\t\tself.p++\n\t\tcase '}':\n\t\t\tself.p++\n\t\t\treturn newObject(ret), 0\n\t\tdefault:\n\t\t\t// if val.isLazy() {\n\t\t\t// return newLazyObject(self, ret), 0\n\t\t\t// }\n\t\t\treturn Node{}, types.ERR_INVALID_CHAR\n\t\t}\n\t}\n}\n\nfunc (self *Parser) decodeString(iv int64, ep int) (Node, types.ParsingError) {\n\tp := self.p - 1\n\ts := self.s[iv:p]\n\n\t/* fast path: no escape sequence */\n\tif ep == -1 {\n\t\treturn NewString(s), 0\n\t}\n\n\t/* unquote the string */\n\tout, err := unquote.String(s)\n\n\t/* check for errors */\n\tif err != 0 {\n\t\treturn Node{}, err\n\t} else {\n\t\treturn newBytes(rt.Str2Mem(out)), 0\n\t}\n}\n\n/** Parser Interface **/\n\nfunc (self *Parser) Pos() int {\n\treturn self.p\n}\n\n// Parse returns a ast.Node representing the parser's JSON.\n// NOTICE: the specific parsing lazy dependens parser's option\n// It only parse first layer and first child for Object or Array be default\nfunc (self *Parser) Parse() (Node, types.ParsingError) {\n\tswitch val := self.decodeValue(); val.Vt {\n\tcase types.V_EOF:\n\t\treturn Node{}, types.ERR_EOF\n\tcase types.V_NULL:\n\t\treturn nullNode, 0\n\tcase types.V_TRUE:\n\t\treturn trueNode, 0\n\tcase types.V_FALSE:\n\t\treturn falseNode, 0\n\tcase types.V_STRING:\n\t\treturn self.decodeString(val.Iv, val.Ep)\n\tcase types.V_ARRAY:\n\t\ts := self.p - 1\n\t\tif p := skipBlank(self.s, self.p); p >= self.p && self.s[p] == ']' {\n\t\t\tself.p = p + 1\n\t\t\treturn Node{t: types.V_ARRAY}, 0\n\t\t}\n\t\tif self.noLazy {\n\t\t\tif self.loadOnce {\n\t\t\t\tself.noLazy = false\n\t\t\t}\n\t\t\treturn self.decodeArray(new(linkedNodes))\n\t\t}\n\t\t// NOTICE: loadOnce always keep raw json for object or array\n\t\tif self.loadOnce {\n\t\t\tself.p = s\n\t\t\ts, e := self.skipFast()\n\t\t\tif e != 0 {\n\t\t\t\treturn Node{}, e\n\t\t\t}\n\t\t\treturn newRawNode(self.s[s:self.p], types.V_ARRAY, true), 0\n\t\t}\n\t\treturn newLazyArray(self), 0\n\tcase types.V_OBJECT:\n\t\ts := self.p - 1\n\t\tif p := skipBlank(self.s, self.p); p >= self.p && self.s[p] == '}' {\n\t\t\tself.p = p + 1\n\t\t\treturn Node{t: types.V_OBJECT}, 0\n\t\t}\n\t\t// NOTICE: loadOnce always keep raw json for object or array\n\t\tif self.noLazy {\n\t\t\tif self.loadOnce {\n\t\t\t\tself.noLazy = false\n\t\t\t}\n\t\t\treturn self.decodeObject(new(linkedPairs))\n\t\t}\n\t\tif self.loadOnce {\n\t\t\tself.p = s\n\t\t\ts, e := self.skipFast()\n\t\t\tif e != 0 {\n\t\t\t\treturn Node{}, e\n\t\t\t}\n\t\t\treturn newRawNode(self.s[s:self.p], types.V_OBJECT, true), 0\n\t\t}\n\t\treturn newLazyObject(self), 0\n\tcase types.V_DOUBLE:\n\t\treturn NewNumber(self.s[val.Ep:self.p]), 0\n\tcase types.V_INTEGER:\n\t\treturn NewNumber(self.s[val.Ep:self.p]), 0\n\tdefault:\n\t\treturn Node{}, types.ParsingError(-val.Vt)\n\t}\n}\n\nfunc (self *Parser) searchKey(match string) types.ParsingError {\n\tns := len(self.s)\n\tif err := self.object(); err != 0 {\n\t\treturn err\n\t}\n\n\t/* check for EOF */\n\tif self.p = self.lspace(self.p); self.p >= ns {\n\t\treturn types.ERR_EOF\n\t}\n\n\t/* check for empty object */\n\tif self.s[self.p] == '}' {\n\t\tself.p++\n\t\treturn _ERR_NOT_FOUND\n\t}\n\n\tvar njs types.JsonState\n\tvar err types.ParsingError\n\t/* decode each pair */\n\tfor {\n\n\t\t/* decode the key */\n\t\tif njs = self.decodeValue(); njs.Vt != types.V_STRING {\n\t\t\treturn types.ERR_INVALID_CHAR\n\t\t}\n\n\t\t/* extract the key */\n\t\tidx := self.p - 1\n\t\tkey := self.s[njs.Iv:idx]\n\n\t\t/* check for escape sequence */\n\t\tif njs.Ep != -1 {\n\t\t\tif key, err = unquote.String(key); err != 0 {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\t/* expect a ':' delimiter */\n\t\tif err = self.delim(); err != 0 {\n\t\t\treturn err\n\t\t}\n\n\t\t/* skip value */\n\t\tif key != match {\n\t\t\tif _, err = self.skipFast(); err != 0 {\n\t\t\t\treturn err\n\t\t\t}\n\t\t} else {\n\t\t\treturn 0\n\t\t}\n\n\t\t/* check for EOF */\n\t\tself.p = self.lspace(self.p)\n\t\tif self.p >= ns {\n\t\t\treturn types.ERR_EOF\n\t\t}\n\n\t\t/* check for the next character */\n\t\tswitch self.s[self.p] {\n\t\tcase ',':\n\t\t\tself.p++\n\t\tcase '}':\n\t\t\tself.p++\n\t\t\treturn _ERR_NOT_FOUND\n\t\tdefault:\n\t\t\treturn types.ERR_INVALID_CHAR\n\t\t}\n\t}\n}\n\nfunc (self *Parser) searchIndex(idx int) types.ParsingError {\n\tns := len(self.s)\n\tif err := self.array(); err != 0 {\n\t\treturn err\n\t}\n\n\t/* check for EOF */\n\tif self.p = self.lspace(self.p); self.p >= ns {\n\t\treturn types.ERR_EOF\n\t}\n\n\t/* check for empty array */\n\tif self.s[self.p] == ']' {\n\t\tself.p++\n\t\treturn _ERR_NOT_FOUND\n\t}\n\n\tvar err types.ParsingError\n\t/* allocate array space and parse every element */\n\tfor i := 0; i < idx; i++ {\n\n\t\t/* decode the value */\n\t\tif _, err = self.skipFast(); err != 0 {\n\t\t\treturn err\n\t\t}\n\n\t\t/* check for EOF */\n\t\tself.p = self.lspace(self.p)\n\t\tif self.p >= ns {\n\t\t\treturn types.ERR_EOF\n\t\t}\n\n\t\t/* check for the next character */\n\t\tswitch self.s[self.p] {\n\t\tcase ',':\n\t\t\tself.p++\n\t\tcase ']':\n\t\t\tself.p++\n\t\t\treturn _ERR_NOT_FOUND\n\t\tdefault:\n\t\t\treturn types.ERR_INVALID_CHAR\n\t\t}\n\t}\n\n\treturn 0\n}\n\nfunc (self *Node) skipNextNode() *Node {\n\tif !self.isLazy() {\n\t\treturn nil\n\t}\n\n\tparser, stack := self.getParserAndArrayStack()\n\tret := &stack.v\n\tsp := parser.p\n\tns := len(parser.s)\n\n\t/* check for EOF */\n\tif parser.p = parser.lspace(sp); parser.p >= ns {\n\t\treturn newSyntaxError(parser.syntaxError(types.ERR_EOF))\n\t}\n\n\t/* check for empty array */\n\tif parser.s[parser.p] == ']' {\n\t\tparser.p++\n\t\tself.setArray(ret)\n\t\treturn nil\n\t}\n\n\tvar val Node\n\t/* skip the value */\n\tif start, err := parser.skipFast(); err != 0 {\n\t\treturn newSyntaxError(parser.syntaxError(err))\n\t} else {\n\t\tt := switchRawType(parser.s[start])\n\t\tif t == _V_NONE {\n\t\t\treturn newSyntaxError(parser.syntaxError(types.ERR_INVALID_CHAR))\n\t\t}\n\t\tval = newRawNode(parser.s[start:parser.p], t, false)\n\t}\n\n\t/* add the value to result */\n\tret.Push(val)\n\tself.l++\n\tparser.p = parser.lspace(parser.p)\n\n\t/* check for EOF */\n\tif parser.p >= ns {\n\t\treturn newSyntaxError(parser.syntaxError(types.ERR_EOF))\n\t}\n\n\t/* check for the next character */\n\tswitch parser.s[parser.p] {\n\tcase ',':\n\t\tparser.p++\n\t\treturn ret.At(ret.Len() - 1)\n\tcase ']':\n\t\tparser.p++\n\t\tself.setArray(ret)\n\t\treturn ret.At(ret.Len() - 1)\n\tdefault:\n\t\treturn newSyntaxError(parser.syntaxError(types.ERR_INVALID_CHAR))\n\t}\n}\n\nfunc (self *Node) skipNextPair() *Pair {\n\tif !self.isLazy() {\n\t\treturn nil\n\t}\n\n\tparser, stack := self.getParserAndObjectStack()\n\tret := &stack.v\n\tsp := parser.p\n\tns := len(parser.s)\n\n\t/* check for EOF */\n\tif parser.p = parser.lspace(sp); parser.p >= ns {\n\t\treturn newErrorPair(parser.syntaxError(types.ERR_EOF))\n\t}\n\n\t/* check for empty object */\n\tif parser.s[parser.p] == '}' {\n\t\tparser.p++\n\t\tself.setObject(ret)\n\t\treturn nil\n\t}\n\n\t/* decode one pair */\n\tvar val Node\n\tvar njs types.JsonState\n\tvar err types.ParsingError\n\n\t/* decode the key */\n\tif njs = parser.decodeValue(); njs.Vt != types.V_STRING {\n\t\treturn newErrorPair(parser.syntaxError(types.ERR_INVALID_CHAR))\n\t}\n\n\t/* extract the key */\n\tidx := parser.p - 1\n\tkey := parser.s[njs.Iv:idx]\n\n\t/* check for escape sequence */\n\tif njs.Ep != -1 {\n\t\tif key, err = unquote.String(key); err != 0 {\n\t\t\treturn newErrorPair(parser.syntaxError(err))\n\t\t}\n\t}\n\n\t/* expect a ':' delimiter */\n\tif err = parser.delim(); err != 0 {\n\t\treturn newErrorPair(parser.syntaxError(err))\n\t}\n\n\t/* skip the value */\n\tif start, err := parser.skipFast(); err != 0 {\n\t\treturn newErrorPair(parser.syntaxError(err))\n\t} else {\n\t\tt := switchRawType(parser.s[start])\n\t\tif t == _V_NONE {\n\t\t\treturn newErrorPair(parser.syntaxError(types.ERR_INVALID_CHAR))\n\t\t}\n\t\tval = newRawNode(parser.s[start:parser.p], t, false)\n\t}\n\n\t/* add the value to result */\n\tret.Push(NewPair(key, val))\n\tself.l++\n\tparser.p = parser.lspace(parser.p)\n\n\t/* check for EOF */\n\tif parser.p >= ns {\n\t\treturn newErrorPair(parser.syntaxError(types.ERR_EOF))\n\t}\n\n\t/* check for the next character */\n\tswitch parser.s[parser.p] {\n\tcase ',':\n\t\tparser.p++\n\t\treturn ret.At(ret.Len() - 1)\n\tcase '}':\n\t\tparser.p++\n\t\tself.setObject(ret)\n\t\treturn ret.At(ret.Len() - 1)\n\tdefault:\n\t\treturn newErrorPair(parser.syntaxError(types.ERR_INVALID_CHAR))\n\t}\n}\n\n/** Parser Factory **/\n\n// Loads parse all json into interface{}\nfunc Loads(src string) (int, interface{}, error) {\n\tps := &Parser{s: src}\n\tnp, err := ps.Parse()\n\n\t/* check for errors */\n\tif err != 0 {\n\t\treturn 0, nil, ps.ExportError(err)\n\t} else {\n\t\tx, err := np.Interface()\n\t\tif err != nil {\n\t\t\treturn 0, nil, err\n\t\t}\n\t\treturn ps.Pos(), x, nil\n\t}\n}\n\n// LoadsUseNumber parse all json into interface{}, with numeric nodes cast to json.Number\nfunc LoadsUseNumber(src string) (int, interface{}, error) {\n\tps := &Parser{s: src}\n\tnp, err := ps.Parse()\n\n\t/* check for errors */\n\tif err != 0 {\n\t\treturn 0, nil, err\n\t} else {\n\t\tx, err := np.InterfaceUseNumber()\n\t\tif err != nil {\n\t\t\treturn 0, nil, err\n\t\t}\n\t\treturn ps.Pos(), x, nil\n\t}\n}\n\n// NewParser returns pointer of new allocated parser\nfunc NewParser(src string) *Parser {\n\treturn &Parser{s: src}\n}\n\n// NewParser returns new allocated parser\nfunc NewParserObj(src string) Parser {\n\treturn Parser{s: src}\n}\n\n// decodeNumber controls if parser decodes the number values instead of skip them\n//\n//\tWARN: once you set decodeNumber(true), please set decodeNumber(false) before you drop the parser\n//\totherwise the memory CANNOT be reused\nfunc (self *Parser) decodeNumber(decode bool) {\n\tif !decode && self.dbuf != nil {\n\t\ttypes.FreeDbuf(self.dbuf)\n\t\tself.dbuf = nil\n\t\treturn\n\t}\n\tif decode && self.dbuf == nil {\n\t\tself.dbuf = types.NewDbuf()\n\t}\n}\n\n// ExportError converts types.ParsingError to std Error\nfunc (self *Parser) ExportError(err types.ParsingError) error {\n\tif err == _ERR_NOT_FOUND {\n\t\treturn ErrNotExist\n\t}\n\treturn fmt.Errorf(\"%q\", SyntaxError{\n\t\tPos: self.p,\n\t\tSrc: self.s,\n\t\tCode: err,\n\t}.Description())\n}\n\nfunc backward(src string, i int) int {\n\tfor ; i >= 0 && utils.IsSpace(src[i]); i-- {\n\t}\n\treturn i\n}\n\nfunc newRawNode(str string, typ types.ValueType, lock bool) Node {\n\tret := Node{\n\t\tt: typ | _V_RAW,\n\t\tp: rt.StrPtr(str),\n\t\tl: uint(len(str)),\n\t}\n\tif lock {\n\t\tret.m = new(sync.RWMutex)\n\t}\n\treturn ret\n}\n\nvar typeJumpTable = [256]types.ValueType{\n\t'\"': types.V_STRING,\n\t'-': _V_NUMBER,\n\t'0': _V_NUMBER,\n\t'1': _V_NUMBER,\n\t'2': _V_NUMBER,\n\t'3': _V_NUMBER,\n\t'4': _V_NUMBER,\n\t'5': _V_NUMBER,\n\t'6': _V_NUMBER,\n\t'7': _V_NUMBER,\n\t'8': _V_NUMBER,\n\t'9': _V_NUMBER,\n\t'[': types.V_ARRAY,\n\t'f': types.V_FALSE,\n\t'n': types.V_NULL,\n\t't': types.V_TRUE,\n\t'{': types.V_OBJECT,\n}\n\nfunc switchRawType(c byte) types.ValueType {\n\treturn typeJumpTable[c]\n}\n\nfunc (self *Node) loadt() types.ValueType {\n\treturn (types.ValueType)(atomic.LoadInt64(&self.t))\n}\n\nfunc (self *Node) lock() bool {\n\tif m := self.m; m != nil {\n\t\tm.Lock()\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc (self *Node) unlock() {\n\tif m := self.m; m != nil {\n\t\tm.Unlock()\n\t}\n}\n\nfunc (self *Node) rlock() bool {\n\tif m := self.m; m != nil {\n\t\tm.RLock()\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc (self *Node) runlock() {\n\tif m := self.m; m != nil {\n\t\tm.RUnlock()\n\t}\n}\n" }, { "path": "ast/parser_test.go", "content": "/*\n * Copyright 2021 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage ast\n\nimport (\n\t\"encoding/json\"\n\t\"os\"\n\t\"runtime\"\n\t\"runtime/debug\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nvar (\n\tdebugSyncGC = os.Getenv(\"SONIC_SYNC_GC\") != \"\"\n\tdebugAsyncGC = os.Getenv(\"SONIC_NO_ASYNC_GC\") == \"\"\n)\n\nfunc TestMain(m *testing.M) {\n\tgo func() {\n\t\tif !debugAsyncGC {\n\t\t\treturn\n\t\t}\n\t\tprintln(\"Begin GC looping...\")\n\t\tfor {\n\t\t\truntime.GC()\n\t\t\tdebug.FreeOSMemory()\n\t\t}\n\t\tprintln(\"stop GC looping!\")\n\t}()\n\ttime.Sleep(time.Millisecond)\n\tm.Run()\n}\n\nfunc TestGC_Parse(t *testing.T) {\n\tif debugSyncGC {\n\t\treturn\n\t}\n\t_, _, err := Loads(_TwitterJson)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\twg := &sync.WaitGroup{}\n\tN := 1000\n\tfor i := 0; i < N; i++ {\n\t\twg.Add(1)\n\t\tgo func(wg *sync.WaitGroup) {\n\t\t\tdefer wg.Done()\n\t\t\t_, _, err := Loads(_TwitterJson)\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\truntime.GC()\n\t\t}(wg)\n\t}\n\twg.Wait()\n}\n\nfunc runDecoderTest(t *testing.T, src string, expect interface{}) {\n\tvv, err := NewParser(src).Parse()\n\tif err != 0 {\n\t\tpanic(err)\n\t}\n\tx, _ := vv.Interface()\n\tassert.Equal(t, expect, x)\n}\n\nfunc runDecoderTestUseNumber(t *testing.T, src string, expect interface{}) {\n\tvv, err := NewParser(src).Parse()\n\tif err != 0 {\n\t\tpanic(err)\n\t}\n\tvvv, _ := vv.InterfaceUseNumber()\n\tswitch vvv.(type) {\n\tcase json.Number:\n\t\tassert.Equal(t, expect, n2f64(vvv.(json.Number)))\n\tcase []interface{}:\n\t\tx := vvv.([]interface{})\n\t\tfor i, e := range x {\n\t\t\tif ev, ok := e.(json.Number); ok {\n\t\t\t\tx[i] = n2f64(ev)\n\t\t\t}\n\t\t}\n\t\tassert.Equal(t, expect, x)\n\tcase map[string]interface{}:\n\t\tx := vvv.(map[string]interface{})\n\t\tfor k, v := range x {\n\t\t\tif ev, ok := v.(json.Number); ok {\n\t\t\t\tx[k] = n2f64(ev)\n\t\t\t}\n\t\t}\n\t\tassert.Equal(t, expect, x)\n\t}\n}\n\nfunc n2f64(i json.Number) float64 {\n\tx, err := i.Float64()\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn x\n}\n\nfunc TestParser_Basic(t *testing.T) {\n\trunDecoderTest(t, `null`, nil)\n\trunDecoderTest(t, `true`, true)\n\trunDecoderTest(t, `false`, false)\n\trunDecoderTest(t, `\"hello, world \\\\ \\/ \\b \\f \\n \\r \\t \\u666f 测试中文\"`, \"hello, world \\\\ / \\b \\f \\n \\r \\t \\u666f 测试中文\")\n\trunDecoderTest(t, `\"\\ud83d\\ude00\"`, \"😀\")\n\trunDecoderTest(t, `0`, float64(0))\n\trunDecoderTest(t, `-0`, float64(0))\n\trunDecoderTest(t, `123456`, float64(123456))\n\trunDecoderTest(t, `-12345`, float64(-12345))\n\trunDecoderTest(t, `0.2`, 0.2)\n\trunDecoderTest(t, `1.2`, 1.2)\n\trunDecoderTest(t, `-0.2`, -0.2)\n\trunDecoderTest(t, `-1.2`, -1.2)\n\trunDecoderTest(t, `0e12`, 0e12)\n\trunDecoderTest(t, `0e+12`, 0e+12)\n\trunDecoderTest(t, `0e-12`, 0e-12)\n\trunDecoderTest(t, `-0e12`, -0e12)\n\trunDecoderTest(t, `-0e+12`, -0e+12)\n\trunDecoderTest(t, `-0e-12`, -0e-12)\n\trunDecoderTest(t, `2e12`, 2e12)\n\trunDecoderTest(t, `2E12`, 2e12)\n\trunDecoderTest(t, `2e+12`, 2e+12)\n\trunDecoderTest(t, `2e-12`, 2e-12)\n\trunDecoderTest(t, `-2e12`, -2e12)\n\trunDecoderTest(t, `-2e+12`, -2e+12)\n\trunDecoderTest(t, `-2e-12`, -2e-12)\n\trunDecoderTest(t, `0.2e12`, 0.2e12)\n\trunDecoderTest(t, `0.2e+12`, 0.2e+12)\n\trunDecoderTest(t, `0.2e-12`, 0.2e-12)\n\trunDecoderTest(t, `-0.2e12`, -0.2e12)\n\trunDecoderTest(t, `-0.2e+12`, -0.2e+12)\n\trunDecoderTest(t, `-0.2e-12`, -0.2e-12)\n\trunDecoderTest(t, `1.2e12`, 1.2e12)\n\trunDecoderTest(t, `1.2e+12`, 1.2e+12)\n\trunDecoderTest(t, `1.2e-12`, 1.2e-12)\n\trunDecoderTest(t, `-1.2e12`, -1.2e12)\n\trunDecoderTest(t, `-1.2e+12`, -1.2e+12)\n\trunDecoderTest(t, `-1.2e-12`, -1.2e-12)\n\trunDecoderTest(t, `-1.2E-12`, -1.2e-12)\n\trunDecoderTest(t, `[]`, []interface{}{})\n\trunDecoderTest(t, `{}`, map[string]interface{}{})\n\trunDecoderTest(t, `[\"asd\", \"123\", true, false, null, 2.4, 1.2e15]`, []interface{}{\"asd\", \"123\", true, false, nil, 2.4, 1.2e15})\n\trunDecoderTest(t, `{\"asdf\": \"qwer\", \"zxcv\": true}`, map[string]interface{}{\"asdf\": \"qwer\", \"zxcv\": true})\n\trunDecoderTest(t, `{\"a\": \"123\", \"b\": true, \"c\": false, \"d\": null, \"e\": 2.4, \"f\": 1.2e15, \"g\": 1}`, map[string]interface{}{\"a\": \"123\", \"b\": true, \"c\": false, \"d\": nil, \"e\": 2.4, \"f\": 1.2e15, \"g\": float64(1)})\n\n\trunDecoderTestUseNumber(t, `null`, nil)\n\trunDecoderTestUseNumber(t, `true`, true)\n\trunDecoderTestUseNumber(t, `false`, false)\n\trunDecoderTestUseNumber(t, `\"hello, world \\\\ \\/ \\b \\f \\n \\r \\t \\u666f 测试中文\"`, \"hello, world \\\\ / \\b \\f \\n \\r \\t \\u666f 测试中文\")\n\trunDecoderTestUseNumber(t, `\"\\ud83d\\ude00\"`, \"😀\")\n\trunDecoderTestUseNumber(t, `0`, float64(0))\n\trunDecoderTestUseNumber(t, `-0`, float64(0))\n\trunDecoderTestUseNumber(t, `123456`, float64(123456))\n\trunDecoderTestUseNumber(t, `-12345`, float64(-12345))\n\trunDecoderTestUseNumber(t, `0.2`, 0.2)\n\trunDecoderTestUseNumber(t, `1.2`, 1.2)\n\trunDecoderTestUseNumber(t, `-0.2`, -0.2)\n\trunDecoderTestUseNumber(t, `-1.2`, -1.2)\n\trunDecoderTestUseNumber(t, `0e12`, 0e12)\n\trunDecoderTestUseNumber(t, `0e+12`, 0e+12)\n\trunDecoderTestUseNumber(t, `0e-12`, 0e-12)\n\trunDecoderTestUseNumber(t, `-0e12`, -0e12)\n\trunDecoderTestUseNumber(t, `-0e+12`, -0e+12)\n\trunDecoderTestUseNumber(t, `-0e-12`, -0e-12)\n\trunDecoderTestUseNumber(t, `2e12`, 2e12)\n\trunDecoderTestUseNumber(t, `2E12`, 2e12)\n\trunDecoderTestUseNumber(t, `2e+12`, 2e+12)\n\trunDecoderTestUseNumber(t, `2e-12`, 2e-12)\n\trunDecoderTestUseNumber(t, `-2e12`, -2e12)\n\trunDecoderTestUseNumber(t, `-2e+12`, -2e+12)\n\trunDecoderTestUseNumber(t, `-2e-12`, -2e-12)\n\trunDecoderTestUseNumber(t, `0.2e12`, 0.2e12)\n\trunDecoderTestUseNumber(t, `0.2e+12`, 0.2e+12)\n\trunDecoderTestUseNumber(t, `0.2e-12`, 0.2e-12)\n\trunDecoderTestUseNumber(t, `-0.2e12`, -0.2e12)\n\trunDecoderTestUseNumber(t, `-0.2e+12`, -0.2e+12)\n\trunDecoderTestUseNumber(t, `-0.2e-12`, -0.2e-12)\n\trunDecoderTestUseNumber(t, `1.2e12`, 1.2e12)\n\trunDecoderTestUseNumber(t, `1.2e+12`, 1.2e+12)\n\trunDecoderTestUseNumber(t, `1.2e-12`, 1.2e-12)\n\trunDecoderTestUseNumber(t, `-1.2e12`, -1.2e12)\n\trunDecoderTestUseNumber(t, `-1.2e+12`, -1.2e+12)\n\trunDecoderTestUseNumber(t, `-1.2e-12`, -1.2e-12)\n\trunDecoderTestUseNumber(t, `-1.2E-12`, -1.2e-12)\n\trunDecoderTestUseNumber(t, `[\"asd\", \"123\", true, false, null, 2.4, 1.2e15, 1]`, []interface{}{\"asd\", \"123\", true, false, nil, 2.4, 1.2e15, float64(1)})\n\trunDecoderTestUseNumber(t, `{\"a\": \"123\", \"b\": true, \"c\": false, \"d\": null, \"e\": 2.4, \"f\": 1.2e15, \"g\": 1}`, map[string]interface{}{\"a\": \"123\", \"b\": true, \"c\": false, \"d\": nil, \"e\": 2.4, \"f\": 1.2e15, \"g\": float64(1)})\n}\n\nfunc TestLoads(t *testing.T) {\n\t_, i, e := Loads(`{\"a\": \"123\", \"b\": true, \"c\": false, \"d\": null, \"e\": 2.4, \"f\": 1.2e15, \"g\": 1}`)\n\tif e != nil {\n\t\tt.Fatal(e)\n\t}\n\tassert.Equal(t, map[string]interface{}{\"a\": \"123\", \"b\": true, \"c\": false, \"d\": nil, \"e\": 2.4, \"f\": 1.2e15, \"g\": float64(1)}, i)\n\t_, i, e = LoadsUseNumber(`{\"a\": \"123\", \"b\": true, \"c\": false, \"d\": null, \"e\": 2.4, \"f\": 1.2e15, \"g\": 1}`)\n\tif e != nil {\n\t\tt.Fatal(e)\n\t}\n\tassert.Equal(t, map[string]interface{}{\"a\": \"123\", \"b\": true, \"c\": false, \"d\": nil, \"e\": json.Number(\"2.4\"), \"f\": json.Number(\"1.2e15\"), \"g\": json.Number(\"1\")}, i)\n}\n\nfunc TestParsehNotExist(t *testing.T) {\n\ts, err := NewParser(` { \"xx\" : [ 0, \"\" ] ,\"yy\" :{ \"2\": \"\" } } `).Parse()\n\tif err != 0 {\n\t\tt.Fatal(err)\n\t}\n\tnode := s.GetByPath(\"xx\", 2)\n\tif node.Exists() {\n\t\tt.Fatalf(\"node: %v\", node)\n\t}\n\tnode = s.GetByPath(\"xx\", 1)\n\tif !node.Exists() {\n\t\tt.Fatalf(\"node: %v\", nil)\n\t}\n\tnode = s.GetByPath(\"yy\", \"3\")\n\tif node.Exists() {\n\t\tt.Fatalf(\"node: %v\", node)\n\t}\n\tnode = s.GetByPath(\"yy\", \"2\")\n\tif !node.Exists() {\n\t\tt.Fatalf(\"node: %v\", nil)\n\t}\n}\n\nfunc BenchmarkParser_Sonic(b *testing.B) {\n\tr, err := NewParser(_TwitterJson).Parse()\n\tif err != 0 {\n\t\tb.Fatal(err)\n\t}\n\tif err := r.LoadAll(); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tb.SetBytes(int64(len(_TwitterJson)))\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tr, _ = NewParser(_TwitterJson).Parse()\n\t\t_ = r.LoadAll()\n\t}\n}\n\nfunc BenchmarkParser_Parallel_Sonic(b *testing.B) {\n\tr, _ := NewParser(_TwitterJson).Parse()\n\tif err := r.LoadAll(); err != nil {\n\t\tb.Fatal(err)\n\t}\n\tb.SetBytes(int64(len(_TwitterJson)))\n\tb.ResetTimer()\n\tb.RunParallel(func(pb *testing.PB) {\n\t\tfor pb.Next() {\n\t\t\tr, _ := NewParser(_TwitterJson).Parse()\n\t\t\t_ = r.LoadAll()\n\t\t}\n\t})\n}\n\nfunc BenchmarkParseEmpty_Sonic(b *testing.B) {\n\tvar emptySample = `{\"a\":[],\"b\":{},\"c\":[{},{},{},{}],\"d\":{\"e\":[],\"f\":[],\"g\":[],\"h\":[]}}`\n\tp := NewParserObj(emptySample)\n\tast, _ := p.Parse()\n\trequire.NoError(b, ast.LoadAll())\n\tb.SetBytes(int64(len(_TwitterJson)))\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tp := NewParserObj(emptySample)\n\t\tast, _ := p.Parse()\n\t\t_ = ast.LoadAll()\n\t}\n}\n\nfunc BenchmarkParseOne_Sonic(b *testing.B) {\n\tast, _ := NewParser(_TwitterJson).Parse()\n\tnode, _ := ast.Get(\"statuses\").Index(2).Get(\"id\").Int64()\n\tif node != 249289491129438208 {\n\t\tb.Fail()\n\t}\n\tb.SetBytes(int64(len(_TwitterJson)))\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tast, _ := NewParser(_TwitterJson).Parse()\n\t\t_, _ = ast.Get(\"statuses\").Index(2).Get(\"id\").Int64()\n\t}\n}\n\nfunc BenchmarkParseOne_Parallel_Sonic(b *testing.B) {\n\tast, _ := NewParser(_TwitterJson).Parse()\n\tnode, _ := ast.Get(\"statuses\").Index(2).Get(\"id\").Int64()\n\tif node != 249289491129438208 {\n\t\tb.Fail()\n\t}\n\tb.SetBytes(int64(len(_TwitterJson)))\n\tb.ResetTimer()\n\tb.RunParallel(func(pb *testing.PB) {\n\t\tfor pb.Next() {\n\t\t\tast, _ := NewParser(_TwitterJson).Parse()\n\t\t\t_, _ = ast.Get(\"statuses\").Index(2).Get(\"id\").Int64()\n\t\t}\n\t})\n}\n\nfunc BenchmarkParseNoLazy_Sonic(b *testing.B) {\n\tb.SetBytes(int64(len(_TwitterJson)))\n\tb.ResetTimer()\n\tast := NewRawConcurrentRead(_TwitterJson)\n\tfor i := 0; i < b.N; i++ {\n\t\tnode := ast.GetByPath(\"statuses\", 3)\n\t\tif node.Check() != nil {\n\t\t\tb.Fail()\n\t\t}\n\t}\n}\n\nfunc BenchmarkParseNoLazy_Parallel_Sonic(b *testing.B) {\n\tb.SetBytes(int64(len(_TwitterJson)))\n\tb.ResetTimer()\n\tast := NewRawConcurrentRead(_TwitterJson)\n\tb.RunParallel(func(p *testing.PB) {\n\t\tfor p.Next() {\n\t\t\tnode := ast.GetByPath(\"statuses\", 3)\n\t\t\tif node.Check() != nil {\n\t\t\t\tb.Fail()\n\t\t\t}\n\t\t}\n\t})\n}\n\nfunc BenchmarkNodeRaw_Parallel_Sonic(b *testing.B) {\n\tb.SetBytes(int64(len(_TwitterJson)))\n\tb.ResetTimer()\n\tast := NewRawConcurrentRead(_TwitterJson)\n\tb.RunParallel(func(p *testing.PB) {\n\t\tfor p.Next() {\n\t\t\tnode := ast.GetByPath(\"statuses\", 3)\n\t\t\tif _, e := node.Raw(); e != nil {\n\t\t\t\tb.Fatal(e)\n\t\t\t}\n\t\t}\n\t})\n}\n\nfunc BenchmarkParseSeven_Sonic(b *testing.B) {\n\tb.SetBytes(int64(len(_TwitterJson)))\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tast, _ := NewParser(_TwitterJson).Parse()\n\t\tnode := ast.GetByPath(\"statuses\", 3, \"id\")\n\t\tnode = ast.GetByPath(\"statuses\", 3, \"user\", \"entities\", \"description\")\n\t\tnode = ast.GetByPath(\"statuses\", 3, \"user\", \"entities\", \"url\", \"urls\")\n\t\tnode = ast.GetByPath(\"statuses\", 3, \"user\", \"entities\", \"url\")\n\t\tnode = ast.GetByPath(\"statuses\", 3, \"user\", \"created_at\")\n\t\tnode = ast.GetByPath(\"statuses\", 3, \"user\", \"name\")\n\t\tnode = ast.GetByPath(\"statuses\", 3, \"text\")\n\t\tif node.Check() != nil {\n\t\t\tb.Fail()\n\t\t}\n\t}\n}\n\nfunc BenchmarkParseSeven_Parallel_Sonic(b *testing.B) {\n\tb.SetBytes(int64(len(_TwitterJson)))\n\tb.ResetTimer()\n\tb.RunParallel(func(pb *testing.PB) {\n\t\tfor pb.Next() {\n\t\t\tast, _ := NewParser(_TwitterJson).Parse()\n\t\t\tnode := ast.GetByPath(\"statuses\", 3, \"id\")\n\t\t\tnode = ast.GetByPath(\"statuses\", 3, \"user\", \"entities\", \"description\")\n\t\t\tnode = ast.GetByPath(\"statuses\", 3, \"user\", \"entities\", \"url\", \"urls\")\n\t\t\tnode = ast.GetByPath(\"statuses\", 3, \"user\", \"entities\", \"url\")\n\t\t\tnode = ast.GetByPath(\"statuses\", 3, \"user\", \"created_at\")\n\t\t\tnode = ast.GetByPath(\"statuses\", 3, \"user\", \"name\")\n\t\t\tnode = ast.GetByPath(\"statuses\", 3, \"text\")\n\t\t\tif node.Check() != nil {\n\t\t\t\tb.Fail()\n\t\t\t}\n\t\t}\n\t})\n}\n" }, { "path": "ast/search.go", "content": "/*\n * Copyright 2021 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage ast\n\nimport (\n\t\"github.com/bytedance/sonic/internal/native/types\"\n\t\"github.com/bytedance/sonic/internal/rt\"\n)\n\n// SearchOptions controls Searcher's behavior\ntype SearchOptions struct {\n\t// ValidateJSON indicates the searcher to validate the entire JSON\n\tValidateJSON bool\n\n\t// CopyReturn indicates the searcher to copy the result JSON instead of refer from the input\n\t// This can help to reduce memory usage if you cache the results\n\tCopyReturn bool\n\n\t// ConcurrentRead indicates the searcher to return a concurrently-READ-safe node,\n\t// including: GetByPath/Get/Index/GetOrIndex/Int64/Bool/Float64/String/Number/Interface/Array/Map/Raw/MarshalJSON\n\tConcurrentRead bool\n}\n\ntype Searcher struct {\n\tparser Parser\n\tSearchOptions\n}\n\nfunc NewSearcher(str string) *Searcher {\n\treturn &Searcher{\n\t\tparser: Parser{\n\t\t\ts: str,\n\t\t\tnoLazy: false,\n\t\t},\n\t\tSearchOptions: SearchOptions{\n\t\t\tValidateJSON: true,\n\t\t},\n\t}\n}\n\n// GetByPathCopy search in depth from top json and returns a **Copied** json node at the path location\nfunc (self *Searcher) GetByPathCopy(path ...interface{}) (Node, error) {\n\tself.CopyReturn = true\n\treturn self.getByPath(path...)\n}\n\n// GetByPathNoCopy search in depth from top json and returns a **Referenced** json node at the path location\n//\n// WARN: this search directly refer partial json from top json, which has faster speed,\n// may consumes more memory.\nfunc (self *Searcher) GetByPath(path ...interface{}) (Node, error) {\n\treturn self.getByPath(path...)\n}\n\nfunc (self *Searcher) getByPath(path ...interface{}) (Node, error) {\n\tvar err types.ParsingError\n\tvar start int\n\n\tself.parser.p = 0\n\tstart, err = self.parser.getByPath(self.ValidateJSON, path...)\n\tif err != 0 {\n\t\t// for compatibility with old version\n\t\tif err == types.ERR_NOT_FOUND {\n\t\t\treturn Node{}, ErrNotExist\n\t\t}\n\t\tif err == types.ERR_UNSUPPORT_TYPE {\n\t\t\tpanic(\"path must be either int(>=0) or string\")\n\t\t}\n\t\treturn Node{}, self.parser.syntaxError(err)\n\t}\n\n\tt := switchRawType(self.parser.s[start])\n\tif t == _V_NONE {\n\t\treturn Node{}, self.parser.ExportError(err)\n\t}\n\n\t// copy string to reducing memory usage\n\tvar raw string\n\tif self.CopyReturn {\n\t\traw = rt.Mem2Str([]byte(self.parser.s[start:self.parser.p]))\n\t} else {\n\t\traw = self.parser.s[start:self.parser.p]\n\t}\n\treturn newRawNode(raw, t, self.ConcurrentRead), nil\n}\n\n// GetByPath searches a path and returns relaction and types of target\nfunc _GetByPath(src string, path ...interface{}) (start int, end int, typ int, err error) {\n\tp := NewParserObj(src)\n\ts, e := p.getByPath(false, path...)\n\tif e != 0 {\n\t\t// for compatibility with old version\n\t\tif e == types.ERR_NOT_FOUND {\n\t\t\treturn -1, -1, 0, ErrNotExist\n\t\t}\n\t\tif e == types.ERR_UNSUPPORT_TYPE {\n\t\t\tpanic(\"path must be either int(>=0) or string\")\n\t\t}\n\t\treturn -1, -1, 0, p.syntaxError(e)\n\t}\n\n\tt := switchRawType(p.s[s])\n\tif t == _V_NONE {\n\t\treturn -1, -1, 0, ErrNotExist\n\t}\n\tif t == _V_NUMBER {\n\t\tp.p = 1 + backward(p.s, p.p-1)\n\t}\n\treturn s, p.p, int(t), nil\n}\n\n// ValidSyntax check if a json has a valid JSON syntax,\n// while not validate UTF-8 charset\nfunc _ValidSyntax(json string) bool {\n\tp := NewParserObj(json)\n\t_, e := p.skip()\n\tif e != 0 {\n\t\treturn false\n\t}\n\tif skipBlank(p.s, p.p) != -int(types.ERR_EOF) {\n\t\treturn false\n\t}\n\treturn true\n}\n\n// SkipFast skip a json value in fast-skip algs,\n// while not strictly validate JSON syntax and UTF-8 charset.\nfunc _SkipFast(src string, i int) (int, int, error) {\n\tp := NewParserObj(src)\n\tp.p = i\n\ts, e := p.skipFast()\n\tif e != 0 {\n\t\treturn -1, -1, p.ExportError(e)\n\t}\n\tt := switchRawType(p.s[s])\n\tif t == _V_NONE {\n\t\treturn -1, -1, ErrNotExist\n\t}\n\tif t == _V_NUMBER {\n\t\tp.p = 1 + backward(p.s, p.p-1)\n\t}\n\treturn s, p.p, nil\n}\n" }, { "path": "ast/search_test.go", "content": "/*\n * Copyright 2021 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage ast\n\nimport (\n\t\"encoding/json\"\n\t\"math\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestGC_Search(t *testing.T) {\n\tif debugSyncGC {\n\t\treturn\n\t}\n\t_, err := NewSearcher(_TwitterJson).GetByPath(\"statuses\", 0, \"id\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\twg := &sync.WaitGroup{}\n\t// A limitation of the race detecting is 8128.\n\t// See https://github.com/golang/go/issues/43898\n\tN := 5000\n\tfor i := 0; i < N; i++ {\n\t\twg.Add(1)\n\t\tgo func(wg *sync.WaitGroup) {\n\t\t\tdefer wg.Done()\n\t\t\t_, err := NewSearcher(_TwitterJson).GetByPath(\"statuses\", 0, \"id\")\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\truntime.GC()\n\t\t}(wg)\n\t}\n\twg.Wait()\n}\n\nfunc TestNodeRace(t *testing.T) {\n\n\tsrc := `{\"1\":1,\"2\": [ 1 , 1 , { \"3\" : 1 , \"4\" : [] } ] }`\n\ts := NewSearcher(src)\n\ts.ConcurrentRead = true\n\tnode, _ := s.GetByPath()\n\n\tcases := []struct {\n\t\tpath []interface{}\n\t\texp []string\n\t\tscalar bool\n\t\tlv int\n\t}{\n\t\t{[]interface{}{\"1\"}, []string{`1`}, true, 0},\n\t\t{[]interface{}{\"2\"}, []string{`[ 1 , 1 , { \"3\" : 1 , \"4\" : [] } ]`, `[1,1,{ \"3\" : 1 , \"4\" : [] }]`, `[1,1,{\"3\":1,\"4\":[]}]`}, false, 3},\n\t\t{[]interface{}{\"2\", 1}, []string{`1`}, true, 1},\n\t\t{[]interface{}{\"2\", 2}, []string{`{ \"3\" : 1 , \"4\" : [] }`, `{\"3\":1,\"4\":[]}`}, false, 2},\n\t\t{[]interface{}{\"2\", 2, \"3\"}, []string{`1`}, true, 0},\n\t\t{[]interface{}{\"2\", 2, \"4\"}, []string{`[]`}, false, 0},\n\t}\n\n\twg := sync.WaitGroup{}\n\tstart := sync.RWMutex{}\n\tstart.Lock()\n\n\tP := 100\n\tfor i := range cases {\n\t\t// println(i)\n\t\tc := cases[i]\n\t\tfor j := 0; j < P; j++ {\n\t\t\twg.Add(1)\n\t\t\tgo func() {\n\t\t\t\tdefer wg.Done()\n\t\t\t\tstart.RLock()\n\t\t\t\tn := node.GetByPath(c.path...)\n\t\t\t\t_ = n.TypeSafe()\n\t\t\t\t_ = n.isAny()\n\t\t\t\tv, err := n.Raw()\n\t\t\t\tiv, _ := n.Int64()\n\t\t\t\tlv, _ := n.Len()\n\t\t\t\t_, e := n.Interface()\n\t\t\t\te2 := n.SortKeys(false)\n\t\t\t\trequire.NoError(t, err)\n\t\t\t\trequire.NoError(t, e)\n\t\t\t\trequire.NoError(t, e2)\n\t\t\t\tif c.scalar {\n\t\t\t\t\trequire.Equal(t, int64(1), iv)\n\t\t\t\t} else {\n\t\t\t\t\trequire.Equal(t, c.lv, lv)\n\t\t\t\t}\n\t\t\t\teq := false\n\t\t\t\tfor _, exp := range c.exp {\n\t\t\t\t\tif exp == v {\n\t\t\t\t\t\teq = true\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\trequire.True(t, eq)\n\t\t\t}()\n\t\t}\n\t}\n\n\tstart.Unlock()\n\twg.Wait()\n}\n\nfunc TestExportErrorInvalidChar(t *testing.T) {\n\tdata := `{\"a\":]`\n\tp := NewSearcher(data)\n\t_, err := p.GetByPath(\"a\")\n\tif err == nil {\n\t\tt.Fatal()\n\t}\n\tif strings.Index(err.Error(), `\"Syntax error at `) != 0 {\n\t\tt.Fatal(err)\n\t}\n\n\tdata = `:\"b\"]`\n\tp = NewSearcher(data)\n\t_, err = p.GetByPath(\"a\")\n\tif err == nil {\n\t\tt.Fatal()\n\t}\n\tif err.Error() != `\"Syntax error at index 0: invalid char\\n\\n\\t:\\\"b\\\"]\\n\\t^....\\n\"` {\n\t\tt.Fatal(err)\n\t}\n\n\tdata = `{:\"b\"]`\n\tp = NewSearcher(data)\n\t_, err = p.GetByPath(\"a\")\n\tif err == nil {\n\t\tt.Fatal()\n\t}\n\tif err.Error() != `\"Syntax error at index 1: invalid char\\n\\n\\t{:\\\"b\\\"]\\n\\t.^....\\n\"` {\n\t\tt.Fatal(err)\n\t}\n\n\tdata = `{`\n\tp = NewSearcher(data)\n\t_, err = p.GetByPath(\"he\")\n\tif err == nil {\n\t\tt.Fatal()\n\t}\n\tif err == ErrNotExist {\n\t\tt.Fatal(err)\n\t}\n\n\tdata = `[`\n\tp = NewSearcher(data)\n\t_, err = p.GetByPath(0)\n\tif err == nil {\n\t\tt.Fatal()\n\t}\n\tif err == ErrNotExist {\n\t\tt.Fatal(err)\n\t}\n}\n\ntype testExportError struct {\n\tdata string\n\tpath []interface{}\n\terr error\n}\n\nfunc TestExportErrNotExist(t *testing.T) {\n\ttests := []testExportError{\n\t\t// object\n\t\t{`{}`, []interface{}{\"b\"}, ErrNotExist},\n\t\t{` { } `, []interface{}{\"b\"}, ErrNotExist},\n\t\t{`{\"a\":null}`, []interface{}{\"b\"}, ErrNotExist},\n\t\t// This should be invalid char errors.\n\t\t// {`{\"a\":null}`, []interface{}{\"a\", \"b\"}, ErrNotExist},\n\t\t// {`{\"a\":null}`, []interface{}{\"a\", 0}, ErrNotExist},\n\t\t// {`{\"a\":null}`, []interface{}{\"a\", \"b\", 0}, ErrNotExist},\n\t\t{`{\"\":{\"b\":123}}`, []interface{}{\"b\"}, ErrNotExist},\n\t\t{`{\"\":{\"b\":123}}`, []interface{}{\"\", \"\"}, ErrNotExist},\n\t\t{`{\"a\":{\"b\":123}}`, []interface{}{\"b\"}, ErrNotExist},\n\t\t{`{\"a\":{\"b\":123}}`, []interface{}{\"a\", \"c\"}, ErrNotExist},\n\t\t{`{\"a\":{\"c\": null, \"b\":{}}}`, []interface{}{\"a\", \"b\", \"c\"}, ErrNotExist},\n\t\t{`{\"a\":{\"b\":123}}`, []interface{}{\"b\", \"b\"}, ErrNotExist},\n\t\t{`{\"\\\"\\\\\":{\"b\":123}}`, []interface{}{\"\\\"\", \"b\"}, ErrNotExist},\n\t\t{`{\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\\\"\\\\\":{\"b\":123}}`,\n\t\t\t[]interface{}{\"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\\\"\", \"b\"}, ErrNotExist},\n\n\t\t// array\n\t\t{`[]`, []interface{}{0}, ErrNotExist},\n\t\t{`[]`, []interface{}{1}, ErrNotExist},\n\t\t{` [ ] `, []interface{}{0}, ErrNotExist},\n\t\t{`[null]`, []interface{}{1}, ErrNotExist},\n\t\t{`[null, [\"null\", 123]]`, []interface{}{2}, ErrNotExist},\n\t\t{`[null, true , false, 14, 2.35, -46, \"hello7\", \"\\\"8\"]`, []interface{}{8}, ErrNotExist},\n\t\t{`[{}]`, []interface{}{1}, ErrNotExist},\n\t\t{`[[]]`, []interface{}{1}, ErrNotExist},\n\t\t{`[[],[{},{}, []],{}]`, []interface{}{3}, ErrNotExist},\n\t}\n\n\tfor _, test := range tests {\n\t\tf := func(t *testing.T) {\n\t\t\tp := NewSearcher(test.data)\n\t\t\tnode, err := p.GetByPath(test.path...)\n\t\t\tif err != test.err || node.Exists() {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t}\n\t\tt.Run(test.data, f)\n\t}\n}\n\nfunc TestSearcher_GetByPath(t *testing.T) {\n\ts := NewSearcher(` { \"xx\" : [] ,\"yy\" :{ }, \"test\" : [ true , 0.1 , \"abc\", [\"h\"], {\"a\":\"bc\"} ] } `)\n\n\tnode, e := s.GetByPath(\"test\", 0)\n\ta, _ := node.Bool()\n\tif e != nil || a != true {\n\t\tt.Fatalf(\"node: %v, err: %v\", node, e)\n\t}\n\n\tnode, e = s.GetByPath(\"test\", 1)\n\tb, _ := node.Float64()\n\tif e != nil || b != 0.1 {\n\t\tt.Fatalf(\"node: %v, err: %v\", node, e)\n\t}\n\n\tnode, e = s.GetByPath(\"test\", 2)\n\tc, _ := node.String()\n\tif e != nil || c != \"abc\" {\n\t\tt.Fatalf(\"node: %v, err: %v\", node, e)\n\t}\n\n\tnode, e = s.GetByPath(\"test\", 3)\n\tarr, _ := node.Array()\n\tif e != nil || arr[0] != \"h\" {\n\t\tt.Fatalf(\"node: %v, err: %v\", node, e)\n\t}\n\n\tnode, e = s.GetByPath(\"test\", 4, \"a\")\n\td, _ := node.String()\n\tif e != nil || d != \"bc\" {\n\t\tt.Fatalf(\"node: %v, err: %v\", node, e)\n\t}\n}\n\nfunc TestSearch_LoadRawNumber(t *testing.T) {\n\ts := NewSearcher(`[\n\t1,\n\t2.34\n\t]`)\n\n\tnode, err := s.getByPath(1)\n\trequire.NoError(t, err)\n\traw, err := node.Raw()\n\trequire.NoError(t, err)\n\trequire.Equal(t, raw, \"2.34\")\n\n\tnode, err = s.getByPath()\n\trequire.NoError(t, err)\n\n\telem := node.Index(1)\n\t// FIXME: raw is `2.34` in aarch64\n\t// raw, err = elem.Raw()\n\t// require.NoError(t, err)\n\t// require.Equal(t, raw, \"2.34\\n\\t\")\n\n\tnum, err := elem.Number()\n\trequire.NoError(t, err)\n\trequire.Equal(t, num, json.Number(\"2.34\"))\n}\n\ntype testGetByPath struct {\n\tjson string\n\tpath []interface{}\n\tvalue interface{}\n\tok bool\n}\n\nfunc TestSearcher_GetByPathSingle(t *testing.T) {\n\ttype Path = []interface{}\n\tconst Ok = true\n\tconst Error = false\n\ttests := []testGetByPath{\n\t\t{`true`, Path{}, true, Ok},\n\t\t{`false`, Path{}, false, Ok},\n\t\t{`null`, Path{}, nil, Ok},\n\t\t{`12345`, Path{}, 12345.0, Ok},\n\t\t{`12345.6789`, Path{}, 12345.6789, Ok},\n\t\t{`\"abc\"`, Path{}, \"abc\", Ok},\n\t\t{`\"a\\\"\\\\bc\"`, Path{}, \"a\\\"\\\\bc\", Ok},\n\t\t{`{\"a\":1}`, Path{\"a\"}, 1.0, Ok},\n\t\t{`{\"\":1}`, Path{\"\"}, 1.0, Ok},\n\t\t{`{\"\":{\"\":1}}`, Path{\"\", \"\"}, 1.0, Ok},\n\t\t{`[1,2,3]`, Path{0}, 1.0, Ok},\n\t\t{`[1,2,3]`, Path{1}, 2.0, Ok},\n\t\t{`[1,2,3]`, Path{2}, 3.0, Ok},\n\n\t\t{`tru`, Path{}, nil, Error},\n\t\t{`fal`, Path{}, nil, Error},\n\t\t{`nul`, Path{}, nil, Error},\n\t\t{`{\"a\":1`, Path{}, nil, Error},\n\t\t{`x12345.6789`, Path{}, nil, Error},\n\t\t{`\"abc`, Path{}, nil, Error},\n\t\t{`\"a\\\"\\\\bc`, Path{}, nil, Error},\n\t\t{`\"a\\\"\\`, Path{}, nil, Error},\n\t\t{`{\"a\":`, Path{\"a\"}, nil, Error},\n\t\t{`[1,2,3]`, Path{4}, nil, Error},\n\t\t{`[1,2,3]`, Path{\"a\"}, nil, Error},\n\t}\n\tfor _, test := range tests {\n\t\tt.Run(test.json, func(t *testing.T) {\n\t\t\ts := NewSearcher(test.json)\n\t\t\tnode, err1 := s.GetByPath(test.path...)\n\t\t\tassert.Equal(t, test.ok, err1 == nil)\n\n\t\t\tvalue, err2 := node.Interface()\n\t\t\tassert.Equal(t, test.value, value)\n\t\t\tassert.Equal(t, test.ok, err2 == nil)\n\t\t})\n\t}\n}\n\nfunc TestSearcher_GetByPathErr(t *testing.T) {\n\ts := NewSearcher(` { \"xx\" : [] ,\"yy\" :{ }, \"test\" : [ true , 0.1 , \"abc\", [\"h\"], {\"a\":\"bc\"} ], \"err1\":[a, ] , \"err2\":{ ,\"x\":\"xx\"} } `)\n\tnode, e := s.GetByPath(\"zz\")\n\tif e == nil {\n\t\tt.Fatalf(\"node: %v, err: %v\", node, e)\n\t}\n\ts.parser.p = 0\n\tnode, e = s.GetByPath(\"xx\", 4)\n\tif e == nil {\n\t\tt.Fatalf(\"node: %v, err: %v\", node, e)\n\t}\n\ts.parser.p = 0\n\tnode, e = s.GetByPath(\"yy\", \"a\")\n\tif e == nil {\n\t\tt.Fatalf(\"node: %v, err: %v\", node, e)\n\t}\n\ts.parser.p = 0\n\tnode, e = s.GetByPath(\"test\", 2, \"x\")\n\tif e == nil {\n\t\tt.Fatalf(\"node: %v, err: %v\", node, e)\n\t}\n\ts.parser.p = 0\n\tnode, e = s.GetByPath(\"err1\", 0)\n\tif e == nil {\n\t\tt.Fatalf(\"node: %v, err: %v\", node, e)\n\t}\n\ts.parser.p = 0\n\tnode, e = s.GetByPath(\"err2\", \"x\")\n\tif e == nil {\n\t\tt.Fatalf(\"node: %v, err: %v\", node, e)\n\t}\n}\n\nfunc TestLoadIndex(t *testing.T) {\n\tnode, err := NewSearcher(`{\"a\":[-0, 1, -1.2, -1.2e-10]}`).GetByPath(\"a\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\ta, _ := node.Index(3).Float64()\n\tassert.Equal(t, -1.2e-10, a)\n\tm, _ := node.Array()\n\tassert.Equal(t, m, []interface{}{\n\t\tfloat64(0),\n\t\tfloat64(1),\n\t\t-1.2,\n\t\t-1.2e-10,\n\t})\n}\n\nfunc TestSearchNotExist(t *testing.T) {\n\ts := NewSearcher(` { \"xx\" : [ 0, \"\" ] ,\"yy\" :{ \"2\": \"\" } } `)\n\tnode, e := s.GetByPath(\"xx\", 2)\n\tif node.Exists() {\n\t\tt.Fatalf(\"node: %v, err: %v\", node, e)\n\t}\n\tnode, e = s.GetByPath(\"xx\", 1)\n\tif e != nil || !node.Exists() {\n\t\tt.Fatalf(\"node: %v, err: %v\", node, e)\n\t}\n\tnode, e = s.GetByPath(\"yy\", \"3\")\n\tif node.Exists() {\n\t\tt.Fatalf(\"node: %v, err: %v\", node, e)\n\t}\n\tnode, e = s.GetByPath(\"yy\", \"2\")\n\tif e != nil || !node.Exists() {\n\t\tt.Fatalf(\"node: %v, err: %v\", node, e)\n\t}\n}\n\nfunc BenchmarkGetOne_Sonic(b *testing.B) {\n\tb.SetBytes(int64(len(_TwitterJson)))\n\tast := NewSearcher(_TwitterJson)\n\tfor i := 0; i < b.N; i++ {\n\t\tnode, err := ast.GetByPath(\"statuses\", 3, \"id\")\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t\tx, _ := node.Int64()\n\t\tif x != 249279667666817024 {\n\t\t\tb.Fatal(node.Interface())\n\t\t}\n\t}\n}\n\nfunc BenchmarkGetOneSafe_Sonic(b *testing.B) {\n\tb.SetBytes(int64(len(_TwitterJson)))\n\tast := NewSearcher(_TwitterJson)\n\tast.ConcurrentRead = true\n\tfor i := 0; i < b.N; i++ {\n\t\tnode, err := ast.GetByPath(\"statuses\", 3, \"id\")\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t\tx, _ := node.Int64()\n\t\tif x != 249279667666817024 {\n\t\t\tb.Fatal(node.Interface())\n\t\t}\n\t}\n}\n\nfunc BenchmarkGetFull_Sonic(b *testing.B) {\n\tast := NewSearcher(_TwitterJson)\n\tb.SetBytes(int64(len(_TwitterJson)))\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tnode, err := ast.GetByPath()\n\t\tif err != nil || node.Type() != V_OBJECT {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkGetWithManyCompare_Sonic(b *testing.B) {\n\tb.SetBytes(int64(len(_LotsCompare)))\n\tast := NewSearcher(_LotsCompare)\n\tfor i := 0; i < b.N; i++ {\n\t\tnode, err := ast.GetByPath(\"is\")\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t\tx, _ := node.Int64()\n\t\tif x != 1 {\n\t\t\tb.Fatal(node.Interface())\n\t\t}\n\t}\n}\n\nfunc BenchmarkGetOne_Parallel_Sonic(b *testing.B) {\n\tb.SetBytes(int64(len(_TwitterJson)))\n\tb.RunParallel(func(pb *testing.PB) {\n\t\tast := NewSearcher(_TwitterJson)\n\t\tfor pb.Next() {\n\t\t\tnode, err := ast.GetByPath(\"statuses\", 3, \"id\")\n\t\t\tif err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\t\t\tx, _ := node.Int64()\n\t\t\tif x != 249279667666817024 {\n\t\t\t\tb.Fatal(node.Interface())\n\t\t\t}\n\t\t}\n\t})\n}\n\nfunc BenchmarkGetOneSafe_Parallel_Sonic(b *testing.B) {\n\tb.SetBytes(int64(len(_TwitterJson)))\n\tb.RunParallel(func(pb *testing.PB) {\n\t\tast := NewSearcher(_TwitterJson)\n\t\tast.ConcurrentRead = true\n\t\tfor pb.Next() {\n\t\t\tnode, err := ast.GetByPath(\"statuses\", 3, \"id\")\n\t\t\tif err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\t\t\tx, _ := node.Int64()\n\t\t\tif x != 249279667666817024 {\n\t\t\t\tb.Fatal(node.Interface())\n\t\t\t}\n\t\t}\n\t})\n}\n\nfunc BenchmarkSetOne_Sonic(b *testing.B) {\n\tnode, err := NewSearcher(_TwitterJson).GetByPath(\"statuses\", 3)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\tn := NewNumber(strconv.Itoa(math.MaxInt32))\n\t_, err = node.Set(\"id\", n)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\tb.SetBytes(int64(len(_TwitterJson)))\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tnode, _ := NewSearcher(_TwitterJson).GetByPath(\"statuses\", 3)\n\t\t_, _ = node.Set(\"id\", n)\n\t}\n}\n\nfunc BenchmarkSetOne_Parallel_Sonic(b *testing.B) {\n\tnode, err := NewSearcher(_TwitterJson).GetByPath(\"statuses\", 3)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\tn := NewNumber(strconv.Itoa(math.MaxInt32))\n\t_, err = node.Set(\"id\", n)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\tb.SetBytes(int64(len(_TwitterJson)))\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\tb.RunParallel(func(pb *testing.PB) {\n\t\tfor pb.Next() {\n\t\t\tnode, _ := NewSearcher(_TwitterJson).GetByPath(\"statuses\", 3)\n\t\t\t_, _ = node.Set(\"id\", n)\n\t\t}\n\t})\n}\n" }, { "path": "ast/stubs.go", "content": "/*\n * Copyright 2021 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage ast\n\nimport (\n\t\"unsafe\"\n\n\t\"github.com/bytedance/sonic/internal/rt\"\n)\n\n//go:nosplit\nfunc mem2ptr(s []byte) unsafe.Pointer {\n\treturn (*rt.GoSlice)(unsafe.Pointer(&s)).Ptr\n}\n" }, { "path": "ast/testdata_test.go", "content": "/*\n * Copyright 2021 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage ast\n\nconst _TwitterJson = `{\n \"statuses\": [\n {\n \"coordinates\": null,\n \"favorited\": false,\n \"truncated\": false,\n \"created_at\": \"Mon Sep 24 03:35:21 +0000 2012\",\n \"id_str\": \"250075927172759552\",\n \"entities\": {\n \"urls\": [\n \n ],\n \"hashtags\": [\n {\n \"text\": \"freebandnames\",\n \"indices\": [\n 20,\n 34\n ]\n }\n ],\n \"user_mentions\": [\n \n ]\n },\n \"in_reply_to_user_id_str\": null,\n \"contributors\": null,\n \"text\": \"Aggressive Ponytail #freebandnames\",\n \"metadata\": {\n \"iso_language_code\": \"en\",\n \"result_type\": \"recent\"\n },\n \"retweet_count\": 0,\n \"in_reply_to_status_id_str\": null,\n \"id\": 250075927172759552,\n \"geo\": null,\n \"retweeted\": false,\n \"in_reply_to_user_id\": null,\n \"place\": null,\n \"user\": {\n \"profile_sidebar_fill_color\": \"DDEEF6\",\n \"profile_sidebar_border_color\": \"C0DEED\",\n \"profile_background_tile\": false,\n \"name\": \"Sean Cummings\",\n \"profile_image_url\": \"http://a0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg\",\n \"created_at\": \"Mon Apr 26 06:01:55 +0000 2010\",\n \"location\": \"LA, CA\",\n \"follow_request_sent\": null,\n \"profile_link_color\": \"0084B4\",\n \"is_translator\": false,\n \"id_str\": \"137238150\",\n \"entities\": {\n \"url\": {\n \"urls\": [\n {\n \"expanded_url\": null,\n \"url\": \"\",\n \"indices\": [\n 0,\n 0\n ]\n }\n ]\n },\n \"description\": {\n \"urls\": [\n \n ]\n }\n },\n \"default_profile\": true,\n \"contributors_enabled\": false,\n \"favourites_count\": 0,\n \"url\": null,\n \"profile_image_url_https\": \"https://si0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg\",\n \"utc_offset\": -28800,\n \"id\": 137238150,\n \"profile_use_background_image\": true,\n \"listed_count\": 2,\n \"profile_text_color\": \"333333\",\n \"lang\": \"en\",\n \"followers_count\": 70,\n \"protected\": false,\n \"notifications\": null,\n \"profile_background_image_url_https\": \"https://si0.twimg.com/images/themes/theme1/bg.png\",\n \"profile_background_color\": \"C0DEED\",\n \"verified\": false,\n \"geo_enabled\": true,\n \"time_zone\": \"Pacific Time (US & Canada)\",\n \"description\": \"Born 330 Live 310\",\n \"default_profile_image\": false,\n \"profile_background_image_url\": \"http://a0.twimg.com/images/themes/theme1/bg.png\",\n \"statuses_count\": 579,\n \"friends_count\": 110,\n \"following\": null,\n \"show_all_inline_media\": false,\n \"screen_name\": \"sean_cummings\"\n },\n \"in_reply_to_screen_name\": null,\n \"source\": \"Twitter for Mac\",\n \"in_reply_to_status_id\": null\n },\n {\n \"coordinates\": null,\n \"favorited\": false,\n \"truncated\": false,\n \"created_at\": \"Fri Sep 21 23:40:54 +0000 2012\",\n \"id_str\": \"249292149810667520\",\n \"entities\": {\n \"urls\": [\n \n ],\n \"hashtags\": [\n {\n \"text\": \"FreeBandNames\",\n \"indices\": [\n 20,\n 34\n ]\n }\n ],\n \"user_mentions\": [\n \n ]\n },\n \"in_reply_to_user_id_str\": null,\n \"contributors\": null,\n \"text\": \"Thee Namaste Nerdz. #FreeBandNames\",\n \"metadata\": {\n \"iso_language_code\": \"pl\",\n \"result_type\": \"recent\"\n },\n \"retweet_count\": 0,\n \"in_reply_to_status_id_str\": null,\n \"id\": 249292149810667520,\n \"geo\": null,\n \"retweeted\": false,\n \"in_reply_to_user_id\": null,\n \"place\": null,\n \"user\": {\n \"profile_sidebar_fill_color\": \"DDFFCC\",\n \"profile_sidebar_border_color\": \"BDDCAD\",\n \"profile_background_tile\": true,\n \"name\": \"Chaz Martenstein\",\n \"profile_image_url\": \"http://a0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg\",\n \"created_at\": \"Tue Apr 07 19:05:07 +0000 2009\",\n \"location\": \"Durham, NC\",\n \"follow_request_sent\": null,\n \"profile_link_color\": \"0084B4\",\n \"is_translator\": false,\n \"id_str\": \"29516238\",\n \"entities\": {\n \"url\": {\n \"urls\": [\n {\n \"expanded_url\": null,\n \"url\": \"http://bullcityrecords.com/wnng/\",\n \"indices\": [\n 0,\n 32\n ]\n }\n ]\n },\n \"description\": {\n \"urls\": [\n \n ]\n }\n },\n \"default_profile\": false,\n \"contributors_enabled\": false,\n \"favourites_count\": 8,\n \"url\": \"http://bullcityrecords.com/wnng/\",\n \"profile_image_url_https\": \"https://si0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg\",\n \"utc_offset\": -18000,\n \"id\": 29516238,\n \"profile_use_background_image\": true,\n \"listed_count\": 118,\n \"profile_text_color\": \"333333\",\n \"lang\": \"en\",\n \"followers_count\": 2052,\n \"protected\": false,\n \"notifications\": null,\n \"profile_background_image_url_https\": \"https://si0.twimg.com/profile_background_images/9423277/background_tile.bmp\",\n \"profile_background_color\": \"9AE4E8\",\n \"verified\": false,\n \"geo_enabled\": false,\n \"time_zone\": \"Eastern Time (US & Canada)\",\n \"description\": \"You will come to Durham, North Carolina. I will sell you some records then, here in Durham, North Carolina. Fun will happen.\",\n \"default_profile_image\": false,\n \"profile_background_image_url\": \"http://a0.twimg.com/profile_background_images/9423277/background_tile.bmp\",\n \"statuses_count\": 7579,\n \"friends_count\": 348,\n \"following\": null,\n \"show_all_inline_media\": true,\n \"screen_name\": \"bullcityrecords\"\n },\n \"in_reply_to_screen_name\": null,\n \"source\": \"web\",\n \"in_reply_to_status_id\": null\n },\n {\n \"coordinates\": null,\n \"favorited\": false,\n \"truncated\": false,\n \"created_at\": \"Fri Sep 21 23:30:20 +0000 2012\",\n \"id_str\": \"249289491129438208\",\n \"entities\": {\n \"urls\": [\n \n ],\n \"hashtags\": [\n {\n \"text\": \"freebandnames\",\n \"indices\": [\n 29,\n 43\n ]\n }\n ],\n \"user_mentions\": [\n \n ]\n },\n \"in_reply_to_user_id_str\": null,\n \"contributors\": null,\n \"text\": \"Mexican Heaven, Mexican Hell #freebandnames\",\n \"metadata\": {\n \"iso_language_code\": \"en\",\n \"result_type\": \"recent\"\n },\n \"retweet_count\": 0,\n \"in_reply_to_status_id_str\": null,\n \"id\": 249289491129438208,\n \"geo\": null,\n \"retweeted\": false,\n \"in_reply_to_user_id\": null,\n \"place\": null,\n \"user\": {\n \"profile_sidebar_fill_color\": \"99CC33\",\n \"profile_sidebar_border_color\": \"829D5E\",\n \"profile_background_tile\": false,\n \"name\": \"Thomas John Wakeman\",\n \"profile_image_url\": \"http://a0.twimg.com/profile_images/2219333930/Froggystyle_normal.png\",\n \"created_at\": \"Tue Sep 01 21:21:35 +0000 2009\",\n \"location\": \"Kingston New York\",\n \"follow_request_sent\": null,\n \"profile_link_color\": \"D02B55\",\n \"is_translator\": false,\n \"id_str\": \"70789458\",\n \"entities\": {\n \"url\": {\n \"urls\": [\n {\n \"expanded_url\": null,\n \"url\": \"\",\n \"indices\": [\n 0,\n 0\n ]\n }\n ]\n },\n \"description\": {\n \"urls\": [\n \n ]\n }\n },\n \"default_profile\": false,\n \"contributors_enabled\": false,\n \"favourites_count\": 19,\n \"url\": null,\n \"profile_image_url_https\": \"https://si0.twimg.com/profile_images/2219333930/Froggystyle_normal.png\",\n \"utc_offset\": -18000,\n \"id\": 70789458,\n \"profile_use_background_image\": true,\n \"listed_count\": 1,\n \"profile_text_color\": \"3E4415\",\n \"lang\": \"en\",\n \"followers_count\": 63,\n \"protected\": false,\n \"notifications\": null,\n \"profile_background_image_url_https\": \"https://si0.twimg.com/images/themes/theme5/bg.gif\",\n \"profile_background_color\": \"352726\",\n \"verified\": false,\n \"geo_enabled\": false,\n \"time_zone\": \"Eastern Time (US & Canada)\",\n \"description\": \"Science Fiction Writer, sort of. Likes Superheroes, Mole People, Alt. Timelines.\",\n \"default_profile_image\": false,\n \"profile_background_image_url\": \"http://a0.twimg.com/images/themes/theme5/bg.gif\",\n \"statuses_count\": 1048,\n \"friends_count\": 63,\n \"following\": null,\n \"show_all_inline_media\": false,\n \"screen_name\": \"MonkiesFist\"\n },\n \"in_reply_to_screen_name\": null,\n \"source\": \"web\",\n \"in_reply_to_status_id\": null\n },\n {\n \"coordinates\": null,\n \"favorited\": false,\n \"truncated\": false,\n \"created_at\": \"Fri Sep 21 22:51:18 +0000 2012\",\n \"id_str\": \"249279667666817024\",\n \"entities\": {\n \"urls\": [\n \n ],\n \"hashtags\": [\n {\n \"text\": \"freebandnames\",\n \"indices\": [\n 20,\n 34\n ]\n }\n ],\n \"user_mentions\": [\n \n ]\n },\n \"in_reply_to_user_id_str\": null,\n \"contributors\": null,\n \"text\": \"The Foolish Mortals #freebandnames\",\n \"metadata\": {\n \"iso_language_code\": \"en\",\n \"result_type\": \"recent\"\n },\n \"retweet_count\": 0,\n \"in_reply_to_status_id_str\": null,\n \"id\": 249279667666817024,\n \"geo\": null,\n \"retweeted\": false,\n \"in_reply_to_user_id\": null,\n \"place\": null,\n \"user\": {\n \"profile_sidebar_fill_color\": \"BFAC83\",\n \"profile_sidebar_border_color\": \"615A44\",\n \"profile_background_tile\": true,\n \"name\": \"Marty Elmer\",\n \"profile_image_url\": \"http://a0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png\",\n \"created_at\": \"Mon May 04 00:05:00 +0000 2009\",\n \"location\": \"Wisconsin, USA\",\n \"follow_request_sent\": null,\n \"profile_link_color\": \"3B2A26\",\n \"is_translator\": false,\n \"id_str\": \"37539828\",\n \"entities\": {\n \"url\": {\n \"urls\": [\n {\n \"expanded_url\": null,\n \"url\": \"http://www.omnitarian.me\",\n \"indices\": [\n 0,\n 24\n ]\n }\n ]\n },\n \"description\": {\n \"urls\": [\n \n ]\n }\n },\n \"default_profile\": false,\n \"contributors_enabled\": false,\n \"favourites_count\": 647,\n \"url\": \"http://www.omnitarian.me\",\n \"profile_image_url_https\": \"https://si0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png\",\n \"utc_offset\": -21600,\n \"id\": 37539828,\n \"profile_use_background_image\": true,\n \"listed_count\": 52,\n \"profile_text_color\": \"000000\",\n \"lang\": \"en\",\n \"followers_count\": 608,\n \"protected\": false,\n \"notifications\": null,\n \"profile_background_image_url_https\": \"https://si0.twimg.com/profile_background_images/106455659/rect6056-9.png\",\n \"profile_background_color\": \"EEE3C4\",\n \"verified\": false,\n \"geo_enabled\": false,\n \"time_zone\": \"Central Time (US & Canada)\",\n \"description\": \"Cartoonist, Illustrator, and T-Shirt connoisseur\",\n \"default_profile_image\": false,\n \"profile_background_image_url\": \"http://a0.twimg.com/profile_background_images/106455659/rect6056-9.png\",\n \"statuses_count\": 3575,\n \"friends_count\": 249,\n \"following\": null,\n \"show_all_inline_media\": true,\n \"screen_name\": \"Omnitarian\"\n },\n \"in_reply_to_screen_name\": null,\n \"source\": \"Twitter for iPhone\",\n \"in_reply_to_status_id\": null\n }\n ],\n \"search_metadata\": {\n \"max_id\": 250126199840518145,\n \"since_id\": 24012619984051000,\n \"refresh_url\": \"?since_id=250126199840518145&q=%23freebandnames&result_type=mixed&include_entities=1\",\n \"next_results\": \"?max_id=249279667666817023&q=%23freebandnames&count=4&include_entities=1&result_type=mixed\",\n \"count\": 4,\n \"completed_in\": 0.035,\n \"since_id_str\": \"24012619984051000\",\n \"query\": \"%23freebandnames\",\n \"max_id_str\": \"250126199840518145\"\n }\n}`\n\nconst _LotsCompare = `{\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"hi\":0,\"is\":1}`\n\ntype _TwitterStruct struct {\n\tStatuses []struct {\n\t\tCoordinates interface{} `json:\"coordinates\"`\n\t\tFavorited bool `json:\"favorited\"`\n\t\tTruncated bool `json:\"truncated\"`\n\t\tCreatedAt string `json:\"created_at\"`\n\t\tIDStr string `json:\"id_str\"`\n\t\tEntities struct {\n\t\t\tUrls []interface{} `json:\"urls\"`\n\t\t\tHashtags []struct {\n\t\t\t\tText string `json:\"text\"`\n\t\t\t\tIndices []int `json:\"indices\"`\n\t\t\t} `json:\"hashtags\"`\n\t\t\tUserMentions []interface{} `json:\"user_mentions\"`\n\t\t} `json:\"entities\"`\n\t\tInReplyToUserIDStr interface{} `json:\"in_reply_to_user_id_str\"`\n\t\tContributors interface{} `json:\"contributors\"`\n\t\tText string `json:\"text\"`\n\t\tMetadata struct {\n\t\t\tIsoLanguageCode string `json:\"iso_language_code\"`\n\t\t\tResultType string `json:\"result_type\"`\n\t\t} `json:\"metadata\"`\n\t\tRetweetCount int `json:\"retweet_count\"`\n\t\tInReplyToStatusIDStr interface{} `json:\"in_reply_to_status_id_str\"`\n\t\tID int64 `json:\"id\"`\n\t\tGeo interface{} `json:\"geo\"`\n\t\tRetweeted bool `json:\"retweeted\"`\n\t\tInReplyToUserID interface{} `json:\"in_reply_to_user_id\"`\n\t\tPlace interface{} `json:\"place\"`\n\t\tUser struct {\n\t\t\tProfileSidebarFillColor string `json:\"profile_sidebar_fill_color\"`\n\t\t\tProfileSidebarBorderColor string `json:\"profile_sidebar_border_color\"`\n\t\t\tProfileBackgroundTile bool `json:\"profile_background_tile\"`\n\t\t\tName string `json:\"name\"`\n\t\t\tProfileImageURL string `json:\"profile_image_url\"`\n\t\t\tCreatedAt string `json:\"created_at\"`\n\t\t\tLocation string `json:\"location\"`\n\t\t\tFollowRequestSent interface{} `json:\"follow_request_sent\"`\n\t\t\tProfileLinkColor string `json:\"profile_link_color\"`\n\t\t\tIsTranslator bool `json:\"is_translator\"`\n\t\t\tIDStr string `json:\"id_str\"`\n\t\t\tEntities struct {\n\t\t\t\tURL struct {\n\t\t\t\t\tUrls []struct {\n\t\t\t\t\t\tExpandedURL interface{} `json:\"expanded_url\"`\n\t\t\t\t\t\tURL string `json:\"url\"`\n\t\t\t\t\t\tIndices []int `json:\"indices\"`\n\t\t\t\t\t} `json:\"urls\"`\n\t\t\t\t} `json:\"url\"`\n\t\t\t\tDescription struct {\n\t\t\t\t\tUrls []interface{} `json:\"urls\"`\n\t\t\t\t} `json:\"description\"`\n\t\t\t} `json:\"entities\"`\n\t\t\tDefaultProfile bool `json:\"default_profile\"`\n\t\t\tContributorsEnabled bool `json:\"contributors_enabled\"`\n\t\t\tFavouritesCount int `json:\"favourites_count\"`\n\t\t\tURL interface{} `json:\"url\"`\n\t\t\tProfileImageURLHTTPS string `json:\"profile_image_url_https\"`\n\t\t\tUtcOffset int `json:\"utc_offset\"`\n\t\t\tID int `json:\"id\"`\n\t\t\tProfileUseBackgroundImage bool `json:\"profile_use_background_image\"`\n\t\t\tListedCount int `json:\"listed_count\"`\n\t\t\tProfileTextColor string `json:\"profile_text_color\"`\n\t\t\tLang string `json:\"lang\"`\n\t\t\tFollowersCount int `json:\"followers_count\"`\n\t\t\tProtected bool `json:\"protected\"`\n\t\t\tNotifications interface{} `json:\"notifications\"`\n\t\t\tProfileBackgroundImageURLHTTPS string `json:\"profile_background_image_url_https\"`\n\t\t\tProfileBackgroundColor string `json:\"profile_background_color\"`\n\t\t\tVerified bool `json:\"verified\"`\n\t\t\tGeoEnabled bool `json:\"geo_enabled\"`\n\t\t\tTimeZone string `json:\"time_zone\"`\n\t\t\tDescription string `json:\"description\"`\n\t\t\tDefaultProfileImage bool `json:\"default_profile_image\"`\n\t\t\tProfileBackgroundImageURL string `json:\"profile_background_image_url\"`\n\t\t\tStatusesCount int `json:\"statuses_count\"`\n\t\t\tFriendsCount int `json:\"friends_count\"`\n\t\t\tFollowing interface{} `json:\"following\"`\n\t\t\tShowAllInlineMedia bool `json:\"show_all_inline_media\"`\n\t\t\tScreenName string `json:\"screen_name\"`\n\t\t} `json:\"user\"`\n\t\tInReplyToScreenName interface{} `json:\"in_reply_to_screen_name\"`\n\t\tSource string `json:\"source\"`\n\t\tInReplyToStatusID interface{} `json:\"in_reply_to_status_id\"`\n\t} `json:\"statuses\"`\n\tSearchMetadata struct {\n\t\tMaxID int64 `json:\"max_id\"`\n\t\tSinceID int64 `json:\"since_id\"`\n\t\tRefreshURL string `json:\"refresh_url\"`\n\t\tNextResults string `json:\"next_results\"`\n\t\tCount int `json:\"count\"`\n\t\tCompletedIn float64 `json:\"completed_in\"`\n\t\tSinceIDStr string `json:\"since_id_str\"`\n\t\tQuery string `json:\"query\"`\n\t\tMaxIDStr string `json:\"max_id_str\"`\n\t} `json:\"search_metadata\"`\n}\n" }, { "path": "ast/visitor.go", "content": "/*\n * Copyright 2021 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage ast\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\n\t\"github.com/bytedance/sonic/internal/native/types\"\n\t\"github.com/bytedance/sonic/unquote\"\n)\n\n// Visitor handles the callbacks during preorder traversal of a JSON AST.\n//\n// According to the JSON RFC8259, a JSON AST can be defined by\n// the following rules without separator / whitespace tokens.\n//\n//\tJSON-AST = value\n//\tvalue = false / null / true / object / array / number / string\n//\tobject = begin-object [ member *( member ) ] end-object\n//\tmember = string value\n//\tarray = begin-array [ value *( value ) ] end-array\ntype Visitor interface {\n\n\t// OnNull handles a JSON null value.\n\tOnNull() error\n\n\t// OnBool handles a JSON true / false value.\n\tOnBool(v bool) error\n\n\t// OnString handles a JSON string value.\n\tOnString(v string) error\n\n\t// OnInt64 handles a JSON number value with int64 type.\n\tOnInt64(v int64, n json.Number) error\n\n\t// OnFloat64 handles a JSON number value with float64 type.\n\tOnFloat64(v float64, n json.Number) error\n\n\t// OnObjectBegin handles the beginning of a JSON object value with a\n\t// suggested capacity that can be used to make your custom object container.\n\t//\n\t// After this point the visitor will receive a sequence of callbacks like\n\t// [string, value, string, value, ......, ObjectEnd].\n\t//\n\t// Note:\n\t// 1. This is a recursive definition which means the value can\n\t// also be a JSON object / array described by a sequence of callbacks.\n\t// 2. The suggested capacity will be 0 if current object is empty.\n\t// 3. Currently sonic use a fixed capacity for non-empty object (keep in\n\t// sync with ast.Node) which might not be very suitable. This may be\n\t// improved in future version.\n\tOnObjectBegin(capacity int) error\n\n\t// OnObjectKey handles a JSON object key string in member.\n\tOnObjectKey(key string) error\n\n\t// OnObjectEnd handles the ending of a JSON object value.\n\tOnObjectEnd() error\n\n\t// OnArrayBegin handles the beginning of a JSON array value with a\n\t// suggested capacity that can be used to make your custom array container.\n\t//\n\t// After this point the visitor will receive a sequence of callbacks like\n\t// [value, value, value, ......, ArrayEnd].\n\t//\n\t// Note:\n\t// 1. This is a recursive definition which means the value can\n\t// also be a JSON object / array described by a sequence of callbacks.\n\t// 2. The suggested capacity will be 0 if current array is empty.\n\t// 3. Currently sonic use a fixed capacity for non-empty array (keep in\n\t// sync with ast.Node) which might not be very suitable. This may be\n\t// improved in future version.\n\tOnArrayBegin(capacity int) error\n\n\t// OnArrayEnd handles the ending of a JSON array value.\n\tOnArrayEnd() error\n}\n\n// VisitorOptions contains all Visitor's options. The default value is an\n// empty VisitorOptions{}.\ntype VisitorOptions struct {\n\t// OnlyNumber indicates parser to directly return number value without\n\t// conversion, then the first argument of OnInt64 / OnFloat64 will always\n\t// be zero.\n\tOnlyNumber bool\n}\n\nvar defaultVisitorOptions = &VisitorOptions{}\n\n// Preorder decodes the whole JSON string and callbacks each AST node to visitor\n// during preorder traversal. Any visitor method with an error returned will\n// break the traversal and the given error will be directly returned. The opts\n// argument can be reused after every call.\nfunc Preorder(str string, visitor Visitor, opts *VisitorOptions) error {\n\tif opts == nil {\n\t\topts = defaultVisitorOptions\n\t}\n\t// process VisitorOptions first to guarantee that all options will be\n\t// constant during decoding and make options more readable.\n\tvar (\n\t\toptDecodeNumber = !opts.OnlyNumber\n\t)\n\n\ttv := &traverser{\n\t\tparser: Parser{\n\t\t\ts: str,\n\t\t\tnoLazy: true,\n\t\t\tskipValue: false,\n\t\t},\n\t\tvisitor: visitor,\n\t}\n\n\tif optDecodeNumber {\n\t\ttv.parser.decodeNumber(true)\n\t}\n\n\terr := tv.decodeValue()\n\n\tif optDecodeNumber {\n\t\ttv.parser.decodeNumber(false)\n\t}\n\treturn err\n}\n\ntype traverser struct {\n\tparser Parser\n\tvisitor Visitor\n}\n\n// NOTE: keep in sync with (*Parser).Parse method.\nfunc (self *traverser) decodeValue() error {\n\tswitch val := self.parser.decodeValue(); val.Vt {\n\tcase types.V_EOF:\n\t\treturn types.ERR_EOF\n\tcase types.V_NULL:\n\t\treturn self.visitor.OnNull()\n\tcase types.V_TRUE:\n\t\treturn self.visitor.OnBool(true)\n\tcase types.V_FALSE:\n\t\treturn self.visitor.OnBool(false)\n\tcase types.V_STRING:\n\t\treturn self.decodeString(val.Iv, val.Ep)\n\tcase types.V_DOUBLE:\n\t\treturn self.visitor.OnFloat64(val.Dv,\n\t\t\tjson.Number(self.parser.s[val.Ep:self.parser.p]))\n\tcase types.V_INTEGER:\n\t\treturn self.visitor.OnInt64(val.Iv,\n\t\t\tjson.Number(self.parser.s[val.Ep:self.parser.p]))\n\tcase types.V_ARRAY:\n\t\treturn self.decodeArray()\n\tcase types.V_OBJECT:\n\t\treturn self.decodeObject()\n\tdefault:\n\t\treturn types.ParsingError(-val.Vt)\n\t}\n}\n\n// NOTE: keep in sync with (*Parser).decodeArray method.\nfunc (self *traverser) decodeArray() error {\n\tsp := self.parser.p\n\tns := len(self.parser.s)\n\n\t/* allocate array space and parse every element */\n\tif err := self.visitor.OnArrayBegin(_DEFAULT_NODE_CAP); err != nil {\n\t\tif err == VisitOPSkip {\n\t\t\t// NOTICE: for user needs to skip entry object\n\t\t\tself.parser.p -= 1\n\t\t\tif _, e := self.parser.skipFast(); e != 0 {\n\t\t\t\treturn e\n\t\t\t}\n\t\t\treturn self.visitor.OnArrayEnd()\n\t\t}\n\t\treturn err\n\t}\n\n\t/* check for EOF */\n\tself.parser.p = self.parser.lspace(sp)\n\tif self.parser.p >= ns {\n\t\treturn types.ERR_EOF\n\t}\n\n\t/* check for empty array */\n\tif self.parser.s[self.parser.p] == ']' {\n\t\tself.parser.p++\n\t\treturn self.visitor.OnArrayEnd()\n\t}\n\n\tfor {\n\t\t/* decode the value */\n\t\tif err := self.decodeValue(); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tself.parser.p = self.parser.lspace(self.parser.p)\n\n\t\t/* check for EOF */\n\t\tif self.parser.p >= ns {\n\t\t\treturn types.ERR_EOF\n\t\t}\n\n\t\t/* check for the next character */\n\t\tswitch self.parser.s[self.parser.p] {\n\t\tcase ',':\n\t\t\tself.parser.p++\n\t\tcase ']':\n\t\t\tself.parser.p++\n\t\t\treturn self.visitor.OnArrayEnd()\n\t\tdefault:\n\t\t\treturn types.ERR_INVALID_CHAR\n\t\t}\n\t}\n}\n\n// NOTE: keep in sync with (*Parser).decodeObject method.\nfunc (self *traverser) decodeObject() error {\n\tsp := self.parser.p\n\tns := len(self.parser.s)\n\n\t/* allocate object space and decode each pair */\n\tif err := self.visitor.OnObjectBegin(_DEFAULT_NODE_CAP); err != nil {\n\t\tif err == VisitOPSkip {\n\t\t\t// NOTICE: for user needs to skip entry object\n\t\t\tself.parser.p -= 1\n\t\t\tif _, e := self.parser.skipFast(); e != 0 {\n\t\t\t\treturn e\n\t\t\t}\n\t\t\treturn self.visitor.OnObjectEnd()\n\t\t}\n\t\treturn err\n\t}\n\n\t/* check for EOF */\n\tself.parser.p = self.parser.lspace(sp)\n\tif self.parser.p >= ns {\n\t\treturn types.ERR_EOF\n\t}\n\n\t/* check for empty object */\n\tif self.parser.s[self.parser.p] == '}' {\n\t\tself.parser.p++\n\t\treturn self.visitor.OnObjectEnd()\n\t}\n\n\tfor {\n\t\tvar njs types.JsonState\n\t\tvar err types.ParsingError\n\n\t\t/* decode the key */\n\t\tif njs = self.parser.decodeValue(); njs.Vt != types.V_STRING {\n\t\t\treturn types.ERR_INVALID_CHAR\n\t\t}\n\n\t\t/* extract the key */\n\t\tidx := self.parser.p - 1\n\t\tkey := self.parser.s[njs.Iv:idx]\n\n\t\t/* check for escape sequence */\n\t\tif njs.Ep != -1 {\n\t\t\tif key, err = unquote.String(key); err != 0 {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\tif err := self.visitor.OnObjectKey(key); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\t/* expect a ':' delimiter */\n\t\tif err = self.parser.delim(); err != 0 {\n\t\t\treturn err\n\t\t}\n\n\t\t/* decode the value */\n\t\tif err := self.decodeValue(); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tself.parser.p = self.parser.lspace(self.parser.p)\n\n\t\t/* check for EOF */\n\t\tif self.parser.p >= ns {\n\t\t\treturn types.ERR_EOF\n\t\t}\n\n\t\t/* check for the next character */\n\t\tswitch self.parser.s[self.parser.p] {\n\t\tcase ',':\n\t\t\tself.parser.p++\n\t\tcase '}':\n\t\t\tself.parser.p++\n\t\t\treturn self.visitor.OnObjectEnd()\n\t\tdefault:\n\t\t\treturn types.ERR_INVALID_CHAR\n\t\t}\n\t}\n}\n\n// NOTE: keep in sync with (*Parser).decodeString method.\nfunc (self *traverser) decodeString(iv int64, ep int) error {\n\tp := self.parser.p - 1\n\ts := self.parser.s[iv:p]\n\n\t/* fast path: no escape sequence */\n\tif ep == -1 {\n\t\treturn self.visitor.OnString(s)\n\t}\n\n\t/* unquote the string */\n\tout, err := unquote.String(s)\n\tif err != 0 {\n\t\treturn err\n\t}\n\treturn self.visitor.OnString(out)\n}\n\n// If visitor return this error on `OnObjectBegin()` or `OnArrayBegin()`,\n// the traverser will skip entry object or array\nvar VisitOPSkip = errors.New(\"\")\n" }, { "path": "ast/visitor_test.go", "content": "/*\n * Copyright 2021 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage ast\n\nimport (\n\t\"bufio\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"sort\"\n\t\"strings\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nvar visitorTestCases = []struct {\n\tname string\n\tjsonStr string\n}{\n\t{\"default\", _TwitterJson},\n\t{\"issue_case01\", \"[1193.6419677734375]\"},\n\t{\"issue653\", `{\"v0\": 0, \"m0\": {}, \"v1\": 1, \"a0\": [], \"v2\": 2}`},\n}\n\ntype visitorNodeDiffTest struct {\n\tt *testing.T\n\tstr string\n\n\ttracer io.Writer\n\n\tcursor Node\n\tstk visitorNodeStack\n\tsp uint8\n}\n\ntype visitorNodeStack = [256]struct {\n\tNode Node\n\tObject map[string]Node\n\tArray []Node\n\n\tObjectKey string\n}\n\nfunc (self *visitorNodeDiffTest) incrSP() {\n\tself.t.Helper()\n\tself.sp++\n\trequire.NotZero(self.t, self.sp, \"stack overflow\")\n}\n\nfunc (self *visitorNodeDiffTest) debugStack() string {\n\tvar buf strings.Builder\n\tbuf.WriteString(\"[\")\n\tfor i := uint8(0); i < self.sp; i++ {\n\t\tif i != 0 {\n\t\t\tbuf.WriteString(\", \")\n\t\t}\n\t\tif self.stk[i].Array != nil {\n\t\t\tbuf.WriteString(\"Array\")\n\t\t} else if self.stk[i].Object != nil {\n\t\t\tbuf.WriteString(\"Object\")\n\t\t} else {\n\t\t\tfmt.Fprintf(&buf, \"Key(%q)\", self.stk[i].ObjectKey)\n\t\t}\n\t}\n\tbuf.WriteString(\"]\")\n\treturn buf.String()\n}\n\nfunc (self *visitorNodeDiffTest) requireType(got int) {\n\tself.t.Helper()\n\twant := self.cursor.Type()\n\trequire.EqualValues(self.t, want, got)\n}\n\nfunc (self *visitorNodeDiffTest) toArrayIndex(array Node, i int) {\n\t// set cursor to next Value if existed\n\tself.t.Helper()\n\tn, err := array.Len()\n\trequire.NoError(self.t, err)\n\tif i < n {\n\t\tself.cursor = *array.Index(i)\n\t\trequire.NoError(self.t, self.cursor.Check())\n\t}\n}\n\nfunc (self *visitorNodeDiffTest) onValueEnd() {\n\tif self.tracer != nil {\n\t\tfmt.Fprintf(self.tracer, \"OnValueEnd: %s\\n\", self.debugStack())\n\t}\n\t// cursor should point to the Value now\n\tself.t.Helper()\n\tif self.sp == 0 {\n\t\tif self.tracer != nil {\n\t\t\tfmt.Fprintf(self.tracer, \"EOF\\n\\n\")\n\t\t}\n\t\treturn\n\t}\n\t// [..., Array, sp]\n\tif array := self.stk[self.sp-1].Array; array != nil {\n\t\tarray = append(array, self.cursor)\n\t\tself.stk[self.sp-1].Array = array\n\t\tself.toArrayIndex(self.stk[self.sp-1].Node, len(array))\n\t\treturn\n\t}\n\t// [..., Object, ObjectKey, sp]\n\trequire.GreaterOrEqual(self.t, self.sp, uint8(2))\n\trequire.NotNil(self.t, self.stk[self.sp-2].Object)\n\trequire.Nil(self.t, self.stk[self.sp-1].Object)\n\trequire.Nil(self.t, self.stk[self.sp-1].Array)\n\tself.stk[self.sp-2].Object[self.stk[self.sp-1].ObjectKey] = self.cursor\n\tself.cursor = self.stk[self.sp-2].Node // reset cursor to Object\n\tself.sp-- // pop ObjectKey\n}\n\nfunc (self *visitorNodeDiffTest) OnNull() error {\n\tif self.tracer != nil {\n\t\tfmt.Fprintf(self.tracer, \"OnNull\\n\")\n\t}\n\tself.requireType(V_NULL)\n\tself.onValueEnd()\n\treturn nil\n}\n\nfunc (self *visitorNodeDiffTest) OnBool(v bool) error {\n\tif self.tracer != nil {\n\t\tfmt.Fprintf(self.tracer, \"OnBool: %t\\n\", v)\n\t}\n\tif v {\n\t\tself.requireType(V_TRUE)\n\t} else {\n\t\tself.requireType(V_FALSE)\n\t}\n\tself.onValueEnd()\n\treturn nil\n}\n\nfunc (self *visitorNodeDiffTest) OnString(v string) error {\n\tif self.tracer != nil {\n\t\tfmt.Fprintf(self.tracer, \"OnString: %q\\n\", v)\n\t}\n\tself.requireType(V_STRING)\n\twant, err := self.cursor.StrictString()\n\trequire.NoError(self.t, err)\n\trequire.EqualValues(self.t, want, v)\n\tself.onValueEnd()\n\treturn nil\n}\n\nfunc (self *visitorNodeDiffTest) OnInt64(v int64, n json.Number) error {\n\tif self.tracer != nil {\n\t\tfmt.Fprintf(self.tracer, \"OnInt64: %d (%q)\\n\", v, n)\n\t}\n\tself.requireType(V_NUMBER)\n\twant, err := self.cursor.StrictInt64()\n\trequire.NoError(self.t, err)\n\trequire.EqualValues(self.t, want, v)\n\tnv, err := n.Int64()\n\trequire.NoError(self.t, err)\n\trequire.EqualValues(self.t, want, nv)\n\tself.onValueEnd()\n\treturn nil\n}\n\nfunc (self *visitorNodeDiffTest) OnFloat64(v float64, n json.Number) error {\n\tif self.tracer != nil {\n\t\tfmt.Fprintf(self.tracer, \"OnFloat64: %f (%q)\\n\", v, n)\n\t}\n\tself.requireType(V_NUMBER)\n\twant, err := self.cursor.StrictFloat64()\n\trequire.NoError(self.t, err)\n\trequire.EqualValues(self.t, want, v)\n\tnv, err := n.Float64()\n\trequire.NoError(self.t, err)\n\trequire.EqualValues(self.t, want, nv)\n\tself.onValueEnd()\n\treturn nil\n}\n\nfunc (self *visitorNodeDiffTest) OnObjectBegin(capacity int) error {\n\tif self.tracer != nil {\n\t\tfmt.Fprintf(self.tracer, \"OnObjectBegin: %d\\n\", capacity)\n\t}\n\tself.requireType(V_OBJECT)\n\tself.stk[self.sp].Node = self.cursor\n\tself.stk[self.sp].Object = make(map[string]Node, capacity)\n\tself.incrSP()\n\treturn nil\n}\n\nfunc (self *visitorNodeDiffTest) OnObjectKey(key string) error {\n\tif self.tracer != nil {\n\t\tfmt.Fprintf(self.tracer, \"OnObjectKey: %q %s\\n\", key, self.debugStack())\n\t}\n\trequire.NotNil(self.t, self.stk[self.sp-1].Object)\n\tnode := self.stk[self.sp-1].Node\n\tself.stk[self.sp].ObjectKey = key\n\tself.incrSP()\n\tself.cursor = *node.Get(key) // set cursor to Value\n\trequire.NoError(self.t, self.cursor.Check())\n\treturn nil\n}\n\nfunc (self *visitorNodeDiffTest) OnObjectEnd() error {\n\tif self.tracer != nil {\n\t\tfmt.Fprintf(self.tracer, \"OnObjectEnd\\n\")\n\t}\n\tobject := self.stk[self.sp-1].Object\n\trequire.NotNil(self.t, object)\n\n\tnode := self.stk[self.sp-1].Node\n\tpairs, err := node.MapUseNode()\n\trequire.NoError(self.t, err)\n\n\tkeysGot := make([]string, 0, len(object))\n\tfor key := range object {\n\t\tkeysGot = append(keysGot, key)\n\t}\n\tkeysWant := make([]string, 0, len(pairs))\n\tfor key := range pairs {\n\t\tkeysWant = append(keysWant, key)\n\t}\n\tsort.Strings(keysGot)\n\tsort.Strings(keysWant)\n\trequire.EqualValues(self.t, keysWant, keysGot)\n\n\tfor key, pair := range pairs {\n\t\ttypeGot := object[key].Type()\n\t\ttypeWant := pair.Type()\n\t\trequire.EqualValues(self.t, typeWant, typeGot)\n\t}\n\n\t// pop Object\n\tself.sp--\n\tself.stk[self.sp].Node = Node{}\n\tself.stk[self.sp].Object = nil\n\n\tself.cursor = node // set cursor to this Object\n\tself.onValueEnd()\n\treturn nil\n}\n\nfunc (self *visitorNodeDiffTest) OnArrayBegin(capacity int) error {\n\tif self.tracer != nil {\n\t\tfmt.Fprintf(self.tracer, \"OnArrayBegin: %d\\n\", capacity)\n\t}\n\tself.requireType(V_ARRAY)\n\tself.stk[self.sp].Node = self.cursor\n\tself.stk[self.sp].Array = make([]Node, 0, capacity)\n\tself.incrSP()\n\tself.toArrayIndex(self.stk[self.sp-1].Node, 0)\n\treturn nil\n}\n\nfunc (self *visitorNodeDiffTest) OnArrayEnd() error {\n\tif self.tracer != nil {\n\t\tfmt.Fprintf(self.tracer, \"OnArrayEnd\\n\")\n\t}\n\tarray := self.stk[self.sp-1].Array\n\trequire.NotNil(self.t, array)\n\n\tnode := self.stk[self.sp-1].Node\n\tvalues, err := node.ArrayUseNode()\n\trequire.NoError(self.t, err)\n\n\trequire.EqualValues(self.t, len(values), len(array))\n\n\tfor i, n := 0, len(values); i < n; i++ {\n\t\ttypeGot := array[i].Type()\n\t\ttypeWant := values[i].Type()\n\t\trequire.EqualValues(self.t, typeWant, typeGot)\n\t}\n\n\t// pop Array\n\tself.sp--\n\tself.stk[self.sp].Node = Node{}\n\tself.stk[self.sp].Array = nil\n\n\tself.cursor = node // set cursor to this Array\n\tself.onValueEnd()\n\treturn nil\n}\n\nfunc (self *visitorNodeDiffTest) Run(t *testing.T, str string,\n\ttracer io.Writer) {\n\tself.t = t\n\tself.str = str\n\tself.tracer = tracer\n\n\tself.t.Helper()\n\n\tself.cursor = NewRaw(self.str)\n\trequire.NoError(self.t, self.cursor.LoadAll())\n\n\tself.stk = visitorNodeStack{}\n\tself.sp = 0\n\n\trequire.NoError(self.t, Preorder(self.str, self, nil))\n}\n\nfunc TestVisitor_NodeDiff(t *testing.T) {\n\tvar suite visitorNodeDiffTest\n\n\tnewTracer := func(t *testing.T) io.Writer {\n\t\tconst EnableTracer = false\n\t\tif !EnableTracer {\n\t\t\treturn nil\n\t\t}\n\t\tbasename := strings.ReplaceAll(t.Name(), \"/\", \"_\")\n\t\tfp, err := os.Create(fmt.Sprintf(\"../output/%s.log\", basename))\n\t\trequire.NoError(t, err)\n\t\twriter := bufio.NewWriter(fp)\n\t\tt.Cleanup(func() {\n\t\t\t_ = writer.Flush()\n\t\t\t_ = fp.Close()\n\t\t})\n\t\treturn writer\n\t}\n\n\tfor _, c := range visitorTestCases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\tsuite.Run(t, c.jsonStr, newTracer(t))\n\t\t})\n\t}\n}\n\ntype visitorUserNode interface {\n\tUserNode()\n}\n\ntype (\n\tvisitorUserNull struct{}\n\tvisitorUserBool struct{ Value bool }\n\tvisitorUserInt64 struct{ Value int64 }\n\tvisitorUserFloat64 struct{ Value float64 }\n\tvisitorUserString struct{ Value string }\n\tvisitorUserObject struct{ Value map[string]visitorUserNode }\n\tvisitorUserArray struct{ Value []visitorUserNode }\n)\n\nfunc (*visitorUserNull) UserNode() {}\nfunc (*visitorUserBool) UserNode() {}\nfunc (*visitorUserInt64) UserNode() {}\nfunc (*visitorUserFloat64) UserNode() {}\nfunc (*visitorUserString) UserNode() {}\nfunc (*visitorUserObject) UserNode() {}\nfunc (*visitorUserArray) UserNode() {}\n\nfunc compareUserNode(tb testing.TB, lhs, rhs visitorUserNode) bool {\n\tswitch lhs := lhs.(type) {\n\tcase *visitorUserNull:\n\t\t_, ok := rhs.(*visitorUserNull)\n\t\treturn assert.True(tb, ok)\n\tcase *visitorUserBool:\n\t\trhs, ok := rhs.(*visitorUserBool)\n\t\treturn assert.True(tb, ok) && assert.Equal(tb, lhs.Value, rhs.Value)\n\tcase *visitorUserInt64:\n\t\trhs, ok := rhs.(*visitorUserInt64)\n\t\treturn assert.True(tb, ok) && assert.Equal(tb, lhs.Value, rhs.Value)\n\tcase *visitorUserFloat64:\n\t\trhs, ok := rhs.(*visitorUserFloat64)\n\t\treturn assert.True(tb, ok) && assert.Equal(tb, lhs.Value, rhs.Value)\n\tcase *visitorUserString:\n\t\trhs, ok := rhs.(*visitorUserString)\n\t\treturn assert.True(tb, ok) && assert.Equal(tb, lhs.Value, rhs.Value)\n\tcase *visitorUserObject:\n\t\trhs, ok := rhs.(*visitorUserObject)\n\t\tif !(assert.True(tb, ok) && assert.Equal(tb, len(lhs.Value), len(rhs.Value))) {\n\t\t\treturn false\n\t\t}\n\t\tfor key, lhs := range lhs.Value {\n\t\t\trhs, ok := rhs.Value[key]\n\t\t\tif !(assert.True(tb, ok) && assert.True(tb, compareUserNode(tb, lhs, rhs))) {\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t\treturn true\n\tcase *visitorUserArray:\n\t\trhs, ok := rhs.(*visitorUserArray)\n\t\tif !(assert.True(tb, ok) && assert.Equal(tb, len(lhs.Value), len(rhs.Value))) {\n\t\t\treturn false\n\t\t}\n\t\tfor i, n := 0, len(lhs.Value); i < n; i++ {\n\t\t\tif !assert.True(tb, compareUserNode(tb, lhs.Value[i], rhs.Value[i])) {\n\t\t\t\treturn false\n\t\t\t}\n\t\t}\n\t\treturn true\n\tdefault:\n\t\ttb.Fatalf(\"unexpected type of UserNode: %T\", lhs)\n\t\treturn false\n\t}\n}\n\ntype visitorUserNodeDecoder interface {\n\tReset()\n\tDecode(str string) (visitorUserNode, error)\n}\n\nvar _ visitorUserNodeDecoder = (*visitorUserNodeASTDecoder)(nil)\n\ntype visitorUserNodeASTDecoder struct{}\n\nfunc (self *visitorUserNodeASTDecoder) Reset() {}\n\nfunc (self *visitorUserNodeASTDecoder) Decode(str string) (visitorUserNode, error) {\n\troot := NewRaw(str)\n\tif err := root.LoadAll(); err != nil {\n\t\treturn nil, err\n\t}\n\treturn self.decodeValue(&root)\n}\n\nfunc (self *visitorUserNodeASTDecoder) decodeValue(root *Node) (visitorUserNode, error) {\n\tswitch typ := root.Type(); typ {\n\t// embed (*Node).Check\n\tcase V_NONE:\n\t\treturn nil, ErrNotExist\n\tcase V_ERROR:\n\t\treturn nil, root\n\n\tcase V_NULL:\n\t\treturn &visitorUserNull{}, nil\n\tcase V_TRUE:\n\t\treturn &visitorUserBool{Value: true}, nil\n\tcase V_FALSE:\n\t\treturn &visitorUserBool{Value: false}, nil\n\n\tcase V_STRING:\n\t\tvalue, err := root.StrictString()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn &visitorUserString{Value: value}, nil\n\n\tcase V_NUMBER:\n\t\tvalue, err := root.StrictNumber()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\ti64, ierr := value.Int64()\n\t\tif ierr == nil {\n\t\t\treturn &visitorUserInt64{Value: i64}, nil\n\t\t}\n\t\tf64, ferr := value.Float64()\n\t\tif ferr == nil {\n\t\t\treturn &visitorUserFloat64{Value: f64}, nil\n\t\t}\n\t\treturn nil, fmt.Errorf(\"invalid number: %v, ierr: %v, ferr: %v\",\n\t\t\tvalue, ierr, ferr)\n\n\tcase V_ARRAY:\n\t\tnodes, err := root.ArrayUseNode()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tvalues := make([]visitorUserNode, len(nodes))\n\t\tfor i := 0; i < len(nodes); i++ {\n\t\t\tn := &nodes[i]\n\t\t\tvalue, err := self.decodeValue(n)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tvalues[i] = value\n\t\t}\n\t\treturn &visitorUserArray{Value: values}, nil\n\n\tcase V_OBJECT:\n\t\tpairs, err := root.MapUseNode()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tvalues := make(map[string]visitorUserNode, len(pairs))\n\t\tfor k, v := range pairs {\n\t\t\tvalue, err := self.decodeValue(&v)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tvalues[k] = value\n\t\t}\n\t\treturn &visitorUserObject{Value: values}, nil\n\n\tcase V_ANY:\n\t\tfallthrough\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"unexpected Node type: %v\", typ)\n\t}\n}\n\nvar _ visitorUserNodeDecoder = (*visitorUserNodeVisitorDecoder)(nil)\n\ntype visitorUserNodeVisitorDecoder struct {\n\tstk visitorUserNodeStack\n\tsp uint8\n}\n\ntype visitorUserNodeStack = [256]struct {\n\tval visitorUserNode\n\tobj map[string]visitorUserNode\n\tarr []visitorUserNode\n\tkey string\n}\n\nfunc (self *visitorUserNodeVisitorDecoder) Reset() {\n\tself.stk = visitorUserNodeStack{}\n\tself.sp = 0\n}\n\nfunc (self *visitorUserNodeVisitorDecoder) Decode(str string) (visitorUserNode, error) {\n\tif err := Preorder(str, self, nil); err != nil {\n\t\treturn nil, err\n\t}\n\treturn self.result()\n}\n\nfunc (self *visitorUserNodeVisitorDecoder) result() (visitorUserNode, error) {\n\tif self.sp != 1 {\n\t\treturn nil, fmt.Errorf(\"incorrect sp: %d\", self.sp)\n\t}\n\treturn self.stk[0].val, nil\n}\n\nfunc (self *visitorUserNodeVisitorDecoder) incrSP() error {\n\tself.sp++\n\tif self.sp == 0 {\n\t\treturn fmt.Errorf(\"reached max depth: %d\", len(self.stk))\n\t}\n\treturn nil\n}\n\nfunc (self *visitorUserNodeVisitorDecoder) OnNull() error {\n\tself.stk[self.sp].val = &visitorUserNull{}\n\tif err := self.incrSP(); err != nil {\n\t\treturn err\n\t}\n\treturn self.onValueEnd()\n}\n\nfunc (self *visitorUserNodeVisitorDecoder) OnBool(v bool) error {\n\tself.stk[self.sp].val = &visitorUserBool{Value: v}\n\tif err := self.incrSP(); err != nil {\n\t\treturn err\n\t}\n\treturn self.onValueEnd()\n}\n\nfunc (self *visitorUserNodeVisitorDecoder) OnString(v string) error {\n\tself.stk[self.sp].val = &visitorUserString{Value: v}\n\tif err := self.incrSP(); err != nil {\n\t\treturn err\n\t}\n\treturn self.onValueEnd()\n}\n\nfunc (self *visitorUserNodeVisitorDecoder) OnInt64(v int64, n json.Number) error {\n\tself.stk[self.sp].val = &visitorUserInt64{Value: v}\n\tif err := self.incrSP(); err != nil {\n\t\treturn err\n\t}\n\treturn self.onValueEnd()\n}\n\nfunc (self *visitorUserNodeVisitorDecoder) OnFloat64(v float64, n json.Number) error {\n\tself.stk[self.sp].val = &visitorUserFloat64{Value: v}\n\tif err := self.incrSP(); err != nil {\n\t\treturn err\n\t}\n\treturn self.onValueEnd()\n}\n\nfunc (self *visitorUserNodeVisitorDecoder) OnObjectBegin(capacity int) error {\n\tself.stk[self.sp].obj = make(map[string]visitorUserNode, capacity)\n\treturn self.incrSP()\n}\n\nfunc (self *visitorUserNodeVisitorDecoder) OnObjectKey(key string) error {\n\tself.stk[self.sp].key = key\n\treturn self.incrSP()\n}\n\nfunc (self *visitorUserNodeVisitorDecoder) OnObjectEnd() error {\n\tself.stk[self.sp-1].val = &visitorUserObject{Value: self.stk[self.sp-1].obj}\n\tself.stk[self.sp-1].obj = nil\n\treturn self.onValueEnd()\n}\n\nfunc (self *visitorUserNodeVisitorDecoder) OnArrayBegin(capacity int) error {\n\tself.stk[self.sp].arr = make([]visitorUserNode, 0, capacity)\n\treturn self.incrSP()\n}\n\nfunc (self *visitorUserNodeVisitorDecoder) OnArrayEnd() error {\n\tself.stk[self.sp-1].val = &visitorUserArray{Value: self.stk[self.sp-1].arr}\n\tself.stk[self.sp-1].arr = nil\n\treturn self.onValueEnd()\n}\n\nfunc (self *visitorUserNodeVisitorDecoder) onValueEnd() error {\n\tif self.sp == 1 {\n\t\treturn nil\n\t}\n\t// [..., Array, Value, sp]\n\tif self.stk[self.sp-2].arr != nil {\n\t\tself.stk[self.sp-2].arr = append(self.stk[self.sp-2].arr, self.stk[self.sp-1].val)\n\t\tself.sp--\n\t\treturn nil\n\t}\n\t// [..., Object, ObjectKey, Value, sp]\n\tself.stk[self.sp-3].obj[self.stk[self.sp-2].key] = self.stk[self.sp-1].val\n\tself.sp -= 2\n\treturn nil\n}\n\nfunc testUserNodeDiff(t *testing.T, d1, d2 visitorUserNodeDecoder, str string) {\n\tt.Helper()\n\td1.Reset()\n\tn1, err := d1.Decode(_TwitterJson)\n\trequire.NoError(t, err)\n\n\td2.Reset()\n\tn2, err := d2.Decode(_TwitterJson)\n\trequire.NoError(t, err)\n\n\trequire.True(t, compareUserNode(t, n1, n2))\n}\n\nfunc TestVisitor_UserNodeDiff(t *testing.T) {\n\tvar d1 visitorUserNodeASTDecoder\n\tvar d2 visitorUserNodeVisitorDecoder\n\n\tfor _, c := range visitorTestCases {\n\t\tt.Run(c.name, func(t *testing.T) {\n\t\t\ttestUserNodeDiff(t, &d1, &d2, c.jsonStr)\n\t\t})\n\t}\n}\n\ntype skipVisitor struct {\n\tsp int\n\tSkip int\n\tinSkip bool\n\tCountSkip int\n}\n\nfunc (self *skipVisitor) OnNull() error {\n\tif self.sp == self.Skip+1 && self.inSkip {\n\t\tpanic(\"unexpected key\")\n\t}\n\treturn nil\n}\n\nfunc (self *skipVisitor) OnFloat64(v float64, n json.Number) error {\n\tif self.sp == self.Skip+1 && self.inSkip {\n\t\tpanic(\"unexpected key\")\n\t}\n\treturn nil\n}\n\nfunc (self *skipVisitor) OnInt64(v int64, n json.Number) error {\n\tif self.sp == self.Skip+1 && self.inSkip {\n\t\tpanic(\"unexpected key\")\n\t}\n\treturn nil\n}\n\nfunc (self *skipVisitor) OnBool(v bool) error {\n\tif self.sp == self.Skip+1 && self.inSkip {\n\t\tpanic(\"unexpected key\")\n\t}\n\treturn nil\n}\n\nfunc (self *skipVisitor) OnString(v string) error {\n\tif self.sp == self.Skip+1 && self.inSkip {\n\t\tpanic(\"unexpected key\")\n\t}\n\treturn nil\n}\n\nfunc (self *skipVisitor) OnObjectBegin(capacity int) error {\n\tprintln(\"self.sp\", self.sp)\n\tif self.sp == self.Skip {\n\t\tself.inSkip = true\n\t\tself.CountSkip++\n\t\tprintln(\"op skip\")\n\t\tself.sp++\n\t\treturn VisitOPSkip\n\t}\n\tself.sp++\n\treturn nil\n}\n\nfunc (self *skipVisitor) OnObjectKey(key string) error {\n\tif self.sp == self.Skip+1 && self.inSkip {\n\t\tpanic(\"unexpected key\")\n\t}\n\treturn nil\n}\n\nfunc (self *skipVisitor) OnObjectEnd() error {\n\tif self.sp == self.Skip+1 {\n\t\tif !self.inSkip {\n\t\t\tpanic(\"not in skip\")\n\t\t}\n\t\tself.inSkip = false\n\t\tprintln(\"finish op skip\")\n\t}\n\tself.sp--\n\treturn nil\n}\n\nfunc (self *skipVisitor) OnArrayBegin(capacity int) error {\n\tprintln(\"arr self.sp\", self.sp)\n\tif self.sp == self.Skip {\n\t\tself.inSkip = true\n\t\tself.CountSkip++\n\t\tprintln(\"arr op skip\")\n\t\tself.sp++\n\t\treturn VisitOPSkip\n\t}\n\tself.sp++\n\treturn nil\n}\n\nfunc (self *skipVisitor) OnArrayEnd() error {\n\tprintln(\"arr self.sp\", self.sp)\n\tif self.sp == self.Skip+1 {\n\t\tif !self.inSkip {\n\t\t\tpanic(\"arr not in skip\")\n\t\t}\n\t\tself.inSkip = false\n\t\tprintln(\"arr finish op skip\")\n\t}\n\tself.sp--\n\treturn nil\n}\n\nfunc TestVisitor_OpSkip(t *testing.T) {\n\tvar suite skipVisitor\n\tsuite.Skip = 1\n\tPreorder(`{ \"a\": [ null ] , \"b\": 1, \"c\": { \"1\" : 1 } }`, &suite, nil)\n\tif suite.CountSkip != 2 {\n\t\tt.Fatal(suite.CountSkip)\n\t}\n}\n\nfunc BenchmarkVisitor_UserNode(b *testing.B) {\n\tconst str = _TwitterJson\n\tb.Run(\"AST\", func(b *testing.B) {\n\t\tvar d visitorUserNodeASTDecoder\n\t\tb.ResetTimer()\n\t\tfor k := 0; k < b.N; k++ {\n\t\t\td.Reset()\n\t\t\t_, err := d.Decode(str)\n\t\t\trequire.NoError(b, err)\n\t\t\tb.SetBytes(int64(len(str)))\n\t\t}\n\t})\n\tb.Run(\"Visitor\", func(b *testing.B) {\n\t\tvar d visitorUserNodeVisitorDecoder\n\t\tb.ResetTimer()\n\t\tfor k := 0; k < b.N; k++ {\n\t\t\td.Reset()\n\t\t\t_, err := d.Decode(str)\n\t\t\trequire.NoError(b, err)\n\t\t\tb.SetBytes(int64(len(str)))\n\t\t}\n\t})\n}\n" }, { "path": "compat.go", "content": "//go:build (!amd64 && !arm64) || go1.27 || !go1.17 || (arm64 && !go1.20)\n// +build !amd64,!arm64 go1.27 !go1.17 arm64,!go1.20\n\n/*\n * Copyright 2021 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage sonic\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"io\"\n\t\"reflect\"\n\n\t\"github.com/bytedance/sonic/option\"\n)\n\nconst apiKind = UseStdJSON\n\ntype frozenConfig struct {\n\tConfig\n}\n\n// Froze convert the Config to API\nfunc (cfg Config) Froze() API {\n\tapi := &frozenConfig{Config: cfg}\n\treturn api\n}\n\nfunc (cfg frozenConfig) marshalOptions(val interface{}, prefix, indent string) ([]byte, error) {\n\tw := bytes.NewBuffer([]byte{})\n\tenc := json.NewEncoder(w)\n\tenc.SetEscapeHTML(cfg.EscapeHTML)\n\tenc.SetIndent(prefix, indent)\n\terr := enc.Encode(val)\n\tout := w.Bytes()\n\n\t// json.Encoder always appends '\\n' after encoding,\n\t// which is not same with json.Marshal()\n\tif len(out) > 0 && out[len(out)-1] == '\\n' {\n\t\tout = out[:len(out)-1]\n\t}\n\treturn out, err\n}\n\n// Marshal is implemented by sonic\nfunc (cfg frozenConfig) Marshal(val interface{}) ([]byte, error) {\n\tif !cfg.EscapeHTML {\n\t\treturn cfg.marshalOptions(val, \"\", \"\")\n\t}\n\treturn json.Marshal(val)\n}\n\n// MarshalToString is implemented by sonic\nfunc (cfg frozenConfig) MarshalToString(val interface{}) (string, error) {\n\tout, err := cfg.Marshal(val)\n\treturn string(out), err\n}\n\n// MarshalIndent is implemented by sonic\nfunc (cfg frozenConfig) MarshalIndent(val interface{}, prefix, indent string) ([]byte, error) {\n\tif !cfg.EscapeHTML {\n\t\treturn cfg.marshalOptions(val, prefix, indent)\n\t}\n\treturn json.MarshalIndent(val, prefix, indent)\n}\n\n// UnmarshalFromString is implemented by sonic\nfunc (cfg frozenConfig) UnmarshalFromString(buf string, val interface{}) error {\n\tr := bytes.NewBufferString(buf)\n\tdec := json.NewDecoder(r)\n\tif cfg.UseNumber {\n\t\tdec.UseNumber()\n\t}\n\tif cfg.DisallowUnknownFields {\n\t\tdec.DisallowUnknownFields()\n\t}\n\terr := dec.Decode(val)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// check the trailing chars\n\toffset := dec.InputOffset()\n\tif t, err := dec.Token(); !(t == nil && err == io.EOF) {\n\t\treturn &json.SyntaxError{Offset: offset}\n\t}\n\treturn nil\n}\n\n// Unmarshal is implemented by sonic\nfunc (cfg frozenConfig) Unmarshal(buf []byte, val interface{}) error {\n\treturn cfg.UnmarshalFromString(string(buf), val)\n}\n\n// NewEncoder is implemented by sonic\nfunc (cfg frozenConfig) NewEncoder(writer io.Writer) Encoder {\n\tenc := json.NewEncoder(writer)\n\tif !cfg.EscapeHTML {\n\t\tenc.SetEscapeHTML(cfg.EscapeHTML)\n\t}\n\treturn enc\n}\n\n// NewDecoder is implemented by sonic\nfunc (cfg frozenConfig) NewDecoder(reader io.Reader) Decoder {\n\tdec := json.NewDecoder(reader)\n\tif cfg.UseNumber {\n\t\tdec.UseNumber()\n\t}\n\tif cfg.DisallowUnknownFields {\n\t\tdec.DisallowUnknownFields()\n\t}\n\treturn dec\n}\n\n// Valid is implemented by sonic\nfunc (cfg frozenConfig) Valid(data []byte) bool {\n\treturn json.Valid(data)\n}\n\n// Pretouch compiles vt ahead-of-time to avoid JIT compilation on-the-fly, in\n// order to reduce the first-hit latency at **amd64** Arch.\n// Opts are the compile options, for example, \"option.WithCompileRecursiveDepth\" is\n// a compile option to set the depth of recursive compile for the nested struct type.\n// * This is the none implement for !amd64.\n// It will be useful for someone who develop with !amd64 arch,like Mac M1.\nfunc Pretouch(vt reflect.Type, opts ...option.CompileOption) error {\n\treturn nil\n}\n" }, { "path": "compat_test.go", "content": "/*\n * Copyright 2022 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage sonic\n\nimport (\n\t\"bytes\"\n\t\"compress/gzip\"\n\t\"encoding/json\"\n\t\"io\"\n\t\"os\"\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/bytedance/sonic/internal/envs\"\n\t\"github.com/bytedance/sonic/option\"\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestCompatUnmarshalStd(t *testing.T) {\n\tvar sobj = map[string]interface{}{}\n\tvar jobj = map[string]interface{}{}\n\tvar data = []byte(`{\"a\":1.00000001E-10}`)\n\tvar str = string(data)\n\tserr := ConfigStd.UnmarshalFromString(str, &sobj)\n\tjerr := json.Unmarshal(data, &jobj)\n\trequire.Equal(t, jerr, serr)\n\trequire.Equal(t, jobj, sobj)\n\tdata[2] = '0'\n\trequire.Equal(t, jobj, sobj)\n\n\tsobj = map[string]interface{}{}\n\tjobj = map[string]interface{}{}\n\tdata = []byte(`{\"a\":1}`)\n\tcfg := Config{\n\t\tUseNumber: true,\n\t}.Froze()\n\tserr = cfg.Unmarshal(data, &sobj)\n\tdec := json.NewDecoder(bytes.NewBuffer(data))\n\tdec.UseNumber()\n\tjerr = dec.Decode(&jobj)\n\trequire.Equal(t, jerr, serr)\n\trequire.Equal(t, jobj, sobj)\n\n\tx := struct {\n\t\tA json.Number\n\t\tB json.Number\n\t}{}\n\ty := struct {\n\t\tA json.Number\n\t\tB json.Number\n\t}{}\n\tdata = []byte(`{\"A\":\"1\", \"C\":-1, \"B\":1}`)\n\tcfg = Config{\n\t\tDisallowUnknownFields: true,\n\t}.Froze()\n\tserr = cfg.Unmarshal(data, &x)\n\tdec = json.NewDecoder(bytes.NewBuffer(data))\n\tdec.UseNumber()\n\tdec.DisallowUnknownFields()\n\tjerr = dec.Decode(&y)\n\trequire.Equal(t, jerr, serr)\n\t// require.Equal(t, y, x)\n}\n\nfunc TestCompatMarshalStd(t *testing.T) {\n\tt.Parallel()\n\tvar obj = map[string]interface{}{\n\t\t\"c\": json.RawMessage(\" [ \\\"<&>\\\" ] \"),\n\t\t\"b\": json.RawMessage(\" [ ] \"),\n\t}\n\tsout, serr := ConfigStd.Marshal(obj)\n\tjout, jerr := json.Marshal(obj)\n\trequire.Equal(t, jerr, serr)\n\trequire.Equal(t, string(jout), string(sout))\n\n\tobj = map[string]interface{}{\n\t\t\"a\": json.RawMessage(\" [} \"),\n\t}\n\tsout, serr = ConfigStd.Marshal(obj)\n\tjout, jerr = json.Marshal(obj)\n\trequire.NotNil(t, jerr)\n\trequire.NotNil(t, serr)\n\trequire.Equal(t, string(jout), string(sout))\n\n\tobj = map[string]interface{}{\n\t\t\"a\": json.RawMessage(\"1\"),\n\t}\n\tsout, serr = ConfigStd.MarshalIndent(obj, \"xxxx\", \" \")\n\tjout, jerr = json.MarshalIndent(obj, \"xxxx\", \" \")\n\trequire.Equal(t, jerr, serr)\n\trequire.Equal(t, string(jout), string(sout))\n}\n\nfunc TestCompatEncoderStd(t *testing.T) {\n\tvar o = map[string]interface{}{\n\t\t\"a\": \"<>\",\n\t\t\"b\": json.RawMessage(\" [ ] \"),\n\t}\n\tvar w1 = bytes.NewBuffer(nil)\n\tvar w2 = bytes.NewBuffer(nil)\n\tvar enc1 = json.NewEncoder(w1)\n\tvar enc2 = ConfigStd.NewEncoder(w2)\n\n\trequire.Nil(t, enc1.Encode(o))\n\trequire.Nil(t, enc2.Encode(o))\n\trequire.Equal(t, w1.String(), w2.String())\n\n\tenc1.SetEscapeHTML(true)\n\tenc2.SetEscapeHTML(true)\n\tenc1.SetIndent(\"\\n\", \" \")\n\tenc2.SetIndent(\"\\n\", \" \")\n\trequire.Nil(t, enc1.Encode(o))\n\trequire.Nil(t, enc2.Encode(o))\n\trequire.Equal(t, w1.String(), w2.String())\n\n\tenc1.SetEscapeHTML(false)\n\tenc2.SetEscapeHTML(false)\n\tenc1.SetIndent(\"\", \"\")\n\tenc2.SetIndent(\"\", \"\")\n\trequire.Nil(t, enc1.Encode(o))\n\trequire.Nil(t, enc2.Encode(o))\n\trequire.Equal(t, w1.String(), w2.String())\n}\n\nfunc TestCompatDecoderStd(t *testing.T) {\n\tvar o1 = map[string]interface{}{}\n\tvar o2 = map[string]interface{}{}\n\tvar s = `{\"a\":\"b\"} {\"1\":\"2\"} a {}`\n\tvar w1 = bytes.NewBuffer([]byte(s))\n\tvar w2 = bytes.NewBuffer([]byte(s))\n\tvar enc1 = json.NewDecoder(w1)\n\tvar enc2 = ConfigStd.NewDecoder(w2)\n\n\trequire.Equal(t, enc1.More(), enc2.More())\n\trequire.Nil(t, enc1.Decode(&o1))\n\trequire.Nil(t, enc2.Decode(&o2))\n\trequire.Equal(t, w1.String(), w2.String())\n\n\trequire.Equal(t, enc1.More(), enc2.More())\n\trequire.Nil(t, enc1.Decode(&o1))\n\trequire.Nil(t, enc2.Decode(&o2))\n\trequire.Equal(t, w1.String(), w2.String())\n\n\trequire.Equal(t, enc1.More(), enc2.More())\n\trequire.NotNil(t, enc1.Decode(&o1))\n\trequire.NotNil(t, enc2.Decode(&o2))\n\trequire.Equal(t, w1.String(), w2.String())\n}\n\nfunc TestPretouch(t *testing.T) {\n\tvar v map[string]interface{}\n\tif err := Pretouch(reflect.TypeOf(v)); err != nil {\n\t\tt.Errorf(\"err:%v\", err)\n\t}\n\n\tif err := Pretouch(reflect.TypeOf(v),\n\t\toption.WithCompileRecursiveDepth(1),\n\t\toption.WithCompileMaxInlineDepth(2),\n\t); err != nil {\n\t\tt.Errorf(\"err:%v\", err)\n\t}\n}\n\nfunc TestGet(t *testing.T) {\n\tvar data = `{\"a\":\"b\"}`\n\tr, err := GetFromString(data, \"a\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tv, err := r.String()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif v != \"b\" {\n\t\tt.Fatal(v)\n\t}\n}\n\nfunc TestUnmarshalWithTrailingChars(t *testing.T) {\n\tfor i, str := range []string{\n\t\t\"123\",\n\t\t\"123 \",\n\t\t\"{} [] \",\n\t\t\"{} [ \",\n\t\t\"[],\",\n\t\t\"[]null\",\n\t\t\"false null\",\n\t} {\n\t\tvar msg1, msg2 json.RawMessage\n\t\terr1 := json.Unmarshal([]byte(str), &msg1)\n\t\terr2 := ConfigStd.UnmarshalFromString(str, &msg2)\n\t\trequire.Equal(t, err1 == nil, err2 == nil, i)\n\t\t// sonic will not clear the unmarshaled value here, but encoding/json will\n\t\t// require.Equal(t, msg1, msg2)\n\t}\n}\n\nfunc TestUnmarshalJSONSuite(t *testing.T) {\n\tif envs.UseOptDec {\n\t\tt.Skip(\"this test still fails in OPTDEC\") // FIXME: fix the optdec issues\n\t}\n\n\tgzFile, err := os.Open(\"testdata/JSONTestSuite/testdata.json.gz\")\n\trequire.NoError(t, err)\n\tdefer gzFile.Close()\n\n\tgzReader, err := gzip.NewReader(gzFile)\n\trequire.NoError(t, err)\n\tdefer gzReader.Close()\n\n\tdata, err := io.ReadAll(gzReader)\n\trequire.NoError(t, err)\n\n\tvar tests map[string]string\n\terr = json.Unmarshal(data, &tests)\n\trequire.NoError(t, err)\n\n\tfor name, tt := range tests {\n\t\tb := []byte(tt)\n\t\tt.Run(name, func(t *testing.T) {\n\t\t\tserr := ConfigStd.Unmarshal(b, new(json.RawMessage))\n\t\t\tjerr := json.Unmarshal(b, new(json.RawMessage))\n\t\t\tassert.Equal(t, jerr != nil, serr != nil, \"json: %v, sonic: %v\", jerr, serr)\n\n\t\t\tserr = ConfigStd.Unmarshal(b, new(interface{}))\n\t\t\tjerr = json.Unmarshal(b, new(interface{}))\n\t\t\tassert.Equal(t, jerr != nil, serr != nil, \"json: %v, sonic: %v\", jerr, serr)\n\t\t})\n\t}\n}\n" }, { "path": "decode_test.go", "content": "//go:build (amd64 && go1.17 && !go1.27) || (arm64 && go1.20 && !go1.27)\n// +build amd64,go1.17,!go1.27 arm64,go1.20,!go1.27\n\n/*\n * Copyright 2021 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage sonic\n\nimport (\n\t\"bytes\"\n\t\"encoding\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"image\"\n\t\"math\"\n\t\"math/big\"\n\t\"math/rand\"\n\t\"net\"\n\t\"reflect\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\t\"unsafe\"\n\n\t\"github.com/bytedance/sonic/decoder\"\n\t\"github.com/bytedance/sonic/internal/native/types\"\n\t\"github.com/bytedance/sonic/internal/utils\"\n\t\"github.com/davecgh/go-spew/spew\"\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\ntype T struct {\n\tX string\n\tY int\n\tZ int `json:\"-\"`\n}\n\ntype U struct {\n\tAlphabet string `json:\"alpha\"`\n}\n\ntype V struct {\n\tF1 interface{}\n\tF2 int32\n\tF3 json.Number\n\tF4 *VOuter\n}\n\ntype VOuter struct {\n\tV V\n}\n\ntype W struct {\n\tS SS\n}\n\ntype P struct {\n\tPP PP\n}\n\ntype PP struct {\n\tT T\n\tTs []T\n}\n\ntype SS string\n\nfunc (*SS) UnmarshalJSON(_ []byte) error {\n\treturn &json.UnmarshalTypeError{Value: \"number\", Type: reflect.TypeOf(SS(\"\"))}\n}\n\n// ifaceNumAsFloat64/ifaceNumAsNumber are used to test unmarshaling with and\n// without UseNumber\nvar ifaceNumAsFloat64 = map[string]interface{}{\n\t\"k1\": float64(1),\n\t\"k2\": \"s\",\n\t\"k3\": []interface{}{float64(1), 2.0, 3e-3},\n\t\"k4\": map[string]interface{}{\"kk1\": \"s\", \"kk2\": float64(2)},\n}\n\nvar ifaceNumAsNumber = map[string]interface{}{\n\t\"k1\": json.Number(\"1\"),\n\t\"k2\": \"s\",\n\t\"k3\": []interface{}{json.Number(\"1\"), json.Number(\"2.0\"), json.Number(\"3e-3\")},\n\t\"k4\": map[string]interface{}{\"kk1\": \"s\", \"kk2\": json.Number(\"2\")},\n}\n\ntype tx struct {\n\tx int\n}\n\ntype u8 uint8\n\n// A type that can unmarshal itself.\n\ntype unmarshaler struct {\n\tT bool\n}\n\nfunc (u *unmarshaler) UnmarshalJSON(_ []byte) error {\n\t*u = unmarshaler{true} // All we need to see that UnmarshalJSON is called.\n\treturn nil\n}\n\ntype ustruct struct {\n\tM unmarshaler\n}\n\ntype unmarshalerText struct {\n\tA, B string\n}\n\n// needed for re-marshaling tests\nfunc (u unmarshalerText) MarshalText() ([]byte, error) {\n\treturn []byte(u.A + \":\" + u.B), nil\n}\n\nfunc (u *unmarshalerText) UnmarshalText(b []byte) error {\n\tpos := bytes.IndexByte(b, ':')\n\tif pos == -1 {\n\t\treturn errors.New(\"missing separator\")\n\t}\n\tu.A, u.B = string(b[:pos]), string(b[pos+1:])\n\treturn nil\n}\n\nvar _ encoding.TextUnmarshaler = (*unmarshalerText)(nil)\n\ntype ustructText struct {\n\tM unmarshalerText\n}\n\n// u8marshal is an integer type that can marshal/unmarshal itself.\ntype u8marshal uint8\n\nfunc (u8 u8marshal) MarshalText() ([]byte, error) {\n\treturn []byte(fmt.Sprintf(\"u%d\", u8)), nil\n}\n\nvar errMissingU8Prefix = errors.New(\"missing 'u' prefix\")\n\nfunc (u8 *u8marshal) UnmarshalText(b []byte) error {\n\tif !bytes.HasPrefix(b, []byte{'u'}) {\n\t\treturn errMissingU8Prefix\n\t}\n\tn, err := strconv.Atoi(string(b[1:]))\n\tif err != nil {\n\t\treturn err\n\t}\n\t*u8 = u8marshal(n)\n\treturn nil\n}\n\nvar _ encoding.TextUnmarshaler = (*u8marshal)(nil)\n\nvar (\n\tumtrue = unmarshaler{true}\n\tumslice = []unmarshaler{{true}}\n\tumstruct = ustruct{unmarshaler{true}}\n\n\tumtrueXY = unmarshalerText{\"x\", \"y\"}\n\tumsliceXY = []unmarshalerText{{\"x\", \"y\"}}\n\tumstructXY = ustructText{unmarshalerText{\"x\", \"y\"}}\n\n\tummapXY = map[unmarshalerText]bool{{\"x\", \"y\"}: true}\n)\n\n// Test data structures for anonymous fields.\n\ntype Point struct {\n\tZ int\n}\n\ntype Top struct {\n\tLevel0 int\n\tEmbed0\n\t*Embed0a\n\t*Embed0b `json:\"e,omitempty\"` // treated as named\n\tEmbed0c `json:\"-\"` // ignored\n\tLoop\n\tEmbed0p // has Point with X, Y, used\n\tEmbed0q // has Point with Z, used\n\tembed // contains exported field\n}\n\ntype Embed0 struct {\n\tLevel1a int // overridden by Embed0a's Level1a with json tag\n\tLevel1b int // used because Embed0a's Level1b is renamed\n\tLevel1c int // used because Embed0a's Level1c is ignored\n\tLevel1d int // annihilated by Embed0a's Level1d\n\tLevel1e int `json:\"x\"` // annihilated by Embed0a.Level1e\n}\n\ntype Embed0a struct {\n\tLevel1a int `json:\"Level1a,omitempty\"`\n\tLevel1b int `json:\"LEVEL1B,omitempty\"`\n\tLevel1c int `json:\"-\"`\n\tLevel1d int // annihilated by Embed0's Level1d\n\tLevel1f int `json:\"x\"` // annihilated by Embed0's Level1e\n}\n\ntype Embed0b Embed0\n\ntype Embed0c Embed0\n\ntype Embed0p struct {\n\timage.Point\n}\n\ntype Embed0q struct {\n\tPoint\n}\n\ntype embed struct {\n\tQ int\n}\n\ntype Loop struct {\n\tLoop1 int `json:\",omitempty\"`\n\tLoop2 int `json:\",omitempty\"`\n\t*Loop\n}\n\n// From reflect test:\n// The X in S6 and S7 annihilate, but they also block the X in S8.S9.\ntype S5 struct {\n\tS6\n\tS7\n\tS8\n}\n\ntype S6 struct {\n\tX int\n}\n\ntype S7 S6\n\ntype S8 struct {\n\tS9\n}\n\ntype S9 struct {\n\tX int\n\tY int\n}\n\n// From reflect test:\n// The X in S11.S6 and S12.S6 annihilate, but they also block the X in S13.S8.S9.\ntype S10 struct {\n\tS11\n\tS12\n\tS13\n}\n\ntype S11 struct {\n\tS6\n}\n\ntype S12 struct {\n\tS6\n}\n\ntype S13 struct {\n\tS8\n}\n\ntype Ambig struct {\n\t// Given \"hello\", the first match should win.\n\tFirst int `json:\"HELLO\"`\n\tSecond int `json:\"Hello\"`\n}\n\ntype XYZ struct {\n\tX interface{}\n\tY interface{}\n\tZ interface{}\n}\n\ntype byteWithMarshalJSON byte\n\nfunc (b byteWithMarshalJSON) MarshalJSON() ([]byte, error) {\n\treturn []byte(fmt.Sprintf(`\"Z%.2x\"`, byte(b))), nil\n}\n\nfunc (b *byteWithMarshalJSON) UnmarshalJSON(data []byte) error {\n\tif len(data) != 5 || data[0] != '\"' || data[1] != 'Z' || data[4] != '\"' {\n\t\treturn fmt.Errorf(\"bad quoted string\")\n\t}\n\ti, err := strconv.ParseInt(string(data[2:4]), 16, 8)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"bad hex\")\n\t}\n\t*b = byteWithMarshalJSON(i)\n\treturn nil\n}\n\ntype byteWithPtrMarshalJSON byte\n\nfunc (b *byteWithPtrMarshalJSON) MarshalJSON() ([]byte, error) {\n\treturn byteWithMarshalJSON(*b).MarshalJSON()\n}\n\nfunc (b *byteWithPtrMarshalJSON) UnmarshalJSON(data []byte) error {\n\treturn (*byteWithMarshalJSON)(b).UnmarshalJSON(data)\n}\n\ntype byteWithMarshalText byte\n\nfunc (b byteWithMarshalText) MarshalText() ([]byte, error) {\n\treturn []byte(fmt.Sprintf(`Z%.2x`, byte(b))), nil\n}\n\nfunc (b *byteWithMarshalText) UnmarshalText(data []byte) error {\n\tif len(data) != 3 || data[0] != 'Z' {\n\t\treturn fmt.Errorf(\"bad quoted string\")\n\t}\n\ti, err := strconv.ParseInt(string(data[1:3]), 16, 8)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"bad hex\")\n\t}\n\t*b = byteWithMarshalText(i)\n\treturn nil\n}\n\ntype byteWithPtrMarshalText byte\n\nfunc (b *byteWithPtrMarshalText) MarshalText() ([]byte, error) {\n\treturn byteWithMarshalText(*b).MarshalText()\n}\n\nfunc (b *byteWithPtrMarshalText) UnmarshalText(data []byte) error {\n\treturn (*byteWithMarshalText)(b).UnmarshalText(data)\n}\n\ntype intWithMarshalJSON int\n\nfunc (b intWithMarshalJSON) MarshalJSON() ([]byte, error) {\n\treturn []byte(fmt.Sprintf(`\"Z%.2x\"`, int(b))), nil\n}\n\nfunc (b *intWithMarshalJSON) UnmarshalJSON(data []byte) error {\n\tif len(data) != 5 || data[0] != '\"' || data[1] != 'Z' || data[4] != '\"' {\n\t\treturn fmt.Errorf(\"bad quoted string\")\n\t}\n\ti, err := strconv.ParseInt(string(data[2:4]), 16, 8)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"bad hex\")\n\t}\n\t*b = intWithMarshalJSON(i)\n\treturn nil\n}\n\ntype intWithPtrMarshalJSON int\n\nfunc (b *intWithPtrMarshalJSON) MarshalJSON() ([]byte, error) {\n\treturn intWithMarshalJSON(*b).MarshalJSON()\n}\n\nfunc (b *intWithPtrMarshalJSON) UnmarshalJSON(data []byte) error {\n\treturn (*intWithMarshalJSON)(b).UnmarshalJSON(data)\n}\n\ntype intWithMarshalText int\n\nfunc (b intWithMarshalText) MarshalText() ([]byte, error) {\n\treturn []byte(fmt.Sprintf(`Z%.2x`, int(b))), nil\n}\n\nfunc (b *intWithMarshalText) UnmarshalText(data []byte) error {\n\tif len(data) != 3 || data[0] != 'Z' {\n\t\treturn fmt.Errorf(\"bad quoted string\")\n\t}\n\ti, err := strconv.ParseInt(string(data[1:3]), 16, 8)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"bad hex\")\n\t}\n\t*b = intWithMarshalText(i)\n\treturn nil\n}\n\ntype intWithPtrMarshalText int\n\nfunc (b *intWithPtrMarshalText) MarshalText() ([]byte, error) {\n\treturn intWithMarshalText(*b).MarshalText()\n}\n\nfunc (b *intWithPtrMarshalText) UnmarshalText(data []byte) error {\n\treturn (*intWithMarshalText)(b).UnmarshalText(data)\n}\n\ntype mapStringToStringData struct {\n\tData map[string]string `json:\"data\"`\n}\n\ntype unmarshalTest struct {\n\tin string\n\tptr interface{} // new(type)\n\tout interface{}\n\terr error\n\tuseNumber bool\n\tgolden bool\n\tdisallowUnknownFields bool\n\tvalidateString bool\n}\n\ntype B struct {\n\tB bool `json:\",string\"`\n}\n\ntype DoublePtr struct {\n\tI **int\n\tJ **int\n}\n\ntype JsonSyntaxError struct {\n\tMsg string\n\tOffset int64\n}\n\nfunc (self *JsonSyntaxError) err() *json.SyntaxError {\n\treturn (*json.SyntaxError)(unsafe.Pointer(self))\n}\n\nvar unmarshalTests = []unmarshalTest{\n\t// basic types\n\t{in: `true`, ptr: new(bool), out: true},\n\t{in: `1`, ptr: new(int), out: 1},\n\t{in: `1.2`, ptr: new(float64), out: 1.2},\n\t{in: `-5`, ptr: new(int16), out: int16(-5)},\n\t{in: `2`, ptr: new(json.Number), out: json.Number(\"2\"), useNumber: true},\n\t{in: `2`, ptr: new(json.Number), out: json.Number(\"2\")},\n\t{in: `2`, ptr: new(interface{}), out: 2.0},\n\t{in: `2`, ptr: new(interface{}), out: json.Number(\"2\"), useNumber: true},\n\t{in: `\"a\\u1234\"`, ptr: new(string), out: \"a\\u1234\"},\n\t{in: `\"http:\\/\\/\"`, ptr: new(string), out: \"http://\"},\n\t{in: `\"g-clef: \\uD834\\uDD1E\"`, ptr: new(string), out: \"g-clef: \\U0001D11E\"},\n\t{in: `\"invalid: \\uD834x\\uDD1E\"`, ptr: new(string), out: \"invalid: \\uFFFDx\\uFFFD\"},\n\t{in: \"null\", ptr: new(interface{}), out: nil},\n\t{in: `{\"X\": [1,2,3], \"Y\": 4}`, ptr: new(T), out: T{Y: 4}, err: &json.UnmarshalTypeError{Value: \"array\", Type: reflect.TypeOf(\"\"), Offset: 7, Struct: \"T\", Field: \"X\"}},\n\t{in: `{\"X\": 23}`, ptr: new(T), out: T{}, err: &json.UnmarshalTypeError{Value: \"number\", Type: reflect.TypeOf(\"\"), Offset: 8, Struct: \"T\", Field: \"X\"}},\n\t{in: `{\"x\": 1}`, ptr: new(tx), out: tx{}},\n\t{in: `{\"x\": 1}`, ptr: new(tx), err: fmt.Errorf(\"json: unknown field \\\"x\\\"\"), disallowUnknownFields: true},\n\t{in: `{\"S\": 23}`, ptr: new(W), out: W{}, err: &json.UnmarshalTypeError{Value: \"number\", Type: reflect.TypeOf(SS(\"\")), Struct: \"W\", Field: \"S\"}},\n\t{in: `{\"F1\":1,\"F2\":2,\"F3\":3}`, ptr: new(V), out: V{F1: float64(1), F2: int32(2), F3: json.Number(\"3\")}},\n\t{in: `{\"F1\":1,\"F2\":2,\"F3\":3}`, ptr: new(V), out: V{F1: json.Number(\"1\"), F2: int32(2), F3: json.Number(\"3\")}, useNumber: true},\n\t{in: `{\"k1\":1,\"k2\":\"s\",\"k3\":[1,2.0,3e-3],\"k4\":{\"kk1\":\"s\",\"kk2\":2}}`, ptr: new(interface{}), out: ifaceNumAsFloat64},\n\t{in: `{\"k1\":1,\"k2\":\"s\",\"k3\":[1,2.0,3e-3],\"k4\":{\"kk1\":\"s\",\"kk2\":2}}`, ptr: new(interface{}), out: ifaceNumAsNumber, useNumber: true},\n\t{in: `{\"\":\"\"}`, ptr: new(struct{}), out: struct{}{}},\n\t{in: `{\"x\":\"\"}`, ptr: new(struct{ X json.Number }), err: errors.New(\"empty string into json number\")},\n\n\t// raw values with whitespace\n\t{in: \"\\n true \", ptr: new(bool), out: true},\n\t{in: \"\\t 1 \", ptr: new(int), out: 1},\n\t{in: \"\\r 1.2 \", ptr: new(float64), out: 1.2},\n\t{in: \"\\t -5 \\n\", ptr: new(int16), out: int16(-5)},\n\t{in: \"\\t \\\"a\\\\u1234\\\" \\n\", ptr: new(string), out: \"a\\u1234\"},\n\n\t// Z has a \"-\" tag.\n\t{in: `{\"Y\": 1, \"Z\": 2}`, ptr: new(T), out: T{Y: 1}},\n\t{in: `{\"Y\": 1, \"Z\": 2}`, ptr: new(T), err: fmt.Errorf(\"json: unknown field \\\"Z\\\"\"), disallowUnknownFields: true},\n\n\t{in: `{\"alpha\": \"abc\", \"alphabet\": \"xyz\"}`, ptr: new(U), out: U{Alphabet: \"abc\"}},\n\t{in: `{\"alpha\": \"abc\", \"alphabet\": \"xyz\"}`, ptr: new(U), err: fmt.Errorf(\"json: unknown field \\\"alphabet\\\"\"), disallowUnknownFields: true},\n\t{in: `{\"alpha\": \"abc\"}`, ptr: new(U), out: U{Alphabet: \"abc\"}},\n\t{in: `{\"alphabet\": \"xyz\"}`, ptr: new(U), out: U{}},\n\t{in: `{\"alphabet\": \"xyz\"}`, ptr: new(U), err: fmt.Errorf(\"json: unknown field \\\"alphabet\\\"\"), disallowUnknownFields: true},\n\n\t// syntax errors\n\t{in: `{\"X\": \"foo\", \"Y\"}`, err: (&JsonSyntaxError{\"invalid character '}' after object key\", 17}).err()},\n\t{in: `[1, 2, 3+]`, err: (&JsonSyntaxError{\"invalid character '+' after array element\", 9}).err()},\n\t{in: `{\"X\":12x}`, err: (&JsonSyntaxError{\"invalid character 'x' after object key:value pair\", 8}).err(), useNumber: true},\n\t{in: `[2, 3`, err: (&JsonSyntaxError{Msg: \"unexpected end of JSON input\", Offset: 5}).err()},\n\t{in: `{\"F3\": -}`, ptr: new(V), out: V{F3: json.Number(\"-\")}, err: (&JsonSyntaxError{Msg: \"invalid character '}' in numeric literal\", Offset: 9}).err()},\n\n\t// raw value errors\n\t{in: \"\\x01 42\", err: (&JsonSyntaxError{\"invalid character '\\\\x01' looking for beginning of value\", 1}).err()},\n\t{in: \" 42 \\x01\", err: (&JsonSyntaxError{\"invalid character '\\\\x01' after top-level value\", 5}).err()},\n\t{in: \"\\x01 true\", err: (&JsonSyntaxError{\"invalid character '\\\\x01' looking for beginning of value\", 1}).err()},\n\t{in: \" false \\x01\", err: (&JsonSyntaxError{\"invalid character '\\\\x01' after top-level value\", 8}).err()},\n\t{in: \"\\x01 1.2\", err: (&JsonSyntaxError{\"invalid character '\\\\x01' looking for beginning of value\", 1}).err()},\n\t{in: \" 3.4 \\x01\", err: (&JsonSyntaxError{\"invalid character '\\\\x01' after top-level value\", 6}).err()},\n\t{in: \"\\x01 \\\"string\\\"\", err: (&JsonSyntaxError{\"invalid character '\\\\x01' looking for beginning of value\", 1}).err()},\n\t{in: \" \\\"string\\\" \\x01\", err: (&JsonSyntaxError{\"invalid character '\\\\x01' after top-level value\", 11}).err()},\n\n\t// array tests\n\t{in: `[1, 2, 3]`, ptr: new([3]int), out: [3]int{1, 2, 3}},\n\t{in: `[1, 2, 3]`, ptr: new([1]int), out: [1]int{1}},\n\t{in: `[1, 2, 3]`, ptr: new([5]int), out: [5]int{1, 2, 3, 0, 0}},\n\t{in: `[1, 2, 3]`, ptr: new(MustNotUnmarshalJSON), err: errors.New(\"MustNotUnmarshalJSON was used\")},\n\n\t// empty array to interface test\n\t{in: `[]`, ptr: new([]interface{}), out: []interface{}{}},\n\t{in: `null`, ptr: new([]interface{}), out: []interface{}(nil)},\n\t{in: `{\"T\":[]}`, ptr: new(map[string]interface{}), out: map[string]interface{}{\"T\": []interface{}{}}},\n\t{in: `{\"T\":null}`, ptr: new(map[string]interface{}), out: map[string]interface{}{\"T\": interface{}(nil)}},\n\n\t// composite tests\n\t{in: allValueIndent, ptr: new(All), out: allValue},\n\t{in: allValueCompact, ptr: new(All), out: allValue},\n\t{in: allValueIndent, ptr: new(*All), out: &allValue},\n\t{in: allValueCompact, ptr: new(*All), out: &allValue},\n\t{in: pallValueIndent, ptr: new(All), out: pallValue},\n\t{in: pallValueCompact, ptr: new(All), out: pallValue},\n\t{in: pallValueIndent, ptr: new(*All), out: &pallValue},\n\t{in: pallValueCompact, ptr: new(*All), out: &pallValue},\n\n\t// unmarshal interface test\n\t{in: `{\"T\":false}`, ptr: new(unmarshaler), out: umtrue}, // use \"false\" so test will fail if custom unmarshaler is not called\n\t{in: `{\"T\":false}`, ptr: new(*unmarshaler), out: &umtrue},\n\t{in: `[{\"T\":false}]`, ptr: new([]unmarshaler), out: umslice},\n\t{in: `[{\"T\":false}]`, ptr: new(*[]unmarshaler), out: &umslice},\n\t{in: `{\"M\":{\"T\":\"x:y\"}}`, ptr: new(ustruct), out: umstruct},\n\n\t// UnmarshalText interface test\n\t{in: `\"x:y\"`, ptr: new(unmarshalerText), out: umtrueXY},\n\t{in: `\"x:y\"`, ptr: new(*unmarshalerText), out: &umtrueXY},\n\t{in: `[\"x:y\"]`, ptr: new([]unmarshalerText), out: umsliceXY},\n\t{in: `[\"x:y\"]`, ptr: new(*[]unmarshalerText), out: &umsliceXY},\n\t{in: `{\"M\":\"x:y\"}`, ptr: new(ustructText), out: umstructXY},\n\n\t// integer-keyed map test\n\t{\n\t\tin: `{\"-1\":\"a\",\"0\":\"b\",\"1\":\"c\"}`,\n\t\tptr: new(map[int]string),\n\t\tout: map[int]string{-1: \"a\", 0: \"b\", 1: \"c\"},\n\t},\n\t{\n\t\tin: `{\"0\":\"a\",\"10\":\"c\",\"9\":\"b\"}`,\n\t\tptr: new(map[u8]string),\n\t\tout: map[u8]string{0: \"a\", 9: \"b\", 10: \"c\"},\n\t},\n\t{\n\t\tin: `{\"-9223372036854775808\":\"min\",\"9223372036854775807\":\"max\"}`,\n\t\tptr: new(map[int64]string),\n\t\tout: map[int64]string{math.MinInt64: \"min\", math.MaxInt64: \"max\"},\n\t},\n\t{\n\t\tin: `{\"18446744073709551615\":\"max\"}`,\n\t\tptr: new(map[uint64]string),\n\t\tout: map[uint64]string{math.MaxUint64: \"max\"},\n\t},\n\t{\n\t\tin: `{\"0\":false,\"10\":true}`,\n\t\tptr: new(map[uintptr]bool),\n\t\tout: map[uintptr]bool{0: false, 10: true},\n\t},\n\n\t// Check that MarshalText and UnmarshalText take precedence\n\t// over default integer handling in map keys.\n\t{\n\t\tin: `{\"u2\":4}`,\n\t\tptr: new(map[u8marshal]int),\n\t\tout: map[u8marshal]int{2: 4},\n\t},\n\t{\n\t\tin: `{\"2\":4}`,\n\t\tptr: new(map[u8marshal]int),\n\t\terr: errMissingU8Prefix,\n\t},\n\n\t// integer-keyed map errors\n\t{\n\t\tin: `{\"abc\":\"abc\"}`,\n\t\tptr: new(map[int]string),\n\t\terr: &json.UnmarshalTypeError{Value: \"number abc\", Type: reflect.TypeOf(0), Offset: 2},\n\t},\n\t{\n\t\tin: `{\"256\":\"abc\"}`,\n\t\tptr: new(map[uint8]string),\n\t\terr: &json.UnmarshalTypeError{Value: \"number 256\", Type: reflect.TypeOf(uint8(0)), Offset: 2},\n\t},\n\t{\n\t\tin: `{\"128\":\"abc\"}`,\n\t\tptr: new(map[int8]string),\n\t\terr: &json.UnmarshalTypeError{Value: \"number 128\", Type: reflect.TypeOf(int8(0)), Offset: 2},\n\t},\n\t{\n\t\tin: `{\"-1\":\"abc\"}`,\n\t\tptr: new(map[uint8]string),\n\t\terr: &json.UnmarshalTypeError{Value: \"number -1\", Type: reflect.TypeOf(uint8(0)), Offset: 2},\n\t},\n\t{\n\t\tin: `{\"F\":{\"a\":2,\"3\":4}}`,\n\t\tptr: new(map[string]map[int]int),\n\t\terr: &json.UnmarshalTypeError{Value: \"number a\", Type: reflect.TypeOf(0), Offset: 7},\n\t},\n\t{\n\t\tin: `{\"F\":{\"a\":2,\"3\":4}}`,\n\t\tptr: new(map[string]map[uint]int),\n\t\terr: &json.UnmarshalTypeError{Value: \"number a\", Type: reflect.TypeOf(uint(0)), Offset: 7},\n\t},\n\n\t// Map keys can be encoding.TextUnmarshalers.\n\t{in: `{\"x:y\":true}`, ptr: new(map[unmarshalerText]bool), out: ummapXY},\n\t// If multiple values for the same key exists, only the most recent value is used.\n\t{in: `{\"x:y\":false,\"x:y\":true}`, ptr: new(map[unmarshalerText]bool), out: ummapXY},\n\n\t{\n\t\tin: `{\n \"Level0\": 1,\n \"Level1b\": 2,\n \"Level1c\": 3,\n \"x\": 4,\n \"Level1a\": 5,\n \"LEVEL1B\": 6,\n \"e\": {\n \"Level1a\": 8,\n \"Level1b\": 9,\n \"Level1c\": 10,\n \"Level1d\": 11,\n \"x\": 12\n },\n \"Loop1\": 13,\n \"Loop2\": 14,\n \"X\": 15,\n \"Y\": 16,\n \"Z\": 17,\n \"Q\": 18\n }`,\n\t\tptr: new(Top),\n\t\tout: Top{\n\t\t\tLevel0: 1,\n\t\t\tEmbed0: Embed0{\n\t\t\t\tLevel1b: 2,\n\t\t\t\tLevel1c: 3,\n\t\t\t},\n\t\t\tEmbed0a: &Embed0a{\n\t\t\t\tLevel1a: 5,\n\t\t\t\tLevel1b: 6,\n\t\t\t},\n\t\t\tEmbed0b: &Embed0b{\n\t\t\t\tLevel1a: 8,\n\t\t\t\tLevel1b: 9,\n\t\t\t\tLevel1c: 10,\n\t\t\t\tLevel1d: 11,\n\t\t\t\tLevel1e: 12,\n\t\t\t},\n\t\t\tLoop: Loop{\n\t\t\t\tLoop1: 13,\n\t\t\t\tLoop2: 14,\n\t\t\t},\n\t\t\tEmbed0p: Embed0p{\n\t\t\t\tPoint: image.Point{X: 15, Y: 16},\n\t\t\t},\n\t\t\tEmbed0q: Embed0q{\n\t\t\t\tPoint: Point{Z: 17},\n\t\t\t},\n\t\t\tembed: embed{\n\t\t\t\tQ: 18,\n\t\t\t},\n\t\t},\n\t},\n\t{\n\t\tin: `{\"hello\": 1}`,\n\t\tptr: new(Ambig),\n\t\tout: Ambig{First: 1},\n\t},\n\n\t{\n\t\tin: `{\"X\": 1,\"Y\":2}`,\n\t\tptr: new(S5),\n\t\tout: S5{S8: S8{S9: S9{Y: 2}}},\n\t},\n\t{\n\t\tin: `{\"X\": 1,\"Y\":2}`,\n\t\tptr: new(S5),\n\t\terr: fmt.Errorf(\"json: unknown field \\\"X\\\"\"),\n\t\tdisallowUnknownFields: true,\n\t},\n\t{\n\t\tin: `{\"X\": 1,\"Y\":2}`,\n\t\tptr: new(S10),\n\t\tout: S10{S13: S13{S8: S8{S9: S9{Y: 2}}}},\n\t},\n\t{\n\t\tin: `{\"X\": 1,\"Y\":2}`,\n\t\tptr: new(S10),\n\t\terr: fmt.Errorf(\"json: unknown field \\\"X\\\"\"),\n\t\tdisallowUnknownFields: true,\n\t},\n\t{\n\t\tin: `{\"I\": 0, \"I\": null, \"J\": null}`,\n\t\tptr: new(DoublePtr),\n\t\tout: DoublePtr{I: nil, J: nil},\n\t},\n\n\t// invalid UTF-8 is coerced to valid UTF-8.\n\t{\n\t\tin: \"\\\"hello\\xffworld\\\"\",\n\t\tptr: new(string),\n\t\tout: \"hello\\xffworld\",\n\t\tvalidateString: false,\n\t},\n\t{\n\t\tin: \"\\\"hello\\xc2\\xc2world\\\"\",\n\t\tptr: new(string),\n\t\tout: \"hello\\xc2\\xc2world\",\n\t\tvalidateString: false,\n\t},\n\t{\n\t\tin: \"\\\"hello\\xc2\\xffworld\\\"\",\n\t\tptr: new(string),\n\t\tout: \"hello\\xc2\\xffworld\",\n\t},\n\t{\n\t\tin: \"\\\"hello\\\\ud800world\\\"\",\n\t\tptr: new(string),\n\t\tout: \"hello\\ufffdworld\",\n\t},\n\t{\n\t\tin: \"\\\"hello\\\\ud800\\\\ud800world\\\"\",\n\t\tptr: new(string),\n\t\tout: \"hello\\ufffd\\ufffdworld\",\n\t},\n\t{\n\t\tin: \"\\\"hello\\xed\\xa0\\x80\\xed\\xb0\\x80world\\\"\",\n\t\tptr: new(string),\n\t\tout: \"hello\\xed\\xa0\\x80\\xed\\xb0\\x80world\",\n\t},\n\n\t// Used to be issue 8305, but time.Time implements encoding.TextUnmarshaler so this works now.\n\t{\n\t\tin: `{\"2009-11-10T23:00:00Z\": \"hello world\"}`,\n\t\tptr: new(map[time.Time]string),\n\t\tout: map[time.Time]string{time.Date(2009, 11, 10, 23, 0, 0, 0, time.UTC): \"hello world\"},\n\t},\n\n\t// issue 8305\n\t{\n\t\tin: `{\"2009-11-10T23:00:00Z\": \"hello world\"}`,\n\t\tptr: new(map[Point]string),\n\t\terr: &json.UnmarshalTypeError{Value: \"object\", Type: reflect.TypeOf(map[Point]string{}), Offset: 1},\n\t},\n\t{\n\t\tin: `{\"asdf\": \"hello world\"}`,\n\t\tptr: new(map[unmarshaler]string),\n\t\terr: &json.UnmarshalTypeError{Value: \"object\", Type: reflect.TypeOf(map[unmarshaler]string{}), Offset: 1},\n\t},\n\n\t// related to issue 13783.\n\t// Go 1.7 changed marshaling a slice of typed byte to use the methods on the byte type,\n\t// similar to marshaling a slice of typed int.\n\t// These tests check that, assuming the byte type also has valid decoding methods,\n\t// either the old base64 string encoding or the new per-element encoding can be\n\t// successfully unmarshaled. The custom unmarshalers were accessible in earlier\n\t// versions of Go, even though the custom marshaler was not.\n\t{\n\t\tin: `\"AQID\"`,\n\t\tptr: new([]byteWithMarshalJSON),\n\t\tout: []byteWithMarshalJSON{1, 2, 3},\n\t},\n\t{\n\t\tin: `[\"Z01\",\"Z02\",\"Z03\"]`,\n\t\tptr: new([]byteWithMarshalJSON),\n\t\tout: []byteWithMarshalJSON{1, 2, 3},\n\t\tgolden: true,\n\t},\n\t{\n\t\tin: `\"AQID\"`,\n\t\tptr: new([]byteWithMarshalText),\n\t\tout: []byteWithMarshalText{1, 2, 3},\n\t},\n\t{\n\t\tin: `[\"Z01\",\"Z02\",\"Z03\"]`,\n\t\tptr: new([]byteWithMarshalText),\n\t\tout: []byteWithMarshalText{1, 2, 3},\n\t\tgolden: true,\n\t},\n\t{\n\t\tin: `\"AQID\"`,\n\t\tptr: new([]byteWithPtrMarshalJSON),\n\t\tout: []byteWithPtrMarshalJSON{1, 2, 3},\n\t},\n\t{\n\t\tin: `[\"Z01\",\"Z02\",\"Z03\"]`,\n\t\tptr: new([]byteWithPtrMarshalJSON),\n\t\tout: []byteWithPtrMarshalJSON{1, 2, 3},\n\t\tgolden: true,\n\t},\n\t{\n\t\tin: `\"AQID\"`,\n\t\tptr: new([]byteWithPtrMarshalText),\n\t\tout: []byteWithPtrMarshalText{1, 2, 3},\n\t},\n\t{\n\t\tin: `[\"Z01\",\"Z02\",\"Z03\"]`,\n\t\tptr: new([]byteWithPtrMarshalText),\n\t\tout: []byteWithPtrMarshalText{1, 2, 3},\n\t\tgolden: true,\n\t},\n\n\t// ints work with the marshaler but not the base64 []byte case\n\t{\n\t\tin: `[\"Z01\",\"Z02\",\"Z03\"]`,\n\t\tptr: new([]intWithMarshalJSON),\n\t\tout: []intWithMarshalJSON{1, 2, 3},\n\t\tgolden: true,\n\t},\n\t{\n\t\tin: `[\"Z01\",\"Z02\",\"Z03\"]`,\n\t\tptr: new([]intWithMarshalText),\n\t\tout: []intWithMarshalText{1, 2, 3},\n\t\tgolden: true,\n\t},\n\t{\n\t\tin: `[\"Z01\",\"Z02\",\"Z03\"]`,\n\t\tptr: new([]intWithPtrMarshalJSON),\n\t\tout: []intWithPtrMarshalJSON{1, 2, 3},\n\t\tgolden: true,\n\t},\n\t{\n\t\tin: `[\"Z01\",\"Z02\",\"Z03\"]`,\n\t\tptr: new([]intWithPtrMarshalText),\n\t\tout: []intWithPtrMarshalText{1, 2, 3},\n\t\tgolden: true,\n\t},\n\n\t{in: `0.000001`, ptr: new(float64), out: 0.000001, golden: true},\n\t{in: `1e-7`, ptr: new(float64), out: 1e-7, golden: true},\n\t{in: `100000000000000000000`, ptr: new(float64), out: 100000000000000000000.0, golden: true},\n\t{in: `1e+21`, ptr: new(float64), out: 1e21, golden: true},\n\t{in: `-0.000001`, ptr: new(float64), out: -0.000001, golden: true},\n\t{in: `-1e-7`, ptr: new(float64), out: -1e-7, golden: true},\n\t{in: `-100000000000000000000`, ptr: new(float64), out: -100000000000000000000.0, golden: true},\n\t{in: `-1e+21`, ptr: new(float64), out: -1e21, golden: true},\n\t{in: `999999999999999900000`, ptr: new(float64), out: 999999999999999900000.0, golden: true},\n\t{in: `9007199254740992`, ptr: new(float64), out: 9007199254740992.0, golden: true},\n\t{in: `9007199254740993`, ptr: new(float64), out: 9007199254740992.0, golden: false},\n\n\t{\n\t\tin: `{\"V\": {\"F2\": \"hello\"}}`,\n\t\tptr: new(VOuter),\n\t\terr: &json.UnmarshalTypeError{\n\t\t\tValue: \"string\",\n\t\t\tStruct: \"V\",\n\t\t\tField: \"V.F2\",\n\t\t\tType: reflect.TypeOf(int32(0)),\n\t\t\tOffset: 20,\n\t\t},\n\t},\n\t{\n\t\tin: `{\"V\": {\"F4\": {}, \"F2\": \"hello\"}}`,\n\t\tptr: new(VOuter),\n\t\terr: &json.UnmarshalTypeError{\n\t\t\tValue: \"string\",\n\t\t\tStruct: \"V\",\n\t\t\tField: \"V.F2\",\n\t\t\tType: reflect.TypeOf(int32(0)),\n\t\t\tOffset: 30,\n\t\t},\n\t},\n\n\t// issue 15146.\n\t// invalid inputs in wrongStringTests below.\n\t{in: `{\"B\":\"true\"}`, ptr: new(B), out: B{true}, golden: true},\n\t{in: `{\"B\":\"false\"}`, ptr: new(B), out: B{false}, golden: true},\n\t{in: `{\"B\": \"maybe\"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal \"maybe\" into bool`)},\n\t{in: `{\"B\": \"tru\"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal \"tru\" into bool`)},\n\t{in: `{\"B\": \"False\"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal \"False\" into bool`)},\n\t{in: `{\"B\": \"null\"}`, ptr: new(B), out: B{false}},\n\t{in: `{\"B\": \"nul\"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal \"nul\" into bool`)},\n\t{in: `{\"B\": [2, 3]}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal unquoted value into bool`)},\n\n\t// additional tests for disallowUnknownFields\n\t{\n\t\tin: `{\n \"Level0\": 1,\n \"Level1b\": 2,\n \"Level1c\": 3,\n \"x\": 4,\n \"Level1a\": 5,\n \"LEVEL1B\": 6,\n \"e\": {\n \"Level1a\": 8,\n \"Level1b\": 9,\n \"Level1c\": 10,\n \"Level1d\": 11,\n \"x\": 12\n },\n \"Loop1\": 13,\n \"Loop2\": 14,\n \"X\": 15,\n \"Y\": 16,\n \"Z\": 17,\n \"Q\": 18,\n \"extra\": true\n }`,\n\t\tptr: new(Top),\n\t\terr: fmt.Errorf(\"json: unknown field \\\"extra\\\"\"),\n\t\tdisallowUnknownFields: true,\n\t},\n\t{\n\t\tin: `{\n \"Level0\": 1,\n \"Level1b\": 2,\n \"Level1c\": 3,\n \"x\": 4,\n \"Level1a\": 5,\n \"LEVEL1B\": 6,\n \"e\": {\n \"Level1a\": 8,\n \"Level1b\": 9,\n \"Level1c\": 10,\n \"Level1d\": 11,\n \"x\": 12,\n \"extra\": null\n },\n \"Loop1\": 13,\n \"Loop2\": 14,\n \"X\": 15,\n \"Y\": 16,\n \"Z\": 17,\n \"Q\": 18\n }`,\n\t\tptr: new(Top),\n\t\terr: fmt.Errorf(\"json: unknown field \\\"extra\\\"\"),\n\t\tdisallowUnknownFields: true,\n\t},\n\t// issue 26444\n\t// json.UnmarshalTypeError without field & struct values\n\t{\n\t\tin: `{\"data\":{\"test1\": \"bob\", \"test2\": 123}}`,\n\t\tptr: new(mapStringToStringData),\n\t\terr: &json.UnmarshalTypeError{Value: \"number\", Type: reflect.TypeOf(\"\"), Offset: 37, Struct: \"mapStringToStringData\", Field: \"data\"},\n\t},\n\t{\n\t\tin: `{\"data\":{\"test1\": 123, \"test2\": \"bob\"}}`,\n\t\tptr: new(mapStringToStringData),\n\t\terr: &json.UnmarshalTypeError{Value: \"number\", Type: reflect.TypeOf(\"\"), Offset: 21, Struct: \"mapStringToStringData\", Field: \"data\"},\n\t},\n\n\t// trying to decode JSON arrays or objects via TextUnmarshaler\n\t{\n\t\tin: `[1, 2, 3]`,\n\t\tptr: new(MustNotUnmarshalText),\n\t\terr: &json.UnmarshalTypeError{Value: \"array\", Type: reflect.TypeOf(&MustNotUnmarshalText{}), Offset: 1},\n\t},\n\t{\n\t\tin: `{\"foo\": \"bar\"}`,\n\t\tptr: new(MustNotUnmarshalText),\n\t\terr: &json.UnmarshalTypeError{Value: \"object\", Type: reflect.TypeOf(&MustNotUnmarshalText{}), Offset: 1},\n\t},\n\t// #22369\n\t{\n\t\tin: `{\"PP\": {\"T\": {\"Y\": \"bad-type\"}}}`,\n\t\tptr: new(P),\n\t\terr: &json.UnmarshalTypeError{\n\t\t\tValue: \"string\",\n\t\t\tStruct: \"T\",\n\t\t\tField: \"PP.T.Y\",\n\t\t\tType: reflect.TypeOf(0),\n\t\t\tOffset: 29,\n\t\t},\n\t},\n\t{\n\t\tin: `{\"Ts\": [{\"Y\": 1}, {\"Y\": 2}, {\"Y\": \"bad-type\"}]}`,\n\t\tptr: new(PP),\n\t\terr: &json.UnmarshalTypeError{\n\t\t\tValue: \"string\",\n\t\t\tStruct: \"T\",\n\t\t\tField: \"Ts.Y\",\n\t\t\tType: reflect.TypeOf(0),\n\t\t\tOffset: 29,\n\t\t},\n\t},\n\t// #14702\n\t{\n\t\tin: `invalid`,\n\t\tptr: new(json.Number),\n\t\terr: (&JsonSyntaxError{\n\t\t\tMsg: \"invalid character 'i' looking for beginning of value\",\n\t\t\tOffset: 1,\n\t\t}).err(),\n\t},\n\t{\n\t\tin: `\"invalid\"`,\n\t\tptr: new(json.Number),\n\t\terr: fmt.Errorf(\"json: invalid number literal, trying to unmarshal %q into Number\", `\"invalid\"`),\n\t},\n\t{\n\t\tin: `{\"A\":\"invalid\"}`,\n\t\tptr: new(struct{ A json.Number }),\n\t\terr: fmt.Errorf(\"json: invalid number literal, trying to unmarshal %q into Number\", `\"invalid\"`),\n\t},\n\t{\n\t\tin: `{\"A\":\"invalid\"}`,\n\t\tptr: new(struct {\n\t\t\tA json.Number `json:\",string\"`\n\t\t}),\n\t\terr: fmt.Errorf(\"json: invalid use of ,string struct tag, trying to unmarshal %q into json.Number\", `invalid`),\n\t},\n\t{\n\t\tin: `{\"A\":\"invalid\"}`,\n\t\tptr: new(map[string]json.Number),\n\t\terr: fmt.Errorf(\"json: invalid number literal, trying to unmarshal %q into Number\", `\"invalid\"`),\n\t},\n\n\t// UTF-8 and string validation tests\n\t{in: `\\u`, ptr: new(interface{}), err: fmt.Errorf(\"json: invald char\"), validateString: true},\n\t{in: `\\u`, ptr: new(string), err: fmt.Errorf(\"json: invald char\"), validateString: true},\n\n\t{in: \"\\\"\\x00\\\"\", ptr: new(interface{}), err: fmt.Errorf(\"json: invald char\"), validateString: true},\n\t{in: \"\\\"\\x00\\\"\", ptr: new(string), err: fmt.Errorf(\"json: invald char\"), validateString: true},\n\t{in: \"\\\"\\xff\\\"\", ptr: new(interface{}), out: interface{}(\"\\ufffd\"), validateString: true},\n\t{in: \"\\\"\\xff\\\"\", ptr: new(string), out: \"\\ufffd\", validateString: true},\n\t{in: \"\\\"\\x00\\\"\", ptr: new(interface{}), out: interface{}(\"\\x00\"), validateString: false},\n\t{in: \"\\\"\\x00\\\"\", ptr: new(string), out: \"\\x00\", validateString: false},\n\t{in: \"\\\"\\xff\\\"\", ptr: new(interface{}), out: interface{}(\"\\xff\"), validateString: false},\n\t{in: \"\\\"\\xff\\\"\", ptr: new(string), out: \"\\xff\", validateString: false},\n\n\t// cases found by fuzz\n\t{\n\t\tin: `{\"H\":{\"A\": {}}}`,\n\t\tptr: new(struct {\n\t\t\tF0 struct {\n\t\t\t\tF1 json.Number \"json:\\\"a,omitempty\\\"\"\n\t\t\t} \"json:\\\"H,\\\"\"\n\t\t}),\n\t\terr: fmt.Errorf(\"Mismatch type json.Number with value object..\"),\n\t},\n}\n\nfunc trim(b []byte) []byte {\n\tif len(b) > 20 {\n\t\treturn b[0:20]\n\t}\n\treturn b\n}\n\nfunc diff(t *testing.T, a, b []byte) {\n\tfor i := 0; ; i++ {\n\t\tif i >= len(a) || i >= len(b) || a[i] != b[i] {\n\t\t\tj := i - 10\n\t\t\tif j < 0 {\n\t\t\t\tj = 0\n\t\t\t}\n\t\t\tt.Errorf(\"diverge at %d: «%s» vs «%s»\", i, trim(a[j:]), trim(b[j:]))\n\t\t\treturn\n\t\t}\n\t}\n}\n\nfunc TestMarshal(t *testing.T) {\n\tb, err := Marshal(allValue)\n\tif err != nil {\n\t\tt.Fatalf(\"Marshal allValue: %v\", err)\n\t}\n\tif string(b) != allValueCompact {\n\t\tt.Errorf(\"Marshal allValueCompact\")\n\t\tdiff(t, b, []byte(allValueCompact))\n\t\treturn\n\t}\n\n\tb, err = Marshal(pallValue)\n\tif err != nil {\n\t\tt.Fatalf(\"Marshal pallValue: %v\", err)\n\t}\n\tif string(b) != pallValueCompact {\n\t\tt.Errorf(\"Marshal pallValueCompact\")\n\t\tdiff(t, b, []byte(pallValueCompact))\n\t\treturn\n\t}\n}\n\nfunc TestMarshalNumberZeroVal(t *testing.T) {\n\tvar n json.Number\n\tout, err := Marshal(n)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\toutStr := string(out)\n\tif outStr != \"0\" {\n\t\tt.Fatalf(\"Invalid zero val for json.Number: %q\", outStr)\n\t}\n}\n\nfunc TestMarshalEmbeds(t *testing.T) {\n\ttop := &Top{\n\t\tLevel0: 1,\n\t\tEmbed0: Embed0{\n\t\t\tLevel1b: 2,\n\t\t\tLevel1c: 3,\n\t\t},\n\t\tEmbed0a: &Embed0a{\n\t\t\tLevel1a: 5,\n\t\t\tLevel1b: 6,\n\t\t},\n\t\tEmbed0b: &Embed0b{\n\t\t\tLevel1a: 8,\n\t\t\tLevel1b: 9,\n\t\t\tLevel1c: 10,\n\t\t\tLevel1d: 11,\n\t\t\tLevel1e: 12,\n\t\t},\n\t\tLoop: Loop{\n\t\t\tLoop1: 13,\n\t\t\tLoop2: 14,\n\t\t},\n\t\tEmbed0p: Embed0p{\n\t\t\tPoint: image.Point{X: 15, Y: 16},\n\t\t},\n\t\tEmbed0q: Embed0q{\n\t\t\tPoint: Point{Z: 17},\n\t\t},\n\t\tembed: embed{\n\t\t\tQ: 18,\n\t\t},\n\t}\n\tb, err := Marshal(top)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\twant := \"{\\\"Level0\\\":1,\\\"Level1b\\\":2,\\\"Level1c\\\":3,\\\"Level1a\\\":5,\\\"LEVEL1B\\\":6,\\\"e\\\":{\\\"Level1a\\\":8,\\\"Level1b\\\":9,\\\"Level1c\\\":10,\\\"Level1d\\\":11,\\\"x\\\":12},\\\"Loop1\\\":13,\\\"Loop2\\\":14,\\\"X\\\":15,\\\"Y\\\":16,\\\"Z\\\":17,\\\"Q\\\":18}\"\n\tif string(b) != want {\n\t\tt.Errorf(\"Wrong marshal result.\\n got: %q\\nwant: %q\", b, want)\n\t}\n}\n\nfunc TestUnmarshal(t *testing.T) {\n\tfor i, tt := range unmarshalTests {\n\t\t// construct decoder\n\t\tdec := decoder.NewDecoder(tt.in)\n\t\tif tt.useNumber {\n\t\t\tdec.UseNumber()\n\t\t}\n\t\tif tt.disallowUnknownFields {\n\t\t\tdec.DisallowUnknownFields()\n\t\t}\n\t\tif tt.validateString {\n\t\t\tdec.ValidateString()\n\t\t}\n\n\t\t// check decode result for invalid jsons\n\t\tif !json.Valid([]byte(tt.in)) || tt.ptr == nil {\n\t\t\tvar sv interface{}\n\t\t\terr := dec.Decode(&sv)\n\t\t\tif err == nil && tt.err != nil {\n\t\t\t\terr = dec.CheckTrailings()\n\t\t\t\tif err == nil && tt.err != nil {\n\t\t\t\t\tt.Errorf(\"test json #%d: %v, %v, want %v\", i, tt.in, err, tt.err)\n\t\t\t\t}\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\t\ttyp := reflect.TypeOf(tt.ptr)\n\t\tif typ.Kind() != reflect.Ptr {\n\t\t\tt.Errorf(\"#%d: unmarshalTest.ptr %T is not a pointer type\", i, tt.ptr)\n\t\t\tcontinue\n\t\t}\n\t\ttyp = typ.Elem()\n\n\t\tv := reflect.New(typ)\n\n\t\tif !reflect.DeepEqual(tt.ptr, v.Interface()) {\n\t\t\t// There's no reason for ptr to point to non-zero data,\n\t\t\t// as we decode into new(right-type), so the data is\n\t\t\t// discarded.\n\t\t\t// This can easily mean tests that silently don't test\n\t\t\t// what they should. To test decoding into existing\n\t\t\t// data, see TestPrefilled.\n\t\t\tt.Errorf(\"#%d: unmarshalTest.ptr %#v is not a pointer to a zero value\", i, tt.ptr)\n\t\t\tcontinue\n\t\t}\n\n\t\tif err := dec.Decode(v.Interface()); (err == nil) != (tt.err == nil) {\n\t\t\tspew.Dump(tt)\n\t\t\tt.Fatalf(\"#%d: %v, want %v\", i, err, tt.err)\n\t\t\tcontinue\n\t\t} else if err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tif !reflect.DeepEqual(v.Elem().Interface(), tt.out) {\n\t\t\trequire.Equal(t, v.Elem().Interface(), tt.out)\n\t\t\tt.Errorf(\"#%d: mismatch\\nhave: %#+v\\nwant: %#+v\", i, v.Elem().Interface(), tt.out)\n\t\t\tdata, _ := Marshal(v.Elem().Interface())\n\t\t\tprintln(string(data))\n\t\t\tdata, _ = Marshal(tt.out)\n\t\t\tprintln(string(data))\n\t\t\tcontinue\n\t\t}\n\n\t\t// Check round trip also decodes correctly.\n\t\tif tt.err == nil {\n\t\t\tenc, err := Marshal(v.Interface())\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"#%d: error re-marshaling: %v\", i, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif tt.golden && !bytes.Equal(enc, []byte(tt.in)) {\n\t\t\t\tt.Errorf(\"#%d: remarshal mismatch:\\nhave: %s\\nwant: %s\", i, enc, []byte(tt.in))\n\t\t\t}\n\t\t\tvv := reflect.New(reflect.TypeOf(tt.ptr).Elem())\n\t\t\tdec = decoder.NewDecoder(string(enc))\n\t\t\tif tt.useNumber {\n\t\t\t\tdec.UseNumber()\n\t\t\t}\n\t\t\tif err := dec.Decode(vv.Interface()); err != nil {\n\t\t\t\tt.Errorf(\"#%d: error re-unmarshaling %#q: %v\", i, enc, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(v.Elem().Interface(), vv.Elem().Interface()) {\n\t\t\t\tt.Errorf(\"#%d: mismatch\\nhave: %#+v\\nwant: %#+v\", i, v.Elem().Interface(), vv.Elem().Interface())\n\t\t\t\tt.Errorf(\" In: %q\", strings.Map(noSpace, tt.in))\n\t\t\t\tt.Errorf(\"Marshal: %q\", strings.Map(noSpace, string(enc)))\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t}\n}\n\nvar jsonBig []byte\n\nfunc initBig() {\n\tn := 10000\n\tif testing.Short() {\n\t\tn = 100\n\t}\n\tb, err := Marshal(genValue(n))\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tjsonBig = b\n}\n\nfunc genValue(n int) interface{} {\n\tif n > 1 {\n\t\tswitch rand.Intn(2) {\n\t\tcase 0:\n\t\t\treturn genArray(n)\n\t\tcase 1:\n\t\t\treturn genMap(n)\n\t\t}\n\t}\n\tswitch rand.Intn(3) {\n\tcase 0:\n\t\treturn rand.Intn(2) == 0\n\tcase 1:\n\t\treturn float64(rand.Uint64()) / 4294967296\n\tcase 2:\n\t\treturn genString(30)\n\t}\n\tpanic(\"unreachable\")\n}\n\nfunc genString(stddev float64) string {\n\tn := int(math.Abs(rand.NormFloat64()*stddev + stddev/2))\n\tc := make([]rune, n)\n\tfor i := range c {\n\t\tf := math.Abs(rand.NormFloat64()*64 + 32)\n\t\tif f > 0x10ffff {\n\t\t\tf = 0x10ffff\n\t\t}\n\t\tc[i] = rune(f)\n\t}\n\treturn string(c)\n}\n\nfunc genArray(n int) []interface{} {\n\tf := int(math.Abs(rand.NormFloat64()) * math.Min(10, float64(n/2)))\n\tif f > n {\n\t\tf = n\n\t}\n\tif f < 1 {\n\t\tf = 1\n\t}\n\tx := make([]interface{}, f)\n\tfor i := range x {\n\t\tx[i] = genValue(((i+1)*n)/f - (i*n)/f)\n\t}\n\treturn x\n}\n\nfunc genMap(n int) map[string]interface{} {\n\tf := int(math.Abs(rand.NormFloat64()) * math.Min(10, float64(n/2)))\n\tif f > n {\n\t\tf = n\n\t}\n\tif n > 0 && f == 0 {\n\t\tf = 1\n\t}\n\tx := make(map[string]interface{})\n\tx[genString(10)] = genValue(n / f)\n\treturn x\n}\n\nfunc TestUnmarshalMarshal(t *testing.T) {\n\tinitBig()\n\tvar v interface{}\n\tif err := Unmarshal(jsonBig, &v); err != nil {\n\t\tif e, ok := err.(decoder.SyntaxError); ok {\n\t\t\tprintln(e.Description())\n\t\t}\n\t\tt.Fatalf(\"Unmarshal: %v\", err)\n\t}\n\tb, err := Marshal(v)\n\tif err != nil {\n\t\tt.Fatalf(\"Marshal: %v\", err)\n\t}\n\tif !bytes.Equal(jsonBig, b) {\n\t\tt.Errorf(\"Marshal jsonBig\")\n\t\tdiff(t, b, jsonBig)\n\t\tprintln(string(b))\n\t\tprintln(string(jsonBig))\n\t\treturn\n\t}\n}\n\nvar numberTests = []struct {\n\tin string\n\ti int64\n\tintErr string\n\tf float64\n\tfloatErr string\n}{\n\t{in: \"-1.23e1\", intErr: \"strconv.ParseInt: parsing \\\"-1.23e1\\\": invalid syntax\", f: -1.23e1},\n\t{in: \"-12\", i: -12, f: -12.0},\n\t{in: \"1e1000\", intErr: \"strconv.ParseInt: parsing \\\"1e1000\\\": invalid syntax\", floatErr: \"strconv.ParseFloat: parsing \\\"1e1000\\\": value out of range\"},\n}\n\n// Independent of Decode, basic coverage of the accessors in json.Number\nfunc TestNumberAccessors(t *testing.T) {\n\tfor _, tt := range numberTests {\n\t\tn := json.Number(tt.in)\n\t\tif s := n.String(); s != tt.in {\n\t\t\tt.Errorf(\"json.Number(%q).String() is %q\", tt.in, s)\n\t\t}\n\t\tif i, err := n.Int64(); err == nil && tt.intErr == \"\" && i != tt.i {\n\t\t\tt.Errorf(\"json.Number(%q).Int64() is %d\", tt.in, i)\n\t\t} else if (err == nil && tt.intErr != \"\") || (err != nil && err.Error() != tt.intErr) {\n\t\t\tt.Errorf(\"json.Number(%q).Int64() wanted error %q but got: %v\", tt.in, tt.intErr, err)\n\t\t}\n\t\tif f, err := n.Float64(); err == nil && tt.floatErr == \"\" && f != tt.f {\n\t\t\tt.Errorf(\"json.Number(%q).Float64() is %g\", tt.in, f)\n\t\t} else if (err == nil && tt.floatErr != \"\") || (err != nil && err.Error() != tt.floatErr) {\n\t\t\tt.Errorf(\"json.Number(%q).Float64() wanted error %q but got: %v\", tt.in, tt.floatErr, err)\n\t\t}\n\t}\n}\n\nfunc TestLargeByteSlice(t *testing.T) {\n\ts0 := make([]byte, 2000)\n\tfor i := range s0 {\n\t\ts0[i] = byte(i)\n\t}\n\tb, err := Marshal(s0)\n\tif err != nil {\n\t\tt.Fatalf(\"Marshal: %v\", err)\n\t}\n\tvar s1 []byte\n\tif err := Unmarshal(b, &s1); err != nil {\n\t\tt.Fatalf(\"Unmarshal: %v\", err)\n\t}\n\tif !bytes.Equal(s0, s1) {\n\t\tt.Errorf(\"Marshal large byte slice\")\n\t\tdiff(t, s0, s1)\n\t}\n}\n\ntype Xint struct {\n\tX int\n}\n\nfunc TestUnmarshalPtrPtr(t *testing.T) {\n\tvar xint Xint\n\tpxint := &xint\n\tif err := Unmarshal([]byte(`{\"X\":1}`), &pxint); err != nil {\n\t\tt.Fatalf(\"Unmarshal: %v\", err)\n\t}\n\tif xint.X != 1 {\n\t\tt.Fatalf(\"Did not write to xint\")\n\t}\n}\n\nfunc noSpace(c rune) rune {\n\tif utils.IsSpace(byte(c)) { // only used for ascii\n\t\treturn -1\n\t}\n\treturn c\n}\n\ntype All struct {\n\tBool bool\n\tInt int\n\tInt8 int8\n\tInt16 int16\n\tInt32 int32\n\tInt64 int64\n\tUint uint\n\tUint8 uint8\n\tUint16 uint16\n\tUint32 uint32\n\tUint64 uint64\n\tUintptr uintptr\n\tFloat32 float32\n\tFloat64 float64\n\n\tFoo string `json:\"bar\"`\n\tFoo2 string `json:\"bar2,dummyopt\"`\n\n\tIntStr int64 `json:\",string\"`\n\tUintptrStr uintptr `json:\",string\"`\n\n\tPBool *bool\n\tPInt *int\n\tPInt8 *int8\n\tPInt16 *int16\n\tPInt32 *int32\n\tPInt64 *int64\n\tPUint *uint\n\tPUint8 *uint8\n\tPUint16 *uint16\n\tPUint32 *uint32\n\tPUint64 *uint64\n\tPUintptr *uintptr\n\tPFloat32 *float32\n\tPFloat64 *float64\n\n\tString string\n\tPString *string\n\n\tMap map[string]Small\n\tMapP map[string]*Small\n\tMapPNil map[string]*Small\n\tPMap *map[string]Small\n\tPMapP *map[string]*Small\n\tPMapPNil *map[string]*Small\n\n\tEmptyMap map[string]Small\n\tNilMap map[string]Small\n\n\tSlice []Small\n\tSliceP []*Small\n\tPSlice *[]Small\n\tPSliceP *[]*Small\n\n\tEmptySlice []Small\n\tNilSlice []Small\n\n\tStringSlice []string\n\tByteSlice []byte\n\n\tSmall Small\n\tPSmall *Small\n\tPPSmall **Small\n\n\tInterface interface{}\n\tPInterface *interface{}\n\n\tunexported int\n}\n\ntype Small struct {\n\tTag string\n}\n\nvar allValue = All{\n\tBool: true,\n\tInt: 2,\n\tInt8: 3,\n\tInt16: 4,\n\tInt32: 5,\n\tInt64: 6,\n\tUint: 7,\n\tUint8: 8,\n\tUint16: 9,\n\tUint32: 10,\n\tUint64: 11,\n\tUintptr: 12,\n\tFloat32: 14.25,\n\tFloat64: 15.25,\n\tFoo: \"foo\",\n\tFoo2: \"foo2\",\n\tIntStr: 42,\n\tUintptrStr: 44,\n\tString: \"16\",\n\tMap: map[string]Small{\n\t\t\"17\": {Tag: \"tag17\"},\n\t},\n\tMapP: map[string]*Small{\n\t\t\"18\": {Tag: \"tag19\"},\n\t},\n\tMapPNil: map[string]*Small{\n\t\t\"19\": nil,\n\t},\n\tEmptyMap: map[string]Small{},\n\tSlice: []Small{{Tag: \"tag20\"}, {Tag: \"tag21\"}},\n\tSliceP: []*Small{{Tag: \"tag22\"}, nil, {Tag: \"tag23\"}},\n\tEmptySlice: []Small{},\n\tStringSlice: []string{\"str24\", \"str25\", \"str26\"},\n\tByteSlice: []byte{27, 28, 29},\n\tSmall: Small{Tag: \"tag30\"},\n\tPSmall: &Small{Tag: \"tag31\"},\n\tInterface: 5.2,\n}\n\nvar pallValue = All{\n\tPBool: &allValue.Bool,\n\tPInt: &allValue.Int,\n\tPInt8: &allValue.Int8,\n\tPInt16: &allValue.Int16,\n\tPInt32: &allValue.Int32,\n\tPInt64: &allValue.Int64,\n\tPUint: &allValue.Uint,\n\tPUint8: &allValue.Uint8,\n\tPUint16: &allValue.Uint16,\n\tPUint32: &allValue.Uint32,\n\tPUint64: &allValue.Uint64,\n\tPUintptr: &allValue.Uintptr,\n\tPFloat32: &allValue.Float32,\n\tPFloat64: &allValue.Float64,\n\tPString: &allValue.String,\n\tPMap: &allValue.Map,\n\tPMapP: &allValue.MapP,\n\tPMapPNil: &allValue.MapPNil,\n\tPSlice: &allValue.Slice,\n\tPSliceP: &allValue.SliceP,\n\tPPSmall: &allValue.PSmall,\n\tPInterface: &allValue.Interface,\n}\n\nvar allValueIndent = `{\n \"Bool\": true,\n \"Int\": 2,\n \"Int8\": 3,\n \"Int16\": 4,\n \"Int32\": 5,\n \"Int64\": 6,\n \"Uint\": 7,\n \"Uint8\": 8,\n \"Uint16\": 9,\n \"Uint32\": 10,\n \"Uint64\": 11,\n \"Uintptr\": 12,\n \"Float32\": 14.25,\n \"Float64\": 15.25,\n \"bar\": \"foo\",\n \"bar2\": \"foo2\",\n \"IntStr\": \"42\",\n \"UintptrStr\": \"44\",\n \"PBool\": null,\n \"PInt\": null,\n \"PInt8\": null,\n \"PInt16\": null,\n \"PInt32\": null,\n \"PInt64\": null,\n \"PUint\": null,\n \"PUint8\": null,\n \"PUint16\": null,\n \"PUint32\": null,\n \"PUint64\": null,\n \"PUintptr\": null,\n \"PFloat32\": null,\n \"PFloat64\": null,\n \"String\": \"16\",\n \"PString\": null,\n \"Map\": {\n \"17\": {\n \"Tag\": \"tag17\"\n }\n },\n \"MapP\": {\n \"18\": {\n \"Tag\": \"tag19\"\n }\n },\n \"MapPNil\": {\n \"19\": null\n },\n \"PMap\": null,\n \"PMapP\": null,\n \"PMapPNil\": null,\n \"EmptyMap\": {},\n \"NilMap\": null,\n \"Slice\": [\n {\n \"Tag\": \"tag20\"\n },\n {\n \"Tag\": \"tag21\"\n }\n ],\n \"SliceP\": [\n {\n \"Tag\": \"tag22\"\n },\n null,\n {\n \"Tag\": \"tag23\"\n }\n ],\n \"PSlice\": null,\n \"PSliceP\": null,\n \"EmptySlice\": [],\n \"NilSlice\": null,\n \"StringSlice\": [\n \"str24\",\n \"str25\",\n \"str26\"\n ],\n \"ByteSlice\": \"Gxwd\",\n \"Small\": {\n \"Tag\": \"tag30\"\n },\n \"PSmall\": {\n \"Tag\": \"tag31\"\n },\n \"PPSmall\": null,\n \"Interface\": 5.2,\n \"PInterface\": null\n}`\n\nvar allValueCompact = strings.Map(noSpace, allValueIndent)\n\nvar pallValueIndent = `{\n \"Bool\": false,\n \"Int\": 0,\n \"Int8\": 0,\n \"Int16\": 0,\n \"Int32\": 0,\n \"Int64\": 0,\n \"Uint\": 0,\n \"Uint8\": 0,\n \"Uint16\": 0,\n \"Uint32\": 0,\n \"Uint64\": 0,\n \"Uintptr\": 0,\n \"Float32\": 0,\n \"Float64\": 0,\n \"bar\": \"\",\n \"bar2\": \"\",\n \"IntStr\": \"0\",\n \"UintptrStr\": \"0\",\n \"PBool\": true,\n \"PInt\": 2,\n \"PInt8\": 3,\n \"PInt16\": 4,\n \"PInt32\": 5,\n \"PInt64\": 6,\n \"PUint\": 7,\n \"PUint8\": 8,\n \"PUint16\": 9,\n \"PUint32\": 10,\n \"PUint64\": 11,\n \"PUintptr\": 12,\n \"PFloat32\": 14.25,\n \"PFloat64\": 15.25,\n \"String\": \"\",\n \"PString\": \"16\",\n \"Map\": null,\n \"MapP\": null,\n \"MapPNil\": null,\n \"PMap\": {\n \"17\": {\n \"Tag\": \"tag17\"\n }\n },\n \"PMapP\": {\n \"18\": {\n \"Tag\": \"tag19\"\n }\n },\n \"PMapPNil\": {\n \"19\": null\n },\n \"EmptyMap\": null,\n \"NilMap\": null,\n \"Slice\": null,\n \"SliceP\": null,\n \"PSlice\": [\n {\n \"Tag\": \"tag20\"\n },\n {\n \"Tag\": \"tag21\"\n }\n ],\n \"PSliceP\": [\n {\n \"Tag\": \"tag22\"\n },\n null,\n {\n \"Tag\": \"tag23\"\n }\n ],\n \"EmptySlice\": null,\n \"NilSlice\": null,\n \"StringSlice\": null,\n \"ByteSlice\": null,\n \"Small\": {\n \"Tag\": \"\"\n },\n \"PSmall\": null,\n \"PPSmall\": {\n \"Tag\": \"tag31\"\n },\n \"Interface\": null,\n \"PInterface\": 5.2\n}`\n\nvar pallValueCompact = strings.Map(noSpace, pallValueIndent)\n\nfunc TestRefUnmarshal(t *testing.T) {\n\ttype S struct {\n\t\t// Ref is defined in encode_test.go.\n\t\tR0 Ref\n\t\tR1 *Ref\n\t\tR2 RefText\n\t\tR3 *RefText\n\t}\n\twant := S{\n\t\tR0: 12,\n\t\tR1: new(Ref),\n\t\tR2: 13,\n\t\tR3: new(RefText),\n\t}\n\t*want.R1 = 12\n\t*want.R3 = 13\n\n\tvar got S\n\tif err := Unmarshal([]byte(`{\"R0\":\"ref\",\"R1\":\"ref\",\"R2\":\"ref\",\"R3\":\"ref\"}`), &got); err != nil {\n\t\tt.Fatalf(\"Unmarshal: %v\", err)\n\t}\n\tif !reflect.DeepEqual(got, want) {\n\t\tt.Errorf(\"got %+v, want %+v\", got, want)\n\t}\n}\n\n// Test that the empty string doesn't panic decoding when ,string is specified\n// Issue 3450\nfunc TestEmptyString(t *testing.T) {\n\ttype T2 struct {\n\t\tNumber1 int `json:\",string\"`\n\t\tNumber2 string `json:\",string\"`\n\t\tPass bool `json:\",string\"`\n\t}\n\tdata := `{\"Number1\":\"1\", \"Number2\":\"\",\"Pass\":\"true\"}`\n\tvar t2, t3 T2\n\tt2.Number2 = \"a\"\n\tt3.Number2 = \"a\"\n\terr := Unmarshal([]byte(data), &t2)\n\tif err == nil {\n\t\tt.Fatal(\"Decode: did not return error\")\n\t}\n\tprintln(err.Error())\n\terr2 := json.Unmarshal([]byte(data), &t3)\n\tassert.Equal(t, err == nil, err2 == nil)\n\tassert.Equal(t, t3, t2)\n}\n\n// Test that a null for ,string is not replaced with the previous quoted string (issue 7046).\n// It should also not be an error (issue 2540, issue 8587).\nfunc TestNullString(t *testing.T) {\n\ttype T struct {\n\t\tA int `json:\",string\"`\n\t\tB int `json:\",string\"`\n\t\tC *int `json:\",string\"`\n\t}\n\tdata := []byte(`{\"A\": \"1\", \"B\": null, \"C\": null}`)\n\tvar s T\n\ts.B = 1\n\ts.C = new(int)\n\t*s.C = 2\n\terr := Unmarshal(data, &s)\n\tif err != nil {\n\t\tt.Fatalf(\"Unmarshal: %v\", err)\n\t}\n\tif s.B != 1 || s.C != nil {\n\t\tt.Fatalf(\"after Unmarshal, s.B=%d, s.C=%p, want 1, nil\", s.B, s.C)\n\t}\n}\n\ntype NullTest struct {\n\tBool bool\n\tInt int\n\tInt8 int8\n\tInt16 int16\n\tInt32 int32\n\tInt64 int64\n\tUint uint\n\tUint8 uint8\n\tUint16 uint16\n\tUint32 uint32\n\tUint64 uint64\n\tFloat32 float32\n\tFloat64 float64\n\tString string\n\tPBool *bool\n\tMap map[string]string\n\tSlice []string\n\tInterface interface{}\n\n\tPRaw *json.RawMessage\n\tPTime *time.Time\n\tPBigInt *big.Int\n\tPText *MustNotUnmarshalText\n\tPBuffer *bytes.Buffer // has methods, just not relevant ones\n\tPStruct *struct{}\n\n\tRaw json.RawMessage\n\tTime time.Time\n\tBigInt big.Int\n\tText MustNotUnmarshalText\n\tBuffer bytes.Buffer\n\tStruct struct{}\n}\n\n// JSON null values should be ignored for primitives and string values instead of resulting in an error.\n// Issue 2540\nfunc TestUnmarshalNulls(t *testing.T) {\n\t// Unmarshal docs:\n\t// The JSON null value unmarshals into an interface, map, pointer, or slice\n\t// by setting that Go value to nil. Because null is often used in JSON to mean\n\t// ``not present,'' unmarshaling a JSON null into any other Go type has no effect\n\t// on the value and produces no error.\n\n\tjsonData := []byte(`{\n \"Bool\" : null,\n \"Int\" : null,\n \"Int8\" : null,\n \"Int16\" : null,\n \"Int32\" : null,\n \"Int64\" : null,\n \"Uint\" : null,\n \"Uint8\" : null,\n \"Uint16\" : null,\n \"Uint32\" : null,\n \"Uint64\" : null,\n \"Float32\" : null,\n \"Float64\" : null,\n \"String\" : null,\n \"PBool\": null,\n \"Map\": null,\n \"Slice\": null,\n \"Interface\": null,\n \"PRaw\": null,\n \"PTime\": null,\n \"PBigInt\": null,\n \"PText\": null,\n \"PBuffer\": null,\n \"PStruct\": null,\n \"Raw\": null,\n \"Time\": null,\n \"BigInt\": null,\n \"Text\": null,\n \"Buffer\": null,\n \"Struct\": null\n }`)\n\tnulls := NullTest{\n\t\tBool: true,\n\t\tInt: 2,\n\t\tInt8: 3,\n\t\tInt16: 4,\n\t\tInt32: 5,\n\t\tInt64: 6,\n\t\tUint: 7,\n\t\tUint8: 8,\n\t\tUint16: 9,\n\t\tUint32: 10,\n\t\tUint64: 11,\n\t\tFloat32: 12.1,\n\t\tFloat64: 13.1,\n\t\tString: \"14\",\n\t\tPBool: new(bool),\n\t\tMap: map[string]string{},\n\t\tSlice: []string{},\n\t\tInterface: new(MustNotUnmarshalJSON),\n\t\tPRaw: new(json.RawMessage),\n\t\tPTime: new(time.Time),\n\t\tPBigInt: new(big.Int),\n\t\tPText: new(MustNotUnmarshalText),\n\t\tPStruct: new(struct{}),\n\t\tPBuffer: new(bytes.Buffer),\n\t\tRaw: json.RawMessage(\"123\"),\n\t\tTime: time.Unix(123456789, 0),\n\t\tBigInt: *big.NewInt(123),\n\t}\n\n\tbefore := nulls.Time.String()\n\n\terr := Unmarshal(jsonData, &nulls)\n\tif err != nil {\n\t\tt.Errorf(\"Unmarshal of null values failed: %v\", err)\n\t}\n\tif !nulls.Bool || nulls.Int != 2 || nulls.Int8 != 3 || nulls.Int16 != 4 || nulls.Int32 != 5 || nulls.Int64 != 6 ||\n\t\tnulls.Uint != 7 || nulls.Uint8 != 8 || nulls.Uint16 != 9 || nulls.Uint32 != 10 || nulls.Uint64 != 11 ||\n\t\tnulls.Float32 != 12.1 || nulls.Float64 != 13.1 || nulls.String != \"14\" {\n\t\tt.Errorf(\"Unmarshal of null values affected primitives\")\n\t}\n\n\tif nulls.PBool != nil {\n\t\tt.Errorf(\"Unmarshal of null did not clear nulls.PBool\")\n\t}\n\tif nulls.Map != nil {\n\t\tt.Errorf(\"Unmarshal of null did not clear nulls.Map\")\n\t}\n\tif nulls.Slice != nil {\n\t\tt.Errorf(\"Unmarshal of null did not clear nulls.Slice\")\n\t}\n\tif nulls.Interface != nil {\n\t\tt.Errorf(\"Unmarshal of null did not clear nulls.Interface\")\n\t}\n\tif nulls.PRaw != nil {\n\t\tt.Errorf(\"Unmarshal of null did not clear nulls.PRaw\")\n\t}\n\tif nulls.PTime != nil {\n\t\tt.Errorf(\"Unmarshal of null did not clear nulls.PTime\")\n\t}\n\tif nulls.PBigInt != nil {\n\t\tt.Errorf(\"Unmarshal of null did not clear nulls.PBigInt\")\n\t}\n\tif nulls.PText != nil {\n\t\tt.Errorf(\"Unmarshal of null did not clear nulls.PText\")\n\t}\n\tif nulls.PBuffer != nil {\n\t\tt.Errorf(\"Unmarshal of null did not clear nulls.PBuffer\")\n\t}\n\tif nulls.PStruct != nil {\n\t\tt.Errorf(\"Unmarshal of null did not clear nulls.PStruct\")\n\t}\n\n\tif string(nulls.Raw) != \"null\" {\n\t\tt.Errorf(\"Unmarshal of json.RawMessage null did not record null: %v\", string(nulls.Raw))\n\t}\n\tif nulls.Time.String() != before {\n\t\tt.Errorf(\"Unmarshal of time.Time null set time to %v\", nulls.Time.String())\n\t}\n\tif nulls.BigInt.String() != \"123\" {\n\t\tt.Errorf(\"Unmarshal of big.Int null set int to %v\", nulls.BigInt.String())\n\t}\n}\n\ntype MustNotUnmarshalJSON struct{}\n\nfunc (x MustNotUnmarshalJSON) UnmarshalJSON(_ []byte) error {\n\treturn errors.New(\"MustNotUnmarshalJSON was used\")\n}\n\ntype MustNotUnmarshalText struct{}\n\nfunc (x MustNotUnmarshalText) UnmarshalText(_ []byte) error {\n\treturn errors.New(\"MustNotUnmarshalText was used\")\n}\n\nfunc TestStringKind(t *testing.T) {\n\ttype stringKind string\n\n\tvar m1, m2 map[stringKind]int\n\tm1 = map[stringKind]int{\n\t\t\"foo\": 42,\n\t}\n\n\tdata, err := Marshal(m1)\n\tif err != nil {\n\t\tt.Errorf(\"Unexpected error marshaling: %v\", err)\n\t}\n\n\terr = Unmarshal(data, &m2)\n\tif err != nil {\n\t\tt.Errorf(\"Unexpected error unmarshaling: %v\", err)\n\t}\n\n\tif !reflect.DeepEqual(m1, m2) {\n\t\tt.Error(\"Items should be equal after encoding and then decoding\")\n\t}\n}\n\n// Custom types with []byte as underlying type could not be marshaled\n// and then unmarshaled.\n// Issue 8962.\nfunc TestByteKind(t *testing.T) {\n\ttype byteKind []byte\n\n\ta := byteKind(\"hello\")\n\n\tdata, err := Marshal(a)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tvar b byteKind\n\terr = Unmarshal(data, &b)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif !reflect.DeepEqual(a, b) {\n\t\tt.Errorf(\"expected %v == %v\", a, b)\n\t}\n}\n\n// The fix for issue 8962 introduced a regression.\n// Issue 12921.\nfunc TestSliceOfCustomByte(t *testing.T) {\n\ttype Uint8 uint8\n\n\ta := []Uint8(\"hello\")\n\n\tdata, err := Marshal(a)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tvar b []Uint8\n\terr = Unmarshal(data, &b)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif !reflect.DeepEqual(a, b) {\n\t\tt.Fatalf(\"expected %v == %v\", a, b)\n\t}\n}\n\nvar decodeTypeErrorTests = []struct {\n\tdest interface{}\n\tsrc string\n}{\n\t{new(error), `{}`}, // issue 4222\n\t{new(error), `[]`},\n\t{new(error), `\"\"`},\n\t{new(error), `123`},\n\t{new(error), `true`},\n}\n\nfunc TestUnmarshalTypeError(t *testing.T) {\n\tfor _, item := range decodeTypeErrorTests {\n\t\terr := Unmarshal([]byte(item.src), item.dest)\n\t\tif _, ok := err.(*json.UnmarshalTypeError); !ok {\n\t\t\tif _, ok := err.(*decoder.MismatchTypeError); !ok {\n\t\t\t\tif _, ok = err.(decoder.SyntaxError); !ok {\n\t\t\t\t\tt.Errorf(\"expected type error for Unmarshal(%q, type %T): got %T\",\n\t\t\t\t\t\titem.src, item.dest, err)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nvar decodeMismatchErrorTests = []struct {\n\tdest interface{}\n\tsrc string\n}{\n\t{new(int), `{}`},\n\t{new(string), `{}`},\n\t{new(bool), `{}`},\n\t{new([]byte), `{}`},\n}\n\nfunc TestMismatchTypeError(t *testing.T) {\n\tfor _, item := range decodeMismatchErrorTests {\n\t\terr := Unmarshal([]byte(item.src), item.dest)\n\t\tif _, ok := err.(*decoder.MismatchTypeError); !ok {\n\t\t\tif _, ok = err.(decoder.SyntaxError); !ok {\n\t\t\t\tt.Errorf(\"expected mismatch error for Unmarshal(%q, type %T): got %T\",\n\t\t\t\t\titem.src, item.dest, err)\n\t\t\t}\n\t\t}\n\t}\n}\n\nvar unmarshalSyntaxTests = []string{\n\t\"tru\",\n\t\"fals\",\n\t\"nul\",\n\t\"123e\",\n\t`\"hello`,\n\t`[1,2,3`,\n\t`{\"key\":1`,\n\t`{\"key\":1,`,\n}\n\nfunc TestUnmarshalSyntax(t *testing.T) {\n\tvar x interface{}\n\tfor _, src := range unmarshalSyntaxTests {\n\t\terr := Unmarshal([]byte(src), &x)\n\t\tif _, ok := err.(decoder.SyntaxError); !ok {\n\t\t\tt.Errorf(\"expected syntax error for Unmarshal(%q): got %T\", src, err)\n\t\t}\n\t}\n}\n\n// Test handling of unexported fields that should be ignored.\n// Issue 4660\n//\n//goland:noinspection GoVetStructTag\ntype unexportedFields struct {\n\tName string\n\tm map[string]interface{} `json:\"-\"`\n\tm2 map[string]interface{} `json:\"abcd\"`\n\n\ts []int `json:\"-\"`\n}\n\nfunc TestUnmarshalUnexported(t *testing.T) {\n\tinput := `{\"Name\": \"Bob\", \"m\": {\"x\": 123}, \"m2\": {\"y\": 456}, \"abcd\": {\"z\": 789}, \"s\": [2, 3]}`\n\twant := &unexportedFields{Name: \"Bob\"}\n\n\tout := &unexportedFields{}\n\terr := Unmarshal([]byte(input), out)\n\tif err != nil {\n\t\tt.Errorf(\"got error %v, expected nil\", err)\n\t}\n\tif !reflect.DeepEqual(out, want) {\n\t\tt.Errorf(\"got %q, want %q\", out, want)\n\t}\n}\n\n// Time3339 is a time.Time which encodes to and from JSON\n// as an RFC 3339 time in UTC.\ntype Time3339 time.Time\n\nfunc (t *Time3339) UnmarshalJSON(b []byte) error {\n\tif len(b) < 2 || b[0] != '\"' || b[len(b)-1] != '\"' {\n\t\treturn fmt.Errorf(\"types: failed to unmarshal non-string value %q as an RFC 3339 time\", b)\n\t}\n\ttm, err := time.Parse(time.RFC3339, string(b[1:len(b)-1]))\n\tif err != nil {\n\t\treturn err\n\t}\n\t*t = Time3339(tm)\n\treturn nil\n}\n\nfunc TestUnmarshalJSONLiteralError(t *testing.T) {\n\tvar t3 Time3339\n\terr := Unmarshal([]byte(`\"0000-00-00T00:00:00Z\"`), &t3)\n\tif err == nil {\n\t\tt.Fatalf(\"expected error; got time %v\", time.Time(t3))\n\t}\n\tif !strings.Contains(err.Error(), \"range\") {\n\t\tt.Errorf(\"got err = %v; want out of range error\", err)\n\t}\n}\n\n// Test that extra object elements in an array do not result in a\n// \"data changing underfoot\" error.\n// Issue 3717\nfunc TestSkipArrayObjects(t *testing.T) {\n\ts := `[{}]`\n\tvar dest [0]interface{}\n\n\terr := Unmarshal([]byte(s), &dest)\n\tif err != nil {\n\t\tt.Errorf(\"got error %q, want nil\", err)\n\t}\n}\n\n// Test semantics of pre-filled data, such as struct fields, map elements,\n// slices, and arrays.\n// Issues 4900 and 8837, among others.\nfunc TestPrefilled(t *testing.T) {\n\tvar one int = 1\n\tvar two int = 2\n\t// Values here change, cannot reuse table across runs.\n\tvar prefillTests = []struct {\n\t\tin string\n\t\tptr interface{}\n\t\tout interface{}\n\t}{\n\t\t{\n\t\t\tin: `{\"X\": 1, \"Y\": 2}`,\n\t\t\tptr: &XYZ{X: float32(3), Y: int16(4), Z: 1.5},\n\t\t\tout: &XYZ{X: float64(1), Y: float64(2), Z: 1.5},\n\t\t},\n\t\t{\n\t\t\tin: `{\"X\": 1, \"Y\": 2}`,\n\t\t\tptr: &map[string]interface{}{\"X\": float32(3), \"Y\": int16(4), \"Z\": 1.5},\n\t\t\tout: &map[string]interface{}{\"X\": float64(1), \"Y\": float64(2), \"Z\": 1.5},\n\t\t},\n\t\t{\n\t\t\tin: `[2]`,\n\t\t\tptr: &[]int{1},\n\t\t\tout: &[]int{2},\n\t\t},\n\t\t{\n\t\t\tin: `[2]`,\n\t\t\tptr: &[]int{1, 3},\n\t\t\tout: &[]int{2},\n\t\t},\n\t\t{\n\t\t\tin: `[2, 3]`,\n\t\t\tptr: &[]int{1},\n\t\t\tout: &[]int{2, 3},\n\t\t},\n\t\t{\n\t\t\tin: `[2, 3]`,\n\t\t\tptr: &[...]int{1},\n\t\t\tout: &[...]int{2},\n\t\t},\n\t\t{\n\t\t\tin: `[3]`,\n\t\t\tptr: &[...]int{1, 2},\n\t\t\tout: &[...]int{3, 0},\n\t\t},\n\t\t{\n\t\t\tin: `[2]`,\n\t\t\tptr: &[]interface{}{&one, 2.0},\n\t\t\tout: &[]interface{}{&two},\n\t\t},\n\t\t// TODO(rfc): different with encoding/json\n\t\t// {\n\t\t// in: `{\"a\": 2}`,\n\t\t// ptr: &map[string]interface{}{\"a\": &one, \"b\": int(2)},\n\t\t// out: &map[string]interface{}{\"a\": 2.0, \"b\": int(2)},\n\t\t// },\n\t}\n\n\tfor _, tt := range prefillTests {\n\t\tptrstr := fmt.Sprintf(\"%v\", tt.ptr)\n\t\terr := Unmarshal([]byte(tt.in), tt.ptr) // tt.ptr edited here\n\t\tif err != nil {\n\t\t\tt.Errorf(\"Unmarshal: %v\", err)\n\t\t}\n\t\tif !reflect.DeepEqual(tt.ptr, tt.out) {\n\t\t\tt.Errorf(\"Unmarshal(%#q, %s): have %v, want %v\", tt.in, ptrstr, tt.ptr, tt.out)\n\t\t}\n\t}\n}\n\nvar invalidUnmarshalTests = []struct {\n\tv interface{}\n\twant string\n}{\n\t{nil, \"json: Unmarshal(nil)\"},\n\t{struct{}{}, \"json: Unmarshal(non-pointer struct {})\"},\n\t{(*int)(nil), \"json: Unmarshal(nil *int)\"},\n}\n\nfunc TestInvalidUnmarshal(t *testing.T) {\n\tbuf := []byte(`{\"a\":\"1\"}`)\n\tfor _, tt := range invalidUnmarshalTests {\n\t\terr := Unmarshal(buf, tt.v)\n\t\tif err == nil {\n\t\t\tt.Errorf(\"Unmarshal expecting error, got nil\")\n\t\t\tcontinue\n\t\t}\n\t\tif got := err.Error(); got != tt.want {\n\t\t\tt.Errorf(\"Unmarshal = %q; want %q\", got, tt.want)\n\t\t}\n\t}\n}\n\nvar invalidUnmarshalTextTests = []struct {\n\tv interface{}\n\twant string\n}{\n\t{nil, \"json: Unmarshal(nil)\"},\n\t{struct{}{}, \"json: Unmarshal(non-pointer struct {})\"},\n\t{(*int)(nil), \"json: Unmarshal(nil *int)\"},\n\t{new(net.IP), \"json: cannot unmarshal number into Go value of type *net.IP\"},\n}\n\nfunc TestInvalidUnmarshalText(t *testing.T) {\n\tbuf := []byte(`123`)\n\tfor _, tt := range invalidUnmarshalTextTests {\n\t\terr := Unmarshal(buf, tt.v)\n\t\tif err == nil {\n\t\t\tt.Errorf(\"Unmarshal expecting error, got nil\")\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\n// Test that string option is ignored for invalid types.\n// Issue 9812.\nfunc TestInvalidStringOption(t *testing.T) {\n\tnum := 0\n\titem := struct {\n\t\tT time.Time `json:\",string\"`\n\t\tM map[string]string `json:\",string\"`\n\t\tS []string `json:\",string\"`\n\t\tA [1]string `json:\",string\"`\n\t\tI interface{} `json:\",string\"`\n\t\tP *int `json:\",string\"`\n\t}{M: make(map[string]string), S: make([]string, 0), I: num, P: &num}\n\n\tdata, err := Marshal(item)\n\tif err != nil {\n\t\tt.Fatalf(\"Marshal: %v\", err)\n\t}\n\terr = Unmarshal(data, &item)\n\tif err != nil {\n\t\tt.Fatalf(\"Unmarshal: %v\", err)\n\t}\n}\n\nfunc TestUnmarshalErrorAfterMultipleJSON(t *testing.T) {\n\ttests := []struct {\n\t\tin string\n\t\terr error\n\t}{{\n\t\tin: `1 false null :`,\n\t\terr: (&JsonSyntaxError{\"invalid character ':' looking for beginning of value\", 13}).err(),\n\t}, {\n\t\tin: `1 [] [,]`,\n\t\terr: (&JsonSyntaxError{\"invalid character ',' looking for beginning of value\", 6}).err(),\n\t}, {\n\t\tin: `1 [] [true:]`,\n\t\terr: (&JsonSyntaxError{\"invalid character ':' after array element\", 10}).err(),\n\t}, {\n\t\tin: `1 {} {\"x\"=}`,\n\t\terr: (&JsonSyntaxError{\"invalid character '=' after object key\", 13}).err(),\n\t}, {\n\t\tin: `falsetruenul#`,\n\t\terr: (&JsonSyntaxError{\"invalid character '#' in literal null (expecting 'l')\", 12}).err(),\n\t}}\n\tfor i, tt := range tests {\n\t\tdec := decoder.NewDecoder(tt.in)\n\t\tvar err error\n\t\tfor {\n\t\t\tvar v interface{}\n\t\t\tif err = dec.Decode(&v); err != nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t\tif v, ok := err.(decoder.SyntaxError); !ok {\n\t\t\tt.Errorf(\"#%d: got %#v, want %#v\", i, err, tt.err)\n\t\t} else if v.Pos != int(tt.err.(*json.SyntaxError).Offset) {\n\t\t\tt.Errorf(\"#%d: got %#v, want %#v\", i, err, tt.err)\n\t\t\tprintln(v.Description())\n\t\t}\n\t}\n}\n\ntype unmarshalPanic struct{}\n\nfunc (unmarshalPanic) UnmarshalJSON([]byte) error { panic(0xdead) }\n\nfunc TestUnmarshalPanic(t *testing.T) {\n\tdefer func() {\n\t\tif got := recover(); !reflect.DeepEqual(got, 0xdead) {\n\t\t\tt.Errorf(\"panic() = (%T)(%v), want 0xdead\", got, got)\n\t\t}\n\t}()\n\t_ = Unmarshal([]byte(\"{}\"), &unmarshalPanic{})\n\tt.Fatalf(\"Unmarshal should have panicked\")\n}\n\n// The decoder used to hang if decoding into an interface pointing to its own address.\n// See golang.org/issues/31740.\nfunc TestUnmarshalRecursivePointer(t *testing.T) {\n\tvar v interface{}\n\tv = &v\n\tdata := []byte(`{\"a\": \"b\"}`)\n\n\tif err := Unmarshal(data, v); err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\ntype textUnmarshalerString string\n\nfunc (m *textUnmarshalerString) UnmarshalText(text []byte) error {\n\t*m = textUnmarshalerString(strings.ToLower(string(text)))\n\treturn nil\n}\n\n// Test unmarshal to a map, where the map key is a user defined type.\n// See golang.org/issues/34437.\nfunc TestUnmarshalMapWithTextUnmarshalerStringKey(t *testing.T) {\n\tvar p map[textUnmarshalerString]string\n\tif err := Unmarshal([]byte(`{\"FOO\": \"1\"}`), &p); err != nil {\n\t\tt.Fatalf(\"Unmarshal unexpected error: %v\", err)\n\t}\n\n\tif _, ok := p[\"foo\"]; !ok {\n\t\tt.Errorf(`Key \"foo\" does not exist in map: %v`, p)\n\t}\n}\n\nfunc TestUnmarshalRescanLiteralMangledUnquote(t *testing.T) {\n\t// See golang.org/issues/38105.\n\tvar p map[textUnmarshalerString]string\n\tif err := Unmarshal([]byte(`{\"开源\":\"12345开源\"}`), &p); err != nil {\n\t\tt.Fatalf(\"Unmarshal unexpected error: %v\", err)\n\t}\n\tif _, ok := p[\"开源\"]; !ok {\n\t\tt.Errorf(`Key \"开源\" does not exist in map: %v`, p)\n\t}\n\n\t// See golang.org/issues/38126.\n\ttype T struct {\n\t\tF1 string `json:\"F1,string\"`\n\t}\n\tt1 := T{\"aaa\\tbbb\"}\n\n\tb, err := Marshal(t1)\n\tif err != nil {\n\t\tt.Fatalf(\"Marshal unexpected error: %v\", err)\n\t}\n\tvar t2 T\n\tif err := Unmarshal(b, &t2); err != nil {\n\t\tt.Fatalf(\"Unmarshal unexpected error: %v\", err)\n\t}\n\tif t1 != t2 {\n\t\tt.Errorf(\"Marshal and Unmarshal roundtrip mismatch: want %q got %q\", t1, t2)\n\t}\n\n\t// See golang.org/issues/39555.\n\tinput := map[textUnmarshalerString]string{\"FOO\": \"\", `\"`: \"\"}\n\n\tencoded, err := Marshal(input)\n\tif err != nil {\n\t\tt.Fatalf(\"Marshal unexpected error: %v\", err)\n\t}\n\tvar got map[textUnmarshalerString]string\n\tif err := Unmarshal(encoded, &got); err != nil {\n\t\tt.Fatalf(\"Unmarshal unexpected error: %v\", err)\n\t}\n\twant := map[textUnmarshalerString]string{\"foo\": \"\", `\"`: \"\"}\n\tif !reflect.DeepEqual(want, got) {\n\t\tt.Fatalf(\"Unexpected roundtrip result:\\nwant: %q\\ngot: %q\", want, got)\n\t}\n}\n\nfunc TestUnmarshalMaxDepth(t *testing.T) {\n\tconst (\n\t\t_MaxDepth = types.MAX_RECURSE\n\t\t_OverMaxDepth = types.MAX_RECURSE + 1\n\t\t_UnderMaxDepth = types.MAX_RECURSE - 2\n\t)\n\ttestcases := []struct {\n\t\tname string\n\t\tdata string\n\t\terrMaxDepth bool\n\t}{\n\t\t{\n\t\t\tname: \"ArrayUnderMaxNestingDepth\",\n\t\t\tdata: `{\"a\":` + strings.Repeat(`[`, _UnderMaxDepth) + `0` + strings.Repeat(`]`, _UnderMaxDepth) + `}`,\n\t\t\terrMaxDepth: false,\n\t\t},\n\t\t{\n\t\t\tname: \"ArrayOverMaxNestingDepth\",\n\t\t\tdata: `{\"a\":` + strings.Repeat(`[`, _OverMaxDepth) + `0` + strings.Repeat(`]`, _OverMaxDepth) + `}`,\n\t\t\terrMaxDepth: true,\n\t\t},\n\t\t{\n\t\t\tname: \"ArrayOverStackDepth\",\n\t\t\tdata: `{\"a\":` + strings.Repeat(`[`, 3000000) + `0` + strings.Repeat(`]`, 3000000) + `}`,\n\t\t\terrMaxDepth: true,\n\t\t},\n\t\t{\n\t\t\tname: \"ObjectUnderMaxNestingDepth\",\n\t\t\tdata: `{\"a\":` + strings.Repeat(`{\"a\":`, _UnderMaxDepth) + `0` + strings.Repeat(`}`, _UnderMaxDepth) + `}`,\n\t\t\terrMaxDepth: false,\n\t\t},\n\t\t{\n\t\t\tname: \"ObjectOverMaxNestingDepth\",\n\t\t\tdata: `{\"a\":` + strings.Repeat(`{\"a\":`, _OverMaxDepth) + `0` + strings.Repeat(`}`, _OverMaxDepth) + `}`,\n\t\t\terrMaxDepth: true,\n\t\t},\n\t\t{\n\t\t\tname: \"ObjectOverStackDepth\",\n\t\t\tdata: `{\"a\":` + strings.Repeat(`{\"a\":`, 3000000) + `0` + strings.Repeat(`}`, 3000000) + `}`,\n\t\t\terrMaxDepth: true,\n\t\t},\n\t}\n\n\ttargets := []struct {\n\t\tname string\n\t\tnewValue func() interface{}\n\t}{\n\t\t{\n\t\t\tname: \"unstructured\",\n\t\t\tnewValue: func() interface{} {\n\t\t\t\tvar v interface{}\n\t\t\t\treturn &v\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"typed named field\",\n\t\t\tnewValue: func() interface{} {\n\t\t\t\tv := struct {\n\t\t\t\t\tA interface{} `json:\"a\"`\n\t\t\t\t}{}\n\t\t\t\treturn &v\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"typed missing field\",\n\t\t\tnewValue: func() interface{} {\n\t\t\t\tv := struct {\n\t\t\t\t\tB interface{} `json:\"b\"`\n\t\t\t\t}{}\n\t\t\t\treturn &v\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"custom unmarshaler\",\n\t\t\tnewValue: func() interface{} {\n\t\t\t\tv := unmarshaler{}\n\t\t\t\treturn &v\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tc := range testcases {\n\t\tfor _, target := range targets {\n\t\t\tt.Run(target.name+\"-\"+tc.name, func(t *testing.T) {\n\t\t\t\terr := Unmarshal([]byte(tc.data), target.newValue())\n\t\t\t\tif !tc.errMaxDepth {\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tt.Errorf(\"unexpected error: %v\", err)\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\tif err == nil {\n\t\t\t\t\t\tt.Errorf(\"expected error containing 'exceeded max depth', got none\")\n\t\t\t\t\t} else if !strings.Contains(err.Error(), \"exceeded max depth\") {\n\t\t\t\t\t\tt.Errorf(\"expected error containing 'exceeded max depth', got: %v\", err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t})\n\t\t}\n\t}\n}\n\n// Issues: map value type larger than 128 bytes are stored by pointer\ntype ChargeToolPacingBucketItemTcc struct {\n\t_ [128]byte\n\tT string `json:\"T\"`\n}\n\ntype ChargeToolPacingParamsForDataRead struct {\n\tBucket2Item map[int64]ChargeToolPacingBucketItemTcc `json:\"bucket_to_item\"`\n}\n\nvar panicStr = `\n{\n \"bucket_to_item\": {\n \"102\" : {\n \"T\": \"xxxx\"\n }\n }\n}\n`\n\nfunc TestChangeTool(t *testing.T) {\n\tdataForRaw := ChargeToolPacingParamsForDataRead{}\n\n\terr := Unmarshal([]byte(panicStr), &dataForRaw)\n\tif err != nil {\n\t\tt.Fatalf(\"err %+v\\n\", err)\n\t}\n\tt.Logf(\"%#v\\n\", dataForRaw)\n\tt.Logf(\"%#v\\n\", &dataForRaw.Bucket2Item)\n\ta := dataForRaw.Bucket2Item[102]\n\tif a.T != \"xxxx\" {\n\t\tt.Fatalf(\"exp:%v, got:%v\", \"xxxx\", a.T)\n\t}\n\n}\n\nfunc TestDecoder_LongestInvalidUtf8(t *testing.T) {\n\tfor _, data := range []string{\n\t\t\"\\\"\" + strings.Repeat(\"\\x80\", 4096) + \"\\\"\",\n\t\t\"\\\"\" + strings.Repeat(\"\\x80\", 4095) + \"\\\"\",\n\t\t\"\\\"\" + strings.Repeat(\"\\x80\", 4097) + \"\\\"\",\n\t\t\"\\\"\" + strings.Repeat(\"\\x80\", 12345) + \"\\\"\",\n\t} {\n\t\ttestDecodeInvalidUtf8(t, []byte(data))\n\t}\n}\n\nfunc testDecodeInvalidUtf8(t *testing.T, data []byte) {\n\tvar sgot, jgot string\n\tserr := ConfigStd.Unmarshal(data, &sgot)\n\tjerr := json.Unmarshal(data, &jgot)\n\tassert.Equal(t, serr != nil, jerr != nil)\n\tif jerr == nil {\n\t\tassert.Equal(t, sgot, jgot)\n\t}\n}\n\nfunc needEscape(b byte) bool {\n\treturn b == '\"' || b == '\\\\' || b < '\\x20'\n}\n\nfunc genRandJsonBytes(length int) []byte {\n\tvar buf bytes.Buffer\n\tbuf.WriteByte('\"')\n\tfor j := 0; j < length; j++ {\n\t\tr := rand.Intn(0xff + 1)\n\t\tif needEscape(byte(r)) {\n\t\t\tbuf.WriteByte('\\\\')\n\t\t}\n\t\tbuf.WriteByte(byte(r))\n\t}\n\tbuf.WriteByte('\"')\n\treturn buf.Bytes()\n}\n\nfunc genRandJsonRune(length int) []byte {\n\tvar buf bytes.Buffer\n\tbuf.WriteByte('\"')\n\tfor j := 0; j < length; j++ {\n\t\tr := rand.Intn(0x10FFFF + 1)\n\t\tif r < 0x80 && needEscape(byte(r)) {\n\t\t\tbuf.WriteByte('\\\\')\n\t\t\tbuf.WriteByte(byte(r))\n\t\t} else {\n\t\t\tbuf.WriteRune(rune(r))\n\t\t}\n\t}\n\tbuf.WriteByte('\"')\n\treturn buf.Bytes()\n}\n\nfunc TestDecoder_RandomInvalidUtf8(t *testing.T) {\n\tnums := 1000\n\tmaxLen := 1000\n\tfor i := 0; i < nums; i++ {\n\t\tlength := rand.Intn(maxLen)\n\t\ttestDecodeInvalidUtf8(t, genRandJsonBytes(length))\n\t\ttestDecodeInvalidUtf8(t, genRandJsonRune(length))\n\t}\n}\n\ntype atofTest struct {\n\tin string\n\tout string\n\terr error\n}\n\n// Tests from Go strconv package, https://github.com/golang/go/blob/master/src/strconv/atof_test.go\n// All tests are passed in Go encoding/json.\nvar atoftests = []atofTest{\n\t{\"1.234e\", \"\", nil}, // error\n\t{\"1i\", \"1\", nil}, // pass\n\t{\"1\", \"1\", nil},\n\t{\"1e23\", \"1e+23\", nil},\n\t{\"1E23\", \"1e+23\", nil},\n\t{\"100000000000000000000000\", \"1e+23\", nil},\n\t{\"1e-100\", \"1e-100\", nil},\n\t{\"123456700\", \"1.234567e+08\", nil},\n\t{\"99999999999999974834176\", \"9.999999999999997e+22\", nil},\n\t{\"100000000000000000000001\", \"1.0000000000000001e+23\", nil},\n\t{\"100000000000000008388608\", \"1.0000000000000001e+23\", nil},\n\t{\"100000000000000016777215\", \"1.0000000000000001e+23\", nil},\n\t{\"100000000000000016777216\", \"1.0000000000000003e+23\", nil},\n\t{\"-1\", \"-1\", nil},\n\t{\"-0.1\", \"-0.1\", nil},\n\t{\"-0\", \"-0\", nil},\n\t{\"1e-20\", \"1e-20\", nil},\n\t{\"625e-3\", \"0.625\", nil},\n\n\t// zeros\n\t{\"0\", \"0\", nil},\n\t{\"0e0\", \"0\", nil},\n\t{\"-0e0\", \"-0\", nil},\n\t{\"0e-0\", \"0\", nil},\n\t{\"-0e-0\", \"-0\", nil},\n\t{\"0e+0\", \"0\", nil},\n\t{\"-0e+0\", \"-0\", nil},\n\t{\"0e+01234567890123456789\", \"0\", nil},\n\t{\"0.00e-01234567890123456789\", \"0\", nil},\n\t{\"-0e+01234567890123456789\", \"-0\", nil},\n\t{\"-0.00e-01234567890123456789\", \"-0\", nil},\n\n\t{\"0e291\", \"0\", nil}, // issue 15364\n\t{\"0e292\", \"0\", nil}, // issue 15364\n\t{\"0e347\", \"0\", nil}, // issue 15364\n\t{\"0e348\", \"0\", nil}, // issue 15364\n\t{\"-0e291\", \"-0\", nil},\n\t{\"-0e292\", \"-0\", nil},\n\t{\"-0e347\", \"-0\", nil},\n\t{\"-0e348\", \"-0\", nil},\n\n\t// largest float64\n\t{\"1.7976931348623157e308\", \"1.7976931348623157e+308\", nil},\n\t{\"-1.7976931348623157e308\", \"-1.7976931348623157e+308\", nil},\n\n\t// the border is ...158079\n\t// borderline - okay\n\t{\"1.7976931348623158e308\", \"1.7976931348623157e+308\", nil},\n\t{\"-1.7976931348623158e308\", \"-1.7976931348623157e+308\", nil},\n\n\t// a little too large\n\t{\"1e308\", \"1e+308\", nil},\n\n\t// denormalized\n\t{\"1e-305\", \"1e-305\", nil},\n\t{\"1e-306\", \"1e-306\", nil},\n\t{\"1e-307\", \"1e-307\", nil},\n\t{\"1e-308\", \"1e-308\", nil},\n\t{\"1e-309\", \"1e-309\", nil},\n\t{\"1e-310\", \"1e-310\", nil},\n\t{\"1e-322\", \"1e-322\", nil},\n\t// smallest denormal\n\t{\"5e-324\", \"5e-324\", nil},\n\t{\"4e-324\", \"5e-324\", nil},\n\t{\"3e-324\", \"5e-324\", nil},\n\t// too small\n\t{\"2e-324\", \"0\", nil},\n\t// way too small\n\t{\"1e-350\", \"0\", nil},\n\t{\"1e-400000\", \"0\", nil},\n\n\t// try to overflow exponent\n\t{\"1e-4294967296\", \"0\", nil},\n\t{\"1e-18446744073709551616\", \"0\", nil},\n\n\t// https://www.exploringbinary.com/java-hangs-when-converting-2-2250738585072012e-308/\n\t{\"2.2250738585072012e-308\", \"2.2250738585072014e-308\", nil},\n\t// https://www.exploringbinary.com/php-hangs-on-numeric-value-2-2250738585072011e-308/\n\t{\"2.2250738585072011e-308\", \"2.225073858507201e-308\", nil},\n\n\t// A very large number (initially wrongly parsed by the fast algorithm).\n\t{\"4.630813248087435e+307\", \"4.630813248087435e+307\", nil},\n\n\t// A different kind of very large number.\n\t{\"22.222222222222222\", \"22.22222222222222\", nil},\n\t{\"2.\" + strings.Repeat(\"2\", 800) + \"e+1\", \"22.22222222222222\", nil},\n\n\t// Exactly halfway between 1 and math.Nextafter(1, 2).\n\t// Round to even (down).\n\t{\"1.00000000000000011102230246251565404236316680908203125\", \"1\", nil},\n\t// Slightly lower; still round down.\n\t{\"1.00000000000000011102230246251565404236316680908203124\", \"1\", nil},\n\t// Slightly higher; round up.\n\t{\"1.00000000000000011102230246251565404236316680908203126\", \"1.0000000000000002\", nil},\n\t// Slightly higher, but you have to read all the way to the end.\n\t{\"1.00000000000000011102230246251565404236316680908203125\" + strings.Repeat(\"0\", 10000) + \"1\", \"1.0000000000000002\", nil},\n\n\t// Halfway between x := math.Nextafter(1, 2) and math.Nextafter(x, 2)\n\t// Round to even (up).\n\t{\"1.00000000000000033306690738754696212708950042724609375\", \"1.0000000000000004\", nil},\n\n\t// Halfway between 1090544144181609278303144771584 and 1090544144181609419040633126912\n\t// (15497564393479157p+46, should round to even 15497564393479156p+46, issue 36657)\n\t{\"1090544144181609348671888949248\", \"1.0905441441816093e+30\", nil},\n\n\t// Corner case between int64 and float64 for the input\n\t{\"9223372036854775807\", \"9223372036854775807\", nil}, // max int64: (1 << 63) - 1\n\t{\"9223372036854775808\", \"9223372036854775808\", nil},\n\t{\"-9223372036854775808\", \"-9223372036854775808\", nil}, // min int64: 1 << 63\n\t{\"-9223372036854775809\", \"-9223372036854775809\", nil},\n}\n\nfunc TestDecodeFloat(t *testing.T) {\n\tfor i, tt := range atoftests {\n\t\t// default float64\n\t\tvar sonicout, stdout float64\n\t\tsonicerr := decoder.NewDecoder(tt.in).Decode(&sonicout)\n\t\tstderr := json.NewDecoder(strings.NewReader(tt.in)).Decode(&stdout)\n\t\tif !reflect.DeepEqual(sonicout, stdout) {\n\t\t\tt.Fatalf(\"Test %d, %#v\\ngot:\\n %#v\\nexp:\\n %#v\\n\", i, tt.in, sonicout, stdout)\n\t\t}\n\t\tif !reflect.DeepEqual(sonicerr == nil, stderr == nil) {\n\t\t\tt.Fatalf(\"Test %d, %#v\\ngot:\\n %#v\\nexp:\\n %#v\\n\", i, tt.in, sonicerr, stderr)\n\t\t}\n\t}\n}\n\ntype useInt64Test struct {\n\tin string\n\tout int64\n}\n\ntype useFloatTest struct {\n\tin string\n\tout float64\n}\n\nvar useinttest = []useInt64Test{\n\t// int64\n\t{\"0\", 0},\n\t{\"1\", 1},\n\t{\"-1\", -1},\n\t{\"100\", 100},\n\n\t{\"-9223372036854775807\", -9223372036854775807},\n\t{\"-9223372036854775808\", -9223372036854775808}, //min int64\n\t{\"9223372036854775807\", 9223372036854775807}, //max int64\n\t{\"9223372036854775806\", 9223372036854775806},\n}\n\nvar usefloattest = []useFloatTest{\n\t// float64\n\t{\"-9223372036854775809\", -9223372036854775809}, // int64 overflow\n\t{\"9223372036854775808\", 9223372036854775808}, // int64 overflow\n\t{\"1e2\", 1e2},\n\t{\"1e-20\", 1e-20},\n\t{\"1.0\", 1},\n}\n\nfunc TestUseInt64(t *testing.T) {\n\tfor i, tt := range useinttest {\n\t\tvar sout interface{}\n\t\tdc := decoder.NewDecoder(tt.in)\n\t\tdc.UseInt64()\n\t\tserr := dc.Decode(&sout)\n\t\tif !reflect.DeepEqual(sout, tt.out) {\n\t\t\tt.Errorf(\"Test %d, %#v\\ngot:\\n %#v\\nexp:\\n %#v\\n\", i, tt.in, sout, tt.in)\n\t\t}\n\t\tif serr != nil {\n\t\t\tt.Errorf(\"Test %d, %#v\\ngot:\\n %#v\\nexp:\\n nil\\n\", i, tt, serr)\n\t\t}\n\t}\n\n\tfor i, tt := range usefloattest {\n\t\tvar sout interface{}\n\t\tdc := decoder.NewDecoder(tt.in)\n\t\tdc.UseInt64()\n\t\t//the input string is not int64, still return float64\n\t\tserr := dc.Decode(&sout)\n\t\tif !reflect.DeepEqual(sout, tt.out) {\n\t\t\tt.Errorf(\"Test %d, %#v\\ngot:\\n %#v\\nexp:\\n %#v\\n\", i, tt.in, sout, tt.in)\n\t\t}\n\t\tif serr != nil {\n\t\t\tt.Errorf(\"Test %d, %#v\\ngot:\\n %#v\\nexp:\\n nil\\n\", i, tt, serr)\n\t\t}\n\t}\n}\n\nfunc TestUseNumber(t *testing.T) {\n\tfor i, tt := range useinttest {\n\t\tvar sout interface{}\n\t\tdc := decoder.NewDecoder(tt.in)\n\t\tdc.UseNumber()\n\t\tserr := dc.Decode(&sout)\n\t\tif !reflect.DeepEqual(sout, json.Number(tt.in)) {\n\t\t\tt.Errorf(\"Test %d, %#v\\ngot:\\n %#v\\nexp:\\n %#v\\n\", i, tt.in, sout, tt.out)\n\t\t}\n\t\tif serr != nil {\n\t\t\tt.Errorf(\"Test %d, %#v\\ngot:\\n %#v\\nexp:\\n nil\\n\", i, tt, serr)\n\t\t}\n\t}\n\n\tfor i, tt := range usefloattest {\n\t\tvar sout interface{}\n\t\tdc := decoder.NewDecoder(tt.in)\n\t\tdc.UseNumber()\n\t\tserr := dc.Decode(&sout)\n\t\tif !reflect.DeepEqual(sout, json.Number(tt.in)) {\n\t\t\tt.Errorf(\"Test %d, %#v\\ngot:\\n %#v\\nexp:\\n %#v\\n\", i, tt.in, sout, tt.out)\n\t\t}\n\t\tif serr != nil {\n\t\t\tt.Errorf(\"Test %d, %#v\\ngot:\\n %#v\\nexp:\\n nil\\n\", i, tt, serr)\n\t\t}\n\t}\n}\n\nfunc BenchmarkDecoderRawMessage(b *testing.B) {\n\tdata := ` {\n \"coordinates\": null,\n \"favorited\": false,\n \"truncated\": false,\n \"created_at\": \"Mon Sep 24 03:35:21 +0000 2012\",\n \"id_str\": \"250075927172759552\",\n \"entities\": {\n \"urls\": [\n \n ],\n \"hashtags\": [\n {\n \"text\": \"freebandnames\",\n \"indices\": [\n 20,\n 34\n ]\n }\n ],\n \"user_mentions\": [\n \n ]\n },\n \"in_reply_to_user_id_str\": null,\n \"contributors\": null,\n \"text\": \"Aggressive Ponytail #freebandnames\",\n \"metadata\": {\n \"iso_language_code\": \"en\",\n \"result_type\": \"recent\"\n },\n \"retweet_count\": 0,\n \"in_reply_to_status_id_str\": null,\n \"id\": 250075927172759552,\n \"geo\": null,\n \"retweeted\": false,\n \"in_reply_to_user_id\": null,\n \"place\": null,\n \"user\": {\n \"profile_sidebar_fill_color\": \"DDEEF6\",\n \"profile_sidebar_border_color\": \"C0DEED\",\n \"profile_background_tile\": false,\n \"name\": \"Sean Cummings\",\n \"profile_image_url\": \"https://a0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg\",\n \"created_at\": \"Mon Apr 26 06:01:55 +0000 2010\",\n \"location\": \"LA, CA\",\n \"follow_request_sent\": null,\n \"profile_link_color\": \"0084B4\",\n \"is_translator\": false,\n \"id_str\": \"137238150\",\n \"entities\": {\n \"url\": {\n \"urls\": [\n {\n \"expanded_url\": null,\n \"url\": \"\",\n \"indices\": [\n 0,\n 0\n ]\n }\n ]\n },\n \"description\": {\n \"urls\": [\n \n ]\n }\n },\n \"default_profile\": true,\n \"contributors_enabled\": false,\n \"favourites_count\": 0,\n \"url\": null,\n \"profile_image_url_https\": \"https://si0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg\",\n \"utc_offset\": -28800,\n \"id\": 137238150,\n \"profile_use_background_image\": true,\n \"listed_count\": 2,\n \"profile_text_color\": \"333333\",\n \"lang\": \"en\",\n \"followers_count\": 70,\n \"protected\": false,\n \"notifications\": null,\n \"profile_background_image_url_https\": \"https://si0.twimg.com/images/themes/theme1/bg.png\",\n \"profile_background_color\": \"C0DEED\",\n \"verified\": false,\n \"geo_enabled\": true,\n \"time_zone\": \"Pacific Time (US & Canada)\",\n \"description\": \"Born 330 Live 310\",\n \"default_profile_image\": false,\n \"profile_background_image_url\": \"https://a0.twimg.com/images/themes/theme1/bg.png\",\n \"statuses_count\": 579,\n \"friends_count\": 110,\n \"following\": null,\n \"show_all_inline_media\": false,\n \"screen_name\": \"sean_cummings\"\n },\n \"in_reply_to_screen_name\": null,\n \"source\": \"Twitter for Mac\",\n \"in_reply_to_status_id\": null\n }`\n\n\tbench := func(b *testing.B, run func(b *testing.B)) {\n\t\tb.ResetTimer()\n\t\tb.ReportAllocs()\n\t\tb.SetBytes(int64(len(data)))\n\t\tfor i := 0; i < b.N; i++ {\n\t\t\trun(b)\n\t\t}\n\t\truntime.GC()\n\t}\n\n\tb.Run(\"StdRawMessage\", func(b *testing.B) {\n\t\tbench(b, func(b *testing.B) {\n\t\t\tvar obj map[string]json.RawMessage\n\t\t\tdc := decoder.NewDecoder(data)\n\t\t\tdc.SetOptions(decoder.OptionUseNumber)\n\t\t\tif err := dc.Decode(&obj); err != nil {\n\t\t\t\tb.Fatal(err.Error())\n\t\t\t}\n\t\t\t_ = obj\n\t\t})\n\t})\n\n\tb.Run(\"NocopyRawMessage\", func(b *testing.B) {\n\t\tbench(b, func(b *testing.B) {\n\t\t\tvar obj map[string]NoCopyRawMessage\n\t\t\tdc := decoder.NewDecoder(data)\n\t\t\tdc.SetOptions(decoder.OptionUseNumber)\n\t\t\tif err := dc.Decode(&obj); err != nil {\n\t\t\t\tb.Fatal(err.Error())\n\t\t\t}\n\t\t\t_ = obj\n\t\t})\n\t})\n\n\tb.Run(\"NocopyRawMessageWithoutValidation\", func(b *testing.B) {\n\t\tbench(b, func(b *testing.B) {\n\t\t\tvar obj map[string]NoCopyRawMessage\n\t\t\tdc := decoder.NewDecoder(data)\n\t\t\tdc.SetOptions(decoder.OptionNoValidateJSON | decoder.OptionUseNumber)\n\t\t\tif err := dc.Decode(&obj); err != nil {\n\t\t\t\tb.Fatal(err.Error())\n\t\t\t}\n\t\t\t_ = obj\n\t\t})\n\t})\n}\n\nfunc TestJsonNumber(t *testing.T) {\n\tapi := Config{\n\t\tUseNumber: true,\n\t}.Froze()\n\n\ttype Foo struct {\n\t\tA json.Number `json:\"a\"`\n\t\tB json.Number `json:\"b\"`\n\t\tC json.Number `json:\"c\"`\n\t}\n\n\tdata := []byte(`{\"a\": 1 , \"b\": \"123\", \"c\": \"0.4e+56\"}`)\n\tvar foo1, foo2 Foo\n\tserr := api.Unmarshal(data, &foo1)\n\tjerr := json.Unmarshal(data, &foo2)\n\tassert.Equal(t, jerr, serr)\n\tassert.Equal(t, foo2, foo1)\n}\n" }, { "path": "decoder/decoder_compat.go", "content": "//go:build (!amd64 && !arm64) || go1.27 || !go1.17 || (arm64 && !go1.20)\n// +build !amd64,!arm64 go1.27 !go1.17 arm64,!go1.20\n\n/*\n* Copyright 2023 ByteDance Inc.\n*\n* Licensed under the Apache License, Version 2.0 (the \"License\");\n* you may not use this file except in compliance with the License.\n* You may obtain a copy of the License at\n*\n* http://www.apache.org/licenses/LICENSE-2.0\n*\n* Unless required by applicable law or agreed to in writing, software\n* distributed under the License is distributed on an \"AS IS\" BASIS,\n* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n* See the License for the specific language governing permissions and\n* limitations under the License.\n */\n\npackage decoder\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"io\"\n\t\"reflect\"\n\t\"unsafe\"\n\n\t\"github.com/bytedance/sonic/internal/compat\"\n\t\"github.com/bytedance/sonic/internal/decoder/consts\"\n\t\"github.com/bytedance/sonic/internal/native/types\"\n\t\"github.com/bytedance/sonic/option\"\n)\n\nfunc init() {\n\tcompat.Warn(\"sonic/decoder\")\n}\n\nconst (\n\t_F_use_int64 = consts.F_use_int64\n\t_F_disable_urc = consts.F_disable_unknown\n\t_F_disable_unknown = consts.F_disable_unknown\n\t_F_copy_string = consts.F_copy_string\n\n\t_F_use_number = consts.F_use_number\n\t_F_validate_string = consts.F_validate_string\n\t_F_allow_control = consts.F_allow_control\n\t_F_no_validate_json = consts.F_no_validate_json\n\t_F_case_sensitive = consts.F_case_sensitive\n)\n\ntype Options uint64\n\nconst (\n\tOptionUseInt64 Options = 1 << _F_use_int64\n\tOptionUseNumber Options = 1 << _F_use_number\n\tOptionUseUnicodeErrors Options = 1 << _F_disable_urc\n\tOptionDisableUnknown Options = 1 << _F_disable_unknown\n\tOptionCopyString Options = 1 << _F_copy_string\n\tOptionValidateString Options = 1 << _F_validate_string\n\tOptionNoValidateJSON Options = 1 << _F_no_validate_json\n\tOptionCaseSensitive Options = 1 << _F_case_sensitive\n)\n\nfunc (self *Decoder) SetOptions(opts Options) {\n\tif (opts&OptionUseNumber != 0) && (opts&OptionUseInt64 != 0) {\n\t\tpanic(\"can't set OptionUseInt64 and OptionUseNumber both!\")\n\t}\n\tself.f = uint64(opts)\n}\n\n// Decoder is the decoder context object\ntype Decoder struct {\n\ti int\n\tf uint64\n\ts string\n}\n\n// NewDecoder creates a new decoder instance.\nfunc NewDecoder(s string) *Decoder {\n\treturn &Decoder{s: s}\n}\n\n// Pos returns the current decoding position.\nfunc (self *Decoder) Pos() int {\n\treturn self.i\n}\n\nfunc (self *Decoder) Reset(s string) {\n\tself.s = s\n\tself.i = 0\n\t// self.f = 0\n}\n\n// NOTE: api fallback do nothing\nfunc (self *Decoder) CheckTrailings() error {\n\tpos := self.i\n\tbuf := self.s\n\t/* skip all the trailing spaces */\n\tif pos != len(buf) {\n\t\tfor pos < len(buf) && (types.SPACE_MASK&(1< c.expTime, fmt.Sprintf(\"%v/%v=%v\", avg1, avg2, float64(avg1)/float64(avg2)))\n\t\t})\n\t}\n}\n\nfunc TestSkipMismatchTypeAmd64Error(t *testing.T) {\n\t// t.Run(\"struct\", func(t *testing.T) {\n\t// println(\"TestSkipError\")\n\t// type skiptype struct {\n\t// A int `json:\"a\"`\n\t// B string `json:\"b\"`\n\n\t// Pass *int `json:\"pass\"`\n\n\t// C struct{\n\n\t// Pass4 interface{} `json:\"pass4\"`\n\n\t// D struct{\n\t// E float32 `json:\"e\"`\n\t// } `json:\"d\"`\n\n\t// Pass2 int `json:\"pass2\"`\n\n\t// } `json:\"c\"`\n\n\t// E bool `json:\"e\"`\n\t// F []int `json:\"f\"`\n\t// G map[string]int `json:\"g\"`\n\t// H bool `json:\"h,string\"`\n\n\t// Pass3 int `json:\"pass2\"`\n\n\t// I json.Number `json:\"i\"`\n\t// }\n\t// var obj, obj2 = &skiptype{Pass:new(int)}, &skiptype{Pass:new(int)}\n\t// var data = `{\"a\":\"\",\"b\":1,\"c\":{\"d\":true,\"pass2\":1,\"pass4\":true},\"e\":{},\"f\":\"\",\"g\":[],\"pass\":null,\"h\":\"1.0\",\"i\":true,\"pass3\":1}`\n\t// d := NewDecoder(data)\n\t// err := d.Decode(obj)\n\t// err2 := json.Unmarshal([]byte(data), obj2)\n\t// println(err2.Error())\n\t// assert.Equal(t, err2 == nil, err == nil)\n\t// // assert.Equal(t, len(data), d.i)\n\t// assert.Equal(t, obj2, obj)\n\t// if te, ok := err.(*MismatchTypeError); ok {\n\t// assert.Equal(t, reflect.TypeOf(obj.I), te.Type)\n\t// assert.Equal(t, strings.Index(data, `\"i\":t`)+4, te.Pos)\n\t// println(err.Error())\n\t// } else {\n\t// t.Fatal(\"invalid error\")\n\t// }\n\t// })\n\tt.Run(\"short array\", func(t *testing.T) {\n\t\tvar obj, obj2 = &[]int{}, &[]int{}\n\t\tvar data = `[\"\"]`\n\t\td := NewDecoder(data)\n\t\terr := d.Decode(obj)\n\t\terr2 := json.Unmarshal([]byte(data), obj2)\n\t\t// println(err2.Error())\n\t\tassert.Equal(t, err2 == nil, err == nil)\n\t\t// assert.Equal(t, len(data), d.i)\n\t\tassert.Equal(t, obj2, obj)\n\t})\n\n\tt.Run(\"int \", func(t *testing.T) {\n\t\tvar obj int = 123\n\t\tvar obj2 int = 123\n\t\tvar data = `[\"\"]`\n\t\td := NewDecoder(data)\n\t\terr := d.Decode(&obj)\n\t\terr2 := json.Unmarshal([]byte(data), &obj2)\n\t\tprintln(err.Error(), obj, obj2)\n\t\tassert.Equal(t, err2 == nil, err == nil)\n\t\t// assert.Equal(t, len(data), d.i)\n\t\tassert.Equal(t, obj2, obj)\n\t})\n\n\tt.Run(\"array\", func(t *testing.T) {\n\t\tvar obj, obj2 = &[]int{}, &[]int{}\n\t\tvar data = `[\"\",true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true]`\n\t\td := NewDecoder(data)\n\t\terr := d.Decode(obj)\n\t\terr2 := json.Unmarshal([]byte(data), obj2)\n\t\t// println(err2.Error())\n\t\tassert.Equal(t, err2 == nil, err == nil)\n\t\t// assert.Equal(t, len(data), d.i)\n\t\tassert.Equal(t, obj2, obj)\n\t})\n\n\tt.Run(\"map\", func(t *testing.T) {\n\t\tvar obj, obj2 = &map[int]int{}, &map[int]int{}\n\t\tvar data = `{\"true\" : { },\"1\":1,\"2\" : true,\"3\":3}`\n\t\td := NewDecoder(data)\n\t\terr := d.Decode(obj)\n\t\terr2 := json.Unmarshal([]byte(data), obj2)\n\t\tassert.Equal(t, err2 == nil, err == nil)\n\t\t// assert.Equal(t, len(data), d.i)\n\t\tassert.Equal(t, obj2, obj)\n\t})\n\tt.Run(\"map error\", func(t *testing.T) {\n\t\tvar obj, obj2 = &map[int]int{}, &map[int]int{}\n\t\tvar data = `{\"true\" : { ],\"1\":1,\"2\" : true,\"3\":3}`\n\t\td := NewDecoder(data)\n\t\terr := d.Decode(obj)\n\t\terr2 := json.Unmarshal([]byte(data), obj2)\n\t\tprintln(err.Error())\n\t\tprintln(err2.Error())\n\t\tassert.Equal(t, err2 == nil, err == nil)\n\t\t// assert.Equal(t, len(data), d.i)\n\t\t// assert.Equal(t, obj2, obj)\n\t})\n}\n\nfunc TestCopyString(t *testing.T) {\n\tvar data []byte\n\tvar dc *Decoder\n\tvar err error\n\tdata = []byte(`{\"A\":\"0\",\"B\":\"1\"}`)\n\tdc = NewDecoder(rt.Mem2Str(data))\n\tdc.UseNumber()\n\tdc.CopyString()\n\tvar obj struct {\n\t\tA string\n\t\tB string\n\t}\n\terr = dc.Decode(&obj)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdata[6] = '1'\n\tif obj.A != \"0\" {\n\t\tt.Fatal(obj)\n\t}\n\tdata[14] = '0'\n\tif obj.B != \"1\" {\n\t\tt.Fatal(obj)\n\t}\n\n\tdata = []byte(`{\"A\":\"0\",\"B\":\"1\"}`)\n\tdc = NewDecoder(rt.Mem2Str(data))\n\tdc.UseNumber()\n\terr = dc.Decode(&obj)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdata[6] = '1'\n\tif obj.A != \"1\" {\n\t\tt.Fatal(obj)\n\t}\n\tdata[14] = '0'\n\tif obj.B != \"0\" {\n\t\tt.Fatal(obj)\n\t}\n\n\tdata = []byte(`{\"A\":\"0\",\"B\":\"1\"}`)\n\tdc = NewDecoder(rt.Mem2Str(data))\n\tdc.UseNumber()\n\tdc.CopyString()\n\tm := map[string]interface{}{}\n\terr = dc.Decode(&m)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdata[2] = 'C'\n\tdata[6] = '1'\n\tif m[\"A\"] != \"0\" {\n\t\tt.Fatal(m)\n\t}\n\tdata[10] = 'D'\n\tdata[14] = '0'\n\tif m[\"B\"] != \"1\" {\n\t\tt.Fatal(m)\n\t}\n\n\tdata = []byte(`{\"A\":\"0\",\"B\":\"1\"}`)\n\tdc = NewDecoder(rt.Mem2Str(data))\n\tdc.UseNumber()\n\tm = map[string]interface{}{}\n\terr = dc.Decode(&m)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdata[6] = '1'\n\tif m[\"A\"] != \"1\" {\n\t\tt.Fatal(m)\n\t}\n\tdata[14] = '0'\n\tif m[\"B\"] != \"0\" {\n\t\tt.Fatal(m)\n\t}\n\n\tdata = []byte(`{\"A\":\"0\",\"B\":\"1\"}`)\n\tdc = NewDecoder(rt.Mem2Str(data))\n\tdc.UseNumber()\n\tdc.CopyString()\n\tvar x interface{}\n\terr = dc.Decode(&x)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdata[2] = 'C'\n\tdata[6] = '1'\n\tm = x.(map[string]interface{})\n\tif m[\"A\"] != \"0\" {\n\t\tt.Fatal(m)\n\t}\n\tdata[10] = 'D'\n\tdata[14] = '0'\n\tif m[\"B\"] != \"1\" {\n\t\tt.Fatal(m)\n\t}\n\n\tdata = []byte(`{\"A\":\"0\",\"B\":\"1\"}`)\n\tdc = NewDecoder(rt.Mem2Str(data))\n\tdc.UseNumber()\n\tvar y interface{}\n\terr = dc.Decode(&y)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tm = y.(map[string]interface{})\n\tdata[6] = '1'\n\tif m[\"A\"] != \"1\" {\n\t\tt.Fatal(m)\n\t}\n\tdata[14] = '0'\n\tif m[\"B\"] != \"0\" {\n\t\tt.Fatal(m)\n\t}\n}\n\nfunc TestDecoder_SetOption(t *testing.T) {\n\tvar v interface{}\n\td := NewDecoder(\"123\")\n\td.SetOptions(OptionUseInt64)\n\terr := d.Decode(&v)\n\tassert.NoError(t, err)\n\tassert.Equal(t, v, int64(123))\n}\n\nfunc BenchmarkSkip_Sonic(b *testing.B) {\n\tvar data = rt.Str2Mem(TwitterJson)\n\tif ret, _ := Skip(data); ret < 0 {\n\t\tb.Fatal()\n\t}\n\tb.SetBytes(int64(len(TwitterJson)))\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\t_, _ = Skip(data)\n\t}\n}\n" }, { "path": "decoder/decoder_test.go", "content": "/*\n * Copyright 2021 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage decoder\n\nimport (\n\t\"encoding/json\"\n\t\"runtime\"\n\t\"runtime/debug\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestMain(m *testing.M) {\n\tgo func() {\n\t\tif !debugAsyncGC {\n\t\t\treturn\n\t\t}\n\t\tprintln(\"Begin GC looping...\")\n\t\tfor {\n\t\t\truntime.GC()\n\t\t\tdebug.FreeOSMemory()\n\t\t}\n\t\tprintln(\"stop GC looping!\")\n\t}()\n\ttime.Sleep(time.Millisecond)\n\tm.Run()\n}\n\nfunc TestGC(t *testing.T) {\n\tif debugSyncGC {\n\t\treturn\n\t}\n\tvar w interface{}\n\tout, err := decode(TwitterJson, &w, true)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif out != len(TwitterJson) {\n\t\tt.Fatal(out)\n\t}\n\twg := &sync.WaitGroup{}\n\tN := 10000\n\tfor i := 0; i < N; i++ {\n\t\twg.Add(1)\n\t\tgo func(wg *sync.WaitGroup) {\n\t\t\tdefer wg.Done()\n\t\t\tvar w interface{}\n\t\t\tout, err := decode(TwitterJson, &w, true)\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif out != len(TwitterJson) {\n\t\t\t\tt.Error(out)\n\t\t\t\treturn\n\t\t\t}\n\t\t\truntime.GC()\n\t\t}(wg)\n\t}\n\twg.Wait()\n}\n\nvar _BindingValue TwitterStruct\n\nfunc init() {\n\t_ = json.Unmarshal([]byte(TwitterJson), &_BindingValue)\n}\n\nfunc TestSkipMismatchTypeError(t *testing.T) {\n\tt.Run(\"struct\", func(t *testing.T) {\n\t\tprintln(\"TestSkipError\")\n\t\ttype skiptype struct {\n\t\t\tA int `json:\"a\"`\n\t\t\tB string `json:\"b\"`\n\n\t\t\tPass *int `json:\"pass\"`\n\n\t\t\tC struct {\n\t\t\t\tPass4 interface{} `json:\"pass4\"`\n\n\t\t\t\tD struct {\n\t\t\t\t\tE float32 `json:\"e\"`\n\t\t\t\t} `json:\"d\"`\n\n\t\t\t\tPass2 int `json:\"pass2\"`\n\t\t\t} `json:\"c\"`\n\n\t\t\tE bool `json:\"e\"`\n\t\t\tF []int `json:\"f\"`\n\t\t\tG map[string]int `json:\"g\"`\n\t\t\tH bool `json:\"h,string\"`\n\n\t\t\tPass3 int `json:\"pass2\"`\n\n\t\t\tI json.Number `json:\"i\"`\n\t\t}\n\t\tvar obj, obj2 = &skiptype{Pass: new(int)}, &skiptype{Pass: new(int)}\n\t\tvar data = `{\"a\":\"\",\"b\":1,\"c\":{\"d\":true,\"pass2\":1,\"pass4\":true},\"e\":{},\"f\":\"\",\"g\":[],\"pass\":null,\"h\":\"1.0\",\"i\":true,\"pass3\":1}`\n\t\td := NewDecoder(data)\n\t\terr := d.Decode(obj)\n\t\terr2 := json.Unmarshal([]byte(data), obj2)\n\t\tprintln(err2.Error())\n\t\tassert.Equal(t, err2 == nil, err == nil)\n\t\t// assert.Equal(t, len(data), d.i)\n\t\tassert.Equal(t, obj2, obj)\n\t\tif err == nil {\n\t\t\tt.Fatal(\"invalid error\")\n\t\t}\n\t})\n\tt.Run(\"short array\", func(t *testing.T) {\n\t\tvar obj, obj2 = &[]int{}, &[]int{}\n\t\tvar data = `[\"\"]`\n\t\td := NewDecoder(data)\n\t\terr := d.Decode(obj)\n\t\terr2 := json.Unmarshal([]byte(data), obj2)\n\t\t// println(err2.Error())\n\t\tassert.Equal(t, err2 == nil, err == nil)\n\t\t// assert.Equal(t, len(data), d.i)\n\t\tassert.Equal(t, obj2, obj)\n\t})\n\n\tt.Run(\"int \", func(t *testing.T) {\n\t\tvar obj int = 123\n\t\tvar obj2 int = 123\n\t\tvar data = `[\"\"]`\n\t\td := NewDecoder(data)\n\t\terr := d.Decode(&obj)\n\t\terr2 := json.Unmarshal([]byte(data), &obj2)\n\t\tprintln(err.Error(), obj, obj2)\n\t\tassert.Equal(t, err2 == nil, err == nil)\n\t\t// assert.Equal(t, len(data), d.i)\n\t\tassert.Equal(t, obj2, obj)\n\t})\n\n\tt.Run(\"array\", func(t *testing.T) {\n\t\tvar obj, obj2 = &[]int{}, &[]int{}\n\t\tvar data = `[\"\",true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true,true]`\n\t\td := NewDecoder(data)\n\t\terr := d.Decode(obj)\n\t\terr2 := json.Unmarshal([]byte(data), obj2)\n\t\t// println(err2.Error())\n\t\tassert.Equal(t, err2 == nil, err == nil)\n\t\t// assert.Equal(t, len(data), d.i)\n\t\tassert.Equal(t, obj2, obj)\n\t})\n\n\tt.Run(\"map\", func(t *testing.T) {\n\t\tvar obj, obj2 = &map[int]int{}, &map[int]int{}\n\t\tvar data = `{\"true\" : { },\"1\":1,\"2\" : true,\"3\":3}`\n\t\td := NewDecoder(data)\n\t\terr := d.Decode(obj)\n\t\terr2 := json.Unmarshal([]byte(data), obj2)\n\t\tassert.Equal(t, err2 == nil, err == nil)\n\t\t// assert.Equal(t, len(data), d.i)\n\t\tassert.Equal(t, obj2, obj)\n\t})\n\tt.Run(\"map error\", func(t *testing.T) {\n\t\tvar obj, obj2 = &map[int]int{}, &map[int]int{}\n\t\tvar data = `{\"true\" : { ],\"1\":1,\"2\" : true,\"3\":3}`\n\t\td := NewDecoder(data)\n\t\terr := d.Decode(obj)\n\t\terr2 := json.Unmarshal([]byte(data), obj2)\n\t\tprintln(err.Error())\n\t\tprintln(err2.Error())\n\t\tassert.Equal(t, err2 == nil, err == nil)\n\t\t// assert.Equal(t, len(data), d.i)\n\t\t// assert.Equal(t, obj2, obj)\n\t})\n}\n\nfunc TestDecodeCorrupt(t *testing.T) {\n\tvar ds = []string{\n\t\t`{,}`,\n\t\t`{,\"a\"}`,\n\t\t`{\"a\":}`,\n\t\t`{\"a\":1,}`,\n\t\t`{\"a\":1,\"b\"}`,\n\t\t`{\"a\":1,\"b\":}`,\n\t\t`{,\"a\":1 \"b\":2}`,\n\t\t`{\"a\",:1 \"b\":2}`,\n\t\t`{\"a\":,1 \"b\":2}`,\n\t\t`{\"a\":1 \"b\",:2}`,\n\t\t`{\"a\":1 \"b\":,2}`,\n\t\t`{\"a\":1 \"b\":2,}`,\n\t\t`{\"a\":1 \"b\":2}`,\n\t\t`[,]`,\n\t\t`[,1]`,\n\t\t`[1,]`,\n\t\t`[,1,2]`,\n\t\t`[1,2,]`,\n\t}\n\tfor _, d := range ds {\n\t\tvar o interface{}\n\t\t_, err := decode(d, &o, false)\n\t\tif err == nil {\n\t\t\tt.Fatalf(\"%#v\", d)\n\t\t}\n\t\tif !(strings.Contains(err.Error(), \"Syntax error\") || strings.Contains(err.Error(), \"invalid character\")) {\n\t\t\tt.Fatal(err.Error())\n\t\t}\n\t}\n}\n\nfunc TestDecodeOption(t *testing.T) {\n\tvar s string\n\tvar d *Decoder\n\tvar out interface{}\n\tvar out2 struct{}\n\tvar err error\n\n\ts = \"123\"\n\td = NewDecoder(s)\n\td.SetOptions(OptionUseNumber)\n\terr = d.Decode(&out)\n\tassert.NoError(t, err)\n\tassert.Equal(t, out.(json.Number), json.Number(\"123\"))\n\n\td = NewDecoder(s)\n\terr = d.Decode(&out)\n\tassert.NoError(t, err)\n\tassert.Equal(t, out.(float64), float64(123))\n\n\ts = `{\"un\": 123}`\n\td = NewDecoder(s)\n\td.SetOptions(OptionDisableUnknown)\n\terr = d.Decode(&out2)\n\tassert.Error(t, err)\n\n\td = NewDecoder(s)\n\terr = d.Decode(&out2)\n\tassert.NoError(t, err)\n}\n\nfunc decode(s string, v interface{}, copy bool) (int, error) {\n\td := NewDecoder(s)\n\tif copy {\n\t\td.CopyString()\n\t}\n\terr := d.Decode(v)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\treturn len(s), err\n}\n\nfunc TestDecoder_Basic(t *testing.T) {\n\tvar v int\n\tpos, err := decode(\"12345\", &v, false)\n\tassert.NoError(t, err)\n\tassert.Equal(t, 5, pos)\n\tassert.Equal(t, 12345, v)\n}\n\nfunc TestDecoder_Generic(t *testing.T) {\n\tvar v interface{}\n\tpos, err := decode(TwitterJson, &v, false)\n\tassert.NoError(t, err)\n\tassert.Equal(t, len(TwitterJson), pos)\n}\n\nfunc TestDecoder_Binding(t *testing.T) {\n\tvar v TwitterStruct\n\tpos, err := decode(TwitterJson, &v, false)\n\tassert.NoError(t, err)\n\tassert.Equal(t, len(TwitterJson), pos)\n\tassert.Equal(t, _BindingValue, v, 0)\n}\n\nfunc BenchmarkDecoder_Generic_Sonic(b *testing.B) {\n\tvar w interface{}\n\t_, _ = decode(TwitterJson, &w, true)\n\tb.SetBytes(int64(len(TwitterJson)))\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tvar v interface{}\n\t\t_, _ = decode(TwitterJson, &v, true)\n\t}\n}\n\nfunc BenchmarkDecoder_Generic_Sonic_Fast(b *testing.B) {\n\tvar w interface{}\n\t_, _ = decode(TwitterJson, &w, false)\n\tb.SetBytes(int64(len(TwitterJson)))\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tvar v interface{}\n\t\t_, _ = decode(TwitterJson, &v, false)\n\t}\n}\n\nfunc BenchmarkDecoder_Generic_StdLib(b *testing.B) {\n\tvar w interface{}\n\tm := []byte(TwitterJson)\n\t_ = json.Unmarshal(m, &w)\n\tb.SetBytes(int64(len(TwitterJson)))\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tvar v interface{}\n\t\t_ = json.Unmarshal(m, &v)\n\t}\n}\n\nfunc BenchmarkDecoder_Binding_Sonic(b *testing.B) {\n\tvar w TwitterStruct\n\t_, _ = decode(TwitterJson, &w, true)\n\tb.SetBytes(int64(len(TwitterJson)))\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tvar v TwitterStruct\n\t\t_, _ = decode(TwitterJson, &v, true)\n\t}\n}\n\nfunc BenchmarkDecoder_Binding_Sonic_Fast(b *testing.B) {\n\tvar w TwitterStruct\n\t_, _ = decode(TwitterJson, &w, false)\n\tb.SetBytes(int64(len(TwitterJson)))\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tvar v TwitterStruct\n\t\t_, _ = decode(TwitterJson, &v, false)\n\t}\n}\n\nfunc BenchmarkDecoder_Binding_StdLib(b *testing.B) {\n\tvar w TwitterStruct\n\tm := []byte(TwitterJson)\n\t_ = json.Unmarshal(m, &w)\n\tb.SetBytes(int64(len(TwitterJson)))\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tvar v TwitterStruct\n\t\t_ = json.Unmarshal(m, &v)\n\t}\n}\n\nfunc BenchmarkDecoder_Parallel_Generic_Sonic(b *testing.B) {\n\tvar w interface{}\n\t_, _ = decode(TwitterJson, &w, true)\n\tb.SetBytes(int64(len(TwitterJson)))\n\tb.ResetTimer()\n\tb.RunParallel(func(pb *testing.PB) {\n\t\tfor pb.Next() {\n\t\t\tvar v interface{}\n\t\t\t_, _ = decode(TwitterJson, &v, true)\n\t\t}\n\t})\n}\n\nfunc BenchmarkDecoder_Parallel_Generic_Sonic_Fast(b *testing.B) {\n\tvar w interface{}\n\t_, _ = decode(TwitterJson, &w, false)\n\tb.SetBytes(int64(len(TwitterJson)))\n\tb.ResetTimer()\n\tb.RunParallel(func(pb *testing.PB) {\n\t\tfor pb.Next() {\n\t\t\tvar v interface{}\n\t\t\t_, _ = decode(TwitterJson, &v, false)\n\t\t}\n\t})\n}\n\nfunc BenchmarkDecoder_Parallel_Generic_StdLib(b *testing.B) {\n\tvar w interface{}\n\tm := []byte(TwitterJson)\n\t_ = json.Unmarshal(m, &w)\n\tb.SetBytes(int64(len(TwitterJson)))\n\tb.ResetTimer()\n\tb.RunParallel(func(pb *testing.PB) {\n\t\tfor pb.Next() {\n\t\t\tvar v interface{}\n\t\t\t_ = json.Unmarshal(m, &v)\n\t\t}\n\t})\n}\n\nfunc BenchmarkDecoder_Parallel_Binding_Sonic(b *testing.B) {\n\tvar w TwitterStruct\n\t_, _ = decode(TwitterJson, &w, true)\n\tb.SetBytes(int64(len(TwitterJson)))\n\tb.ResetTimer()\n\tb.RunParallel(func(pb *testing.PB) {\n\t\tfor pb.Next() {\n\t\t\tvar v TwitterStruct\n\t\t\t_, _ = decode(TwitterJson, &v, true)\n\t\t}\n\t})\n}\n\nfunc BenchmarkDecoder_Parallel_Binding_Sonic_Fast(b *testing.B) {\n\tvar w TwitterStruct\n\t_, _ = decode(TwitterJson, &w, false)\n\tb.SetBytes(int64(len(TwitterJson)))\n\tb.ResetTimer()\n\tb.RunParallel(func(pb *testing.PB) {\n\t\tfor pb.Next() {\n\t\t\tvar v TwitterStruct\n\t\t\t_, _ = decode(TwitterJson, &v, false)\n\t\t}\n\t})\n}\n\nfunc BenchmarkDecoder_Parallel_Binding_StdLib(b *testing.B) {\n\tvar w TwitterStruct\n\tm := []byte(TwitterJson)\n\t_ = json.Unmarshal(m, &w)\n\tb.SetBytes(int64(len(TwitterJson)))\n\tb.ResetTimer()\n\tb.RunParallel(func(pb *testing.PB) {\n\t\tfor pb.Next() {\n\t\t\tvar v TwitterStruct\n\t\t\t_ = json.Unmarshal(m, &v)\n\t\t}\n\t})\n}\n" }, { "path": "decoder/testdata_test.go", "content": "/*\n * Copyright 2021 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage decoder\n\nimport (\n\t\"os\"\n)\n\nvar (\n\tdebugSyncGC = os.Getenv(\"SONIC_SYNC_GC\") != \"\"\n\tdebugAsyncGC = os.Getenv(\"SONIC_NO_ASYNC_GC\") == \"\"\n)\n\nconst TwitterJson = `{\n \"statuses\": [\n {\n \"coordinates\": null,\n \"favorited\": false,\n \"truncated\": false,\n \"created_at\": \"Mon Sep 24 03:35:21 +0000 2012\",\n \"id_str\": \"250075927172759552\",\n \"entities\": {\n \"urls\": [\n \n ],\n \"hashtags\": [\n {\n \"text\": \"freebandnames\",\n \"indices\": [\n 20,\n 34\n ]\n }\n ],\n \"user_mentions\": [\n \n ]\n },\n \"in_reply_to_user_id_str\": null,\n \"contributors\": null,\n \"text\": \"Aggressive Ponytail #freebandnames\",\n \"metadata\": {\n \"iso_language_code\": \"en\",\n \"result_type\": \"recent\"\n },\n \"retweet_count\": 0,\n \"in_reply_to_status_id_str\": null,\n \"id\": 250075927172759552,\n \"geo\": null,\n \"retweeted\": false,\n \"in_reply_to_user_id\": null,\n \"place\": null,\n \"user\": {\n \"profile_sidebar_fill_color\": \"DDEEF6\",\n \"profile_sidebar_border_color\": \"C0DEED\",\n \"profile_background_tile\": false,\n \"name\": \"Sean Cummings\",\n \"profile_image_url\": \"https://a0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg\",\n \"created_at\": \"Mon Apr 26 06:01:55 +0000 2010\",\n \"location\": \"LA, CA\",\n \"follow_request_sent\": null,\n \"profile_link_color\": \"0084B4\",\n \"is_translator\": false,\n \"id_str\": \"137238150\",\n \"entities\": {\n \"url\": {\n \"urls\": [\n {\n \"expanded_url\": null,\n \"url\": \"\",\n \"indices\": [\n 0,\n 0\n ]\n }\n ]\n },\n \"description\": {\n \"urls\": [\n \n ]\n }\n },\n \"default_profile\": true,\n \"contributors_enabled\": false,\n \"favourites_count\": 0,\n \"url\": null,\n \"profile_image_url_https\": \"https://si0.twimg.com/profile_images/2359746665/1v6zfgqo8g0d3mk7ii5s_normal.jpeg\",\n \"utc_offset\": -28800,\n \"id\": 137238150,\n \"profile_use_background_image\": true,\n \"listed_count\": 2,\n \"profile_text_color\": \"333333\",\n \"lang\": \"en\",\n \"followers_count\": 70,\n \"protected\": false,\n \"notifications\": null,\n \"profile_background_image_url_https\": \"https://si0.twimg.com/images/themes/theme1/bg.png\",\n \"profile_background_color\": \"C0DEED\",\n \"verified\": false,\n \"geo_enabled\": true,\n \"time_zone\": \"Pacific Time (US & Canada)\",\n \"description\": \"Born 330 Live 310\",\n \"default_profile_image\": false,\n \"profile_background_image_url\": \"https://a0.twimg.com/images/themes/theme1/bg.png\",\n \"statuses_count\": 579,\n \"friends_count\": 110,\n \"following\": null,\n \"show_all_inline_media\": false,\n \"screen_name\": \"sean_cummings\"\n },\n \"in_reply_to_screen_name\": null,\n \"source\": \"Twitter for Mac\",\n \"in_reply_to_status_id\": null\n },\n {\n \"coordinates\": null,\n \"favorited\": false,\n \"truncated\": false,\n \"created_at\": \"Fri Sep 21 23:40:54 +0000 2012\",\n \"id_str\": \"249292149810667520\",\n \"entities\": {\n \"urls\": [\n \n ],\n \"hashtags\": [\n {\n \"text\": \"FreeBandNames\",\n \"indices\": [\n 20,\n 34\n ]\n }\n ],\n \"user_mentions\": [\n \n ]\n },\n \"in_reply_to_user_id_str\": null,\n \"contributors\": null,\n \"text\": \"Thee Namaste Nerdz. #FreeBandNames\",\n \"metadata\": {\n \"iso_language_code\": \"pl\",\n \"result_type\": \"recent\"\n },\n \"retweet_count\": 0,\n \"in_reply_to_status_id_str\": null,\n \"id\": 249292149810667520,\n \"geo\": null,\n \"retweeted\": false,\n \"in_reply_to_user_id\": null,\n \"place\": null,\n \"user\": {\n \"profile_sidebar_fill_color\": \"DDFFCC\",\n \"profile_sidebar_border_color\": \"BDDCAD\",\n \"profile_background_tile\": true,\n \"name\": \"Chaz Martenstein\",\n \"profile_image_url\": \"https://a0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg\",\n \"created_at\": \"Tue Apr 07 19:05:07 +0000 2009\",\n \"location\": \"Durham, NC\",\n \"follow_request_sent\": null,\n \"profile_link_color\": \"0084B4\",\n \"is_translator\": false,\n \"id_str\": \"29516238\",\n \"entities\": {\n \"url\": {\n \"urls\": [\n {\n \"expanded_url\": null,\n \"url\": \"https://bullcityrecords.com/wnng/\",\n \"indices\": [\n 0,\n 32\n ]\n }\n ]\n },\n \"description\": {\n \"urls\": [\n \n ]\n }\n },\n \"default_profile\": false,\n \"contributors_enabled\": false,\n \"favourites_count\": 8,\n \"url\": \"https://bullcityrecords.com/wnng/\",\n \"profile_image_url_https\": \"https://si0.twimg.com/profile_images/447958234/Lichtenstein_normal.jpg\",\n \"utc_offset\": -18000,\n \"id\": 29516238,\n \"profile_use_background_image\": true,\n \"listed_count\": 118,\n \"profile_text_color\": \"333333\",\n \"lang\": \"en\",\n \"followers_count\": 2052,\n \"protected\": false,\n \"notifications\": null,\n \"profile_background_image_url_https\": \"https://si0.twimg.com/profile_background_images/9423277/background_tile.bmp\",\n \"profile_background_color\": \"9AE4E8\",\n \"verified\": false,\n \"geo_enabled\": false,\n \"time_zone\": \"Eastern Time (US & Canada)\",\n \"description\": \"You will come to Durham, North Carolina. I will sell you some records then, here in Durham, North Carolina. Fun will happen.\",\n \"default_profile_image\": false,\n \"profile_background_image_url\": \"https://a0.twimg.com/profile_background_images/9423277/background_tile.bmp\",\n \"statuses_count\": 7579,\n \"friends_count\": 348,\n \"following\": null,\n \"show_all_inline_media\": true,\n \"screen_name\": \"bullcityrecords\"\n },\n \"in_reply_to_screen_name\": null,\n \"source\": \"web\",\n \"in_reply_to_status_id\": null\n },\n {\n \"coordinates\": null,\n \"favorited\": false,\n \"truncated\": false,\n \"created_at\": \"Fri Sep 21 23:30:20 +0000 2012\",\n \"id_str\": \"249289491129438208\",\n \"entities\": {\n \"urls\": [\n \n ],\n \"hashtags\": [\n {\n \"text\": \"freebandnames\",\n \"indices\": [\n 29,\n 43\n ]\n }\n ],\n \"user_mentions\": [\n \n ]\n },\n \"in_reply_to_user_id_str\": null,\n \"contributors\": null,\n \"text\": \"Mexican Heaven, Mexican Hell #freebandnames\",\n \"metadata\": {\n \"iso_language_code\": \"en\",\n \"result_type\": \"recent\"\n },\n \"retweet_count\": 0,\n \"in_reply_to_status_id_str\": null,\n \"id\": 249289491129438208,\n \"geo\": null,\n \"retweeted\": false,\n \"in_reply_to_user_id\": null,\n \"place\": null,\n \"user\": {\n \"profile_sidebar_fill_color\": \"99CC33\",\n \"profile_sidebar_border_color\": \"829D5E\",\n \"profile_background_tile\": false,\n \"name\": \"Thomas John Wakeman\",\n \"profile_image_url\": \"https://a0.twimg.com/profile_images/2219333930/Froggystyle_normal.png\",\n \"created_at\": \"Tue Sep 01 21:21:35 +0000 2009\",\n \"location\": \"Kingston New York\",\n \"follow_request_sent\": null,\n \"profile_link_color\": \"D02B55\",\n \"is_translator\": false,\n \"id_str\": \"70789458\",\n \"entities\": {\n \"url\": {\n \"urls\": [\n {\n \"expanded_url\": null,\n \"url\": \"\",\n \"indices\": [\n 0,\n 0\n ]\n }\n ]\n },\n \"description\": {\n \"urls\": [\n \n ]\n }\n },\n \"default_profile\": false,\n \"contributors_enabled\": false,\n \"favourites_count\": 19,\n \"url\": null,\n \"profile_image_url_https\": \"https://si0.twimg.com/profile_images/2219333930/Froggystyle_normal.png\",\n \"utc_offset\": -18000,\n \"id\": 70789458,\n \"profile_use_background_image\": true,\n \"listed_count\": 1,\n \"profile_text_color\": \"3E4415\",\n \"lang\": \"en\",\n \"followers_count\": 63,\n \"protected\": false,\n \"notifications\": null,\n \"profile_background_image_url_https\": \"https://si0.twimg.com/images/themes/theme5/bg.gif\",\n \"profile_background_color\": \"352726\",\n \"verified\": false,\n \"geo_enabled\": false,\n \"time_zone\": \"Eastern Time (US & Canada)\",\n \"description\": \"Science Fiction Writer, sort of. Likes Superheroes, Mole People, Alt. Timelines.\",\n \"default_profile_image\": false,\n \"profile_background_image_url\": \"https://a0.twimg.com/images/themes/theme5/bg.gif\",\n \"statuses_count\": 1048,\n \"friends_count\": 63,\n \"following\": null,\n \"show_all_inline_media\": false,\n \"screen_name\": \"MonkiesFist\"\n },\n \"in_reply_to_screen_name\": null,\n \"source\": \"web\",\n \"in_reply_to_status_id\": null\n },\n {\n \"coordinates\": null,\n \"favorited\": false,\n \"truncated\": false,\n \"created_at\": \"Fri Sep 21 22:51:18 +0000 2012\",\n \"id_str\": \"249279667666817024\",\n \"entities\": {\n \"urls\": [\n \n ],\n \"hashtags\": [\n {\n \"text\": \"freebandnames\",\n \"indices\": [\n 20,\n 34\n ]\n }\n ],\n \"user_mentions\": [\n \n ]\n },\n \"in_reply_to_user_id_str\": null,\n \"contributors\": null,\n \"text\": \"The Foolish Mortals #freebandnames\",\n \"metadata\": {\n \"iso_language_code\": \"en\",\n \"result_type\": \"recent\"\n },\n \"retweet_count\": 0,\n \"in_reply_to_status_id_str\": null,\n \"id\": 249279667666817024,\n \"geo\": null,\n \"retweeted\": false,\n \"in_reply_to_user_id\": null,\n \"place\": null,\n \"user\": {\n \"profile_sidebar_fill_color\": \"BFAC83\",\n \"profile_sidebar_border_color\": \"615A44\",\n \"profile_background_tile\": true,\n \"name\": \"Marty Elmer\",\n \"profile_image_url\": \"https://a0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png\",\n \"created_at\": \"Mon May 04 00:05:00 +0000 2009\",\n \"location\": \"Wisconsin, USA\",\n \"follow_request_sent\": null,\n \"profile_link_color\": \"3B2A26\",\n \"is_translator\": false,\n \"id_str\": \"37539828\",\n \"entities\": {\n \"url\": {\n \"urls\": [\n {\n \"expanded_url\": null,\n \"url\": \"https://www.omnitarian.me\",\n \"indices\": [\n 0,\n 24\n ]\n }\n ]\n },\n \"description\": {\n \"urls\": [\n \n ]\n }\n },\n \"default_profile\": false,\n \"contributors_enabled\": false,\n \"favourites_count\": 647,\n \"url\": \"https://www.omnitarian.me\",\n \"profile_image_url_https\": \"https://si0.twimg.com/profile_images/1629790393/shrinker_2000_trans_normal.png\",\n \"utc_offset\": -21600,\n \"id\": 37539828,\n \"profile_use_background_image\": true,\n \"listed_count\": 52,\n \"profile_text_color\": \"000000\",\n \"lang\": \"en\",\n \"followers_count\": 608,\n \"protected\": false,\n \"notifications\": null,\n \"profile_background_image_url_https\": \"https://si0.twimg.com/profile_background_images/106455659/rect6056-9.png\",\n \"profile_background_color\": \"EEE3C4\",\n \"verified\": false,\n \"geo_enabled\": false,\n \"time_zone\": \"Central Time (US & Canada)\",\n \"description\": \"Cartoonist, Illustrator, and T-Shirt connoisseur\",\n \"default_profile_image\": false,\n \"profile_background_image_url\": \"https://a0.twimg.com/profile_background_images/106455659/rect6056-9.png\",\n \"statuses_count\": 3575,\n \"friends_count\": 249,\n \"following\": null,\n \"show_all_inline_media\": true,\n \"screen_name\": \"Omnitarian\"\n },\n \"in_reply_to_screen_name\": null,\n \"source\": \"Twitter for iPhone\",\n \"in_reply_to_status_id\": null\n }\n ],\n \"search_metadata\": {\n \"max_id\": 250126199840518145,\n \"since_id\": 24012619984051000,\n \"refresh_url\": \"?since_id=250126199840518145&q=%23freebandnames&result_type=mixed&include_entities=1\",\n \"next_results\": \"?max_id=249279667666817023&q=%23freebandnames&count=4&include_entities=1&result_type=mixed\",\n \"count\": 4,\n \"completed_in\": 0.035,\n \"since_id_str\": \"24012619984051000\",\n \"query\": \"%23freebandnames\",\n \"max_id_str\": \"250126199840518145\"\n }\n}`\n\ntype TwitterStruct struct {\n\tStatuses []Statuses `json:\"statuses\"`\n\tSearchMetadata SearchMetadata `json:\"search_metadata\"`\n}\n\ntype Hashtags struct {\n\tText string `json:\"text\"`\n\tIndices []int `json:\"indices\"`\n}\n\ntype Entities struct {\n\tUrls []interface{} `json:\"urls\"`\n\tHashtags []Hashtags `json:\"hashtags\"`\n\tUserMentions []interface{} `json:\"user_mentions\"`\n}\n\ntype Metadata struct {\n\tIsoLanguageCode string `json:\"iso_language_code\"`\n\tResultType string `json:\"result_type\"`\n}\n\ntype Urls struct {\n\tExpandedURL interface{} `json:\"expanded_url\"`\n\tURL string `json:\"url\"`\n\tIndices []int `json:\"indices\"`\n}\n\ntype URL struct {\n\tUrls []Urls `json:\"urls\"`\n}\n\ntype Description struct {\n\tUrls []interface{} `json:\"urls\"`\n}\n\ntype UserEntities struct {\n\tURL URL `json:\"url\"`\n\tDescription Description `json:\"description\"`\n}\n\ntype User struct {\n\tProfileSidebarFillColor string `json:\"profile_sidebar_fill_color\"`\n\tProfileSidebarBorderColor string `json:\"profile_sidebar_border_color\"`\n\tProfileBackgroundTile bool `json:\"profile_background_tile\"`\n\tName string `json:\"name\"`\n\tProfileImageURL string `json:\"profile_image_url\"`\n\tCreatedAt string `json:\"created_at\"`\n\tLocation string `json:\"location\"`\n\tFollowRequestSent interface{} `json:\"follow_request_sent\"`\n\tProfileLinkColor string `json:\"profile_link_color\"`\n\tIsTranslator bool `json:\"is_translator\"`\n\tIDStr string `json:\"id_str\"`\n\tEntities UserEntities `json:\"entities\"`\n\tDefaultProfile bool `json:\"default_profile\"`\n\tContributorsEnabled bool `json:\"contributors_enabled\"`\n\tFavouritesCount int `json:\"favourites_count\"`\n\tURL interface{} `json:\"url\"`\n\tProfileImageURLHTTPS string `json:\"profile_image_url_https\"`\n\tUtcOffset int `json:\"utc_offset\"`\n\tID int `json:\"id\"`\n\tProfileUseBackgroundImage bool `json:\"profile_use_background_image\"`\n\tListedCount int `json:\"listed_count\"`\n\tProfileTextColor string `json:\"profile_text_color\"`\n\tLang string `json:\"lang\"`\n\tFollowersCount int `json:\"followers_count\"`\n\tProtected bool `json:\"protected\"`\n\tNotifications interface{} `json:\"notifications\"`\n\tProfileBackgroundImageURLHTTPS string `json:\"profile_background_image_url_https\"`\n\tProfileBackgroundColor string `json:\"profile_background_color\"`\n\tVerified bool `json:\"verified\"`\n\tGeoEnabled bool `json:\"geo_enabled\"`\n\tTimeZone string `json:\"time_zone\"`\n\tDescription string `json:\"description\"`\n\tDefaultProfileImage bool `json:\"default_profile_image\"`\n\tProfileBackgroundImageURL string `json:\"profile_background_image_url\"`\n\tStatusesCount int `json:\"statuses_count\"`\n\tFriendsCount int `json:\"friends_count\"`\n\tFollowing interface{} `json:\"following\"`\n\tShowAllInlineMedia bool `json:\"show_all_inline_media\"`\n\tScreenName string `json:\"screen_name\"`\n}\n\ntype Statuses struct {\n\tCoordinates interface{} `json:\"coordinates\"`\n\tFavorited bool `json:\"favorited\"`\n\tTruncated bool `json:\"truncated\"`\n\tCreatedAt string `json:\"created_at\"`\n\tIDStr string `json:\"id_str\"`\n\tEntities Entities `json:\"entities\"`\n\tInReplyToUserIDStr interface{} `json:\"in_reply_to_user_id_str\"`\n\tContributors interface{} `json:\"contributors\"`\n\tText string `json:\"text\"`\n\tMetadata Metadata `json:\"metadata\"`\n\tRetweetCount int `json:\"retweet_count\"`\n\tInReplyToStatusIDStr interface{} `json:\"in_reply_to_status_id_str\"`\n\tID int64 `json:\"id\"`\n\tGeo interface{} `json:\"geo\"`\n\tRetweeted bool `json:\"retweeted\"`\n\tInReplyToUserID interface{} `json:\"in_reply_to_user_id\"`\n\tPlace interface{} `json:\"place\"`\n\tUser User `json:\"user\"`\n\tInReplyToScreenName interface{} `json:\"in_reply_to_screen_name\"`\n\tSource string `json:\"source\"`\n\tInReplyToStatusID interface{} `json:\"in_reply_to_status_id\"`\n}\n\ntype SearchMetadata struct {\n\tMaxID int64 `json:\"max_id\"`\n\tSinceID int64 `json:\"since_id\"`\n\tRefreshURL string `json:\"refresh_url\"`\n\tNextResults string `json:\"next_results\"`\n\tCount int `json:\"count\"`\n\tCompletedIn float64 `json:\"completed_in\"`\n\tSinceIDStr string `json:\"since_id_str\"`\n\tQuery string `json:\"query\"`\n\tMaxIDStr string `json:\"max_id_str\"`\n}\n" }, { "path": "docs/INTRODUCTION.md", "content": "# Introduction to Sonic\n\nEnglish | [中文](INTRODUCTION_ZH_CN.md)\n\n## Background\nAccording to the overall profiling of production services in Bytedance, we found that the overhead of JSON serialization and deserialization is unexpectedly high: the total is near to 10% CPU, and the extreme one accounts for more than 40% CPU. Therefore, **the performance of JSON lib is a key issue for the promotion of machine utilization**.\n\n## Research\nWe conducted a series of surveys and benchmarks on open-sourced JSON libraries for Golang, but the result is disappointing: **no silver bullet**. First of all, no one can perform at least the top three across various business scenarios. Even the most widely used [json-iterator](https://github.com/json-iterator/go) will severely degrade in generic (no-schema) or big-volume JSON serialization and deserialization. Secondly, compared with other JSON libraries writing in other languages, their speed is generally much slower. For example, [Simdjson-go](https://github.com/minio/simdjson-go) has a 50% reduction in decoding performance compared to [simdjson](https://github.com/simdjson/simdjson). What's more, we barely found JSON libraries which provide API to modify the underlying values.\n\nTherefore, we decided to **develop a brand-new JSON library with high performance as well as wide applicability**.\n\n## Thinking\nBefore starting our design, we need to figure out some questions:\n\n### Why is Json-iterator faster than Standard Library?\nFirst of all, the **schema-based processing mechanism** used by the standard library is commendable, in which the parser can obtain meta information in advance when scanning, thereby shortening the time of branch selection. However, its original implementation did not make good use of this mechanism, instead, **it spent a lot of time reflecting to obtain meta info of schema**. Meanwhile, The approach of json-iterator is: Interpret structure as field-by-field encoding and decoding functions, and then assembled and cached them, minimizing the performance loss cost by reflection. But does it work once and for all? No. In practical tests, we found that **the deeper and larger the input JSON got, the smaller the gap between json-iterator and other libraries gradually became** - eventually event got surpassed:\n![Scalability](./imgs/introduction-1.png) \n\nThe reason is that **this implementation transforms into a large number of interface encapsulations and function calls**, followed by function-call losses:\n1. **Calling interface involves dynamic addressing of itab**\n2. **Assembly functions cannot be inlined**, while Golang's function-call performance is poor (no parameter-passing-by-register)\n\n#### Is there a way to avoid the function-call overhead of dynamic assembly?\nThe first thing we thought about was code generation like [easyjson](https://github.com/mailru/easyjson). But it comes with **schema dependency and convenience losses**. To achieve a real drop-in replacement of the standard library, we turned to another technology - **[JIT](https://en.wikipedia.org/wiki/Just-in-time_compilation) (just-in-time compiling)**. Because the compiled codec function is an integrated function, which can greatly reduce function calls while ensuring flexibility.\n\n### Why is Simdjson-go not fast enough?\n[SIMD](https://en.wikipedia.org/wiki/SIMD) (Single-Instruction-Multi-Data) is a special set of CPU instructions for the parallel processing of vectorized data. At present, it is supported by most CPUs and widely used in image processing and big data computing. Undoubtedly, SIMD is useful in JSON processing (itoa, char-search， and so on are all suitable scenarios). We can see that simdjson-go is very competitive in large JSON scenarios (>100KB). However, for some extremely small or irregular character strings, **the extra load operation required by SIMD will lead to performance degradation**. Therefore, we need to dedicate ourselves to branch predicting and decide which scenarios should use SIMD and which should not (for example, the string length is less than 16 bytes).\n\nThe second problem comes from the Go compiler itself. In order to ensure the compilation speed, **Golang does very little optimization work during the compilation phase** and cannot directly use compiler backends such as [LLVM](https://en.wikipedia.org/wiki/LLVM) (Low-Level Virtual Machine) for optimization.\n\nSo, **can some crucial calculation functions be written in another language with higher execution efficiency**?\nC/Clang is an ideal compilation tool (internal integration LLVM). But the key is how to embed the optimized assembly into Golang.\n\n### How to use Gjson well?\nWe also found that [gjson](https://github.com/tidwall/gjson) has a huge advantage in single-key lookup scenarios. This is because its lookup is implemented by a **lazy-load mechanism**, which subtly skips passing-by values and effectively reduces a lot of unnecessary parsing. Practical application has proved that making good use of this feature in product can indeed bring benefits. But when it comes to multi-key lookup, Gjson does worse event than std, which is a side effect of its skipping mechanism - **searching for the same path leads to repeated parsing** (skip is also a lightweight parsing). Therefore, the accurate adaptation of practical scenarios is the key.\n\n## Design\nBased on the above questions, our design is easy to implement:\n\n1. Aiming at the function-call overhead cost by the codec dynamic-assembly, **`JIT` tech is used to assemble opcodes (asm) corresponding to the schema at runtime**, which is finally cached into the off-heap memory in the form of Golang functions.\n2. For practical scenarios where big data and small data coexist, we **use pre-conditional judgment** (string size, floating precision, etc.) **to combine `SIMD` with scalar instructions** to achieve the best adaptation.\n3. As for insufficiency in compiling optimization of go language, we decided to **use `C/Clang` to write and compile core computational functions**, and **developed a set of [asm2asm](https://github.com/chenzhuoyu/asm2asm) tools to translate the fully optimized x86 assembly into plan9** and finally load it into Golang runtime.\n4. Giving the big speed gap between parsing and skipping, the **`lazy-load` mechanism** is certainly used in our AST parser, but in **a more adaptive and efficient way to reduce the overhead of multiple-key queries**.\n![design](./imgs/introduction-2.png)\n\nIn detail, we conducted some further optimization:\n1. Since the native-asm functions cannot be inlined in Golang, we found that its cost even exceeded the improvement brought by the optimization of the C compiler. So we reimplemented a set of lightweight function-calls in JIT:\n - `Global-function-table + static offset` for calling instruction\n - **Pass parameters using registers**\n2. `Sync.Map` was used to cache the codecs at first, but for our **quasi-static** (read far more than write), **fewer elements** (usually no more than a few dozen) scenarios, its performance is not optimal, so we reimplement a high-performance and concurrent-safe cache with `open-addressing-hash + RCU` tech.\n" }, { "path": "docs/INTRODUCTION_ZH_CN.md", "content": "# Sonic 简介\n\n[English](INTRODUCTION.md) | 中文\n\n## 背景\n\n根据字节跳动生产服务的整体分析，我们发现 JSON 序列化和反序列化的开销意外地很高：CPU 使用率接近 10%，其中极端情况下超过 40%。因此，**JSON 库的性能是提高机器利用率的关键问题**。\n\n## 研究\n\n我们对开源的 Golang JSON 库进行了一系列调研和基准测试，但结果令人失望：**没有万能的解决方案**。首先，没有一个库能够在各种业务场景中至少达到前三名。即使是最广泛使用的 [json-iterator](https://github.com/json-iterator/go) ，在通用（无模式）或大量 JSON 序列化和反序列化的情况下，性能也会严重下降。其次，与其他语言编写的 JSON 库相比，它们的速度通常要慢得多。例如，[Simdjson-go](https://github.com/minio/simdjson-go)的解码性能比[simdjson](https://github.com/simdjson/simdjson)低了 50%。而且，我们几乎找不到支持修改底层值的 JSON 库的 API。\n\n因此，我们决定**开发一个全新的高性能、适用广泛的 JSON 库**。\n\n## 设想\n\n在开始设计之前，我们需要搞清楚一些问题：\n\n### 为什么 Json-iterator 比标准库快？\n\n首先，标准库使用的**基于模式(Schema)的处理机制**是值得称赞的，解析器可以在扫描时提前获取元信息，从而缩短分支选择的时间。然而，它的原始实现没有很好地利用这个机制，而是**花费了大量时间使用反射获取模式的元信息**。与此同时，json-iterator 的方法是：将结构解释为逐个字段的编码和解码函数，然后将它们组装和缓存起来，最小化反射带来的性能损失。但这种方法是否一劳永逸呢？实际测试中，我们发现**随着输入的 JSON 变深、变大，json-iterator 和其他库之间的差距逐渐缩小**——甚至最终被超越：\n![Scalability](./imgs/introduction-1.png) \n\n原因是**该实现转化为大量接口封装和函数调用**，导致了函数调用的性能损失：\n1. **调用接口涉及到对 `itab` 的动态地址获取**\n2. **组装的函数无法内联**，而 Golang 的函数调用性能较差（没有寄存器传参）\n\n#### 有没有办法避免动态组装函数的调用开销？\n\n我们首先考虑的是类似[easyjson](https://github.com/mailru/easyjson)的代码生成。但是这会带来**模式依赖和便利性下降**。为了实现对标准库的真正插拔式替换，我们转向了另一种技术- **[JIT](https://zh.wikipedia.org/zh-cn/即时编译) （即时编译）**。因为编译后的编解码函数是一个集成的函数，它可以大大减少函数调用，同时保证灵活性。\n\n### 为什么 Simdjson-go 速度不够快？\n\n[SIMD](https://en.wikipedia.org/wiki/SIMD) （单指令流多数据流）是一组特殊的 CPU 指令，用于并行处理矢量化数据。目前，大多数 CPU 都支持 SIMD ，并广泛用于图像处理和大数据计算。毫无疑问，SIMD在JSON处理中很有用（整形-字符串转换，字符搜索等都是合适的场景）。我们可以看到， simdjson-go 在大型 JSON 场景 (>100KB) 下非常有竞争力。然而，对于一些很小或不规则的字符字符串， **SIMD 所需的额外加载操作将导致性能下降**。因此，我们需要考虑不同的场景，并决定哪些场景应该使用 SIMD ，哪些不应该使用（例如，长度小于16字节的字符串）。\n\n第二个问题来自 Go 编译器本身。为了保证编译速度， **Golang 在编译阶段几乎不进行任何优化工作**也无法直接使用编译器后端，如 [LLVM](https://en.wikipedia.org/wiki/LLVM) 等进行优化。\n\n那么，**一些关键的计算函数能否用计算效率更高的其他语言编写吗**?\n\nC/Clang 是一种理想的编译工具（内部集成了 LLVM ）。但关键是如何将优化后的汇编嵌入到 Golang 中。\n\n### 如何更好地使用 `Gjson` ？\n我们还发现在单键查找场景中， [gjson](https://github.com/tidwall/gjson)具有巨大的优势。这是因为它的查找是通过**惰性加载机制**实现的，巧妙地跳过了传递的值，并有效的减少了许多不必要的解析。实际应用证明，在产品中充分利用这个特性确实能带来收益。但是，当涉及到多键查找时，Gjson甚至比标准库还要差，这是其跳过机制的副作用——**搜索相同路径会导致重复解析**（跳过解析也是一种轻量的解析）因此，根据实际情况准确的做出调整是关键问题。\n\n## 设计\n\n基于以上问题，我们的设计很好实现：\n\n1. 针对编解码动态汇编的函数调用开销，我们**使用 JIT 技术在运行时组装与模式对应的字节码（汇编指令）**，最终将其以 Golang 函数的形式缓存在堆外内存上。\n2. 针对大数据和小数据共存的实际场景，我们**使用预处理判断**（字符串大小、浮点数精度等）**将 SIMD 与标量指令相结合**，从而实现对实际情况的最佳适应。\n3. 对于 Golang 语言编译优化的不足，我们决定**使用 C/Clang 编写和编译核心计算函数**，并且**开发了一套 [asm2asm](https://github.com/chenzhuoyu/asm2asm) 工具，将经过充分优化的 x86 汇编代码转换为 Plan9 格式**，最终加载到 Golang 运行时中。\n4. 考虑到解析和跳过解析之间的速度差异很大， **惰性加载机制**当然也在我们的 AST 解析器中使用了，但**以一种更具适应性和高效性的方式来降低多键查询的开销**。\n![design](./imgs/introduction-2.png)\n\n在细节上，我们进行了一些进一步的优化：\n1. 由于 Golang 中的原生汇编函数不能被内联，我们发现其成本甚至超过了 C 编译器的优化所带来的改善。所以我们在 JIT 中重新实现了一组轻量级的函数调用：\n - 全局函数表+静态偏移量，用于调用指令\n - **使用寄存器传递参数**\n2. `Sync.Map` 一开始被用来缓存编解码器，但是对于我们的**准静态**（读远多于写），**元素较少**（通常不足几十个）的场景，它的性能并不理想，所以我们使用开放寻址哈希和 RCU 技术重新实现了一个高性能且并发安全的缓存。\n" }, { "path": "encode_test.go", "content": "//go:build (amd64 && go1.17 && !go1.27) || (arm64 && go1.20 && !go1.27)\n// +build amd64,go1.17,!go1.27 arm64,go1.20,!go1.27\n\n/*\n * Copyright 2021 ByteDance Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage sonic\n\nimport (\n\t\"bytes\"\n\t\"encoding\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"log\"\n\t\"math\"\n\t\"os\"\n\t\"reflect\"\n\t\"regexp\"\n\t\"runtime\"\n\t\"runtime/debug\"\n\t\"strconv\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n\t\"unsafe\"\n\n\t\"github.com/bytedance/sonic/encoder\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nvar (\n\tdebugAsyncGC = os.Getenv(\"SONIC_NO_ASYNC_GC\") == \"\"\n)\n\nfunc TestMain(m *testing.M) {\n\tgo func() {\n\t\tif !debugAsyncGC {\n\t\t\treturn\n\t\t}\n\t\tprintln(\"Begin GC looping...\")\n\t\tfor {\n\t\t\truntime.GC()\n\t\t\tdebug.FreeOSMemory()\n\t\t}\n\t}()\n\ttime.Sleep(time.Millisecond)\n\tm.Run()\n}\n\ntype Optionals struct {\n\tSr string `json:\"sr\"`\n\tSo string `json:\"so,omitempty\"`\n\tSw string `json:\"-\"`\n\n\tIr int `json:\"omitempty\"` // actually named omitempty, not an option\n\tIo int `json:\"io,omitempty\"`\n\n\tSlr []string `json:\"slr,random\"`\n\tSlo []string `json:\"slo,omitempty\"`\n\n\tMr map[string]interface{} `json:\"mr\"`\n\tMo map[string]interface{} `json:\",omitempty\"`\n\n\tFr float64 `json:\"fr\"`\n\tFo float64 `json:\"fo,omitempty\"`\n\n\tBr bool `json:\"br\"`\n\tBo bool `json:\"bo,omitempty\"`\n\n\tUr uint `json:\"ur\"`\n\tUo uint `json:\"uo,omitempty\"`\n\n\tStr struct{} `json:\"str\"`\n\tSto struct{} `json:\"sto,omitempty\"`\n}\n\nvar optionalsExpected = `{\n \"sr\": \"\",\n \"omitempty\": 0,\n \"slr\": null,\n \"mr\": {},\n \"fr\": 0,\n \"br\": false,\n \"ur\": 0,\n \"str\": {},\n \"sto\": {}\n}`\n\nfunc TestOmitEmpty(t *testing.T) {\n\tvar o Optionals\n\to.Sw = \"something\"\n\to.Mr = map[string]interface{}{}\n\to.Mo = map[string]interface{}{}\n\n\tgot, err := encoder.EncodeIndented(&o, \"\", \" \", 0)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif got := string(got); got != optionalsExpected {\n\t\tt.Errorf(\" got: %s\\nwant: %s\\n\", got, optionalsExpected)\n\t}\n}\n\ntype StringTag struct {\n\tBoolStr bool `json:\",string\"`\n\tIntStr int64 `json:\",string\"`\n\tUintptrStr uintptr `json:\",string\"`\n\tStrStr string `json:\",string\"`\n\tNumberStr json.Number `json:\",string\"`\n}\n\nfunc TestRoundtripStringTag(t *testing.T) {\n\ttests := []struct {\n\t\tname string\n\t\tin StringTag\n\t\twant string // empty to just test that we roundtrip\n\t}{\n\t\t{\n\t\t\tname: \"AllTypes\",\n\t\t\tin: StringTag{\n\t\t\t\tBoolStr: true,\n\t\t\t\tIntStr: 42,\n\t\t\t\tUintptrStr: 44,\n\t\t\t\tStrStr: \"xzbit\",\n\t\t\t\tNumberStr: \"46\",\n\t\t\t},\n\t\t\twant: `{\n \"BoolStr\": \"true\",\n \"IntStr\": \"42\",\n \"UintptrStr\": \"44\",\n \"StrStr\": \"\\\"xzbit\\\"\",\n \"NumberStr\": \"46\"\n }`,\n\t\t},\n\t\t{\n\t\t\t// See golang.org/issues/38173.\n\t\t\tname: \"StringDoubleEscapes\",\n\t\t\tin: StringTag{\n\t\t\t\tStrStr: \"\\b\\f\\n\\r\\t\\\"\\\\\",\n\t\t\t\tNumberStr: \"0\", // just to satisfy the roundtrip\n\t\t\t},\n\t\t\twant: `{\n \"BoolStr\": \"false\",\n \"IntStr\": \"0\",\n \"UintptrStr\": \"0\",\n \"StrStr\": \"\\\"\\\\u0008\\\\u000c\\\\n\\\\r\\\\t\\\\\\\"\\\\\\\\\\\"\",\n \"NumberStr\": \"0\"\n }`,\n\t\t},\n\t}\n\tfor _, test := range tests {\n\t\tt.Run(test.name, func(t *testing.T) {\n\t\t\t// Indent with a tab prefix to make the multi-line string\n\t\t\t// literals in the table nicer to read.\n\t\t\tgot, err := encoder.EncodeIndented(&test.in, \" \", \" \", 0)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif got := string(got); got != test.want {\n\t\t\t\tt.Fatalf(\" got: %s\\nwant: %s\\n\", got, test.want)\n\t\t\t}\n\n\t\t\t// Verify that it round-trips.\n\t\t\tvar s2 StringTag\n\t\t\tif err := Unmarshal(got, &s2); err != nil {\n\t\t\t\tt.Fatalf(\"Decode: %v\", err)\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(test.in, s2) {\n\t\t\t\tt.Fatalf(\"decode didn't match.\\nsource: %#v\\nEncoded as:\\n%s\\ndecode: %#v\", test.in, string(got), s2)\n\t\t\t}\n\t\t})\n\t}\n}\n\n// byte slices are special even if they're renamed types.\ntype renamedByte byte\ntype renamedByteSlice []byte\ntype renamedRenamedByteSlice []renamedByte\n\nfunc TestEncodeRenamedByteSlice(t *testing.T) {\n\ts := renamedByteSlice(\"abc\")\n\tresult, err := Marshal(s)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\texpect := `\"YWJj\"`\n\tif string(result) != expect {\n\t\tt.Errorf(\" got %s want %s\", result, expect)\n\t}\n\tr := renamedRenamedByteSlice(\"abc\")\n\tresult, err = Marshal(r)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif string(result) != expect {\n\t\tt.Errorf(\" got %s want %s\", result, expect)\n\t}\n}\n\ntype SamePointerNoCycle struct {\n\tPtr1, Ptr2 *SamePointerNoCycle\n}\n\nvar samePointerNoCycle = &SamePointerNoCycle{}\n\ntype PointerCycle struct {\n\tPtr *PointerCycle\n}\n\nvar pointerCycle = &PointerCycle{}\n\ntype PointerCycleIndirect struct {\n\tPtrs []interface{}\n}\n\ntype RecursiveSlice []RecursiveSlice\n\nvar (\n\tpointerCycleIndirect = &PointerCycleIndirect{}\n\tmapCycle = make(map[string]interface{})\n\tsliceCycle = []interface{}{nil}\n\tsliceNoCycle = []interface{}{nil, nil}\n\trecursiveSliceCycle = []RecursiveSlice{nil}\n)\n\nfunc init() {\n\tptr := &SamePointerNoCycle{}\n\tsamePointerNoCycle.Ptr1 = ptr\n\tsamePointerNoCycle.Ptr2 = ptr\n\n\tpointerCycle.Ptr = pointerCycle\n\tpointerCycleIndirect.Ptrs = []interface{}{pointerCycleIndirect}\n\n\tmapCycle[\"x\"] = mapCycle\n\tsliceCycle[0] = sliceCycle\n\tsliceNoCycle[1] = sliceNoCycle[:1]\n\tfor i := 3; i > 0; i-- {\n\t\tsliceNoCycle = []interface{}{sliceNoCycle}\n\t}\n\trecursiveSliceCycle[0] = recursiveSliceCycle\n}\n\nfunc TestSamePointerNoCycle(t *testing.T) {\n\tif _, err := Marshal(samePointerNoCycle); err != nil {\n\t\tt.Fatalf(\"unexpected error: %v\", err)\n\t}\n}\n\nfunc TestSliceNoCycle(t *testing.T) {\n\tif _, err := Marshal(sliceNoCycle); err != nil {\n\t\tt.Fatalf(\"unexpected error: %v\", err)\n\t}\n}\n\nvar unsupportedValues = []interface{}{\n\tmath.NaN(),\n\tmath.Inf(-1),\n\tmath.Inf(1),\n\tpointerCycle,\n\tpointerCycleIndirect,\n\tmapCycle,\n\tsliceCycle,\n\trecursiveSliceCycle,\n}\n\nfunc TestUnsupportedValues(t *testing.T) {\n\tfor _, v := range unsupportedValues {\n\t\tif _, err := Marshal(v); err != nil {\n\t\t\tif _, ok := err.(*json.UnsupportedValueError); !ok {\n\t\t\t\tt.Errorf(\"for %v, got %T want UnsupportedValueError\", v, err)\n\t\t\t}\n\t\t} else {\n\t\t\tt.Errorf(\"for %v, expected error\", v)\n\t\t}\n\t}\n}\n\n// Ref has Marshaler and Unmarshaler methods with pointer receiver.\ntype Ref int\n\nfunc (*Ref) MarshalJSON() ([]byte, error) {\n\treturn []byte(`\"ref\"`), nil\n}\n\nfunc (r *Ref) UnmarshalJSON([]byte) error {\n\t*r = 12\n\treturn nil\n}\n\n// Val has Marshaler methods with value receiver.\ntype Val int\n\nfunc (Val) MarshalJSON() ([]byte, error) {\n\treturn []byte(`\"val\"`), nil\n}\n\n// RefText has Marshaler and Unmarshaler methods with pointer receiver.\ntype RefText int\n\nfunc (*RefText) MarshalText() ([]byte, error) {\n\treturn []byte(`\"ref\"`), nil\n}\n\nfunc (r *RefText) UnmarshalText([]byte) error {\n\t*r = 13\n\treturn nil\n}\n\n// ValText has Marshaler methods with value receiver.\ntype ValText int\n\nfunc (ValText) MarshalText() ([]byte, error) {\n\treturn []byte(`\"val\"`), nil\n}\n\nfunc TestRefValMarshal(t *testing.T) {\n\tvar s = struct {\n\t\tR0 Ref\n\t\tR1 *Ref\n\t\tR2 RefText\n\t\tR3 *RefText\n\t\tV0 Val\n\t\tV1 *Val\n\t\tV2 ValText\n\t\tV3 *ValText\n\t}{\n\t\tR0: 12,\n\t\tR1: new(Ref),\n\t\tR2: 14,\n\t\tR3: new(RefText),\n\t\tV0: 13,\n\t\tV1: new(Val),\n\t\tV2: 15,\n\t\tV3: new(ValText),\n\t}\n\tconst want = `{\"R0\":\"ref\",\"R1\":\"ref\",\"R2\":\"\\\"ref\\\"\",\"R3\":\"\\\"ref\\\"\",\"V0\":\"val\",\"V1\":\"val\",\"V2\":\"\\\"val\\\"\",\"V3\":\"\\\"val\\\"\"}`\n\tb, err := Marshal(&s)\n\tif err != nil {\n\t\tt.Fatalf(\"Marshal: %v\", err)\n\t}\n\tif got := string(b); got != want {\n\t\tt.Errorf(\"got %q, want %q\", got, want)\n\t}\n}\n\n/*\nFIXME: disabling these test cases for now, because Sonic does not implement HTML escape\n I don't think there are real usages of the `HTMLEscape` feature in real code\n\n// C implements Marshaler and returns unescaped JSON.\ntype C int\n\nfunc (C) MarshalJSON() ([]byte, error) {\n return []byte(`\"<&>\"`), nil\n}\n\n// CText implements Marshaler and returns unescaped text.\ntype CText int\n\nfunc (CText) MarshalText() ([]byte, error) {\n return []byte(`\"<&>\"`), nil\n}\n\nfunc TestMarshalerEscaping(t *testing.T) {\n var c C\n want := `\"\\u003c\\u0026\\u003e\"`\n b, err := Marshal(c)\n if err != nil {\n t.Fatalf(\"Marshal(c): %v\", err)\n }\n if got := string(b); got != want {\n t.Errorf(\"Marshal(c) = %#q, want %#q\", got, want)\n }\n\n var ct CText\n want = `\"\\\"\\u003c\\u0026\\u003e\\\"\"`\n b, err = Marshal(ct)\n if err != nil {\n t.Fatalf(\"Marshal(ct): %v\", err)\n }\n if got := string(b); got != want {\n t.Errorf(\"Marshal(ct) = %#q, want %#q\", got, want)\n }\n}\n*/\n\nfunc TestAnonymousFields(t *testing.T) {\n\ttests := []struct {\n\t\tlabel string // Test name\n\t\tmakeInput func() interface{} // Function to create input value\n\t\twant string // Expected JSON output\n\t}{{\n\t\t// Both S1 and S2 have a field named X. From the perspective of S,\n\t\t// it is ambiguous which one X refers to.\n\t\t// This should not serialize either field.\n\t\tlabel: \"AmbiguousField\",\n\t\tmakeInput: func() interface{} {\n\t\t\ttype (\n\t\t\t\tS1 struct{ x, X int }\n\t\t\t\tS2 struct{ x, X int }\n\t\t\t\tS struct {\n\t\t\t\t\tS1\n\t\t\t\t\tS2\n\t\t\t\t}\n\t\t\t)\n\t\t\treturn S{S1{1, 2}, S2{3, 4}}\n\t\t},\n\t\twant: `{}`,\n\t}, {\n\t\tlabel: \"DominantField\",\n\t\t// Both S1 and S2 have a field named X, but since S has an X field as\n\t\t// well, it takes precedence over S1.X and S2.X.\n\t\tmakeInput: func() interface{} {\n\t\t\ttype (\n\t\t\t\tS1 struct{ x, X int }\n\t\t\t\tS2 struct{ x, X int }\n\t\t\t\tS struct {\n\t\t\t\t\tS1\n\t\t\t\t\tS2\n\t\t\t\t\tx, X int\n\t\t\t\t}\n\t\t\t)\n\t\t\treturn S{S1{1, 2}, S2{3, 4}, 5, 6}\n\t\t},\n\t\twant: `{\"X\":6}`,\n\t}, {\n\t\t// Unexported embedded field of non-struct type should not be serialized.\n\t\tlabel: \"UnexportedEmbeddedInt\",\n\t\tmakeInput: func() interface{} {\n\t\t\ttype (\n\t\t\t\tmyInt int\n\t\t\t\tS struct{ myInt }\n\t\t\t)\n\t\t\treturn S{5}\n\t\t},\n\t\twant: `{}`,\n\t}, {\n\t\t// Exported embedded field of non-struct type should be serialized.\n\t\tlabel: \"ExportedEmbeddedInt\",\n\t\tmakeInput: func() interface{} {\n\t\t\ttype (\n\t\t\t\tMyInt int\n\t\t\t\tS struct{ MyInt }\n\t\t\t)\n\t\t\treturn S{5}\n\t\t},\n\t\twant: `{\"MyInt\":5}`,\n\t}, {\n\t\t// Unexported embedded field of pointer to non-struct type\n\t\t// should not be serialized.\n\t\tlabel: \"UnexportedEmbeddedIntPointer\",\n\t\tmakeInput: func() interface{} {\n\t\t\ttype (\n\t\t\t\tmyInt int\n\t\t\t\tS struct{ *myInt }\n\t\t\t)\n\t\t\ts := S{new(myInt)}\n\t\t\t*s.myInt = 5\n\t\t\treturn s\n\t\t},\n\t\twant: `{}`,\n\t}, {\n\t\t// Exported embedded field of pointer to non-struct type\n\t\t// should be serialized.\n\t\tlabel: \"ExportedEmbeddedIntPointer\",\n\t\tmakeInput: func() interface{} {\n\t\t\ttype (\n\t\t\t\tMyInt int\n\t\t\t\tS struct{ *MyInt }\n\t\t\t)\n\t\t\ts := S{new(MyInt)}\n\t\t\t*s.MyInt = 5\n\t\t\treturn s\n\t\t},\n\t\twant: `{\"MyInt\":5}`,\n\t}, {\n\t\t// Exported fields of embedded structs should have their\n\t\t// exported fields be serialized regardless of whether the struct types\n\t\t// themselves are exported.\n\t\tlabel: \"EmbeddedStruct\",\n\t\tmakeInput: func() interface{} {\n\t\t\ttype (\n\t\t\t\ts1 struct{ x, X int }\n\t\t\t\tS2 struct{ y, Y int }\n\t\t\t\tS struct {\n\t\t\t\t\ts1\n\t\t\t\t\tS2\n\t\t\t\t}\n\t\t\t)\n\t\t\treturn S{s1{1, 2}, S2{3, 4}}\n\t\t},\n\t\twant: `{\"X\":2,\"Y\":4}`,\n\t}, {\n\t\t// Exported fields of pointers to embedded structs should have their\n\t\t// exported fields be serialized regardless of whether the struct types\n\t\t// themselves are exported.\n\t\tlabel: \"EmbeddedStructPointer\",\n\t\tmakeInput: func() interface{} {\n\t\t\ttype (\n\t\t\t\ts1 struct{ x, X int }\n\t\t\t\tS2 struct{ y, Y int }\n\t\t\t\tS struct {\n\t\t\t\t\t*s1\n\t\t\t\t\t*S2\n\t\t\t\t}\n\t\t\t)\n\t\t\treturn S{&s1{1, 2}, &S2{3, 4}}\n\t\t},\n\t\twant: `{\"X\":2,\"Y\":4}`,\n\t}, {\n\t\t// Exported fields on embedded unexported structs at multiple levels\n\t\t// of nesting should still be serialized.\n\t\tlabel: \"NestedStructAndInts\",\n\t\tmakeInput: func() interface{} {\n\t\t\ttype (\n\t\t\t\tMyInt1 int\n\t\t\t\tMyInt2 int\n\t\t\t\tmyInt int\n\t\t\t\ts2 struct {\n\t\t\t\t\tMyInt2\n\t\t\t\t\tmyInt\n\t\t\t\t}\n\t\t\t\ts1 struct {\n\t\t\t\t\tMyInt1\n\t\t\t\t\tmyInt\n\t\t\t\t\ts2\n\t\t\t\t}\n\t\t\t\tS struct {\n\t\t\t\t\ts1\n\t\t\t\t\tmyInt\n\t\t\t\t}\n\t\t\t)\n\t\t\treturn S{s1{1, 2, s2{3, 4}}, 6}\n\t\t},\n\t\twant: `{\"MyInt1\":1,\"MyInt2\":3}`,\n\t}, {\n\t\t// If an anonymous struct pointer field is nil, we should ignore\n\t\t// the embedded fields behind it. Not properly doing so may\n\t\t// result in the wrong output or reflect panics.\n\t\tlabel: \"EmbeddedFieldBehindNilPointer\",\n\t\tmakeInput: func() interface{} {\n\t\t\ttype (\n\t\t\t\tS2 struct{ Field string }\n\t\t\t\tS struct{ *S2 }\n\t\t\t)\n\t\t\treturn S{}\n\t\t},\n\t\twant: `{}`,\n\t}}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.label, func(t *testing.T) {\n\t\t\tb, err := Marshal(tt.makeInput())\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"Marshal() = %v, want nil error\", err)\n\t\t\t}\n\t\t\tif string(b) != tt.want {\n\t\t\t\tt.Fatalf(\"Marshal() = %q, want %q\", b, tt.want)\n\t\t\t}\n\t\t})\n\t}\n}\n\ntype BugA struct {\n\tS string\n}\n\ntype BugB struct {\n\tBugA\n\tS string\n}\n\ntype BugC struct {\n\tS string\n}\n\n// Legal Go: We never use the repeated embedded field (S).\ntype BugX struct {\n\tA int\n\tBugA\n\tBugB\n}\n\n// golang.org/issue/16042.\n// Even if a nil interface value is passed in, as long as\n// it implements Marshaler, it should be marshaled.\ntype nilJSONMarshaler string\n\nfunc (nm *nilJSONMarshaler) MarshalJSON() ([]byte, error) {\n\tif nm == nil {\n\t\treturn Marshal(\"0zenil0\")\n\t}\n\treturn Marshal(\"zenil:\" + string(*nm))\n}\n\n// golang.org/issue/34235.\n// Even if a nil interface value is passed in, as long as\n// it implements encoding.TextMarshaler, it should be marshaled.\ntype nilTextMarshaler string\n\nfunc (nm *nilTextMarshaler) MarshalText() ([]byte, error) {\n\tif nm == nil {\n\t\treturn []byte(\"0zenil0\"), nil\n\t}\n\treturn []byte(\"zenil:\" + string(*nm)), nil\n}\n\n// See golang.org/issue/16042 and golang.org/issue/34235.\nfunc TestNilMarshal(t *testing.T) {\n\ttestCases := []struct {\n\t\tv interface{}\n\t\twant string\n\t}{\n\t\t{v: nil, want: `null`},\n\t\t{v: new(float64), want: `0`},\n\t\t{v: []interface{}(nil), want: `null`},\n\t\t{v: []string(nil), want: `null`},\n\t\t{v: map[string]string(nil), want: `null`},\n\t\t{v: []byte(nil), want: `null`},\n\t\t{v: struct{ M string }{\"gopher\"}, want: `{\"M\":\"gopher\"}`},\n\t\t{v: struct{ M json.Marshaler }{}, want: `{\"M\":null}`},\n\t\t{v: struct{ M json.Marshaler }{(*nilJSONMarshaler)(nil)}, want: `{\"M\":\"0zenil0\"}`},\n\t\t{v: struct{ M interface{} }{(*nilJSONMarshaler)(nil)}, want: `{\"M\":null}`},\n\t\t{v: struct{ M encoding.TextMarshaler }{}, want: `{\"M\":null}`},\n\t\t{v: struct{ M encoding.TextMarshaler }{(*nilTextMarshaler)(nil)}, want: `{\"M\":\"0zenil0\"}`},\n\t\t{v: struct{ M interface{} }{(*nilTextMarshaler)(nil)}, want: `{\"M\":null}`},\n\t}\n\n\tfor _, tt := range testCases {\n\t\tout, err := Marshal(tt.v)\n\t\tif err != nil || string(out) != tt.want {\n\t\t\tt.Errorf(\"Marshal(%#v) = %#q, %#v, want %#q, nil\", tt.v, out, err, tt.want)\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\n// Issue 5245.\nfunc TestEmbeddedBug(t *testing.T) {\n\tv := BugB{\n\t\tBugA{\"A\"},\n\t\t\"B\",\n\t}\n\tb, err := Marshal(v)\n\tif err != nil {\n\t\tt.Fatal(\"Marshal:\", err)\n\t}\n\twant := `{\"S\":\"B\"}`\n\tgot := string(b)\n\tif got != want {\n\t\tt.Fatalf(\"Marshal: got %s want %s\", got, want)\n\t}\n\t// Now check that the duplicate field, S, does not appear.\n\tx := BugX{\n\t\tA: 23,\n\t}\n\tb, err = Marshal(x)\n\tif err != nil {\n\t\tt.Fatal(\"Marshal:\", err)\n\t}\n\twant = `{\"A\":23}`\n\tgot = string(b)\n\tif got != want {\n\t\tt.Fatalf(\"Marshal: got %s want %s\", got, want)\n\t}\n}\n\ntype BugD struct { // Same as BugA after tagging.\n\tXXX string `json:\"S\"`\n}\n\n// BugD's tagged S field should dominate BugA's.\ntype BugY struct {\n\tBugA\n\tBugD\n}\n\n// Test that a field with a tag dominates untagged fields.\nfunc TestTaggedFieldDominates(t *testing.T) {\n\tv := BugY{\n\t\tBugA{\"BugA\"},\n\t\tBugD{\"BugD\"},\n\t}\n\tb, err := Marshal(v)\n\tif err != nil {\n\t\tt.Fatal(\"Marshal:\", err)\n\t}\n\twant := `{\"S\":\"BugD\"}`\n\tgot := string(b)\n\tif got != want {\n\t\tt.Fatalf(\"Marshal: got %s want %s\", got, want)\n\t}\n}\n\n// There are no tags here, so S should not appear.\ntype BugZ struct {\n\tBugA\n\tBugC\n\tBugY // Contains a tagged S field through BugD; should not dominate.\n}\n\nfunc TestDuplicatedFieldDisappears(t *testing.T) {\n\tv := BugZ{\n\t\tBugA{\"BugA\"},\n\t\tBugC{\"BugC\"},\n\t\tBugY{\n\t\t\tBugA{\"nested BugA\"},\n\t\t\tBugD{\"nested BugD\"},\n\t\t},\n\t}\n\tb, err := Marshal(v)\n\tif err != nil {\n\t\tt.Fatal(\"Marshal:\", err)\n\t}\n\twant := `{}`\n\tgot := string(b)\n\tif got != want {\n\t\tt.Fatalf(\"Marshal: got %s want %s\", got, want)\n\t}\n}\n\nfunc TestStdLibIssue10281(t *testing.T) {\n\ttype Foo struct {\n\t\tN json.Number\n\t}\n\tx := Foo{json.Number(`invalid`)}\n\n\tb, err := Marshal(&x)\n\tif err == nil {\n\t\tt.Errorf(\"Marshal(&x) = %#q; want error\", b)\n\t}\n}\n\n// golang.org/issue/8582\nfunc TestEncodePointerString(t *testing.T) {\n\ttype stringPointer struct {\n\t\tN *int64 `json:\"n,string\"`\n\t}\n\tvar n int64 = 42\n\tb, err := Marshal(stringPointer{N: &n})\n\tif err != nil {\n\t\tt.Fatalf(\"Marshal: %v\", err)\n\t}\n\tif got, want := string(b), `{\"n\":\"42\"}`; got != want {\n\t\tt.Errorf(\"Marshal = %s, want %s\", got, want)\n\t}\n\tvar back stringPointer\n\terr = Unmarshal(b, &back)\n\tif err != nil {\n\t\tt.Fatalf(\"Unmarshal: %v\", err)\n\t}\n\tif back.N == nil {\n\t\tt.Fatalf(\"Unmarshaled nil N field\")\n\t}\n\tif *back.N != 42 {\n\t\tt.Fatalf(\"*N = %d; want 42\", *back.N)\n\t}\n}\n\nvar encodeStringTests = []struct {\n\tin string\n\tout string\n}{\n\t{\"\\x00\", `\"\\u0000\"`},\n\t{\"\\x01\", `\"\\u0001\"`},\n\t{\"\\x02\", `\"\\u0002\"`},\n\t{\"\\x03\", `\"\\u0003\"`},\n\t{\"\\x04\", `\"\\u0004\"`},\n\t{\"\\x05\", `\"\\u0005\"`},\n\t{\"\\x06\", `\"\\u0006\"`},\n\t{\"\\x07\", `\"\\u0007\"`},\n\t{\"\\x08\", `\"\\u0008\"`},\n\t{\"\\x09\", `\"\\t\"`},\n\t{\"\\x0a\", `\"\\n\"`},\n\t{\"\\x0b\", `\"\\u000b\"`},\n\t{\"\\x0c\", `\"\\u000c\"`},\n\t{\"\\x0d\", `\"\\r\"`},\n\t{\"\\x0e\", `\"\\u000e\"`},\n\t{\"\\x0f\", `\"\\u000f\"`},\n\t{\"\\x10\", `\"\\u0010\"`},\n\t{\"\\x11\", `\"\\u0011\"`},\n\t{\"\\x12\", `\"\\u0012\"`},\n\t{\"\\x13\", `\"\\u0013\"`},\n\t{\"\\x14\", `\"\\u0014\"`},\n\t{\"\\x15\", `\"\\u0015\"`},\n\t{\"\\x16\", `\"\\u0016\"`},\n\t{\"\\x17\", `\"\\u0017\"`},\n\t{\"\\x18\", `\"\\u0018\"`},\n\t{\"\\x19\", `\"\\u0019\"`},\n\t{\"\\x1a\", `\"\\u001a\"`},\n\t{\"\\x1b\", `\"\\u001b\"`},\n\t{\"\\x1c\", `\"\\u001c\"`},\n\t{\"\\x1d\", `\"\\u001d\"`},\n\t{\"\\x1e\", `\"\\u001e\"`},\n\t{\"\\x1f\", `\"\\u001f\"`},\n}\n\nfunc TestEncodeString(t *testing.T) {\n\tfor _, tt := range encodeStringTests {\n\t\tb, err := Marshal(tt.in)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"Marshal(%q): %v\", tt.in, err)\n\t\t\tcontinue\n\t\t}\n\t\tout := string(b)\n\t\tif out != tt.out {\n\t\t\tt.Errorf(\"Marshal(%q) = %#q, want %#q\", tt.in, out, tt.out)\n\t\t}\n\t}\n}\n\ntype jsonbyte byte\n\nfunc (b jsonbyte) MarshalJSON() ([]byte, error) { return tenc(`{\"JB\":%d}`, b) }\n\ntype textbyte byte\n\nfunc (b textbyte) MarshalText() ([]byte, error) { return tenc(`TB:%d`, b) }\n\ntype jsonint int\n\nfunc (i jsonint) MarshalJSON() ([]byte, error) { return tenc(`{\"JI\":%d}`, i) }\n\ntype textint int\n\nfunc (i textint) MarshalText() ([]byte, error) { return tenc(`TI:%d`, i) }\n\nfunc tenc(format string, a ...interface{}) ([]byte, error) {\n\tvar buf bytes.Buffer\n\t_, _ = fmt.Fprintf(&buf, format, a...)\n\treturn buf.Bytes(), nil\n}\n\n// Issue 13783\nfunc TestEncodeBytekind(t *testing.T) {\n\ttestdata := []struct {\n\t\tdata interface{}\n\t\twant string\n\t}{\n\t\t{byte(7), \"7\"},\n\t\t{jsonbyte(7), `{\"JB\":7}`},\n\t\t{textbyte(4), `\"TB:4\"`},\n\t\t{jsonint(5), `{\"JI\":5}`},\n\t\t{textint(1), `\"TI:1\"`},\n\t\t{[]byte{0, 1}, `\"AAE=\"`},\n\t\t{[]jsonbyte{0, 1}, `[{\"JB\":0},{\"JB\":1}]`},\n\t\t{[][]jsonbyte{{0, 1}, {3}}, `[[{\"JB\":0},{\"JB\":1}],[{\"JB\":3}]]`},\n\t\t{[]textbyte{2, 3}, `[\"TB:2\",\"TB:3\"]`},\n\t\t{[]jsonint{5, 4}, `[{\"JI\":5},{\"JI\":4}]`},\n\t\t{[]textint{9, 3}, `[\"TI:9\",\"TI:3\"]`},\n\t\t{[]int{9, 3}, `[9,3]`},\n\t}\n\tfor _, d := range testdata {\n\t\tjs, err := Marshal(d.data)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t\tcontinue\n\t\t}\n\t\tgot, want := string(js), d.want\n\t\tif got != want {\n\t\t\tt.Errorf(\"got %s, want %s\", got, want)\n\t\t}\n\t}\n}\n\n// https://golang.org/issue/33675\nfunc TestNilMarshalerTextMapKey(t *testing.T) {\n\tb, err := Marshal(map[*unmarshalerText]int{\n\t\t(*unmarshalerText)(nil): 1,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"Failed to Marshal *text.Marshaler: %v\", err)\n\t}\n\tconst want = `{\"\":1}`\n\tif string(b) != want {\n\t\tt.Errorf(\"Marshal map with *text.Marshaler keys: got %#q, want %#q\", b, want)\n\t}\n}\n\nvar re = regexp.MustCompile\n\n// syntactic checks on form of marshaled floating point numbers.\nvar badFloatREs = []*regexp.Regexp{\n\tre(`p`), // no binary exponential notation\n\tre(`^\\+`), // no leading + sign\n\tre(`^-?0[^.]`), // no unnecessary leading zeros\n\tre(`^-?\\.`), // leading zero required before decimal point\n\tre(`\\.(e|$)`), // no trailing decimal\n\tre(`\\.[0-9]+0(e|$)`), // no trailing zero in fraction\n\tre(`^-?(0|[0-9]{2,})\\..*e`), // exponential notation must have normalized mantissa\n\tre(`e[+-]0`), // exponent must not have leading zeros\n\tre(`e-[1-6]$`), // not tiny enough for exponential notation\n\tre(`e+(.|1.|20)$`), // not big enough for exponential notation\n\tre(`^-?0\\.0000000`), // too tiny, should use exponential notation\n\tre(`^-?[0-9]{22}`), // too big, should use exponential notation\n\tre(`[1-9][0-9]{16}[1-9]`), // too many significant digits in integer\n\tre(`[1-9][0-9.]{17}[1-9]`), // too many significant digits in decimal\n}\n\nfunc TestMarshalFloat(t *testing.T) {\n\tt.Parallel()\n\tnfail := 0\n\ttest := func(f float64, bits int) {\n\t\tvf := interface{}(f)\n\t\tif bits == 32 {\n\t\t\tf = float64(float32(f)) // round\n\t\t\tvf = float32(f)\n\t\t}\n\t\tbout, err := Marshal(vf)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"Marshal(%T(%g)): %v\", vf, vf, err)\n\t\t\tnfail++\n\t\t\treturn\n\t\t}\n\t\tout := string(bout)\n\n\t\t// result must convert back to the same float\n\t\tg, err := strconv.ParseFloat(out, bits)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"Marshal(%T(%g)) = %q, cannot parse back: %v\", vf, vf, out, err)\n\t\t\tnfail++\n\t\t\treturn\n\t\t}\n\t\tif f != g {\n\t\t\tt.Errorf(\"Marshal(%T(%g)) = %q (is %g, not %g)\", vf, vf, out, float32(g), vf)\n\t\t\tnfail++\n\t\t\treturn\n\t\t}\n\n\t\tfor _, re := range badFloatREs {\n\t\t\tif re.MatchString(out) {\n\t\t\t\tt.Errorf(\"Marshal(%T(%g)) = %q, must not match /%s/\", vf, vf, out, re)\n\t\t\t\tnfail++\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\n\tvar (\n\t\tbigger = math.Inf(+1)\n\t\tsmaller = math.Inf(-1)\n\t)\n\n\tvar digits = \"1.2345678901234567890123\"\n\tfor i := len(digits); i >= 2; i-- {\n\t\tif testing.Short() && i < len(digits)-4 {\n\t\t\tbreak\n\t\t}\n\t\tfor exp := -30; exp <= 30; exp++ {\n\t\t\tfor _, sign := range \"+-\" {\n\t\t\t\tfor bits := 32; bits <= 64; bits += 32 {\n\t\t\t\t\ts := fmt.Sprintf(\"%c%se%d\", sign, digits[:i], exp)\n\t\t\t\t\tf, err := strconv.ParseFloat(s, bits)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tlog.Fatal(err)\n\t\t\t\t\t}\n\t\t\t\t\tnext := math.Nextafter\n\t\t\t\t\tif bits == 32 {\n\t\t\t\t\t\tnext = func(g, h float64) float64 {\n\t\t\t\t\t\t\treturn float64(math.Nextafter32(float32(g), float32(h)))\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\ttest(f, bits)\n\t\t\t\t\ttest(next(f, bigger), bits)\n\t\t\t\t\ttest(next(f, smaller), bits)\n\t\t\t\t\tif nfail > 50 {\n\t\t\t\t\t\tt.Fatalf(\"stopping test early\")\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\ttest(0, 64)\n\ttest(math.Copysign(0, -1), 64)\n\ttest(0, 32)\n\ttest(math.Copysign(0, -1), 32)\n}\n\nfunc TestMarshalRawMessageValue(t *testing.T) {\n\ttype (\n\t\tT1 struct {\n\t\t\tM json.RawMessage `json:\",omitempty\"`\n\t\t}\n\t\tT2 struct {\n\t\t\tM *json.RawMessage `json:\",omitempty\"`\n\t\t}\n\t)\n\n\tvar (\n\t\trawNil = json.RawMessage(nil)\n\t\trawEmpty = json.RawMessage([]byte{})\n\t\trawText = json.RawMessage(`\"foo\"`)\n\t)\n\n\ttests := []struct {\n\t\tin interface{}\n\t\twant string\n\t\tok bool\n\t}{\n\t\t// Test with nil RawMessage.\n\t\t{rawNil, \"null\", true},\n\t\t{&rawNil, \"null\", true},\n\t\t{[]interface{}{rawNil}, \"[null]\", true},\n\t\t{&[]interface{}{rawNil}, \"[null]\", true},\n\t\t{[]interface{}{&rawNil}, \"[null]\", true},\n\t\t{&[]interface{}{&rawNil}, \"[null]\", true},\n\t\t{struct{ M json.RawMessage }{rawNil}, `{\"M\":null}`, true},\n\t\t{&struct{ M json.RawMessage }{rawNil}, `{\"M\":null}`, true},\n\t\t{struct{ M *json.RawMessage }{&rawNil}, `{\"M\":null}`, true},\n\t\t{&struct{ M *json.RawMessage }{&rawNil}, `{\"M\":null}`, true},\n\t\t{map[string]interface{}{\"M\": rawNil}, `{\"M\":null}`, true},\n\t\t{&map[string]interface{}{\"M\": rawNil}, `{\"M\":null}`, true},\n\t\t{map[string]interface{}{\"M\": &rawNil}, `{\"M\":null}`, true},\n\t\t{&map[string]interface{}{\"M\": &rawNil}, `{\"M\":null}`, true},\n\t\t{T1{rawNil}, \"{}\", true},\n\t\t{T2{&rawNil}, `{\"M\":null}`, true},\n\t\t{&T1{rawNil}, \"{}\", true},\n\t\t{&T2{&rawNil}, `{\"M\":null}`, true},\n\n\t\t// Test with empty, but non-nil, RawMessage.\n\t\t{rawEmpty, \"\", false},\n\t\t{&rawEmpty, \"\", false},\n\t\t{[]interface{}{rawEmpty}, \"\", false},\n\t\t{&[]interface{}{rawEmpty}, \"\", false},\n\t\t{[]interface{}{&rawEmpty}, \"\", false},\n\t\t{&[]interface{}{&rawEmpty}, \"\", false},\n\t\t{struct{ X json.RawMessage }{rawEmpty}, \"\", false},\n\t\t{&struct{ X json.RawMessage }{rawEmpty}, \"\", false},\n\t\t{struct{ X *json.RawMessage }{&rawEmpty}, \"\", false},\n\t\t{&struct{ X *json.RawMessage }{&rawEmpty}, \"\", false},\n\t\t{map[string]interface{}{\"nil\": rawEmpty}, \"\", false},\n\t\t{&map[string]interface{}{\"nil\": rawEmpty}, \"\", false},\n\t\t{map[string]interface{}{\"nil\": &rawEmpty}, \"\", false},\n\t\t{&map[string]interface{}{\"nil\": &rawEmpty}, \"\", false},\n\t\t{T1{rawEmpty}, \"{}\", true},\n\t\t{T2{&rawEmpty}, \"\", false},\n\t\t{&T1{rawEmpty}, \"{}\", true},\n\t\t{&T2{&rawEmpty}, \"\", false},\n\n\t\t// Test with RawMessage with some text.\n\t\t//\n\t\t// The tests below marked with Issue6458 used to generate \"ImZvbyI=\" instead \"foo\".\n\t\t// This behavior was intentionally changed in Go 1.8.\n\t\t// See https://golang.org/issues/14493#issuecomment-255857318\n\t\t{rawText, `\"foo\"`, true}, // Issue6458\n\t\t{&rawText, `\"foo\"`, true},\n\t\t{[]interface{}{rawText}, `[\"foo\"]`, true}, // Issue6458\n\t\t{&[]interface{}{rawText}, `[\"foo\"]`, true}, // Issue6458\n\t\t{[]interface{}{&rawText}, `[\"foo\"]`, true},\n\t\t{&[]interface{}{&rawText}, `[\"foo\"]`, true},\n\t\t{struct{ M json.RawMessage }{rawText}, `{\"M\":\"foo\"}`, true}, // Issue6458\n\t\t{&struct{ M json.RawMessage }{rawText}, `{\"M\":\"foo\"}`, true},\n\t\t{struct{ M *json.RawMessage }{&rawText}, `{\"M\":\"foo\"}`, true},\n\t\t{&struct{ M *json.RawMessage }{&rawText}, `{\"M\":\"foo\"}`, true},\n\t\t{map[string]interface{}{\"M\": rawText}, `{\"M\":\"foo\"}`, true}, // Issue6458\n\t\t{&map[string]interface{}{\"M\": rawText}, `{\"M\":\"foo\"}`, true}, // Issue6458\n\t\t{map[string]interface{}{\"M\": &rawText}, `{\"M\":\"foo\"}`, true},\n\t\t{&map[string]interface{}{\"M\": &rawText}, `{\"M\":\"foo\"}`, true},\n\t\t{T1{rawText}, `{\"M\":\"foo\"}`, true}, // Issue6458\n\t\t{T2{&rawText}, `{\"M\":\"foo\"}`, true},\n\t\t{&T1{rawText}, `{\"M\":\"foo\"}`, true},\n\t\t{&T2{&rawText}, `{\"M\":\"foo\"}`, true},\n\t}\n\n\tfor i, tt := range tests {\n\t\tb, err := Marshal(tt.in)\n\t\tif ok := err == nil; ok != tt.ok {\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"test %d, unexpected failure: %v\", i, err)\n\t\t\t} else {\n\t\t\t\tt.Errorf(\"test %d, unexpected success\", i)\n\t\t\t}\n\t\t}\n\t\tif got := string(b); got != tt.want {\n\t\t\tt.Errorf(\"test %d, Marshal(%#v) = %q, want %q\", i, tt.in, got, tt.want)\n\t\t}\n\t}\n}\n\ntype marshalPanic struct{}\n\nfunc (marshalPanic) MarshalJSON() ([]byte, error) { panic(0xdead) }\n\nfunc TestMarshalPanic(t *testing.T) {\n\tdefer func() {\n\t\tif got := recover(); !reflect.DeepEqual(got, 0xdead) {\n\t\t\tt.Errorf(\"panic() = (%T)(%v), want 0xdead\", got, got)\n\t\t}\n\t}()\n\t_, _ = Marshal(&marshalPanic{})\n\tt.Error(\"Marshal should have panicked\")\n}\n\n//goland:noinspection NonAsciiCharacters\nfunc TestMarshalUncommonFieldNames(t *testing.T) {\n\tv := struct {\n\t\tA0, À, Aβ int\n\t}{}\n\tb, err := Marshal(v)\n\tif err != nil {\n\t\tt.Fatal(\"Marshal:\", err)\n\t}\n\twant := `{\"A0\":0,\"À\":0,\"Aβ\":0}`\n\tgot := string(b)\n\tif got != want {\n\t\tt.Fatalf(\"Marshal: got %s want %s\", got, want)\n\t}\n}\n\ntype DummyMarshalerError struct {\n\tType reflect.Type\n\tErr error\n\tSourceFunc string\n}\n\nfunc (self *DummyMarshalerError) err() *json.MarshalerError {\n\treturn (*json.MarshalerError)(unsafe.Pointer(self))\n}\n\nfunc TestMarshalerError(t *testing.T) {\n\ts := \"test variable\"\n\tst := reflect.TypeOf(s)\n\terrText := \"json: test error\"\n\n\ttests := []struct {\n\t\terr *json.MarshalerError\n\t\twant string\n\t}{\n\t\t{\n\t\t\t(&DummyMarshalerError{st, fmt.Errorf(errText), \"\"}).err(),\n\t\t\t\"json: error calling MarshalJSON for type \" + st.String() + \": \" + errText,\n\t\t},\n\t\t{\n\t\t\t(&DummyMarshalerError{st, fmt.Errorf(errText), \"TestMarshalerError\"}).err(),\n\t\t\t\"json: error calling TestMarshalerError for type \" + st.String() + \": \" + errText,\n\t\t},\n\t}\n\n\tfor i, tt := range tests {\n\t\tgot := tt.err.Error()\n\t\tif got != tt.want {\n\t\t\tt.Errorf(\"MarshalerError test %d, got: %s, want: %s\", i, got, tt.want)\n\t\t}\n\t}\n}\n\nfunc TestMarshalNullNil(t *testing.T) {\n\tvar v = struct {\n\t\tA []int\n\t\tB map[string]int\n\t}{}\n\to, e := Marshal(v)\n\tassert.Nil(t, e)\n\tassert.Equal(t, `{\"A\":null,\"B\":null}`, string(o))\n\to, e = Config{\n\t\tNoNullSliceOrMap: true,\n\t}.Froze().Marshal(v)\n\tassert.Nil(t, e)\n\tassert.Equal(t, `{\"A\":[],\"B\":{}}`, string(o))\n}\n\nfunc TestEncoder_LongestInvalidUtf8(t *testing.T) {\n\tfor _, data := range []string{\n\t\t\"\\\"\" + strings.Repeat(\"\\x80\", 4096) + \"\\\"\",\n\t\t\"\\\"\" + strings.Repeat(\"\\x80\", 4095) + \"\\\"\",\n\t\t\"\\\"\" + strings.Repeat(\"\\x80\", 4097) + \"\\\"\",\n\t\t\"\\\"\" + strings.Repeat(\"\\x80\", 12345) + \"\\\"\",\n\t} {\n\t\ttestEncodeInvalidUtf8(t, []byte(data))\n\t}\n}\n\nfunc testEncodeInvalidUtf8(t *testing.T, data []byte) {\n\tjgot, jerr := json.Marshal(data)\n\tsgot, serr := ConfigStd.Marshal(data)\n\tassert.Equal(t, serr != nil, jerr != nil)\n\tif jerr == nil {\n\t\tassert.Equal(t, sgot, jgot)\n\t}\n}\n\nfunc TestEncoder_RandomInvalidUtf8(t *testing.T) {\n\tnums := 1000\n\tmaxLen := 1000\n\tfor i := 0; i < nums; i++ {\n\t\ttestEncodeInvalidUtf8(t, genRandJsonBytes(maxLen))\n\t\ttestEncodeInvalidUtf8(t, genRandJsonRune(maxLen))\n\t}\n}\n\nfunc TestMarshalInfOrNan(t *testing.T) {\n\ttests := []interface{}{\n\t\tmath.Inf(1), math.Inf(-1), math.NaN(), float32(math.Inf(1)), float32(math.Inf(-1)), float32(math.NaN()), []interface{}{math.Inf(1), math.Inf(-1), math.NaN(), float32(math.Inf(1)), float32(math.Inf(-1)), float32(math.NaN())},\n\n\t\t[]float64{math.Inf(1), math.Inf(-1), math.NaN()},\n\t\t[]float32{float32(math.Inf(1)), float32(math.Inf(-1)), float32(math.NaN())},\n\t}\n\n\tallowNanInf := Config{\n\t\tEncodeNullForInfOrNan: true,\n\t}.Froze()\n\tfor _, tt := range tests {\n\t\tb, err := allowNanInf.Marshal(tt)\n\t\tassert.Nil(t, err)\n\t\tif len(b) == 4 {\n\t\t\tassert.Equal(t, string(b), \"null\")\n\t\t} else {\n\t\t\tprintln(string(b))\n\t\t}\n\n\t\tb, err = Marshal(tt)\n\t\tassert.NotNil(t, err)\n\t\tassert.True(t, strings.Contains(err.Error(), \"json: unsupported value: NaN or ±Infinite\"))\n\t}\n}\n" }, { "path": "encoder/encoder_compat.go", "content": "//go:build (!amd64 && !arm64) || go1.27 || !go1.17 || (arm64 && !go1.20)\n// +build !amd64,!arm64 go1.27 !go1.17 arm64,!go1.20\n\n/*\n* Copyright 2023 ByteDance Inc.\n*\n* Licensed under the Apache License, Version 2.0 (the \"License\");\n* you may not use this file except in compliance with the License.\n* You may obtain a copy of the License at\n*\n* http://www.apache.org/licenses/LICENSE-2.0\n*\n* Unless required by applicable law or agreed to in writing, software\n* distributed under the License is distributed on an \"AS IS\" BASIS,\n* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n* See the License for the specific language governing permissions and\n* limitations under the License.\n */\n\npackage encoder\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"io\"\n\t\"reflect\"\n\n\t\"github.com/bytedance/sonic/internal/compat\"\n\t\"github.com/bytedance/sonic/option\"\n)\n\nfunc init() {\n\tcompat.Warn(\"sonic/encoder\")\n}\n\n// EnableFallback indicates if encoder use fallback\nconst EnableFallback = true\n\n// Options is a set of encoding options.\ntype Options uint64\n\nconst (\n\tbitSortMapKeys = iota\n\tbitEscapeHTML\n\tbitCompactMarshaler\n\tbitNoQuoteTextMarshaler\n\tbitNoNullSliceOrMap\n\tbitValidateString\n\tbitNoValidateJSONMarshaler\n\tbitNoEncoderNewline\n\n\t// used for recursive compile\n\tbitPointerValue = 63\n)\n\nconst (\n\t// SortMapKeys indicates that the keys of a map needs to be sorted\n\t// before serializing into JSON.\n\t// WARNING: This hurts performance A LOT, USE WITH CARE.\n\tSortMapKeys Options = 1 << bitSortMapKeys\n\n\t// EscapeHTML indicates encoder to escape all HTML characters\n\t// after serializing into JSON (see https://pkg.go.dev/encoding/json#HTMLEscape).\n\t// WARNING: This hurts performance A LOT, USE WITH CARE.\n\tEscapeHTML Options = 1 << bitEscapeHTML\n\n\t// CompactMarshaler indicates that the output JSON from json.Marshaler\n\t// is always compact and needs no validation\n\tCompactMarshaler Options = 1 << bitCompactMarshaler\n\n\t// NoQuoteTextMarshaler indicates that the output text from encoding.TextMarshaler\n\t// is always escaped string and needs no quoting\n\tNoQuoteTextMarshaler Options = 1 << bitNoQuoteTextMarshaler\n\n\t// NoNullSliceOrMap indicates all empty Array or Object are encoded as '[]' or '{}',\n\t// instead of 'null'\n\tNoNullSliceOrMap Options = 1 << bitNoNullSliceOrMap\n\n\t// ValidateString indicates that encoder should validate the input string\n\t// before encoding it into JSON.\n\tValidateString Options = 1 << bitValidateString\n\n\t// NoValidateJSONMarshaler indicates that the encoder should not validate the output string\n\t// after encoding the JSONMarshaler to JSON.\n\tNoValidateJSONMarshaler Options = 1 << bitNoValidateJSONMarshaler\n\n\t// NoEncoderNewline indicates that the encoder should not add a newline after every message\n\tNoEncoderNewline Options = 1 << bitNoEncoderNewline\n\n\t// CompatibleWithStd is used to be compatible with std encoder.\n\tCompatibleWithStd Options = SortMapKeys | EscapeHTML | CompactMarshaler\n)\n\n// Encoder represents a specific set of encoder configurations.\ntype Encoder struct {\n\tOpts Options\n\tprefix string\n\tindent string\n}\n\n// Encode returns the JSON encoding of v.\nfunc (self *Encoder) Encode(v interface{}) ([]byte, error) {\n\tif self.indent != \"\" || self.prefix != \"\" {\n\t\treturn EncodeIndented(v, self.prefix, self.indent, self.Opts)\n\t}\n\treturn Encode(v, self.Opts)\n}\n\n// SortKeys enables the SortMapKeys option.\nfunc (self *Encoder) SortKeys() *Encoder {\n\tself.Opts |= SortMapKeys\n\treturn self\n}\n\n// SetEscapeHTML specifies if option EscapeHTML opens\nfunc (self *Encoder) SetEscapeHTML(f bool) {\n\tif f {\n\t\tself.Opts |= EscapeHTML\n\t} else {\n\t\tself.Opts &= ^EscapeHTML\n\t}\n}\n\n// SetValidateString specifies if option ValidateString opens\nfunc (self *Encoder) SetValidateString(f bool) {\n\tif f {\n\t\tself.Opts |= ValidateString\n\t} else {\n\t\tself.Opts &= ^ValidateString\n\t}\n}\n\n// SetNoValidateJSONMarshaler specifies if option NoValidateJSONMarshaler opens\nfunc (self *Encoder) SetNoValidateJSONMarshaler(f bool) {\n\tif f {\n\t\tself.Opts |= NoValidateJSONMarshaler\n\t} else {\n\t\tself.Opts &= ^NoValidateJSONMarshaler\n\t}\n}\n\n// SetNoEncoderNewline specifies if option NoEncoderNewline opens\nfunc (self *Encoder) SetNoEncoderNewline(f bool) {\n\tif f {\n\t\tself.Opts |= NoEncoderNewline\n\t} else {\n\t\tself.Opts &= ^NoEncoderNewline\n\t}\n}\n\n// SetCompactMarshaler specifies if option CompactMarshaler opens\nfunc (self *Encoder) SetCompactMarshaler(f bool) {\n\tif f {\n\t\tself.Opts |= CompactMarshaler\n\t} else {\n\t\tself.Opts &= ^CompactMarshaler\n\t}\n}\n\n// SetNoQuoteTextMarshaler specifies if option NoQuoteTextMarshaler opens\nfunc (self *Encoder) SetNoQuoteTextMarshaler(f bool) {\n\tif f {\n\t\tself.Opts |= NoQuoteTextMarshaler\n\t} else {\n\t\tself.Opts &= ^NoQuoteTextMarshaler\n\t}\n}\n\n// SetIndent instructs the encoder to format each subsequent encoded\n// value as if indented by the package-level function EncodeIndent().\n// Calling SetIndent(\"\", \"\") disables indentation.\nfunc (enc *Encoder) SetIndent(prefix, indent string) {\n\tenc.prefix = prefix\n\tenc.indent = indent\n}\n\n// Quote returns the JSON-quoted version of s.\nfunc Quote(s string) string {\n\t/* check for empty string */\n\tif s == \"\" {\n\t\treturn `\"\"`\n\t}\n\n\tout, _ := json.Marshal(s)\n\treturn string(out)\n}\n\n// Encode returns the JSON encoding of val, encoded with opts.\nfunc Encode(val interface{}, opts Options) ([]byte, error) {\n\treturn json.Marshal(val)\n}\n\n// EncodeInto is like Encode but uses a user-supplied buffer instead of allocating\n// a new one.\nfunc EncodeInto(buf *[]byte, val interface{}, opts Options) error {\n\tif buf == nil {\n\t\tpanic(\"user-supplied buffer buf is nil\")\n\t}\n\tw := bytes.NewBuffer(*buf)\n\tenc := json.NewEncoder(w)\n\tenc.SetEscapeHTML((opts & EscapeHTML) != 0)\n\terr := enc.Encode(val)\n\t*buf = w.Bytes()\n\tl := len(*buf)\n\tif l > 0 && (opts&NoEncoderNewline != 0) && (*buf)[l-1] == '\\n' {\n\t\t*buf = (*buf)[:l-1]\n\t}\n\treturn err\n}\n\n// HTMLEscape appends to dst the JSON-encoded src with <, >, &, U+2028 and U+2029\n// characters inside string literals changed to \\u003c, \\u003e, \\u0026, \\u2028, \\u2029\n// so that the JSON will be safe to embed inside HTML