[
  {
    "path": ".github/CODEOWNERS",
    "content": "# For more information, please refer to https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners\n\n*   @cloudwego/netpoll-reviewers @cloudwego/netpoll-approvers @cloudwego/netpoll-maintainers\n"
  },
  {
    "path": ".github/ISSUE_TEMPLATE/bug_report.md",
    "content": "---\nname: Bug report\nabout: Create a report to help us improve\ntitle: ''\nlabels: ''\nassignees: ''\n\n---\n\n**Describe the bug**\nA clear and concise description of what the bug is.\n\n**To Reproduce**\nSteps to reproduce the behavior:\n1. Go to '...'\n2. Click on '....'\n3. Scroll down to '....'\n4. See error\n\n**Expected behavior**\nA clear and concise description of what you expected to happen.\n\n**Screenshots**\nIf applicable, add screenshots to help explain your problem.\n\n**Desktop (please complete the following information):**\n - OS: [e.g. iOS]\n - Browser [e.g. chrome, safari]\n - Version [e.g. 22]\n\n**Smartphone (please complete the following information):**\n - Device: [e.g. iPhone6]\n - OS: [e.g. iOS8.1]\n - Browser [e.g. stock browser, safari]\n - Version [e.g. 22]\n\n**Additional context**\nAdd any other context about the problem here.\n"
  },
  {
    "path": ".github/ISSUE_TEMPLATE/feature_request.md",
    "content": "---\nname: Feature request\nabout: Suggest an idea for this project\ntitle: ''\nlabels: ''\nassignees: ''\n\n---\n\n**Is your feature request related to a problem? Please describe.**\nA clear and concise description of what the problem is. Ex. I'm always frustrated when [...]\n\n**Describe the solution you'd like**\nA clear and concise description of what you want to happen.\n\n**Describe alternatives you've considered**\nA clear and concise description of any alternative solutions or features you've considered.\n\n**Additional context**\nAdd any other context or screenshots about the feature request here.\n"
  },
  {
    "path": ".github/PULL_REQUEST_TEMPLATE.md",
    "content": ""
  },
  {
    "path": ".github/workflows/pr-check.yml",
    "content": "name: Push and Pull Request Check\n\non: [ push, pull_request ]\n\njobs:\n  compatibility-test:\n    strategy:\n      matrix:\n        go: [ 1.18, 1.24 ]\n        os: [ ubuntu-latest, ubuntu-24.04-arm, macos-latest ]\n    runs-on: ${{ matrix.os }}\n    steps:\n      - uses: actions/checkout@v4\n      - name: Set up Go\n        uses: actions/setup-go@v5\n        with:\n          go-version: ${{ matrix.go }}\n      - name: Unit Test\n        run: go test -timeout=2m -race ./...\n      - name: Benchmark\n        run: go test -bench=. -benchmem -run=none ./... -benchtime=100ms\n\n  windows-test:\n    runs-on: windows-latest\n    steps:\n      - uses: actions/checkout@v4\n      - name: Set up Go\n        uses: actions/setup-go@v5\n        with:\n          go-version: stable\n      - name: Build Test\n        run: go vet ./...\n\n  compliant:\n    runs-on: ubuntu-latest\n    steps:\n      - uses: actions/checkout@v4\n\n      - name: Check License Header\n        uses: apache/skywalking-eyes/header@v0.4.0\n        env:\n          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n\n      - name: Check Spell\n        uses: crate-ci/typos@v1.13.14\n\n  golangci-lint:\n    runs-on: ubuntu-latest\n    steps:\n      - uses: actions/checkout@v4\n      - name: Set up Go\n        uses: actions/setup-go@v5\n        with:\n          go-version: stable\n          # for self-hosted, the cache path is shared across projects\n          # and it works well without the cache of github actions\n          # Enable it if we're going to use Github only\n          cache: false\n\n      - name: Golangci Lint\n        # https://golangci-lint.run/\n        uses: golangci/golangci-lint-action@v6\n        with:\n          version: latest\n          only-new-issues: true\n"
  },
  {
    "path": ".gitignore",
    "content": "# Binaries for programs and plugins\n*.exe\n*.exe~\n*.dll\n*.so\n*.dylib\n\n# Test binary, built with `go test -c`\n*.test\n\n# Output of the go coverage tool, specifically when used with LiteIDE\n*.out\n\n# Dependency directories (remove the comment below to include it)\n# vendor/\n.idea/"
  },
  {
    "path": ".golangci.yaml",
    "content": "# Options for analysis running.\nrun:\n  timeout: 3m\n\nlinters: # https://golangci-lint.run/usage/linters/\n  disable-all: true\n  enable:\n    - gosimple\n    - govet\n    - ineffassign\n    - staticcheck\n    - unused\n    - unconvert\n    - goimports\n    - gofumpt\n\n# Refer to https://golangci-lint.run/usage/linters\nlinters-settings:\n  gofumpt:\n    # Choose whether to use the extra rules.\n    # Default: false\n    extra-rules: true\n  goimports:\n    # Put imports beginning with prefix after 3rd-party packages.\n    # It's a comma-separated list of prefixes.\n    local-prefixes: github.com/cloudwego/netpoll\n\nissues:\n  exclude-use-default: true"
  },
  {
    "path": ".licenserc.yaml",
    "content": "header:\n  license:\n    spdx-id: Apache-2.0\n    copyright-owner: CloudWeGo Authors\n\n  paths:\n    - '**/*.go'\n    - '**/*.s'\n\n  paths-ignore:\n    - 'net_netfd.go'\n    - 'net_sock.go'\n    - 'net_tcpsock.go'\n    - 'net_unixsock.go'\n    - 'sys_sockopt_bsd.go'\n    - 'sys_sockopt_linux.go'\n\n  comment: on-failure"
  },
  {
    "path": "CODE_OF_CONDUCT.md",
    "content": "# Contributor Covenant Code of Conduct\n\n## Our Pledge\n\nWe as members, contributors, and leaders pledge to make participation in our\ncommunity a harassment-free experience for everyone, regardless of age, body\nsize, visible or invisible disability, ethnicity, sex characteristics, gender\nidentity and expression, level of experience, education, socio-economic status,\nnationality, personal appearance, race, religion, or sexual identity\nand orientation.\n\nWe pledge to act and interact in ways that contribute to an open, welcoming,\ndiverse, inclusive, and healthy community.\n\n## Our Standards\n\nExamples of behavior that contributes to a positive environment for our\ncommunity include:\n\n* Demonstrating empathy and kindness toward other people\n* Being respectful of differing opinions, viewpoints, and experiences\n* Giving and gracefully accepting constructive feedback\n* Accepting responsibility and apologizing to those affected by our mistakes,\n  and learning from the experience\n* Focusing on what is best not just for us as individuals, but for the\n  overall community\n\nExamples of unacceptable behavior include:\n\n* The use of sexualized language or imagery, and sexual attention or\n  advances of any kind\n* Trolling, insulting or derogatory comments, and personal or political attacks\n* Public or private harassment\n* Publishing others' private information, such as a physical or email\n  address, without their explicit permission\n* Other conduct which could reasonably be considered inappropriate in a\n  professional setting\n\n## Enforcement Responsibilities\n\nCommunity leaders are responsible for clarifying and enforcing our standards of\nacceptable behavior and will take appropriate and fair corrective action in\nresponse to any behavior that they deem inappropriate, threatening, offensive,\nor harmful.\n\nCommunity leaders have the right and responsibility to remove, edit, or reject\ncomments, commits, code, wiki edits, issues, and other contributions that are\nnot aligned to this Code of Conduct, and will communicate reasons for moderation\ndecisions when appropriate.\n\n## Scope\n\nThis Code of Conduct applies within all community spaces, and also applies when\nan individual is officially representing the community in public spaces.\nExamples of representing our community include using an official e-mail address,\nposting via an official social media account, or acting as an appointed\nrepresentative at an online or offline event.\n\n## Enforcement\n\nInstances of abusive, harassing, or otherwise unacceptable behavior may be\nreported to the community leaders responsible for enforcement at\nconduct@cloudwego.io.\nAll complaints will be reviewed and investigated promptly and fairly.\n\nAll community leaders are obligated to respect the privacy and security of the\nreporter of any incident.\n\n## Enforcement Guidelines\n\nCommunity leaders will follow these Community Impact Guidelines in determining\nthe consequences for any action they deem in violation of this Code of Conduct:\n\n### 1. Correction\n\n**Community Impact**: Use of inappropriate language or other behavior deemed\nunprofessional or unwelcome in the community.\n\n**Consequence**: A private, written warning from community leaders, providing\nclarity around the nature of the violation and an explanation of why the\nbehavior was inappropriate. A public apology may be requested.\n\n### 2. Warning\n\n**Community Impact**: A violation through a single incident or series\nof actions.\n\n**Consequence**: A warning with consequences for continued behavior. No\ninteraction with the people involved, including unsolicited interaction with\nthose enforcing the Code of Conduct, for a specified period of time. This\nincludes avoiding interactions in community spaces as well as external channels\nlike social media. Violating these terms may lead to a temporary or\npermanent ban.\n\n### 3. Temporary Ban\n\n**Community Impact**: A serious violation of community standards, including\nsustained inappropriate behavior.\n\n**Consequence**: A temporary ban from any sort of interaction or public\ncommunication with the community for a specified period of time. No public or\nprivate interaction with the people involved, including unsolicited interaction\nwith those enforcing the Code of Conduct, is allowed during this period.\nViolating these terms may lead to a permanent ban.\n\n### 4. Permanent Ban\n\n**Community Impact**: Demonstrating a pattern of violation of community\nstandards, including sustained inappropriate behavior,  harassment of an\nindividual, or aggression toward or disparagement of classes of individuals.\n\n**Consequence**: A permanent ban from any sort of public interaction within\nthe community.\n\n## Attribution\n\nThis Code of Conduct is adapted from the [Contributor Covenant][homepage],\nversion 2.0, available at\nhttps://www.contributor-covenant.org/version/2/0/code_of_conduct.html.\n\nCommunity Impact Guidelines were inspired by [Mozilla's code of conduct\nenforcement ladder](https://github.com/mozilla/diversity).\n\n[homepage]: https://www.contributor-covenant.org\n\nFor answers to common questions about this code of conduct, see the FAQ at\nhttps://www.contributor-covenant.org/faq. Translations are available at\nhttps://www.contributor-covenant.org/translations.\n"
  },
  {
    "path": "CONTRIBUTING.md",
    "content": "# How to Contribute\n\n## Your First Pull Request\nWe use github for our codebase. You can start by reading [How To Pull Request](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests).\n\n## Without Semantic Versioning\nWe keep the stable code in branch `main` like `golang.org/x`. Development base on branch `develop`. And we promise the **Forward Compatibility** by adding new package directory with suffix `v2/v3` when code has break changes.\n\n## Branch Organization\nWe use [git-flow](https://nvie.com/posts/a-successful-git-branching-model/) as our branch organization, as known as [FDD](https://en.wikipedia.org/wiki/Feature-driven_development)\n\n## Bugs\n### 1. How to Find Known Issues\nWe are using [Github Issues](https://github.com/cloudwego/netpoll/issues) for our public bugs. We keep a close eye on this and try to make it clear when we have an internal fix in progress. Before filing a new task, try to make sure your problem doesn’t already exist.\n\n### 2. Reporting New Issues\nProviding a reduced test code is a recommended way for reporting issues. Then can be placed in:\n- Just in issues\n- [Golang Playground](https://play.golang.org/)\n\n### 3. Security Bugs\nPlease do not report the safe disclosure of bugs to public issues. Contact us by [Support Email](mailto:conduct@cloudwego.io)\n\n## How to Get in Touch\n- [Email](mailto:conduct@cloudwego.io)\n\n## Submit a Pull Request\nBefore you submit your Pull Request (PR) consider the following guidelines:\n1. Search [GitHub](https://github.com/cloudwego/netpoll/pulls) for an open or closed PR that relates to your submission. You don't want to duplicate existing efforts.\n2. Be sure that an issue describes the problem you're fixing, or documents the design for the feature you'd like to add. Discussing the design upfront helps to ensure that we're ready to accept your work.\n3. [Fork](https://docs.github.com/en/github/getting-started-with-github/fork-a-repo) the cloudwego/netpoll repo.\n4. In your forked repository, make your changes in a new git branch:\n    ```\n    git checkout -b my-fix-branch main\n    ```\n5. Create your patch, including appropriate test cases.\n6. Follow our [Style Guides](#code-style-guides).\n7. Commit your changes using a descriptive commit message that follows [AngularJS Git Commit Message Conventions](https://docs.google.com/document/d/1QrDFcIiPjSLDn3EL15IJygNPiHORgU1_OOAqWjiDU5Y/edit).\n   Adherence to these conventions is necessary because release notes are automatically generated from these messages.\n8. Push your branch to GitHub:\n    ```\n    git push origin my-fix-branch\n    ```\n9. In GitHub, send a pull request to `netpoll:main`\n\n## Contribution Prerequisites\n- Our development environment keeps up with [Go Official](https://golang.org/project/).\n- You need to fully check with lint tools before submitting your pull request. [gofmt](https://golang.org/pkg/cmd/gofmt/) and [golangci-lint](https://github.com/golangci/golangci-lint)\n- You are familiar with [Github](https://github.com)\n- Maybe you need to be familiar with [Actions](https://github.com/features/actions)(our default workflow tool).\n\n## Code Style Guides\n- [Effective Go](https://golang.org/doc/effective_go)\n- [Go Code Review Comments](https://github.com/golang/go/wiki/CodeReviewComments)\n"
  },
  {
    "path": "CREDITS",
    "content": ""
  },
  {
    "path": "LICENSE",
    "content": "                                 Apache License\n                           Version 2.0, January 2004\n                        http://www.apache.org/licenses/\n\n   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n   1. Definitions.\n\n      \"License\" shall mean the terms and conditions for use, reproduction,\n      and distribution as defined by Sections 1 through 9 of this document.\n\n      \"Licensor\" shall mean the copyright owner or entity authorized by\n      the copyright owner that is granting the License.\n\n      \"Legal Entity\" shall mean the union of the acting entity and all\n      other entities that control, are controlled by, or are under common\n      control with that entity. For the purposes of this definition,\n      \"control\" means (i) the power, direct or indirect, to cause the\n      direction or management of such entity, whether by contract or\n      otherwise, or (ii) ownership of fifty percent (50%) or more of the\n      outstanding shares, or (iii) beneficial ownership of such entity.\n\n      \"You\" (or \"Your\") shall mean an individual or Legal Entity\n      exercising permissions granted by this License.\n\n      \"Source\" form shall mean the preferred form for making modifications,\n      including but not limited to software source code, documentation\n      source, and configuration files.\n\n      \"Object\" form shall mean any form resulting from mechanical\n      transformation or translation of a Source form, including but\n      not limited to compiled object code, generated documentation,\n      and conversions to other media types.\n\n      \"Work\" shall mean the work of authorship, whether in Source or\n      Object form, made available under the License, as indicated by a\n      copyright notice that is included in or attached to the work\n      (an example is provided in the Appendix below).\n\n      \"Derivative Works\" shall mean any work, whether in Source or Object\n      form, that is based on (or derived from) the Work and for which the\n      editorial revisions, annotations, elaborations, or other modifications\n      represent, as a whole, an original work of authorship. For the purposes\n      of this License, Derivative Works shall not include works that remain\n      separable from, or merely link (or bind by name) to the interfaces of,\n      the Work and Derivative Works thereof.\n\n      \"Contribution\" shall mean any work of authorship, including\n      the original version of the Work and any modifications or additions\n      to that Work or Derivative Works thereof, that is intentionally\n      submitted to Licensor for inclusion in the Work by the copyright owner\n      or by an individual or Legal Entity authorized to submit on behalf of\n      the copyright owner. For the purposes of this definition, \"submitted\"\n      means any form of electronic, verbal, or written communication sent\n      to the Licensor or its representatives, including but not limited to\n      communication on electronic mailing lists, source code control systems,\n      and issue tracking systems that are managed by, or on behalf of, the\n      Licensor for the purpose of discussing and improving the Work, but\n      excluding communication that is conspicuously marked or otherwise\n      designated in writing by the copyright owner as \"Not a Contribution.\"\n\n      \"Contributor\" shall mean Licensor and any individual or Legal Entity\n      on behalf of whom a Contribution has been received by Licensor and\n      subsequently incorporated within the Work.\n\n   2. Grant of Copyright License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      copyright license to reproduce, prepare Derivative Works of,\n      publicly display, publicly perform, sublicense, and distribute the\n      Work and such Derivative Works in Source or Object form.\n\n   3. Grant of Patent License. Subject to the terms and conditions of\n      this License, each Contributor hereby grants to You a perpetual,\n      worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n      (except as stated in this section) patent license to make, have made,\n      use, offer to sell, sell, import, and otherwise transfer the Work,\n      where such license applies only to those patent claims licensable\n      by such Contributor that are necessarily infringed by their\n      Contribution(s) alone or by combination of their Contribution(s)\n      with the Work to which such Contribution(s) was submitted. If You\n      institute patent litigation against any entity (including a\n      cross-claim or counterclaim in a lawsuit) alleging that the Work\n      or a Contribution incorporated within the Work constitutes direct\n      or contributory patent infringement, then any patent licenses\n      granted to You under this License for that Work shall terminate\n      as of the date such litigation is filed.\n\n   4. Redistribution. You may reproduce and distribute copies of the\n      Work or Derivative Works thereof in any medium, with or without\n      modifications, and in Source or Object form, provided that You\n      meet the following conditions:\n\n      (a) You must give any other recipients of the Work or\n          Derivative Works a copy of this License; and\n\n      (b) You must cause any modified files to carry prominent notices\n          stating that You changed the files; and\n\n      (c) You must retain, in the Source form of any Derivative Works\n          that You distribute, all copyright, patent, trademark, and\n          attribution notices from the Source form of the Work,\n          excluding those notices that do not pertain to any part of\n          the Derivative Works; and\n\n      (d) If the Work includes a \"NOTICE\" text file as part of its\n          distribution, then any Derivative Works that You distribute must\n          include a readable copy of the attribution notices contained\n          within such NOTICE file, excluding those notices that do not\n          pertain to any part of the Derivative Works, in at least one\n          of the following places: within a NOTICE text file distributed\n          as part of the Derivative Works; within the Source form or\n          documentation, if provided along with the Derivative Works; or,\n          within a display generated by the Derivative Works, if and\n          wherever such third-party notices normally appear. The contents\n          of the NOTICE file are for informational purposes only and\n          do not modify the License. You may add Your own attribution\n          notices within Derivative Works that You distribute, alongside\n          or as an addendum to the NOTICE text from the Work, provided\n          that such additional attribution notices cannot be construed\n          as modifying the License.\n\n      You may add Your own copyright statement to Your modifications and\n      may provide additional or different license terms and conditions\n      for use, reproduction, or distribution of Your modifications, or\n      for any such Derivative Works as a whole, provided Your use,\n      reproduction, and distribution of the Work otherwise complies with\n      the conditions stated in this License.\n\n   5. Submission of Contributions. Unless You explicitly state otherwise,\n      any Contribution intentionally submitted for inclusion in the Work\n      by You to the Licensor shall be under the terms and conditions of\n      this License, without any additional terms or conditions.\n      Notwithstanding the above, nothing herein shall supersede or modify\n      the terms of any separate license agreement you may have executed\n      with Licensor regarding such Contributions.\n\n   6. Trademarks. This License does not grant permission to use the trade\n      names, trademarks, service marks, or product names of the Licensor,\n      except as required for reasonable and customary use in describing the\n      origin of the Work and reproducing the content of the NOTICE file.\n\n   7. Disclaimer of Warranty. Unless required by applicable law or\n      agreed to in writing, Licensor provides the Work (and each\n      Contributor provides its Contributions) on an \"AS IS\" BASIS,\n      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n      implied, including, without limitation, any warranties or conditions\n      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n      PARTICULAR PURPOSE. You are solely responsible for determining the\n      appropriateness of using or redistributing the Work and assume any\n      risks associated with Your exercise of permissions under this License.\n\n   8. Limitation of Liability. In no event and under no legal theory,\n      whether in tort (including negligence), contract, or otherwise,\n      unless required by applicable law (such as deliberate and grossly\n      negligent acts) or agreed to in writing, shall any Contributor be\n      liable to You for damages, including any direct, indirect, special,\n      incidental, or consequential damages of any character arising as a\n      result of this License or out of the use or inability to use the\n      Work (including but not limited to damages for loss of goodwill,\n      work stoppage, computer failure or malfunction, or any and all\n      other commercial damages or losses), even if such Contributor\n      has been advised of the possibility of such damages.\n\n   9. Accepting Warranty or Additional Liability. While redistributing\n      the Work or Derivative Works thereof, You may choose to offer,\n      and charge a fee for, acceptance of support, warranty, indemnity,\n      or other liability obligations and/or rights consistent with this\n      License. However, in accepting such obligations, You may act only\n      on Your own behalf and on Your sole responsibility, not on behalf\n      of any other Contributor, and only if You agree to indemnify,\n      defend, and hold each Contributor harmless for any liability\n      incurred by, or claims asserted against, such Contributor by reason\n      of your accepting any such warranty or additional liability.\n\n   END OF TERMS AND CONDITIONS\n\n   APPENDIX: How to apply the Apache License to your work.\n\n      To apply the Apache License to your work, attach the following\n      boilerplate notice, with the fields enclosed by brackets \"[]\"\n      replaced with your own identifying information. (Don't include\n      the brackets!)  The text should be enclosed in the appropriate\n      comment syntax for the file format. We also recommend that a\n      file or class name and description of purpose be included on the\n      same \"printed page\" as the copyright notice for easier\n      identification within third-party archives.\n\n   Copyright [yyyy] [name of copyright owner]\n\n   Licensed under the Apache License, Version 2.0 (the \"License\");\n   you may not use this file except in compliance with the License.\n   You may obtain a copy of the License at\n\n       http://www.apache.org/licenses/LICENSE-2.0\n\n   Unless required by applicable law or agreed to in writing, software\n   distributed under the License is distributed on an \"AS IS\" BASIS,\n   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n   See the License for the specific language governing permissions and\n   limitations under the License.\n"
  },
  {
    "path": "NOTICE",
    "content": "CloudWeGO\nCopyright 2022 CloudWeGO authors.\n\nGo\nCopyright (c) 2009 The Go Authors."
  },
  {
    "path": "README.md",
    "content": "# CloudWeGo-Netpoll\n\n[中文](README_CN.md)\n\n[![Release](https://img.shields.io/github/v/release/cloudwego/netpoll)](https://github.com/cloudwego/netpoll/releases)\n[![WebSite](https://img.shields.io/website?up_message=cloudwego&url=https%3A%2F%2Fwww.cloudwego.io%2F)](https://www.cloudwego.io/)\n[![License](https://img.shields.io/github/license/cloudwego/netpoll)](https://github.com/cloudwego/netpoll/blob/main/LICENSE)\n[![Go Report Card](https://goreportcard.com/badge/github.com/cloudwego/netpoll)](https://goreportcard.com/report/github.com/cloudwego/netpoll)\n[![OpenIssue](https://img.shields.io/github/issues/cloudwego/netpoll)](https://github.com/cloudwego/netpoll/issues)\n[![ClosedIssue](https://img.shields.io/github/issues-closed/cloudwego/netpoll)](https://github.com/cloudwego/netpoll/issues?q=is%3Aissue+is%3Aclosed)\n![Stars](https://img.shields.io/github/stars/cloudwego/netpoll)\n![Forks](https://img.shields.io/github/forks/cloudwego/netpoll)\n\n## Introduction\n\n[Netpoll][Netpoll] is a high-performance non-blocking I/O networking framework, which\nfocused on RPC scenarios, developed by [ByteDance][ByteDance].\n\nRPC is usually heavy on processing logic and therefore cannot handle I/O serially. But Go's standard\nlibrary [net][net] is designed for blocking I/O APIs, so that the RPC framework can\nonly follow the One Conn One Goroutine design. It will waste a lot of cost for context switching, due to a large number\nof goroutines under high concurrency. Besides, [net.Conn][net.Conn] has\nno API to check Alive, so it is difficult to make an efficient connection pool for RPC framework, because there may be a\nlarge number of failed connections in the pool.\n\nOn the other hand, the open source community currently lacks Go network libraries that focus on RPC scenarios. Similar\nrepositories such as: [evio][evio], [gnet][gnet], etc., are all\nfocus on scenarios like [Redis][Redis], [HAProxy][HAProxy].\n\nBut now, [Netpoll][Netpoll] was born and solved the above problems. It draws inspiration\nfrom the design of [evio][evio] and [netty][netty], has\nexcellent [Performance](#performance), and is more suitable for microservice architecture.\nAlso [Netpoll][Netpoll] provides a number of [Features](#features), and it is recommended\nto replace [net][net] in some RPC scenarios.\n\nWe developed the RPC framework [Kitex][Kitex] and HTTP framework [Hertz][Hertz]\nbased on [Netpoll][Netpoll], both with industry-leading performance.\n\n[Examples][netpoll-examples] show how to build RPC client and server\nusing [Netpoll][Netpoll].\n\nFor more information, please refer to [Document](#document).\n\n## Features\n\n* **Already**\n    - [LinkBuffer][LinkBuffer] provides nocopy API for streaming reading and writing\n    - [gopool][gopool] provides high-performance goroutine pool\n    - [mcache][mcache] provides efficient memory reuse\n    - `IsActive` supports checking whether the connection is alive\n    - `Dialer` supports building clients\n    - `EventLoop` supports building a server\n    - TCP, Unix Domain Socket\n    - Linux, macOS (operating system)\n\n* **Unsupported**\n    - Windows (operating system)\n\n## Performance\n\nBenchmark should meet the requirements of industrial use. \nIn the RPC scenario, concurrency and timeout are necessary support items.\n\nWe provide the [netpoll-benchmark][netpoll-benchmark] project to track and compare \nthe performance of [Netpoll][Netpoll] and other frameworks under different conditions for reference.\n\nMore benchmarks reference [kitex-benchmark][kitex-benchmark] and [hertz-benchmark][hertz-benchmark].\n\n## Reference\n\n* [Official Website](https://www.cloudwego.io)\n* [Getting Started](docs/guide/guide_en.md)\n* [Design](docs/reference/design_en.md)\n* [Why DATA RACE](docs/reference/explain.md)\n\n[Netpoll]: https://github.com/cloudwego/netpoll\n[net]: https://github.com/golang/go/tree/master/src/net\n[net.Conn]: https://github.com/golang/go/blob/master/src/net/net.go\n[evio]: https://github.com/tidwall/evio\n[gnet]: https://github.com/panjf2000/gnet\n[netty]: https://github.com/netty/netty\n[Kitex]: https://github.com/cloudwego/kitex\n[Hertz]: https://github.com/cloudwego/hertz\n\n[netpoll-benchmark]: https://github.com/cloudwego/netpoll-benchmark\n[kitex-benchmark]: https://github.com/cloudwego/kitex-benchmark\n[hertz-benchmark]: https://github.com/cloudwego/hertz-benchmark\n[netpoll-examples]:https://github.com/cloudwego/netpoll-examples\n\n[ByteDance]: https://www.bytedance.com\n[Redis]: https://redis.io\n[HAProxy]: http://www.haproxy.org\n\n[LinkBuffer]: nocopy_linkbuffer.go\n[gopool]: https://github.com/bytedance/gopkg/tree/develop/util/gopool\n[mcache]: https://github.com/bytedance/gopkg/tree/develop/lang/mcache\n"
  },
  {
    "path": "README_CN.md",
    "content": "# CloudWeGo-Netpoll\n\n[English](README.md)\n\n[![Release](https://img.shields.io/github/v/release/cloudwego/netpoll)](https://github.com/cloudwego/netpoll/releases)\n[![WebSite](https://img.shields.io/website?up_message=cloudwego&url=https%3A%2F%2Fwww.cloudwego.io%2F)](https://www.cloudwego.io/)\n[![License](https://img.shields.io/github/license/cloudwego/netpoll)](https://github.com/cloudwego/netpoll/blob/main/LICENSE)\n[![Go Report Card](https://goreportcard.com/badge/github.com/cloudwego/netpoll)](https://goreportcard.com/report/github.com/cloudwego/netpoll)\n[![OpenIssue](https://img.shields.io/github/issues/cloudwego/netpoll)](https://github.com/cloudwego/netpoll/issues)\n[![ClosedIssue](https://img.shields.io/github/issues-closed/cloudwego/netpoll)](https://github.com/cloudwego/netpoll/issues?q=is%3Aissue+is%3Aclosed)\n![Stars](https://img.shields.io/github/stars/cloudwego/netpoll)\n![Forks](https://img.shields.io/github/forks/cloudwego/netpoll)\n\n## 简介\n\n[Netpoll][Netpoll] 是由 [字节跳动][ByteDance] 开发的高性能 NIO(Non-blocking I/O)\n网络库，专注于 RPC 场景。\n\nRPC 通常有较重的处理逻辑，因此无法串行处理 I/O。而 Go 的标准库 [net][net] 设计了 BIO(Blocking I/O) 模式的\nAPI，使得 RPC 框架设计上只能为每个连接都分配一个 goroutine。 这在高并发下，会产生大量的\ngoroutine，大幅增加调度开销。此外，[net.Conn][net.Conn] 没有提供检查连接活性的 API，因此 RPC\n框架很难设计出高效的连接池，池中的失效连接无法及时清理。\n\n另一方面，开源社区目前缺少专注于 RPC 方案的 Go 网络库。类似的项目如：[evio][evio]\n, [gnet][gnet] 等，均面向 [Redis][Redis], [HAProxy][HAProxy] 这样的场景。\n\n因此 [Netpoll][Netpoll] 应运而生，它借鉴了 [evio][evio]\n和 [netty][netty] 的优秀设计，具有出色的 [性能](#性能)，更适用于微服务架构。\n同时，[Netpoll][Netpoll] 还提供了一些 [特性](#特性)，推荐在 RPC 设计中替代\n[net][net] 。\n\n基于 [Netpoll][Netpoll] 开发的 RPC 框架 [Kitex][Kitex] 和 HTTP 框架 [Hertz][Hertz]，性能均业界领先。\n\n[范例][netpoll-examples] 展示了如何使用 [Netpoll][Netpoll]\n构建 RPC Client 和 Server。\n\n更多信息请参阅 [文档](#文档)。\n\n## 特性\n\n* **已经支持**\n    - [LinkBuffer][LinkBuffer] 提供可以流式读写的 nocopy API\n    - [gopool][gopool] 提供高性能的 goroutine 池\n    - [mcache][mcache] 提供高效的内存复用\n    - `IsActive` 支持检查连接是否存活\n    - `Dialer` 支持构建 client\n    - `EventLoop` 支持构建 server\n    - 支持 TCP，Unix Domain Socket\n    - 支持 Linux，macOS（操作系统）\n\n* **不被支持**\n    - Windows（操作系统）\n\n## 性能\n\n性能测试应满足工业级使用要求，在 RPC 场景下，并发请求、等待超时是必要的支持项。\n\n我们提供了 [netpoll-benchmark][netpoll-benchmark] 项目用来长期追踪和比较 [Netpoll][Netpoll] 与其他框架在不同情况下的性能数据以供参考。\n\n更多测试参考 [kitex-benchmark][kitex-benchmark] 和 [hertz-benchmark][hertz-benchmark]\n\n## 参考\n\n* [官方网站](https://www.cloudwego.io)\n* [使用文档](docs/guide/guide_cn.md)\n* [设计文档](docs/reference/design_cn.md)\n* [DATA RACE 说明](docs/reference/explain.md)\n\n[Netpoll]: https://github.com/cloudwego/netpoll\n[net]: https://github.com/golang/go/tree/master/src/net\n[net.Conn]: https://github.com/golang/go/blob/master/src/net/net.go\n[evio]: https://github.com/tidwall/evio\n[gnet]: https://github.com/panjf2000/gnet\n[netty]: https://github.com/netty/netty\n[Kitex]: https://github.com/cloudwego/kitex\n[Hertz]: https://github.com/cloudwego/hertz\n\n[netpoll-benchmark]: https://github.com/cloudwego/netpoll-benchmark\n[kitex-benchmark]: https://github.com/cloudwego/kitex-benchmark\n[hertz-benchmark]: https://github.com/cloudwego/hertz-benchmark\n[netpoll-examples]:https://github.com/cloudwego/netpoll-examples\n\n[ByteDance]: https://www.bytedance.com\n[Redis]: https://redis.io\n[HAProxy]: http://www.haproxy.org\n\n[LinkBuffer]: nocopy_linkbuffer.go\n[gopool]: https://github.com/bytedance/gopkg/tree/develop/util/gopool\n[mcache]: https://github.com/bytedance/gopkg/tree/develop/lang/mcache\n"
  },
  {
    "path": "_typos.toml",
    "content": "# Typo check: https://github.com/crate-ci/typos\n\n[files]\nextend-exclude = [\"go.mod\", \"go.sum\"]\n\n[default.extend-identifiers]\n# *sigh* this just isn't worth the cost of fixing\nnd = \"nd\"\nwrite_datas = \"write_datas\"\n"
  },
  {
    "path": "connection.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage netpoll\n\nimport (\n\t\"net\"\n\t\"time\"\n)\n\n// CloseCallback will be called after the connection is closed.\n// Return: error is unused which will be ignored directly.\ntype CloseCallback func(connection Connection) error\n\n// Connection supports reading and writing simultaneously,\n// but does not support simultaneous reading or writing by multiple goroutines.\n// It maintains its own input/output buffer, and provides nocopy API for reading and writing.\ntype Connection interface {\n\t// Connection extends net.Conn, just for interface compatibility.\n\t// It's not recommended to use net.Conn API except for io.Closer.\n\tnet.Conn\n\n\t// The recommended API for nocopy reading and writing.\n\t// Reader will return nocopy buffer data, or error after timeout which set by SetReadTimeout.\n\tReader() Reader\n\t// Writer will write data to the connection by NIO mode,\n\t// so it will return an error only when the connection isn't Active.\n\tWriter() Writer\n\n\t// IsActive checks whether the connection is active or not.\n\tIsActive() bool\n\n\t// SetReadTimeout sets the timeout for future Read calls wait.\n\t// A zero value for timeout means Reader will not timeout.\n\tSetReadTimeout(timeout time.Duration) error\n\n\t// SetWriteTimeout sets the timeout for future Write calls wait.\n\t// A zero value for timeout means Writer will not timeout.\n\tSetWriteTimeout(timeout time.Duration) error\n\n\t// SetIdleTimeout sets the idle timeout of connections by enabling TCP KeepAlive\n\t// and setting the KeepAlive interval to the given timeout duration.\n\t// NOTE: Despite its name, this does not track application-level idle time.\n\t// It configures OS-level TCP KeepAlive to detect dead peers on idle connections.\n\t// The name is kept for backward compatibility.\n\tSetIdleTimeout(timeout time.Duration) error\n\n\t// SetOnRequest can set or replace the OnRequest method for a connection, but can't be set to nil.\n\t// Although SetOnRequest avoids data race, it should still be used before transmitting data.\n\t// Replacing OnRequest while processing data may cause unexpected behavior and results.\n\t// Generally, the server side should uniformly set the OnRequest method for each connection via NewEventLoop,\n\t// which is set when the connection is initialized.\n\t// On the client side, if necessary, make sure that OnRequest is set before sending data.\n\tSetOnRequest(on OnRequest) error\n\n\t// AddCloseCallback can add hangup callback for a connection, which will be called when connection closing.\n\t// This is very useful for cleaning up idle connections. For instance, you can use callbacks to clean up\n\t// the local resources, which bound to the idle connection, when hangup by the peer. No need another goroutine\n\t// to polling check connection status.\n\tAddCloseCallback(callback CloseCallback) error\n}\n\n// Conn extends net.Conn, but supports getting the conn's fd.\ntype Conn interface {\n\tnet.Conn\n\n\t// Fd return conn's fd, used by poll\n\tFd() (fd int)\n}\n\n// Listener extends net.Listener, but supports getting the listener's fd.\ntype Listener interface {\n\tnet.Listener\n\n\t// Fd return listener's fd, used by poll.\n\tFd() (fd int)\n}\n\n// Dialer extends net.Dialer's API, just for interface compatibility.\n// DialConnection is recommended, but of course all functions are practically the same.\n// The returned net.Conn can be directly asserted as Connection if error is nil.\ntype Dialer interface {\n\tDialConnection(network, address string, timeout time.Duration) (connection Connection, err error)\n\n\tDialTimeout(network, address string, timeout time.Duration) (conn net.Conn, err error)\n}\n"
  },
  {
    "path": "connection_errors.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage netpoll\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"syscall\"\n)\n\n// extends syscall.Errno, the range is set to 0x100-0x1FF\nconst (\n\t// The connection closed when in use.\n\tErrConnClosed = syscall.Errno(0x101)\n\t// Read I/O buffer timeout, called by Connection.Reader\n\tErrReadTimeout = syscall.Errno(0x102)\n\t// Dial timeout\n\tErrDialTimeout = syscall.Errno(0x103)\n\t// Calling dialer without timeout.\n\tErrDialNoDeadline = syscall.Errno(0x104) // TODO: no-deadline support in future\n\t// The calling function not support.\n\tErrUnsupported = syscall.Errno(0x105)\n\t// Same as io.EOF\n\tErrEOF = syscall.Errno(0x106)\n\t// Write I/O buffer timeout, calling by Connection.Writer\n\tErrWriteTimeout = syscall.Errno(0x107)\n\t// Concurrent connection access error\n\tErrConcurrentAccess = syscall.Errno(0x108)\n)\n\nconst ErrnoMask = 0xFF\n\n// wrap Errno, implement xerrors.Wrapper\nfunc Exception(err error, suffix string) error {\n\tno, ok := err.(syscall.Errno)\n\tif !ok {\n\t\tif suffix == \"\" {\n\t\t\treturn err\n\t\t}\n\t\treturn fmt.Errorf(\"%s %s\", err.Error(), suffix)\n\t}\n\treturn &exception{no: no, suffix: suffix}\n}\n\nvar _ net.Error = (*exception)(nil)\n\ntype exception struct {\n\tno     syscall.Errno\n\tsuffix string\n}\n\nfunc (e *exception) Error() string {\n\tvar s string\n\tif int(e.no)&0x100 != 0 {\n\t\ts = errnos[int(e.no)&ErrnoMask]\n\t}\n\tif s == \"\" {\n\t\ts = e.no.Error()\n\t}\n\tif e.suffix != \"\" {\n\t\ts += \" \" + e.suffix\n\t}\n\treturn s\n}\n\nfunc (e *exception) Is(target error) bool {\n\tif e == target {\n\t\treturn true\n\t}\n\tif e.no == target {\n\t\treturn true\n\t}\n\t// TODO: ErrConnClosed contains ErrEOF\n\tif e.no == ErrEOF && target == ErrConnClosed {\n\t\treturn true\n\t}\n\treturn e.no.Is(target)\n}\n\nfunc (e *exception) Unwrap() error {\n\treturn e.no\n}\n\nfunc (e *exception) Timeout() bool {\n\tswitch e.no {\n\tcase ErrDialTimeout, ErrReadTimeout, ErrWriteTimeout:\n\t\treturn true\n\t}\n\treturn e.no.Timeout()\n}\n\nfunc (e *exception) Temporary() bool {\n\treturn e.no.Temporary()\n}\n\n// Errors defined in netpoll\nvar errnos = [...]string{\n\tErrnoMask & ErrConnClosed:       \"connection has been closed\",\n\tErrnoMask & ErrReadTimeout:      \"connection read timeout\",\n\tErrnoMask & ErrDialTimeout:      \"dial wait timeout\",\n\tErrnoMask & ErrDialNoDeadline:   \"dial no deadline\",\n\tErrnoMask & ErrUnsupported:      \"netpoll does not support\",\n\tErrnoMask & ErrEOF:              \"EOF\",\n\tErrnoMask & ErrWriteTimeout:     \"connection write timeout\",\n\tErrnoMask & ErrConcurrentAccess: \"concurrent connection access\",\n}\n"
  },
  {
    "path": "connection_errors_test.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n// +build !windows\n\npackage netpoll\n\nimport (\n\t\"errors\"\n\t\"syscall\"\n\t\"testing\"\n)\n\nfunc TestErrno(t *testing.T) {\n\tvar err1 error = Exception(ErrConnClosed, \"when next\")\n\tMustTrue(t, errors.Is(err1, ErrConnClosed))\n\tEqual(t, err1.Error(), \"connection has been closed when next\")\n\tt.Logf(\"error1=%s\", err1)\n\n\tvar err2 error = Exception(syscall.EPIPE, \"when flush\")\n\tMustTrue(t, errors.Is(err2, syscall.EPIPE))\n\tEqual(t, err2.Error(), \"broken pipe when flush\")\n\tt.Logf(\"error2=%s\", err2)\n}\n"
  },
  {
    "path": "connection_impl.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n// +build !windows\n\npackage netpoll\n\nimport (\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"syscall\"\n\t\"time\"\n)\n\ntype connState = int32\n\nconst (\n\tconnStateNone         = 0\n\tconnStateConnected    = 1\n\tconnStateDisconnected = 2\n)\n\n// connection is the implementation of Connection\ntype connection struct {\n\tnetFD\n\tonEvent\n\tlocker\n\toperator      *FDOperator\n\treadTimeout   time.Duration\n\treadDeadline  int64 // UnixNano(). it overwrites readTimeout. 0 if not set.\n\treadTimer     *time.Timer\n\treadTrigger   chan error\n\twaitReadSize  int64\n\twriteTimeout  time.Duration\n\twriteDeadline int64 // UnixNano(). it overwrites writeTimeout. 0 if not set.\n\twriteTimer    *time.Timer\n\twriteTrigger  chan error\n\tinputBuffer   *LinkBuffer\n\toutputBuffer  *LinkBuffer\n\toutputBarrier *barrier\n\tmaxSize       int       // The maximum size of data between two Release().\n\tbookSize      int       // The size of data that can be read at once.\n\tstate         connState // Connection state should be changed sequentially.\n}\n\nvar (\n\t_ Connection = &connection{}\n\t_ Reader     = &connection{}\n\t_ Writer     = &connection{}\n)\n\n// Reader implements Connection.\nfunc (c *connection) Reader() Reader {\n\treturn c\n}\n\n// Writer implements Connection.\nfunc (c *connection) Writer() Writer {\n\treturn c\n}\n\n// IsActive implements Connection.\nfunc (c *connection) IsActive() bool {\n\treturn c.isCloseBy(none)\n}\n\n// SetIdleTimeout implements Connection.\nfunc (c *connection) SetIdleTimeout(timeout time.Duration) error {\n\tif timeout > 0 {\n\t\treturn c.SetKeepAlive(int(timeout.Seconds()))\n\t}\n\treturn nil\n}\n\n// SetReadTimeout implements Connection.\nfunc (c *connection) SetReadTimeout(timeout time.Duration) error {\n\tif timeout >= 0 {\n\t\tc.readTimeout = timeout\n\t}\n\tc.readDeadline = 0\n\treturn nil\n}\n\n// SetWriteTimeout implements Connection.\nfunc (c *connection) SetWriteTimeout(timeout time.Duration) error {\n\tif timeout >= 0 {\n\t\tc.writeTimeout = timeout\n\t}\n\tc.writeDeadline = 0\n\treturn nil\n}\n\n// SetDeadline implements net.Conn.SetDeadline\nfunc (c *connection) SetDeadline(t time.Time) error {\n\tv := int64(0)\n\tif !t.IsZero() {\n\t\tv = t.UnixNano()\n\t}\n\tc.readDeadline = v\n\tc.writeDeadline = v\n\treturn nil\n}\n\n// SetReadDeadline implements net.Conn.SetReadDeadline\nfunc (c *connection) SetReadDeadline(t time.Time) error {\n\tif t.IsZero() {\n\t\tc.readDeadline = 0\n\t} else {\n\t\tc.readDeadline = t.UnixNano()\n\t}\n\treturn nil\n}\n\n// SetWriteDeadline implements net.Conn.SetWriteDeadline\nfunc (c *connection) SetWriteDeadline(t time.Time) error {\n\tif t.IsZero() {\n\t\tc.writeDeadline = 0\n\t} else {\n\t\tc.writeDeadline = t.UnixNano()\n\t}\n\treturn nil\n}\n\n// ------------------------------------------ implement zero-copy reader ------------------------------------------\n\n// Next implements Connection.\nfunc (c *connection) Next(n int) (p []byte, err error) {\n\tif err = c.waitRead(n); err != nil {\n\t\treturn p, err\n\t}\n\treturn c.inputBuffer.Next(n)\n}\n\n// Peek implements Connection.\nfunc (c *connection) Peek(n int) (buf []byte, err error) {\n\tif err = c.waitRead(n); err != nil {\n\t\treturn buf, err\n\t}\n\treturn c.inputBuffer.Peek(n)\n}\n\n// Skip implements Connection.\nfunc (c *connection) Skip(n int) (err error) {\n\tif err = c.waitRead(n); err != nil {\n\t\treturn err\n\t}\n\treturn c.inputBuffer.Skip(n)\n}\n\n// Release implements Connection.\nfunc (c *connection) Release() (err error) {\n\t// Check inputBuffer length first to reduce contention in mux situation.\n\t// c.operator.do competes with c.inputs/c.inputAck\n\tif c.inputBuffer.Len() == 0 && c.operator.do() {\n\t\tmaxSize := c.inputBuffer.calcMaxSize()\n\t\t// Set the maximum value of maxsize equal to mallocMax to prevent GC pressure.\n\t\tif maxSize > mallocMax {\n\t\t\tmaxSize = mallocMax\n\t\t}\n\n\t\tif maxSize > c.maxSize {\n\t\t\tc.maxSize = maxSize\n\t\t}\n\t\t// Double check length to reset tail node\n\t\tif c.inputBuffer.Len() == 0 {\n\t\t\tc.inputBuffer.resetTail(c.maxSize)\n\t\t}\n\t\tc.operator.done()\n\t}\n\treturn c.inputBuffer.Release()\n}\n\n// Slice implements Connection.\nfunc (c *connection) Slice(n int) (r Reader, err error) {\n\tif err = c.waitRead(n); err != nil {\n\t\treturn nil, err\n\t}\n\treturn c.inputBuffer.Slice(n)\n}\n\n// Len implements Connection.\nfunc (c *connection) Len() (length int) {\n\treturn c.inputBuffer.Len()\n}\n\n// Until implements Connection.\nfunc (c *connection) Until(delim byte) (line []byte, err error) {\n\tvar n, l int\n\tfor {\n\t\tif err = c.waitRead(n + 1); err != nil {\n\t\t\t// return all the data in the buffer\n\t\t\tline, _ = c.inputBuffer.Next(c.inputBuffer.Len())\n\t\t\treturn\n\t\t}\n\n\t\tl = c.inputBuffer.Len()\n\t\ti := c.inputBuffer.indexByte(delim, n)\n\t\tif i < 0 {\n\t\t\tn = l // skip all exists bytes\n\t\t\tcontinue\n\t\t}\n\t\treturn c.Next(i + 1)\n\t}\n}\n\n// ReadString implements Connection.\nfunc (c *connection) ReadString(n int) (s string, err error) {\n\tif err = c.waitRead(n); err != nil {\n\t\treturn s, err\n\t}\n\treturn c.inputBuffer.ReadString(n)\n}\n\n// ReadBinary implements Connection.\nfunc (c *connection) ReadBinary(n int) (p []byte, err error) {\n\tif err = c.waitRead(n); err != nil {\n\t\treturn p, err\n\t}\n\treturn c.inputBuffer.ReadBinary(n)\n}\n\n// ReadByte implements Connection.\nfunc (c *connection) ReadByte() (b byte, err error) {\n\tif err = c.waitRead(1); err != nil {\n\t\treturn b, err\n\t}\n\treturn c.inputBuffer.ReadByte()\n}\n\n// ------------------------------------------ implement zero-copy writer ------------------------------------------\n\n// Malloc implements Connection.\nfunc (c *connection) Malloc(n int) (buf []byte, err error) {\n\tif !c.IsActive() {\n\t\treturn nil, Exception(ErrConnClosed, \"when malloc\")\n\t}\n\treturn c.outputBuffer.Malloc(n)\n}\n\n// MallocLen implements Connection.\nfunc (c *connection) MallocLen() (length int) {\n\treturn c.outputBuffer.MallocLen()\n}\n\n// Flush will send all malloc data to the peer,\n// so must confirm that the allocated bytes have been correctly assigned.\n//\n// Flush first checks whether the out buffer is empty.\n// If empty, it will call syscall.Write to send data directly,\n// otherwise the buffer will be sent asynchronously by the epoll trigger.\nfunc (c *connection) Flush() error {\n\tif !c.IsActive() {\n\t\treturn Exception(ErrConnClosed, \"when flush\")\n\t}\n\n\tif !c.lock(flushing) {\n\t\treturn Exception(ErrConcurrentAccess, \"when flush\")\n\t}\n\tdefer c.unlock(flushing)\n\n\tc.outputBuffer.Flush()\n\treturn c.flush()\n}\n\n// MallocAck implements Connection.\nfunc (c *connection) MallocAck(n int) (err error) {\n\tif !c.IsActive() {\n\t\treturn Exception(ErrConnClosed, \"when malloc ack\")\n\t}\n\treturn c.outputBuffer.MallocAck(n)\n}\n\n// Append implements Connection.\nfunc (c *connection) Append(w Writer) (err error) {\n\tif !c.IsActive() {\n\t\treturn Exception(ErrConnClosed, \"when append\")\n\t}\n\treturn c.outputBuffer.Append(w)\n}\n\n// WriteString implements Connection.\nfunc (c *connection) WriteString(s string) (n int, err error) {\n\tif !c.IsActive() {\n\t\treturn 0, Exception(ErrConnClosed, \"when write string\")\n\t}\n\treturn c.outputBuffer.WriteString(s)\n}\n\n// WriteBinary implements Connection.\nfunc (c *connection) WriteBinary(b []byte) (n int, err error) {\n\tif !c.IsActive() {\n\t\treturn 0, Exception(ErrConnClosed, \"when write binary\")\n\t}\n\treturn c.outputBuffer.WriteBinary(b)\n}\n\n// WriteDirect implements Connection.\nfunc (c *connection) WriteDirect(p []byte, remainCap int) (err error) {\n\tif !c.IsActive() {\n\t\treturn Exception(ErrConnClosed, \"when write direct\")\n\t}\n\treturn c.outputBuffer.WriteDirect(p, remainCap)\n}\n\n// WriteByte implements Connection.\nfunc (c *connection) WriteByte(b byte) (err error) {\n\tif !c.IsActive() {\n\t\treturn Exception(ErrConnClosed, \"when write byte\")\n\t}\n\treturn c.outputBuffer.WriteByte(b)\n}\n\n// ------------------------------------------ implement net.Conn ------------------------------------------\n\n// Read behavior is the same as net.Conn, it will return io.EOF if buffer is empty.\nfunc (c *connection) Read(p []byte) (n int, err error) {\n\tif len(p) == 0 {\n\t\treturn 0, nil\n\t}\n\tif err = c.waitRead(1); err != nil {\n\t\treturn 0, err\n\t}\n\treturn c.inputBuffer.readCopy(p), nil\n}\n\n// Write will Flush soon.\nfunc (c *connection) Write(p []byte) (n int, err error) {\n\tif !c.IsActive() {\n\t\treturn 0, Exception(ErrConnClosed, \"when write\")\n\t}\n\n\tif !c.lock(flushing) {\n\t\treturn 0, Exception(ErrConcurrentAccess, \"when write\")\n\t}\n\tdefer c.unlock(flushing)\n\n\tdst, _ := c.outputBuffer.Malloc(len(p))\n\tn = copy(dst, p)\n\tc.outputBuffer.Flush()\n\terr = c.flush()\n\treturn n, err\n}\n\n// Close implements Connection.\nfunc (c *connection) Close() error {\n\treturn c.onClose()\n}\n\n// Detach detaches the connection from poller but doesn't close it.\nfunc (c *connection) Detach() error {\n\tc.detaching = true\n\treturn c.onClose()\n}\n\n// ------------------------------------------ private ------------------------------------------\n\nvar barrierPool = sync.Pool{\n\tNew: func() interface{} {\n\t\treturn &barrier{\n\t\t\tbs:  make([][]byte, barriercap),\n\t\t\tivs: make([]syscall.Iovec, barriercap),\n\t\t}\n\t},\n}\n\n// init initializes the connection with options\nfunc (c *connection) init(conn Conn, opts *options) (err error) {\n\t// init buffer, barrier, finalizer\n\tc.readTrigger = make(chan error, 1)\n\tc.writeTrigger = make(chan error, 1)\n\tc.bookSize, c.maxSize = defaultLinkBufferSize, defaultLinkBufferSize\n\tc.inputBuffer, c.outputBuffer = NewLinkBuffer(defaultLinkBufferSize), NewLinkBuffer()\n\tc.outputBarrier = barrierPool.Get().(*barrier)\n\tc.state = connStateNone\n\n\tc.initNetFD(conn) // conn must be *netFD{}\n\tc.initFDOperator()\n\tc.initFinalizer()\n\n\tsyscall.SetNonblock(c.fd, true)\n\t// enable TCP_NODELAY by default\n\tswitch c.network {\n\tcase \"tcp\", \"tcp4\", \"tcp6\":\n\t\tsetTCPNoDelay(c.fd, true)\n\t}\n\n\t// connection initialized and prepare options\n\treturn c.onPrepare(opts)\n}\n\nfunc (c *connection) initNetFD(conn Conn) {\n\tif nfd, ok := conn.(*netFD); ok {\n\t\tc.netFD = *nfd\n\t\treturn\n\t}\n\tc.netFD = netFD{\n\t\tfd:         conn.Fd(),\n\t\tlocalAddr:  conn.LocalAddr(),\n\t\tremoteAddr: conn.RemoteAddr(),\n\t}\n}\n\nfunc (c *connection) initFDOperator() {\n\tpoll := pollmanager.Pick()\n\top := poll.Alloc()\n\top.FD = c.fd\n\top.OnRead, op.OnWrite, op.OnHup = nil, nil, c.onHup\n\top.Inputs, op.InputAck = c.inputs, c.inputAck\n\top.Outputs, op.OutputAck = c.outputs, c.outputAck\n\tc.operator = op\n}\n\nfunc (c *connection) initFinalizer() {\n\tc.AddCloseCallback(func(connection Connection) (err error) {\n\t\tc.stop(flushing)\n\t\tc.operator.Free()\n\t\tif err = c.netFD.Close(); err != nil {\n\t\t\tlogger.Printf(\"NETPOLL: netFD close failed: %v\", err)\n\t\t}\n\t\tc.closeBuffer()\n\t\treturn nil\n\t})\n}\n\nfunc (c *connection) triggerRead(err error) {\n\tselect {\n\tcase c.readTrigger <- err:\n\tdefault:\n\t}\n}\n\nfunc (c *connection) triggerWrite(err error) {\n\tselect {\n\tcase c.writeTrigger <- err:\n\tdefault:\n\t}\n}\n\n// waitRead will wait full n bytes.\nfunc (c *connection) waitRead(n int) (err error) {\n\tif n <= c.inputBuffer.Len() {\n\t\treturn nil\n\t}\n\tatomic.StoreInt64(&c.waitReadSize, int64(n))\n\tdefer atomic.StoreInt64(&c.waitReadSize, 0)\n\tif dl := c.readDeadline; dl > 0 {\n\t\ttimeout := time.Duration(dl - time.Now().UnixNano())\n\t\tif timeout <= 0 {\n\t\t\treturn Exception(ErrReadTimeout, c.remoteAddr.String())\n\t\t}\n\t\treturn c.waitReadWithTimeout(n, timeout)\n\t} else if c.readTimeout > 0 {\n\t\treturn c.waitReadWithTimeout(n, c.readTimeout)\n\t}\n\t// wait full n\n\tfor c.inputBuffer.Len() < n {\n\t\tswitch c.status(closing) {\n\t\tcase poller:\n\t\t\treturn Exception(ErrEOF, \"wait read\")\n\t\tcase user:\n\t\t\treturn Exception(ErrConnClosed, \"wait read\")\n\t\tdefault:\n\t\t\terr = <-c.readTrigger\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\treturn nil\n}\n\n// waitReadWithTimeout will wait full n bytes or until timeout.\nfunc (c *connection) waitReadWithTimeout(n int, timeout time.Duration) (err error) {\n\tif c.readTimer == nil {\n\t\tc.readTimer = time.NewTimer(timeout)\n\t} else {\n\t\tc.readTimer.Reset(timeout)\n\t}\n\n\tfor c.inputBuffer.Len() < n {\n\t\tswitch c.status(closing) {\n\t\tcase poller:\n\t\t\t// cannot return directly, stop timer first!\n\t\t\terr = Exception(ErrEOF, \"wait read\")\n\t\t\tgoto RET\n\t\tcase user:\n\t\t\t// cannot return directly, stop timer first!\n\t\t\terr = Exception(ErrConnClosed, \"wait read\")\n\t\t\tgoto RET\n\t\tdefault:\n\t\t\tselect {\n\t\t\tcase <-c.readTimer.C:\n\t\t\t\t// double check if there is enough data to be read\n\t\t\t\tif c.inputBuffer.Len() >= n {\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\treturn Exception(ErrReadTimeout, c.remoteAddr.String())\n\t\t\tcase err = <-c.readTrigger:\n\t\t\t\tif err != nil {\n\t\t\t\t\tgoto RET\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t}\nRET:\n\t// clean timer.C\n\tif !c.readTimer.Stop() {\n\t\t<-c.readTimer.C\n\t}\n\treturn err\n}\n\n// flush writes data directly.\nfunc (c *connection) flush() error {\n\tif c.outputBuffer.IsEmpty() {\n\t\treturn nil\n\t}\n\tbs := c.outputBuffer.GetBytes(c.outputBarrier.bs)\n\tn, err := sendmsg(c.fd, bs, c.outputBarrier.ivs, false)\n\tif err != nil && err != syscall.EAGAIN {\n\t\treturn Exception(err, \"when flush\")\n\t}\n\tif n > 0 {\n\t\terr = c.outputBuffer.Skip(n)\n\t\tc.outputBuffer.Release()\n\t\tif err != nil {\n\t\t\treturn Exception(err, \"when flush\")\n\t\t}\n\t}\n\t// return if write all buffer.\n\tif c.outputBuffer.IsEmpty() {\n\t\treturn nil\n\t}\n\terr = c.operator.Control(PollR2RW)\n\tif err != nil {\n\t\treturn Exception(err, \"when flush\")\n\t}\n\n\treturn c.waitFlush()\n}\n\nfunc (c *connection) waitFlush() (err error) {\n\ttimeout := c.writeTimeout\n\tif dl := c.writeDeadline; dl > 0 {\n\t\ttimeout = time.Duration(dl - time.Now().UnixNano())\n\t\tif timeout <= 0 {\n\t\t\treturn Exception(ErrWriteTimeout, c.remoteAddr.String())\n\t\t}\n\t}\n\tif timeout == 0 {\n\t\treturn <-c.writeTrigger\n\t}\n\n\t// set write timeout\n\tif c.writeTimer == nil {\n\t\tc.writeTimer = time.NewTimer(timeout)\n\t} else {\n\t\tc.writeTimer.Reset(timeout)\n\t}\n\n\tselect {\n\tcase err = <-c.writeTrigger:\n\t\tif !c.writeTimer.Stop() { // clean timer\n\t\t\t<-c.writeTimer.C\n\t\t}\n\t\treturn err\n\tcase <-c.writeTimer.C:\n\t\tselect {\n\t\t// try fetch writeTrigger if both cases fires\n\t\tcase err = <-c.writeTrigger:\n\t\t\treturn err\n\t\tdefault:\n\t\t}\n\t\t// if timeout, remove write event from poller\n\t\t// we cannot flush it again, since we don't if the poller is still process outputBuffer\n\t\tc.operator.Control(PollRW2R)\n\t\treturn Exception(ErrWriteTimeout, c.remoteAddr.String())\n\t}\n}\n\nfunc (c *connection) getState() connState {\n\treturn atomic.LoadInt32(&c.state)\n}\n\nfunc (c *connection) setState(newState connState) {\n\tatomic.StoreInt32(&c.state, newState)\n}\n\nfunc (c *connection) changeState(from, to connState) bool {\n\treturn atomic.CompareAndSwapInt32(&c.state, from, to)\n}\n"
  },
  {
    "path": "connection_lock.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage netpoll\n\nimport (\n\t\"runtime\"\n\t\"sync/atomic\"\n)\n\ntype who = int32\n\nconst (\n\tnone who = iota\n\tuser\n\tpoller\n)\n\ntype key int32\n\n/* State Diagram\n+--------------+         +--------------+\n|  processing  |-------->|   flushing   |\n+-------+------+         +-------+------+\n        |\n        |                +--------------+\n        +--------------->|   closing    |\n                         +--------------+\n\n- \"processing\" locks onRequest handler, and doesn't exist in dialer.\n- \"flushing\" locks outputBuffer\n- \"closing\" should wait for flushing finished and call the closeCallback after that.\n*/\n\nconst (\n\tclosing key = iota\n\tconnecting\n\tprocessing\n\tflushing\n\t// total must be at the bottom.\n\ttotal\n)\n\ntype locker struct {\n\t// keychain used for lock/unlock/stop operation by who.\n\t// 0 means unlock, 1 means locked, 2 means stop.\n\tkeychain [total]int32\n}\n\nfunc (l *locker) closeBy(w who) (success bool) {\n\treturn atomic.CompareAndSwapInt32(&l.keychain[closing], 0, w)\n}\n\nfunc (l *locker) isCloseBy(w who) (yes bool) {\n\treturn atomic.LoadInt32(&l.keychain[closing]) == w\n}\n\nfunc (l *locker) status(k key) int32 {\n\treturn atomic.LoadInt32(&l.keychain[k])\n}\n\nfunc (l *locker) force(k key, v int32) {\n\tatomic.StoreInt32(&l.keychain[k], v)\n}\n\nfunc (l *locker) lock(k key) (success bool) {\n\treturn atomic.CompareAndSwapInt32(&l.keychain[k], 0, 1)\n}\n\nfunc (l *locker) unlock(k key) {\n\tatomic.StoreInt32(&l.keychain[k], 0)\n}\n\nfunc (l *locker) stop(k key) {\n\tfor !atomic.CompareAndSwapInt32(&l.keychain[k], 0, 2) && atomic.LoadInt32(&l.keychain[k]) != 2 {\n\t\truntime.Gosched()\n\t}\n}\n\nfunc (l *locker) isUnlock(k key) bool {\n\treturn atomic.LoadInt32(&l.keychain[k]) == 0\n}\n"
  },
  {
    "path": "connection_onevent.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n// +build !windows\n\npackage netpoll\n\nimport (\n\t\"context\"\n\t\"sync/atomic\"\n\n\t\"github.com/cloudwego/netpoll/internal/runner\"\n)\n\n// ------------------------------------ implement OnPrepare, OnRequest, CloseCallback ------------------------------------\n\ntype gracefulExit interface {\n\tisIdle() (yes bool)\n\tClose() (err error)\n}\n\n// onEvent is the collection of event processing.\n// OnPrepare, OnRequest, CloseCallback share the lock processing,\n// which is a CAS lock and can only be cleared by OnRequest.\ntype onEvent struct {\n\tctx                  context.Context\n\tonConnectCallback    atomic.Value\n\tonDisconnectCallback atomic.Value\n\tonRequestCallback    atomic.Value\n\tcloseCallbacks       atomic.Value // value is latest *callbackNode\n}\n\ntype callbackNode struct {\n\tfn  CloseCallback\n\tpre *callbackNode\n}\n\n// SetOnConnect set the OnConnect callback.\nfunc (c *connection) SetOnConnect(onConnect OnConnect) error {\n\tif onConnect != nil {\n\t\tc.onConnectCallback.Store(onConnect)\n\t}\n\treturn nil\n}\n\n// SetOnDisconnect set the OnDisconnect callback.\nfunc (c *connection) SetOnDisconnect(onDisconnect OnDisconnect) error {\n\tif onDisconnect != nil {\n\t\tc.onDisconnectCallback.Store(onDisconnect)\n\t}\n\treturn nil\n}\n\n// SetOnRequest initialize ctx when setting OnRequest.\nfunc (c *connection) SetOnRequest(onRequest OnRequest) error {\n\tif onRequest == nil {\n\t\treturn nil\n\t}\n\tc.onRequestCallback.Store(onRequest)\n\t// fix: trigger OnRequest if there is already input data.\n\tif !c.inputBuffer.IsEmpty() {\n\t\tc.onRequest()\n\t}\n\treturn nil\n}\n\n// AddCloseCallback adds a CloseCallback to this connection.\nfunc (c *connection) AddCloseCallback(callback CloseCallback) error {\n\tif callback == nil {\n\t\treturn nil\n\t}\n\tcb := &callbackNode{}\n\tcb.fn = callback\n\tif pre := c.closeCallbacks.Load(); pre != nil {\n\t\tcb.pre = pre.(*callbackNode)\n\t}\n\tc.closeCallbacks.Store(cb)\n\treturn nil\n}\n\n// onPrepare supports close connection, but not read/write data.\n// connection will be registered by this call after preparing.\nfunc (c *connection) onPrepare(opts *options) (err error) {\n\tif opts != nil {\n\t\tc.SetOnConnect(opts.onConnect)\n\t\tc.SetOnDisconnect(opts.onDisconnect)\n\t\tc.SetOnRequest(opts.onRequest)\n\t\tc.SetReadTimeout(opts.readTimeout)\n\t\tc.SetWriteTimeout(opts.writeTimeout)\n\t\tc.SetIdleTimeout(opts.idleTimeout)\n\n\t\t// calling prepare first and then register.\n\t\tif opts.onPrepare != nil {\n\t\t\tc.ctx = opts.onPrepare(c)\n\t\t}\n\t}\n\n\tif c.ctx == nil {\n\t\tc.ctx = context.Background()\n\t}\n\t// prepare may close the connection.\n\tif c.IsActive() {\n\t\treturn c.register()\n\t}\n\treturn nil\n}\n\n// onConnect is responsible for executing onRequest if there is new data coming after onConnect callback finished.\nfunc (c *connection) onConnect() {\n\tonConnect, _ := c.onConnectCallback.Load().(OnConnect)\n\tif onConnect == nil {\n\t\tc.changeState(connStateNone, connStateConnected)\n\t\treturn\n\t}\n\tif !c.lock(connecting) {\n\t\t// it never happens because onDisconnect will not lock connecting if c.connected == 0\n\t\treturn\n\t}\n\tonRequest, _ := c.onRequestCallback.Load().(OnRequest)\n\tc.onProcess(onConnect, onRequest)\n}\n\n// when onDisconnect called, c.IsActive() must return false\nfunc (c *connection) onDisconnect() {\n\tonDisconnect, _ := c.onDisconnectCallback.Load().(OnDisconnect)\n\tif onDisconnect == nil {\n\t\treturn\n\t}\n\tonConnect, _ := c.onConnectCallback.Load().(OnConnect)\n\tif onConnect == nil {\n\t\t// no need lock if onConnect is nil\n\t\t// it's ok to force set state to disconnected since onConnect is nil\n\t\tc.setState(connStateDisconnected)\n\t\tonDisconnect(c.ctx, c)\n\t\treturn\n\t}\n\t// check if OnConnect finished when onConnect != nil && onDisconnect != nil\n\tif c.getState() != connStateNone && c.lock(connecting) { // means OnConnect already finished\n\t\t// protect onDisconnect run once\n\t\t// if CAS return false, means OnConnect already helps to run onDisconnect\n\t\tif c.changeState(connStateConnected, connStateDisconnected) {\n\t\t\tonDisconnect(c.ctx, c)\n\t\t}\n\t\tc.unlock(connecting)\n\t\treturn\n\t}\n\t// OnConnect is not finished yet, return and let onConnect helps to call onDisconnect\n}\n\n// onRequest is responsible for executing the closeCallbacks after the connection has been closed.\nfunc (c *connection) onRequest() (needTrigger bool) {\n\tonRequest, ok := c.onRequestCallback.Load().(OnRequest)\n\tif !ok {\n\t\treturn true\n\t}\n\t// wait onConnect finished first\n\tif c.getState() == connStateNone && c.onConnectCallback.Load() != nil {\n\t\t// let onConnect to call onRequest\n\t\treturn\n\t}\n\tprocessed := c.onProcess(nil, onRequest)\n\t// if not processed, should trigger read\n\treturn !processed\n}\n\n// onProcess is responsible for executing the onConnect/onRequest function serially,\n// and make sure the connection has been closed correctly if user call c.Close() in onConnect/onRequest function.\nfunc (c *connection) onProcess(onConnect OnConnect, onRequest OnRequest) (processed bool) {\n\t// task already exists\n\tif !c.lock(processing) {\n\t\treturn false\n\t}\n\n\ttask := func() {\n\t\tpanicked := true\n\t\tdefer func() {\n\t\t\tif !panicked {\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// cannot use recover() here, since we don't want to break the panic stack\n\t\t\tc.unlock(processing)\n\t\t\tif c.IsActive() {\n\t\t\t\tc.Close()\n\t\t\t} else {\n\t\t\t\tc.closeCallback(false, false)\n\t\t\t}\n\t\t}()\n\t\t// trigger onConnect first\n\t\tif onConnect != nil && c.changeState(connStateNone, connStateConnected) {\n\t\t\tc.ctx = onConnect(c.ctx, c)\n\t\t\tif !c.IsActive() && c.changeState(connStateConnected, connStateDisconnected) {\n\t\t\t\t// since we hold connecting lock, so we should help to call onDisconnect here\n\t\t\t\tonDisconnect, _ := c.onDisconnectCallback.Load().(OnDisconnect)\n\t\t\t\tif onDisconnect != nil {\n\t\t\t\t\tonDisconnect(c.ctx, c)\n\t\t\t\t}\n\t\t\t}\n\t\t\tc.unlock(connecting)\n\t\t}\n\tSTART:\n\t\t// The `onRequest` must be executed at least once if conn have any readable data,\n\t\t// which is in order to cover the `send & close by peer` case.\n\t\tif onRequest != nil && c.Reader().Len() > 0 {\n\t\t\t_ = onRequest(c.ctx, c)\n\t\t}\n\t\t// The processing loop must ensure that the connection meets `IsActive`.\n\t\t// `onRequest` must either eventually read all the input data or actively Close the connection,\n\t\t// otherwise the goroutine will fall into a dead loop.\n\t\tvar closedBy who\n\t\tfor {\n\t\t\tclosedBy = c.status(closing)\n\t\t\t// close by user or not processable\n\t\t\tif closedBy == user || onRequest == nil || c.Reader().Len() == 0 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\t_ = onRequest(c.ctx, c)\n\t\t}\n\t\t// handling callback if connection has been closed.\n\t\tif closedBy != none {\n\t\t\t//  if closed by user when processing, it \"may\" needs detach\n\t\t\tneedDetach := closedBy == user\n\t\t\t// Here is a corner case that operator will be detached twice:\n\t\t\t//   If server closed the connection(client OnHup will detach op first and closeBy=poller),\n\t\t\t//   and then client's OnRequest function also closed the connection(closeBy=user).\n\t\t\t// But operator already prevent that detach twice will not cause any problem\n\t\t\tc.closeCallback(false, needDetach)\n\t\t\tpanicked = false\n\t\t\treturn\n\t\t}\n\t\tc.unlock(processing)\n\t\t// Note: Poller's closeCallback call will try to get processing lock failed but here already near to unlock processing.\n\t\t//       So here we need to check connection state again, to avoid connection leak\n\t\t// double check close state\n\t\tif c.status(closing) != 0 && c.lock(processing) {\n\t\t\t// poller will get the processing lock failed, here help poller do closeCallback\n\t\t\t// fd must already detach by poller\n\t\t\tc.closeCallback(false, false)\n\t\t\tpanicked = false\n\t\t\treturn\n\t\t}\n\t\t// double check is processable\n\t\tif onRequest != nil && c.Reader().Len() > 0 && c.lock(processing) {\n\t\t\tgoto START\n\t\t}\n\t\t// task exits\n\t\tpanicked = false\n\t} // end of task closure func\n\n\t// add new task\n\trunner.RunTask(c.ctx, task)\n\treturn true\n}\n\n// closeCallback .\n// It can be confirmed that closeCallback and onRequest will not be executed concurrently.\n// If onRequest is still running, it will trigger closeCallback on exit.\nfunc (c *connection) closeCallback(needLock, needDetach bool) (err error) {\n\tif needLock && !c.lock(processing) {\n\t\treturn nil\n\t}\n\tif needDetach && c.operator.poll != nil { // If Close is called during OnPrepare, poll is not registered.\n\t\t// PollDetach only happen when user call conn.Close() or poller detect error\n\t\tif err := c.operator.Control(PollDetach); err != nil {\n\t\t\tlogger.Printf(\"NETPOLL: closeCallback[%v,%v] detach operator failed: %v\", needLock, needDetach, err)\n\t\t}\n\t}\n\tlatest := c.closeCallbacks.Load()\n\tif latest == nil {\n\t\treturn nil\n\t}\n\tfor callback := latest.(*callbackNode); callback != nil; callback = callback.pre {\n\t\tcallback.fn(c)\n\t}\n\treturn nil\n}\n\n// register only use for connection register into poll.\nfunc (c *connection) register() (err error) {\n\terr = c.operator.Control(PollReadable)\n\tif err != nil {\n\t\tlogger.Printf(\"NETPOLL: connection register failed: %v\", err)\n\t\tc.Close()\n\t\treturn Exception(ErrConnClosed, err.Error())\n\t}\n\treturn nil\n}\n\n// isIdle implements gracefulExit.\nfunc (c *connection) isIdle() (yes bool) {\n\treturn c.isUnlock(processing) &&\n\t\tc.inputBuffer.IsEmpty() &&\n\t\tc.outputBuffer.IsEmpty()\n}\n"
  },
  {
    "path": "connection_reactor.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n// +build !windows\n\npackage netpoll\n\nimport (\n\t\"sync/atomic\"\n)\n\n// ------------------------------------------ implement FDOperator ------------------------------------------\n\n// onHup means close by poller.\nfunc (c *connection) onHup(p Poll) error {\n\tif !c.closeBy(poller) {\n\t\treturn nil\n\t}\n\tc.triggerRead(Exception(ErrEOF, \"peer close\"))\n\tc.triggerWrite(Exception(ErrConnClosed, \"peer close\"))\n\n\t// call Disconnect callback first\n\tc.onDisconnect()\n\n\t// It depends on closing by user if OnConnect and OnRequest is nil, otherwise it needs to be released actively.\n\t// It can be confirmed that the OnRequest goroutine has been exited before closeCallback executing,\n\t// and it is safe to close the buffer at this time.\n\tonConnect := c.onConnectCallback.Load()\n\tonRequest := c.onRequestCallback.Load()\n\tneedCloseByUser := onConnect == nil && onRequest == nil\n\tif !needCloseByUser {\n\t\t// already PollDetach when call OnHup\n\t\tc.closeCallback(true, false)\n\t}\n\treturn nil\n}\n\n// onClose means close by user.\nfunc (c *connection) onClose() error {\n\t// user code close the connection\n\tif c.closeBy(user) {\n\t\tc.triggerRead(Exception(ErrConnClosed, \"self close\"))\n\t\tc.triggerWrite(Exception(ErrConnClosed, \"self close\"))\n\t\t// Detach from poller when processing finished, otherwise it will cause race\n\t\tc.closeCallback(true, true)\n\t\treturn nil\n\t}\n\n\t// closed by poller\n\t// still need to change closing status to `user` since OnProcess should not be processed again\n\tc.force(closing, user)\n\n\t// user code should actively close the connection to recycle resources.\n\t// poller already detached operator\n\treturn c.closeCallback(true, false)\n}\n\n// closeBuffer recycle input & output LinkBuffer.\nfunc (c *connection) closeBuffer() {\n\tonConnect, _ := c.onConnectCallback.Load().(OnConnect)\n\tonRequest, _ := c.onRequestCallback.Load().(OnRequest)\n\t// if client close the connection, we cannot ensure that the poller is not process the buffer,\n\t// so we need to check the buffer length, and if it's an \"unclean\" close operation, let's give up to reuse the buffer\n\tif c.inputBuffer.Len() == 0 || onConnect != nil || onRequest != nil {\n\t\tc.inputBuffer.Close()\n\t}\n\tif c.outputBuffer.Len() == 0 || onConnect != nil || onRequest != nil {\n\t\tc.outputBuffer.Close()\n\t\tbarrierPool.Put(c.outputBarrier)\n\t}\n}\n\n// inputs implements FDOperator.\nfunc (c *connection) inputs(vs [][]byte) (rs [][]byte) {\n\tvs[0] = c.inputBuffer.book(c.bookSize, c.maxSize)\n\treturn vs[:1]\n}\n\n// inputAck implements FDOperator.\nfunc (c *connection) inputAck(n int) (err error) {\n\tif n <= 0 {\n\t\tc.inputBuffer.bookAck(0)\n\t\treturn nil\n\t}\n\n\t// Auto size bookSize.\n\tif n == c.bookSize && c.bookSize < mallocMax {\n\t\tc.bookSize <<= 1\n\t}\n\n\tlength, _ := c.inputBuffer.bookAck(n)\n\tif c.maxSize < length {\n\t\tc.maxSize = length\n\t}\n\tif c.maxSize > mallocMax {\n\t\tc.maxSize = mallocMax\n\t}\n\n\tneedTrigger := true\n\tif length == n { // first start onRequest\n\t\tneedTrigger = c.onRequest()\n\t}\n\tif needTrigger && length >= int(atomic.LoadInt64(&c.waitReadSize)) {\n\t\tc.triggerRead(nil)\n\t}\n\treturn nil\n}\n\n// outputs implements FDOperator.\nfunc (c *connection) outputs(vs [][]byte) (rs [][]byte, _ bool) {\n\tif c.outputBuffer.IsEmpty() {\n\t\tc.rw2r()\n\t\treturn rs, false\n\t}\n\trs = c.outputBuffer.GetBytes(vs)\n\treturn rs, false\n}\n\n// outputAck implements FDOperator.\nfunc (c *connection) outputAck(n int) (err error) {\n\tif n > 0 {\n\t\tc.outputBuffer.Skip(n)\n\t\tc.outputBuffer.Release()\n\t}\n\tif c.outputBuffer.IsEmpty() {\n\t\tc.rw2r()\n\t}\n\treturn nil\n}\n\n// rw2r removed the monitoring of write events.\nfunc (c *connection) rw2r() {\n\tc.operator.Control(PollRW2R)\n\tc.triggerWrite(nil)\n}\n"
  },
  {
    "path": "connection_test.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n// +build !windows\n\npackage netpoll\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"net\"\n\t\"os\"\n\t\"runtime\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"syscall\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc BenchmarkConnectionIO(b *testing.B) {\n\tdataSize := 1024 * 16\n\twriteBuffer := make([]byte, dataSize)\n\trfd, wfd := GetSysFdPairs()\n\trconn, wconn := new(connection), new(connection)\n\trconn.init(&netFD{fd: rfd}, &options{onRequest: func(ctx context.Context, connection Connection) error {\n\t\tread, _ := connection.Reader().Next(dataSize)\n\t\t_ = wconn.Reader().Release()\n\t\t_, _ = connection.Writer().WriteBinary(read)\n\t\t_ = connection.Writer().Flush()\n\t\treturn nil\n\t}})\n\twconn.init(&netFD{fd: wfd}, new(options))\n\n\tb.ResetTimer()\n\tb.ReportAllocs()\n\tfor i := 0; i < b.N; i++ {\n\t\t_, _ = wconn.WriteBinary(writeBuffer)\n\t\t_ = wconn.Flush()\n\t\t_, _ = wconn.Reader().Next(dataSize)\n\t\t_ = wconn.Reader().Release()\n\t}\n}\n\nfunc TestConnectionWrite(t *testing.T) {\n\tcycle, caps := 10000, 256\n\tmsg, buf := make([]byte, caps), make([]byte, caps)\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\tvar count int32\n\texpect := int32(cycle * caps)\n\topts := &options{}\n\topts.onRequest = func(ctx context.Context, connection Connection) error {\n\t\tn, err := connection.Read(buf)\n\t\tMustNil(t, err)\n\t\tif atomic.AddInt32(&count, int32(n)) >= expect {\n\t\t\twg.Done()\n\t\t}\n\t\treturn nil\n\t}\n\n\tr, w := GetSysFdPairs()\n\trconn, wconn := &connection{}, &connection{}\n\trconn.init(&netFD{fd: r}, opts)\n\twconn.init(&netFD{fd: w}, opts)\n\n\tfor i := 0; i < cycle; i++ {\n\t\tn, err := wconn.Write(msg)\n\t\tMustNil(t, err)\n\t\tEqual(t, n, len(msg))\n\t}\n\twg.Wait()\n\tEqual(t, atomic.LoadInt32(&count), expect)\n\trconn.Close()\n}\n\nfunc TestConnectionLargeWrite(t *testing.T) {\n\t// ci machine don't have 4GB memory, so skip test\n\tt.Skipf(\"skip large write test for ci job\")\n\ttotalSize := 1024 * 1024 * 1024 * 4\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\topts := &options{}\n\topts.onRequest = func(ctx context.Context, connection Connection) error {\n\t\tif connection.Reader().Len() < totalSize {\n\t\t\treturn nil\n\t\t}\n\t\t_, err := connection.Reader().Next(totalSize)\n\t\tMustNil(t, err)\n\t\terr = connection.Reader().Release()\n\t\tMustNil(t, err)\n\t\twg.Done()\n\t\treturn nil\n\t}\n\n\tr, w := GetSysFdPairs()\n\trconn, wconn := &connection{}, &connection{}\n\trconn.init(&netFD{fd: r}, opts)\n\twconn.init(&netFD{fd: w}, opts)\n\n\tmsg := make([]byte, totalSize/4)\n\tfor i := 0; i < 4; i++ {\n\t\t_, err := wconn.Writer().WriteBinary(msg)\n\t\tMustNil(t, err)\n\t}\n\twg.Wait()\n\n\trconn.Close()\n}\n\nfunc TestConnectionRead(t *testing.T) {\n\tr, w := GetSysFdPairs()\n\trconn, wconn := &connection{}, &connection{}\n\terr := rconn.init(&netFD{fd: r}, nil)\n\tMustNil(t, err)\n\terr = wconn.init(&netFD{fd: w}, nil)\n\tMustNil(t, err)\n\n\tsize := 256\n\tcycleTime := 1000\n\tmsg := make([]byte, size)\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tfor i := 0; i < cycleTime; i++ {\n\t\t\tbuf, err := rconn.Reader().Next(size)\n\t\t\tMustNil(t, err)\n\t\t\tEqual(t, len(buf), size)\n\t\t\trconn.Reader().Release()\n\t\t}\n\t}()\n\tfor i := 0; i < cycleTime; i++ {\n\t\tn, err := wconn.Write(msg)\n\t\tMustNil(t, err)\n\t\tEqual(t, n, len(msg))\n\t}\n\twg.Wait()\n\trconn.Close()\n}\n\n// TestConnectionIOReader tests the io.Reader Read method which uses readCopy internally.\n// Verifies that Read after Peek preserves exposed buffer until Release.\nfunc TestConnectionIOReader(t *testing.T) {\n\tr, w := GetSysFdPairs()\n\trconn := &connection{}\n\trconn.init(&netFD{fd: r}, nil)\n\n\tmsg := make([]byte, 64)\n\tfor i := range msg {\n\t\tmsg[i] = byte(i)\n\t}\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\t// Peek exposes the underlying buffer\n\t\tpk, err := rconn.Peek(16)\n\t\tMustNil(t, err)\n\t\tEqual(t, len(pk), 16)\n\n\t\t// Read copies without exposing\n\t\tbuf := make([]byte, 64)\n\t\tn, err := rconn.Read(buf)\n\t\tMustNil(t, err)\n\t\tEqual(t, n, 64)\n\t\tfor i := 0; i < 64; i++ {\n\t\t\tEqual(t, buf[i], byte(i))\n\t\t}\n\n\t\t// Peek data still valid before Release\n\t\tfor i := 0; i < 16; i++ {\n\t\t\tEqual(t, pk[i], byte(i))\n\t\t}\n\t\trconn.Release()\n\t}()\n\tsyscall.Write(w, msg)\n\twg.Wait()\n\trconn.Close()\n\tsyscall.Close(w)\n}\n\nfunc TestConnectionReadAfterClosed(t *testing.T) {\n\tr, w := GetSysFdPairs()\n\trconn := &connection{}\n\trconn.init(&netFD{fd: r}, nil)\n\tsize := 256\n\tmsg := make([]byte, size)\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tbuf, err := rconn.Reader().Next(size)\n\t\tMustNil(t, err)\n\t\tEqual(t, len(buf), size)\n\t}()\n\ttime.Sleep(time.Millisecond)\n\tsyscall.Write(w, msg)\n\tsyscall.Close(w)\n\twg.Wait()\n}\n\nfunc TestConnectionWaitReadHalfPacket(t *testing.T) {\n\tr, w := GetSysFdPairs()\n\trconn := &connection{}\n\trconn.init(&netFD{fd: r}, nil)\n\tsize := pagesize * 2\n\tmsg := make([]byte, size)\n\n\t// write half packet\n\tsyscall.Write(w, msg[:size/2])\n\t// wait poller reads buffer\n\tfor rconn.inputBuffer.Len() <= 0 {\n\t\truntime.Gosched()\n\t}\n\n\t// wait read full packet\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tbuf, err := rconn.Reader().Next(size)\n\t\tEqual(t, atomic.LoadInt64(&rconn.waitReadSize), int64(0))\n\t\tMustNil(t, err)\n\t\tEqual(t, len(buf), size)\n\t}()\n\n\t// write left half packet\n\tfor atomic.LoadInt64(&rconn.waitReadSize) <= 0 {\n\t\truntime.Gosched()\n\t}\n\tEqual(t, atomic.LoadInt64(&rconn.waitReadSize), int64(size))\n\tsyscall.Write(w, msg[size/2:])\n\twg.Wait()\n}\n\nfunc TestReadTimer(t *testing.T) {\n\tread := time.NewTimer(time.Second)\n\tMustTrue(t, read.Stop())\n\ttime.Sleep(time.Millisecond)\n\tEqual(t, len(read.C), 0)\n}\n\nfunc TestReadTrigger(t *testing.T) {\n\ttrigger := make(chan int, 1)\n\tselect {\n\tcase trigger <- 0:\n\tdefault:\n\t}\n\tEqual(t, len(trigger), 1)\n}\n\nfunc writeAll(fd int, buf []byte) error {\n\tfor len(buf) > 0 {\n\t\tn, err := syscall.Write(fd, buf)\n\t\tif n < 0 {\n\t\t\treturn err\n\t\t}\n\t\tbuf = buf[n:]\n\t}\n\treturn nil\n}\n\nfunc createTestTCPListener(t *testing.T) net.Listener {\n\tln, err := net.Listen(\"tcp\", \"127.0.0.1:0\")\n\tMustNil(t, err)\n\treturn ln\n}\n\n// Large packet write test. The socket buffer is 2MB by default, here to verify\n// whether Connection.Close can be executed normally after socket output buffer is full.\nfunc TestLargeBufferWrite(t *testing.T) {\n\tln := createTestTCPListener(t)\n\tdefer ln.Close()\n\taddress := ln.Addr().String()\n\tln, err := ConvertListener(ln)\n\tMustNil(t, err)\n\n\ttrigger := make(chan int)\n\tdefer close(trigger)\n\tgo func() {\n\t\tfor {\n\t\t\tconn, err := ln.Accept()\n\t\t\tif conn == nil && err == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\ttrigger <- conn.(*netFD).fd\n\t\t\t<-trigger\n\t\t\terr = ln.Close()\n\t\t\tMustNil(t, err)\n\t\t\treturn\n\t\t}\n\t}()\n\n\tconn, err := DialConnection(\"tcp\", address, time.Second)\n\tMustNil(t, err)\n\trfd := <-trigger\n\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\tbufferSize := 2 * 1024 * 1024 // 2MB\n\tround := 128\n\t// start large buffer writing\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tfor i := 1; i <= round+1; i++ {\n\t\t\t_, err := conn.Writer().Malloc(bufferSize)\n\t\t\tMustNil(t, err)\n\t\t\terr = conn.Writer().Flush()\n\t\t\tif i <= round {\n\t\t\t\tMustNil(t, err)\n\t\t\t}\n\t\t}\n\t}()\n\n\t// wait socket buffer full\n\ttime.Sleep(time.Millisecond * 100)\n\tbuf := make([]byte, 1024)\n\tfor received := 0; received < round*bufferSize; {\n\t\tn, _ := syscall.Read(rfd, buf)\n\t\treceived += n\n\t}\n\t// close success\n\terr = conn.Close()\n\tMustNil(t, err)\n\twg.Wait()\n\ttrigger <- 1\n}\n\nfunc TestConnectionTimeout(t *testing.T) {\n\tln, err := net.Listen(\"tcp\", \"127.0.0.1:0\")\n\tMustNil(t, err)\n\tdefer ln.Close()\n\n\tconst (\n\t\tbufsz    = 1 << 20\n\t\tinterval = 10 * time.Millisecond\n\t)\n\n\tcalcRate := func(n int32) int32 {\n\t\tv := n / int32(time.Second/interval)\n\t\tif v > bufsz {\n\t\t\tpanic(v)\n\t\t}\n\t\tif v < 1 {\n\t\t\treturn 1\n\t\t}\n\t\treturn v\n\t}\n\n\twn := int32(1) // for each Read, must <= bufsz\n\tsetServerWriteRate := func(n int32) {\n\t\tatomic.StoreInt32(&wn, calcRate(n))\n\t}\n\n\trn := int32(1) // for each Write, must <= bufsz\n\tsetServerReadRate := func(n int32) {\n\t\tatomic.StoreInt32(&rn, calcRate(n))\n\t}\n\n\tgo func() {\n\t\tfor {\n\t\t\tconn, err := ln.Accept()\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// set small SO_SNDBUF/SO_RCVBUF buffer for better control timeout test\n\t\t\ttcpconn := conn.(*net.TCPConn)\n\t\t\ttcpconn.SetReadBuffer(512)\n\t\t\ttcpconn.SetWriteBuffer(512)\n\t\t\tgo func() {\n\t\t\t\tbuf := make([]byte, bufsz)\n\t\t\t\tfor {\n\t\t\t\t\tn := atomic.LoadInt32(&rn)\n\t\t\t\t\t_, err := conn.Read(buf[:int(n)])\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tconn.Close()\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\ttime.Sleep(interval)\n\t\t\t\t}\n\t\t\t}()\n\n\t\t\tgo func() {\n\t\t\t\tbuf := make([]byte, bufsz)\n\t\t\t\tfor {\n\t\t\t\t\tn := atomic.LoadInt32(&wn)\n\t\t\t\t\t_, err := conn.Write(buf[:int(n)])\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tconn.Close()\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\ttime.Sleep(interval)\n\t\t\t\t}\n\t\t\t}()\n\t\t}\n\t}()\n\n\tnewConn := func() Connection {\n\t\tconn, err := DialConnection(\"tcp\", ln.Addr().String(), time.Second)\n\t\tMustNil(t, err)\n\t\tfd := conn.(Conn).Fd()\n\t\t// set small SO_SNDBUF/SO_RCVBUF buffer for better control timeout test\n\t\terr = syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_SNDBUF, 512)\n\t\tMustNil(t, err)\n\t\terr = syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_RCVBUF, 512)\n\t\tMustNil(t, err)\n\t\treturn conn\n\t}\n\n\tmallocAndFlush := func(conn Connection, sz int) error {\n\t\t_, err := conn.Writer().Malloc(sz)\n\t\tMustNil(t, err)\n\t\treturn conn.Writer().Flush()\n\t}\n\n\tt.Run(\"TestWriteTimeout\", func(t *testing.T) {\n\t\tsetServerReadRate(10 << 10) // 10KB/s\n\n\t\tconn := newConn()\n\t\tdefer conn.Close()\n\n\t\t// write 1KB without timeout\n\t\terr := mallocAndFlush(conn, 1<<10) // ~100ms\n\t\tMustNil(t, err)\n\n\t\t// write 50ms timeout\n\t\t_ = conn.SetWriteTimeout(50 * time.Millisecond)\n\t\terr = mallocAndFlush(conn, 1<<20)\n\t\tMustTrue(t, errors.Is(err, ErrWriteTimeout))\n\t})\n\n\tt.Run(\"TestReadTimeout\", func(t *testing.T) {\n\t\tsetServerWriteRate(10 << 10) // 10KB/s\n\n\t\tconn := newConn()\n\t\tdefer conn.Close()\n\n\t\t// read 1KB without timeout\n\t\t_, err := conn.Reader().Next(1 << 10) // ~100ms\n\t\tMustNil(t, err)\n\n\t\t// read 20KB ~ 2s, 50ms timeout\n\t\t_ = conn.SetReadTimeout(50 * time.Millisecond)\n\t\t_, err = conn.Reader().Next(20 << 10)\n\t\tMustTrue(t, errors.Is(err, ErrReadTimeout))\n\t})\n\n\tt.Run(\"TestWriteDeadline\", func(t *testing.T) {\n\t\tsetServerReadRate(10 << 10) // 10KB/s\n\n\t\tconn := newConn()\n\t\tdefer conn.Close()\n\n\t\t// write 1KB without deadline\n\t\terr := conn.SetWriteDeadline(time.Now())\n\t\tMustNil(t, err)\n\t\terr = conn.SetDeadline(time.Time{})\n\t\tMustNil(t, err)\n\t\terr = mallocAndFlush(conn, 1<<10) // ~100ms\n\t\tMustNil(t, err)\n\n\t\t// write with deadline\n\t\terr = conn.SetWriteDeadline(time.Now().Add(50 * time.Millisecond))\n\t\tMustNil(t, err)\n\t\tt0 := time.Now()\n\t\terr = mallocAndFlush(conn, 1<<20)\n\t\tMustTrue(t, errors.Is(err, ErrWriteTimeout))\n\t\tMustTrue(t, time.Since(t0)-50*time.Millisecond < 20*time.Millisecond)\n\n\t\t// write deadline exceeded\n\t\tt1 := time.Now()\n\t\terr = mallocAndFlush(conn, 10<<10)\n\t\tMustTrue(t, errors.Is(err, ErrWriteTimeout))\n\t\tMustTrue(t, time.Since(t1) < 20*time.Millisecond)\n\t})\n\n\tt.Run(\"TestReadDeadline\", func(t *testing.T) {\n\t\tsetServerWriteRate(20 << 10) // 20KB/s\n\n\t\tconn := newConn()\n\t\tdefer conn.Close()\n\n\t\t// read 1KB without deadline\n\t\terr := conn.SetReadDeadline(time.Now())\n\t\tMustNil(t, err)\n\t\terr = conn.SetDeadline(time.Time{})\n\t\tMustNil(t, err)\n\t\t_, err = conn.Reader().Next(1 << 10)\n\t\tMustNil(t, err)\n\n\t\t// read 100KB with deadline\n\t\terr = conn.SetReadDeadline(time.Now().Add(50 * time.Millisecond))\n\t\tMustNil(t, err)\n\t\tt0 := time.Now()\n\t\t_, err = conn.Reader().Next(100 << 10)\n\t\tMustTrue(t, errors.Is(err, ErrReadTimeout))\n\t\tMustTrue(t, time.Since(t0)-50*time.Millisecond < 20*time.Millisecond)\n\n\t\t// read 10KB, deadline exceeded\n\t\tt1 := time.Now()\n\t\t_, err = conn.Reader().Next(10 << 10)\n\t\tMustTrue(t, errors.Is(err, ErrReadTimeout))\n\t\tMustTrue(t, time.Since(t1) < 20*time.Millisecond)\n\t})\n}\n\n// TestConnectionLargeMemory is used to verify the memory usage in the large package scenario.\nfunc TestConnectionLargeMemory(t *testing.T) {\n\tvar start, end runtime.MemStats\n\truntime.GC()\n\truntime.ReadMemStats(&start)\n\n\tr, w := GetSysFdPairs()\n\trconn := &connection{}\n\trconn.init(&netFD{fd: r}, nil)\n\n\tvar wg sync.WaitGroup\n\trn, wn := 1024, 1*1024*1024\n\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\t_, err := rconn.Reader().Next(wn)\n\t\tMustNil(t, err)\n\t}()\n\n\tmsg := make([]byte, rn)\n\tfor i := 0; i < wn/rn; i++ {\n\t\tn, err := syscall.Write(w, msg)\n\t\tif err != nil {\n\t\t\tMustNil(t, err)\n\t\t}\n\t\tEqual(t, n, rn)\n\t}\n\n\truntime.ReadMemStats(&end)\n\talloc := end.TotalAlloc - start.TotalAlloc\n\tlimit := uint64(4 * 1024 * 1024)\n\tAssert(t, alloc <= limit, fmt.Sprintf(\"alloc[%d] out of memory %d\", alloc, limit))\n}\n\n// TestSetTCPNoDelay is used to verify the connection initialization set the TCP_NODELAY correctly\nfunc TestSetTCPNoDelay(t *testing.T) {\n\tfd, err := sysSocket(syscall.AF_INET, syscall.SOCK_STREAM, 0)\n\tMustNil(t, err)\n\tconn := &connection{}\n\tconn.init(&netFD{network: \"tcp\", fd: fd}, nil)\n\n\tn, _ := syscall.GetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_NODELAY)\n\tMustTrue(t, n > 0)\n\terr = setTCPNoDelay(fd, false)\n\tMustNil(t, err)\n\tn, _ = syscall.GetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_NODELAY)\n\tMustTrue(t, n == 0)\n}\n\nfunc TestConnectionUntil(t *testing.T) {\n\tr, w := GetSysFdPairs()\n\trconn, wconn := &connection{}, &connection{}\n\trconn.init(&netFD{fd: r}, nil)\n\twconn.init(&netFD{fd: w}, nil)\n\tloopSize := 10000\n\n\tmsg := make([]byte, 1002)\n\tmsg[500], msg[1001] = '\\n', '\\n'\n\tgo func() {\n\t\tfor i := 0; i < loopSize; i++ {\n\t\t\tn, err := wconn.Write(msg)\n\t\t\tMustNil(t, err)\n\t\t\tMustTrue(t, n == len(msg))\n\t\t}\n\t\twconn.Write(msg[:100])\n\t\twconn.Close()\n\t}()\n\n\tfor i := 0; i < loopSize*2; i++ {\n\t\tbuf, err := rconn.Reader().Until('\\n')\n\t\tMustNil(t, err)\n\t\tEqual(t, len(buf), 501)\n\t\trconn.Reader().Release()\n\t}\n\n\tbuf, err := rconn.Reader().Until('\\n')\n\tEqual(t, len(buf), 100)\n\tAssert(t, errors.Is(err, ErrEOF), err)\n}\n\nfunc TestBookSizeLargerThanMaxSize(t *testing.T) {\n\tr, w := GetSysFdPairs()\n\trconn, wconn := &connection{}, &connection{}\n\terr := rconn.init(&netFD{fd: r}, nil)\n\tMustNil(t, err)\n\terr = wconn.init(&netFD{fd: w}, nil)\n\tMustNil(t, err)\n\n\t// prepare data\n\tmaxSize := 1024 * 1024 * 128\n\torigin := make([][]byte, 0)\n\tfor size := maxSize; size > 0; size = size >> 1 {\n\t\tch := 'a' + byte(size%26)\n\t\torigin = append(origin, make([]byte, size))\n\t\tfor i := 0; i < size; i++ {\n\t\t\torigin[len(origin)-1][i] = ch\n\t\t}\n\t}\n\n\t// read\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tidx := 0\n\t\tfor size := maxSize; size > 0; size = size >> 1 {\n\t\t\tbuf, err := rconn.Reader().Next(size)\n\t\t\tMustNil(t, err)\n\t\t\tEqual(t, string(buf), string(origin[idx]))\n\t\t\terr = rconn.Reader().Release()\n\t\t\tMustNil(t, err)\n\t\t\tidx++\n\t\t}\n\t}()\n\n\t// write\n\tfor i := 0; i < len(origin); i++ {\n\t\tn, err := wconn.Write(origin[i])\n\t\tMustNil(t, err)\n\t\tEqual(t, n, len(origin[i]))\n\t}\n\twg.Wait()\n\trconn.Close()\n\twconn.Close()\n}\n\nfunc TestConnDetach(t *testing.T) {\n\tln := createTestTCPListener(t)\n\tdefer ln.Close()\n\taddress := ln.Addr().String()\n\n\t// accept => read => write\n\tvar wg sync.WaitGroup\n\tgo func() {\n\t\tfor {\n\t\t\tconn, err := ln.Accept()\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif conn == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\twg.Add(1)\n\t\t\tgo func() {\n\t\t\t\tdefer wg.Done()\n\t\t\t\tbuf := make([]byte, 1024)\n\t\t\t\t// slow read\n\t\t\t\t_, err := conn.Read(buf)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\ttime.Sleep(10 * time.Millisecond)\n\t\t\t\t_, err = conn.Write(buf)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}()\n\t\t}\n\t}()\n\n\t// dial => detach => write => read\n\tc, err := DialConnection(\"tcp\", address, time.Second)\n\tMustNil(t, err)\n\tconn := c.(*TCPConnection)\n\terr = conn.Detach()\n\tMustNil(t, err)\n\n\tf := os.NewFile(uintptr(conn.fd), \"netpoll-connection\")\n\tdefer f.Close()\n\tgonetconn, err := net.FileConn(f)\n\tMustNil(t, err)\n\tbuf := make([]byte, 1024)\n\t_, err = gonetconn.Write(buf)\n\tMustNil(t, err)\n\t_, err = gonetconn.Read(buf)\n\tMustNil(t, err)\n\n\terr = gonetconn.Close()\n\tMustNil(t, err)\n\terr = ln.Close()\n\tMustNil(t, err)\n\terr = c.Close()\n\tMustNil(t, err)\n\twg.Wait()\n}\n\nfunc TestParallelShortConnection(t *testing.T) {\n\tln := createTestTCPListener(t)\n\tdefer ln.Close()\n\taddress := ln.Addr().String()\n\n\tvar received int64\n\tel, err := NewEventLoop(func(ctx context.Context, connection Connection) error {\n\t\tdata, err := connection.Reader().Next(connection.Reader().Len())\n\t\tatomic.AddInt64(&received, int64(len(data)))\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\t// t.Logf(\"conn[%s] received: %d, active: %v\", connection.RemoteAddr(), len(data), connection.IsActive())\n\t\treturn nil\n\t})\n\tMustNil(t, err)\n\tgo func() {\n\t\tel.Serve(ln)\n\t}()\n\tdefer el.Shutdown(context.Background())\n\n\tconns := 100\n\tsizePerConn := 1024\n\ttotalSize := conns * sizePerConn\n\tvar wg sync.WaitGroup\n\tfor i := 0; i < conns; i++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tconn, err := DialConnection(\"tcp\", address, time.Second)\n\t\t\tMustNil(t, err)\n\t\t\tn, err := conn.Writer().WriteBinary(make([]byte, sizePerConn))\n\t\t\tMustNil(t, err)\n\t\t\tMustTrue(t, n == sizePerConn)\n\t\t\terr = conn.Writer().Flush()\n\t\t\tMustNil(t, err)\n\t\t\terr = conn.Close()\n\t\t\tMustNil(t, err)\n\t\t}()\n\t}\n\twg.Wait()\n\n\tt0 := time.Now()\n\tfor atomic.LoadInt64(&received) < int64(totalSize) {\n\t\ttime.Sleep(time.Millisecond)\n\t\tif time.Since(t0) > 100*time.Millisecond { // max wait 100ms\n\t\t\tbreak\n\t\t}\n\t}\n\tEqual(t, atomic.LoadInt64(&received), int64(totalSize))\n}\n\nfunc TestConnectionServerClose(t *testing.T) {\n\tln := createTestTCPListener(t)\n\tdefer ln.Close()\n\taddress := ln.Addr().String()\n\n\t/*\n\t\tClient              Server\n\t\t- Client --- connect   --> Server\n\t\t- Client <-- [ping]   --- Server\n\t\t- Client --- [pong]   --> Server\n\t\t- Client <-- close     --- Server\n\t\t- Client --- close     --> Server\n\t*/\n\tconst PING, PONG = \"ping\", \"pong\"\n\tvar wg sync.WaitGroup\n\tel, err := NewEventLoop(\n\t\tfunc(ctx context.Context, connection Connection) error {\n\t\t\tt.Logf(\"server.OnRequest: addr=%s\", connection.RemoteAddr())\n\t\t\tdefer wg.Done()\n\t\t\tbuf, err := connection.Reader().Next(len(PONG)) // pong\n\t\t\tEqual(t, string(buf), PONG)\n\t\t\tMustNil(t, err)\n\t\t\terr = connection.Reader().Release()\n\t\t\tMustNil(t, err)\n\t\t\terr = connection.Close()\n\t\t\tMustNil(t, err)\n\t\t\treturn err\n\t\t},\n\t\tWithOnConnect(func(ctx context.Context, connection Connection) context.Context {\n\t\t\tt.Logf(\"server.OnConnect: addr=%s\", connection.RemoteAddr())\n\t\t\tdefer wg.Done()\n\t\t\t// check OnPrepare\n\t\t\tv := ctx.Value(\"prepare\").(string)\n\t\t\tEqual(t, v, \"true\")\n\n\t\t\t_, err := connection.Writer().WriteBinary([]byte(PING))\n\t\t\tMustNil(t, err)\n\t\t\terr = connection.Writer().Flush()\n\t\t\tMustNil(t, err)\n\t\t\tconnection.AddCloseCallback(func(connection Connection) error {\n\t\t\t\tt.Logf(\"server.CloseCallback: addr=%s\", connection.RemoteAddr())\n\t\t\t\twg.Done()\n\t\t\t\treturn nil\n\t\t\t})\n\t\t\treturn ctx\n\t\t}),\n\t\tWithOnPrepare(func(connection Connection) context.Context {\n\t\t\tt.Logf(\"server.OnPrepare: addr=%s\", connection.RemoteAddr())\n\t\t\tdefer wg.Done()\n\t\t\t//nolint:staticcheck // SA1029 no built-in type string as key\n\t\t\treturn context.WithValue(context.Background(), \"prepare\", \"true\")\n\t\t}),\n\t)\n\tMustNil(t, err)\n\n\tdefer el.Shutdown(context.Background())\n\tgo func() {\n\t\terr := el.Serve(ln)\n\t\tif err != nil {\n\t\t\tt.Logf(\"service end with error: %v\", err)\n\t\t}\n\t}()\n\n\tvar clientOnRequest OnRequest = func(ctx context.Context, connection Connection) error {\n\t\tt.Logf(\"client.OnRequest: addr=%s\", connection.LocalAddr())\n\t\tdefer wg.Done()\n\t\tbuf, err := connection.Reader().Next(len(PING))\n\t\tMustNil(t, err)\n\t\tEqual(t, string(buf), PING)\n\n\t\t_, err = connection.Writer().WriteBinary([]byte(PONG))\n\t\tMustNil(t, err)\n\t\terr = connection.Writer().Flush()\n\t\tMustNil(t, err)\n\n\t\t_, err = connection.Reader().Next(1) // server will not send any data, just wait for server close\n\t\tMustTrue(t, errors.Is(err, ErrEOF))  // should get EOF when server close\n\n\t\treturn connection.Close()\n\t}\n\tconns := 10\n\t// server: OnPrepare, OnConnect, OnRequest, CloseCallback\n\t// client: OnRequest, CloseCallback\n\twg.Add(conns * 6)\n\tfor i := 0; i < conns; i++ {\n\t\tgo func() {\n\t\t\tconn, err := DialConnection(\"tcp\", address, time.Second)\n\t\t\tMustNil(t, err)\n\t\t\terr = conn.SetOnRequest(clientOnRequest)\n\t\t\tMustNil(t, err)\n\t\t\tconn.AddCloseCallback(func(connection Connection) error {\n\t\t\t\tt.Logf(\"client.CloseCallback: addr=%s\", connection.LocalAddr())\n\t\t\t\tdefer wg.Done()\n\t\t\t\treturn nil\n\t\t\t})\n\t\t}()\n\t}\n\twg.Wait()\n}\n\nfunc TestWriterAfterClose(t *testing.T) {\n\tr, w := GetSysFdPairs()\n\trconn, wconn := &connection{}, &connection{}\n\trconn.init(&netFD{fd: r}, nil)\n\twconn.init(&netFD{fd: w}, nil)\n\n\terr := wconn.Close()\n\tMustNil(t, err)\n\n\tfor wconn.IsActive() {\n\t\truntime.Gosched()\n\t}\n\n\tmethods := []struct {\n\t\tname string\n\t\tfn   func() error\n\t}{\n\t\t{\"Malloc\", func() error { _, err := wconn.Malloc(1); return err }},\n\t\t{\"MallocAck\", func() error { return wconn.MallocAck(0) }},\n\t\t{\"WriteBinary\", func() error { _, err := wconn.WriteBinary([]byte(\"hi\")); return err }},\n\t\t{\"WriteString\", func() error { _, err := wconn.WriteString(\"hi\"); return err }},\n\t\t{\"WriteByte\", func() error { return wconn.WriteByte('a') }},\n\t\t{\"WriteDirect\", func() error { return wconn.WriteDirect([]byte(\"hi\"), 0) }},\n\t\t{\"Flush\", func() error { return wconn.Flush() }},\n\t}\n\tfor _, tc := range methods {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tdefer func() {\n\t\t\t\tif r := recover(); r != nil {\n\t\t\t\t\tt.Fatalf(\"Writer.%s panicked after Close: %v\", tc.name, r)\n\t\t\t\t}\n\t\t\t}()\n\t\t\terr := tc.fn()\n\t\t\tAssert(t, err != nil, fmt.Sprintf(\"Writer.%s should return error after Close\", tc.name))\n\t\t})\n\t}\n\trconn.Close()\n}\n\nfunc TestConnectionDailTimeoutAndClose(t *testing.T) {\n\tln := createTestTCPListener(t)\n\tdefer ln.Close()\n\n\tgo func() {\n\t\tfor {\n\t\t\tconn, err := ln.Accept()\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\ttime.Sleep(time.Millisecond)\n\t\t\tconn.Close()\n\t\t}\n\t}()\n\n\tvar wg sync.WaitGroup\n\tfor i := 0; i < 100; i++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tconn, err := DialConnection(\"tcp\", ln.Addr().String(), time.Millisecond)\n\t\t\tAssert(t, err == nil || strings.Contains(err.Error(), \"i/o timeout\"), err)\n\t\t\tif err == nil { // XXX: conn is always not nil ...\n\t\t\t\tconn.Close()\n\t\t\t}\n\t\t}()\n\t}\n\twg.Wait()\n}\n"
  },
  {
    "path": "docs/guide/guide_cn.md",
    "content": "# 快速开始\n\n本教程通过一些简单的 [示例][Examples] 帮助您开始使用 [Netpoll][Netpoll]，包括如何使用 [Server](#1-使用-sever)、[Client](#2-使用-dialer) 和 [nocopy API](#3-使用-nocopy-api)。\n\n## 1. 使用 Sever\n\n[这里][server-example] 是一个简单的 server 例子，接下来我们会解释它是如何构建的。\n\n### 1.1 创建 Listener\n\n首先我们需要一个 `Listener`，它可以是 `net.Listener` 或者 `netpoll.Listener`，两者都可以，依据你的代码情况自由选择。\n创建 `Listener` 的过程如下：\n\n```go\npackage main\n\nimport \"net\"\n\nfunc main() {\n\tlistener, err := net.Listen(network, address)\n\tif err != nil {\n\t\tpanic(\"create net listener failed\")\n\t}\n\t...\n}\n```\n\n或者\n\n```go\npackage main\n\nimport \"github.com/cloudwego/netpoll\"\n\nfunc main() {\n\tlistener, err := netpoll.CreateListener(network, address)\n\tif err != nil {\n\t\tpanic(\"create netpoll listener failed\")\n\t}\n\t...\n}\n```\n\n### 1.2 创建 EventLoop\n\n`EventLoop` 是一个事件驱动的调度器，一个真正的 NIO Server，负责连接管理、事件调度等。\n\n参数说明:\n\n* `OnRequest` 是用户应该自己实现来处理业务逻辑的接口。 [注释][netpoll.go] 详细描述了它的行为。\n* `Option` 用于自定义 `EventLoop` 创建时的配置，下面的例子展示了它的用法。更多详情请参考 [options][netpoll_options.go]。\n\n创建过程如下：\n\n```go\npackage main\n\nimport (\n\t\"time\"\n\t\"github.com/cloudwego/netpoll\"\n)\n\nvar eventLoop netpoll.EventLoop\n\nfunc main() {\n\t...\n\teventLoop, _ = netpoll.NewEventLoop(\n\t\thandle,\n\t\tnetpoll.WithOnPrepare(prepare),\n\t\tnetpoll.WithReadTimeout(time.Second),\n\t)\n\t...\n}\n```\n\n### 1.3 运行 Server\n\n`EventLoop` 通过绑定 `Listener` 来提供服务，如下所示。`Serve` 方法为阻塞式调用，直到发生 `panic` 等错误，或者由用户主动调用 `Shutdown` 时触发退出。\n\n```go\npackage main\n\nimport (\n\t\"github.com/cloudwego/netpoll\"\n)\n\nvar eventLoop netpoll.EventLoop\n\nfunc main() {\n\t...\n\t// start listen loop ...\n\teventLoop.Serve(listener)\n}\n```\n\n### 1.4 关闭 Server\n\n`EventLoop` 提供了 `Shutdown` 功能，用于优雅地停止服务器。用法如下：\n\n```go\npackage main\n\nimport (\n\t\"context\"\n\t\"time\"\n\t\"github.com/cloudwego/netpoll\"\n)\n\nvar eventLoop netpoll.EventLoop\n\nfunc main() {\n\t// stop server ...\n\tctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)\n\tdefer cancel()\n\teventLoop.Shutdown(ctx)\n}\n```\n\n## 2. 使用 Dialer\n\n[Netpoll][Netpoll] 也支持在 Client 端使用，提供了 `Dialer`，类似于 `net.Dialer`。同样的，[这里][client-example] 展示了一个简单的 Client 端示例，接下来我们详细介绍一下：\n\n### 2.1 快速方式\n\n与 [Net][net] 类似，[Netpoll][Netpoll] 提供了几个用于直接建立连接的公共方法，可以直接调用。 如：\n\n```go\nDialConnection(network, address string, timeout time.Duration) (connection Connection, err error)\n\nDialTCP(ctx context.Context, network string, laddr, raddr *TCPAddr) (*TCPConnection, error)\n\nDialUnix(network string, laddr, raddr *UnixAddr) (*UnixConnection, error)\n```\n\n### 2.2 创建 Dialer\n\n[Netpoll][Netpoll] 还定义了`Dialer` 接口。 用法如下：（通常推荐使用上一节的快速方式）\n\n```go\npackage main\n\nimport (\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc main() {\n\t// Dial a connection with Dialer.\n\tdialer := netpoll.NewDialer()\n\tconn, err := dialer.DialConnection(network, address, timeout)\n\tif err != nil {\n\t\tpanic(\"dial netpoll connection failed\")\n\t}\n\t...\n}\n```\n\n## 3. 使用 Nocopy API\n\n`Connection` 提供了 Nocopy API —— `Reader` 和 `Writer`，以避免频繁复制。下面介绍一下它们的简单用法。\n\n```go\npackage main\n\ntype Connection interface {\n\t// Recommended nocopy APIs\n\tReader() Reader\n\tWriter() Writer\n\t... // see code comments for more details\n}\n```\n\n### 3.1 简单用法\n\nNocopy API 设计为两步操作。\n\n使用 `Reader` 时，通过 `Next`、`Peek`、`ReadString` 等方法读取数据后，还需要主动调用 `Release` 方法释放 buffer（`Nocopy` 读取 buffer 的原地址，所以您必须主动再次确认 buffer 已经不再使用）。\n\n同样，使用 `Writer` 时，首先需要分配一个 `[]byte` 来写入数据，然后调用 `Flush` 确认所有数据都已经写入。`Writer` 还提供了丰富的 API 来分配 buffer，例如 `Malloc`、`WriteString` 等。\n\n下面是一些简单的读写数据的例子。 更多详情请参考 [说明][nocopy.go]。\n\n```go\npackage main\n\nimport (\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc main() {\n\tvar conn netpoll.Connection\n\tvar reader, writer = conn.Reader(), conn.Writer()\n\t\n\t// reading\n\tbuf, _ := reader.Next(n)\n\t... parse the read data ...\n\treader.Release()\n\t\n\t// writing\n\tvar write_data []byte\n\t... make the write data ...\n\talloc, _ := writer.Malloc(len(write_data))\n\tcopy(alloc, write_data) // write data\n\twriter.Flush()\n}\n```\n\n### 3.2 高阶用法\n\n如果你想使用单个连接来发送（或接收）多组数据（如连接多路复用），那么你将面临数据打包和分包。在 [net][net] 上，这种工作一般都是通过复制来完成的。一个例子如下：\n\n```go\npackage main\n\nimport (\n\t\"net\"\n)\n\nfunc main() {\n\tvar conn net.Conn\n\tvar buf = make([]byte, 8192)\n\t\n\t// reading\n\tfor {\n\t\tn, _ := conn.Read(buf)\n\t\t... unpacking & handling ...\n\t\tvar i int\n\t\tfor i = 0; i <= n-pkgsize; i += pkgsize {\n\t\t\tpkg := append([]byte{}, buf[i:i+pkgsize]...)\n\t\t\tgo func() {\n\t\t\t\t... handling pkg ...\n\t\t\t}\n\t\t}\n\t\tbuf = append(buf[:0], buf[i:n]...)\n\t}\n\t\n\t// writing\n\tvar write_datas <-chan []byte\n\t... packing write ...\n\tfor {\n\t\tpkg := <-write_datas\n\t\tconn.Write(pkg)\n\t}\n}\n```\n\n但是，[Netpoll][Netpoll] 不需要这样做，nocopy APIs 支持对 buffer 进行原地址操作（原地址组包和分包），并通过引用计数实现资源的自动回收和重用。\n\n示例如下（使用方法 `Reader.Slice` 和 `Writer.Append`）：\n\n```go\npackage main\n\nimport (\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc main() {\n\tvar conn netpoll.Connection\n\t\n\t// reading\n\treader := conn.Reader()\n\tfor {\n\t\t... unpacking & handling ...\n\t\tpkg, _ := reader.Slice(pkgsize)\n\t\tgo func() {\n\t\t\t... handling pkg ...\n\t\t\tpkg.Release()\n\t\t}\n\t}\n\t\n\t// writing\n\tvar write_datas <-chan netpoll.Writer\n\t... packing write ...\n\twriter := conn.Writer()\n\tfor {\n\t\tselect {\n\t\tcase pkg := <-write_datas:\n\t\t\twriter.Append(pkg)\n\t\tdefault:\n\t\t\tif writer.MallocLen() > 0 {\n\t\t\t\twriter.Flush()\n\t\t\t}\n\t\t}\n\t}\n}\n```\n\n# 常见用法\n\n## 1. 如何配置 poller 的数量 ？\n\n`NumLoops` 表示 [Netpoll][Netpoll] 创建的 `epoll` 的数量，默认已经根据P的数量自动调整(`runtime.GOMAXPROCS(0)`)，用户一般不需要关心。\n\n但是如果你的服务有大量的 I/O，你可能需要如下配置：\n\n```go\npackage main\n\nimport (\n\t\"runtime\"\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc init() {\n\tnetpoll.SetNumLoops(runtime.GOMAXPROCS(0))\n}\n```\n\n## 2. 如何配置 poller 的连接负载均衡 ？\n\n当 [Netpoll][Netpoll] 中有多个 poller 时，服务进程中的连接会负载均衡到每个 poller。\n\n现在支持以下策略：\n\n1. Random\n   * 新连接将分配给随机选择的轮询器。\n2. RoundRobin\n   * 新连接将按顺序分配给轮询器。\n     \n[Netpoll][Netpoll] 默认使用 `RoundRobin`，用户可以通过以下方式更改：\n     \n```go\npackage main\n\nimport (\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc init() {\n\tnetpoll.SetLoadBalance(netpoll.Random)\n\t\n\t// or\n\tnetpoll.SetLoadBalance(netpoll.RoundRobin)\n}\n```\n\n## 3. 如何配置 [gopool][gopool] ？\n\n[Netpoll][Netpoll] 默认使用 [gopool][gopool] 作为 goroutine 池来优化 `栈扩张` 问题（RPC 服务常见问题）。\n\n[gopool][gopool] 项目中已经详细解释了如何自定义配置，这里不再赘述。\n\n当然，如果你的项目没有 `栈扩张` 问题，建议最好关闭 [gopool][gopool]，关闭方式如下：\n\n```go\npackage main\n\nimport (\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc init() {\n\tnetpoll.DisableGopool()\n}\n```\n\n## 4. 如何初始化新的连接 ？\n\nClient 和 Server 端通过不同的方式初始化新连接。\n\n1. 在 Server 端，定义了 `OnPrepare` 来初始化新链接，同时支持返回一个 `context`，可以传递给后续的业务处理并复用。`WithOnPrepare` 提供方法注册。当 Server 接收新连接时，会自动执行注册的 `OnPrepare` 方法来完成准备工作。示例如下：\n\n```go\npackage main\n\nimport (\n\t\"context\"\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc main() {\n\t// register OnPrepare\n\tvar onPrepare netpoll.OnPrepare = prepare\n\tevl, _ := netpoll.NewEventLoop(handler, netpoll.WithOnPrepare(onPrepare))\n\t...\n}\n\nfunc prepare(connection netpoll.Connection) (ctx context.Context) {\n\t... prepare connection ...\n\treturn\n}\n```\n\n2. 在 Client 端，连接初始化需要由用户自行完成。 一般来说，`Dialer` 创建的新连接是可以由用户自行控制的，这与 Server 端被动接收连接不同。因此，用户不需要依赖触发器，可以自行初始化，如下所示：\n\n```go\npackage main\n\nimport (\n\t\"context\"\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc main() {\n\tconn, err := netpoll.DialConnection(network, address, timeout)\n\tif err != nil {\n\t\tpanic(\"dial netpoll connection failed\")\n\t}\n\t... prepare here directly ...\n\tprepare(conn)\n\t...\n}\n\nfunc prepare(connection netpoll.Connection) (ctx context.Context) {\n\t... prepare connection ...\n\treturn\n}\n```\n\n## 5. 如何配置连接超时 ？\n\n[Netpoll][Netpoll] 现在支持两种类型的超时配置：\n\n1. 读超时（`ReadTimeout`）\n   * 为了保持与 `net.Conn` 相同的操作风格，`Connection.Reader` 也被设计为阻塞读取。 所以提供了读取超时（`ReadTimeout`）。\n   * 读超时（`ReadTimeout`）没有默认值（默认无限等待），可以通过 `Connection` 或 `EventLoop.Option` 进行配置，例如：\n\n```go\npackage main\n\nimport (\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc main() {\n\tvar conn netpoll.Connection\n\t\n\t// 1. setting by Connection\n\tconn.SetReadTimeout(timeout)\n\t\n\t// or\n\t\n\t// 2. setting with Option\n\tnetpoll.NewEventLoop(handler, netpoll.WithReadTimeout(timeout))\n\t...\n}\n```\n\n2. 空闲超时（`IdleTimeout`）\n   * 空闲超时（`IdleTimeout`）利用 `TCP KeepAlive` 机制来踢出死连接并减少维护开销。使用 [Netpoll][Netpoll] 时，一般不需要频繁创建和关闭连接，所以通常来说，空闲连接影响不大。当连接长时间处于非活动状态时，为了防止出现假死、对端挂起、异常断开等造成的死连接，在空闲超时（`IdleTimeout`）后，netpoll 会主动关闭连接。\n   * 空闲超时（`IdleTimeout`）的默认配置为 `10min`，可以通过 `Connection` API 或 `EventLoop.Option` 进行配置，例如：\n\n```go\npackage main\n\nimport (\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc main() {\n\tvar conn netpoll.Connection\n\t\n\t// 1. setting by Connection\n\tconn.SetIdleTimeout(timeout)\n\t\n\t// or\n\t\n\t// 2. setting with Option\n\tnetpoll.NewEventLoop(handler, netpoll.WithIdleTimeout(timeout))\n\t...\n}\n```\n\n## 6. 如何配置连接的读事件回调 ？\n\n`OnRequest` 是指连接上发生读事件时 [Netpoll][Netpoll] 触发的回调。在 Server 端，在创建 `EventLoop` 时，可以注册一个`OnRequest`，在每次连接数据到达时触发，进行业务处理。Client端默认没有 `OnRequest`，需要时可以通过 API 设置。例如：\n\n```go\npackage main\n\nimport (\n\t\"context\"\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc main() {\n\tvar onRequest netpoll.OnRequest = handler\n\t\n\t// 1. on server side\n\tevl, _ := netpoll.NewEventLoop(onRequest, opts...)\n\t...\n\t\n\t// 2. on client side\n\tconn, _ := netpoll.DialConnection(network, address, timeout)\n\tconn.SetOnRequest(handler)\n\t...\n}\n\nfunc handler(ctx context.Context, connection netpoll.Connection) (err error) {\n\t... handling ...\n\treturn nil\n}\n```\n\n## 7. 如何配置连接的关闭回调 ？\n\n`CloseCallback` 是指连接关闭时 [Netpoll][Netpoll] 触发的回调，用于在连接关闭后进行额外的处理。\n[Netpoll][Netpoll] 能够感知连接状态。当连接被对端关闭或被自己清理时，会主动触发 `CloseCallback`，而不是由下一次调用 `Read` 或 `Write` 时返回错误（`net.Conn` 的方式）。\n`Connection` 提供了添加 `CloseCallback` 的 API，已经添加的回调无法删除，支持多个回调。\n\n```go\npackage main\n\nimport (\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc main() {\n\tvar conn netpoll.Connection\n\t\n\t// add close callback\n\tvar cb netpoll.CloseCallback = callback\n\tconn.AddCloseCallback(cb)\n\t...\n}\n\nfunc callback(connection netpoll.Connection) error {\n\treturn nil\n}\n```\n\n# 注意事项\n\n## 1. 错误设置 NumLoops\n\n如果你的服务器运行在物理机上，Go 进程创建的 P 个数就等于机器的 CPU 核心数。 但是 Server 可能不会使用这么多核心。在这种情况下，过多的 poller 会导致性能下降。\n\n这里提供了以下几种解决方案：\n\n1. 使用 `taskset` 命令来限制 CPU 个数，例如：\n\n```shell\ntaskset -c 0-3 $run_your_server\n```\n\n2. 主动设置 P 的个数，例如：\n\n```go\npackage main\n\nimport (\n\t\"runtime\"\n)\n\nfunc init() {\n\truntime.GOMAXPROCS(num_you_want)\n}\n```\n\n3. 主动设置 poller 的个数，例如：\n\n```go\npackage main\n\nimport (\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc init() {\n\tnetpoll.SetNumLoops(num_you_want)\n}\n```\n\n[Netpoll]: https://github.com/cloudwego/netpoll\n\n[net]: https://github.com/golang/go/tree/master/src/net\n\n[gopool]: https://github.com/bytedance/gopkg/tree/develop/util/gopool\n\n[Examples]: https://github.com/cloudwego/netpoll-examples\n\n[server-example]: https://github.com/cloudwego/netpoll-examples/blob/main/server.go\n\n[client-example]: https://github.com/cloudwego/netpoll-examples/blob/main/client.go\n\n[netpoll.go]: https://github.com/cloudwego/netpoll/blob/main/netpoll.go\n\n[netpoll_options.go]: https://github.com/cloudwego/netpoll/blob/main/netpoll_options.go\n\n[nocopy.go]: https://github.com/cloudwego/netpoll/blob/main/nocopy.go\n"
  },
  {
    "path": "docs/guide/guide_en.md",
    "content": "# Tutorial\n\nThis tutorial gets you started with [Netpoll][Netpoll] through some simple [examples][Examples], includes how to\nuse [Server](#1-use-sever), [Client](#2-use-dialer) and [nocopy APIs](#3-use-nocopy-api).\n\n## 1. Use Server\n\n[Here][server-example] is a simple server demo, we will explain how it is constructed next.\n\n### 1.1 Create Listener\n\nFirst we need to get a `Listener`, it can be `net.Listener` or `netpoll.Listener`, which is no difference for server\nusage. Create a `Listener` as shown below:\n\n```go\npackage main\n\nimport \"net\"\n\nfunc main() {\n\tlistener, err := net.Listen(network, address)\n\tif err != nil {\n\t\tpanic(\"create net listener failed\")\n\t}\n\t...\n}\n```\n\nor\n\n```go\npackage main\n\nimport \"github.com/cloudwego/netpoll\"\n\nfunc main() {\n\tlistener, err := netpoll.CreateListener(network, address)\n\tif err != nil {\n\t\tpanic(\"create netpoll listener failed\")\n\t}\n\t...\n}\n```\n\n### 1.2 New EventLoop\n\n`EventLoop` is an event-driven scheduler, a real NIO Server, responsible for connection management, event scheduling,\netc.\n\nparams:\n\n* `OnRequest` is an interface that users should implement by themselves to process business\n  logic. [Code Comment][netpoll.go] describes its behavior in detail.\n* `Option` is used to customize the configuration when creating `EventLoop`, and the following example shows its usage.\n  For more details, please refer to [options][netpoll_options.go].\n\nThe creation process is as follows:\n\n```go\npackage main\n\nimport (\n\t\"time\"\n\t\"github.com/cloudwego/netpoll\"\n)\n\nvar eventLoop netpoll.EventLoop\n\nfunc main() {\n\t...\n\teventLoop, _ := netpoll.NewEventLoop(\n\t\thandle,\n\t\tnetpoll.WithOnPrepare(prepare),\n\t\tnetpoll.WithReadTimeout(time.Second),\n\t)\n\t...\n}\n```\n\n### 1.3 Run Server\n\n`EventLoop` provides services by binding `Listener`, as shown below.\n`Serve` function will block until an error occurs, such as a panic or the user actively calls `Shutdown`.\n\n```go\npackage main\n\nimport (\n\t\"github.com/cloudwego/netpoll\"\n)\n\nvar eventLoop netpoll.EventLoop\n\nfunc main() {\n\t...\n\t// start listen loop ...\n\teventLoop.Serve(listener)\n}\n```\n\n### 1.4 Shutdown Server\n\n`EventLoop` provides the `Shutdown` function, which is used to stop the server gracefully. The usage is as follows.\n\n```go\npackage main\n\nimport (\n\t\"context\"\n\t\"time\"\n\t\"github.com/cloudwego/netpoll\"\n)\n\nvar eventLoop netpoll.EventLoop\n\nfunc main() {\n\t// stop server ...\n\tctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)\n\tdefer cancel()\n\teventLoop.Shutdown(ctx)\n}\n```\n\n## 2. Use Dialer\n\n[Netpoll][Netpoll] also has the ability to be used on the Client side. It provides `Dialer`, similar to `net.Dialer`.\nAgain, [here][client-example] is a simple client demo, and then we introduce it in detail.\n\n### 2.1 The Fast Way\n\nSimilar to [Net][net], [Netpoll][Netpoll] provides several public functions for directly dialing a connection. such as:\n\n```go\nDialConnection(network, address string, timeout time.Duration) (connection Connection, err error)\n\nDialTCP(ctx context.Context, network string, laddr, raddr *TCPAddr) (*TCPConnection, error)\n\nDialUnix(network string, laddr, raddr *UnixAddr) (*UnixConnection, error)\n```\n\n### 2.2 Create Dialer\n\n[Netpoll][Netpoll] also defines the `Dialer` interface. The usage is as follows:\n(of course, you can usually use the fast way)\n\n```go\npackage main\n\nimport (\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc main() {\n\t// Dial a connection with Dialer.\n\tdialer := netpoll.NewDialer()\n\tconn, err := dialer.DialConnection(network, address, timeout)\n\tif err != nil {\n\t\tpanic(\"dial netpoll connection failed\")\n\t}\n\t...\n}\n```\n\n## 3. Use Nocopy API\n\n`Connection` provides Nocopy APIs - `Reader` and `Writer`, to avoid frequent copying. Let’s introduce their simple\nusage.\n\n```go\npackage main\n\ntype Connection interface {\n\t// Recommended nocopy APIs\n\tReader() Reader\n\tWriter() Writer\n\t... // see code comments for more details\n}\n```\n\n### 3.1 Simple Usage\n\nNocopy APIs is designed as a two-step operation.\n\nOn `Reader`, after reading data through `Next`, `Peek`, `ReadString`, etc., you still have to actively call `Release` to\nrelease the buffer(`Nocopy` reads the original address of the buffer, so you must take the initiative to confirm that\nthe buffer is no longer used).\n\nSimilarly, on `Writer`, you first need to allocate a buffer to write data, and then call `Flush` to confirm that all\ndata has been written.\n`Writer` also provides rich APIs to allocate buffers, such as `Malloc`, `WriteString` and so on.\n\nThe following shows some simple examples of reading and writing data. For more details, please refer to\nthe [code comments][nocopy.go].\n\n```go\npackage main\n\nimport (\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc main() {\n\tvar conn netpoll.Connection\n\tvar reader, writer = conn.Reader(), conn.Writer()\n\t\n\t// reading\n\tbuf, _ := reader.Next(n)\n\t... parse the read data ...\n\treader.Release()\n\t\n\t// writing\n\tvar write_data []byte\n\t... make the write data ...\n\talloc, _ := writer.Malloc(len(write_data))\n\tcopy(alloc, write_data) // write data\n\twriter.Flush()\n}\n```\n\n### 3.2 Advanced Usage\n\nIf you want to use the connection to send (or receive) multiple sets of data, then you will face the work of packing and\nunpacking the data.\n\nOn [net][net], this kind of work is generally done by copying. An example is as follows:\n\n```go\npackage main\n\nimport (\n\t\"net\"\n)\n\nfunc main() {\n\tvar conn net.Conn\n\tvar buf = make([]byte, 8192)\n\t\n\t// reading\n\tfor {\n\t\tn, _ := conn.Read(buf)\n\t\t... unpacking & handling ...\n\t\tvar i int\n\t\tfor i = 0; i <= n-pkgsize; i += pkgsize {\n\t\t\tpkg := append([]byte{}, buf[i:i+pkgsize]...)\n\t\t\tgo func() {\n\t\t\t\t... handling pkg ...\n\t\t\t}\n\t\t}\n\t\tbuf = append(buf[:0], buf[i:n]...)\n\t}\n\t\n\t// writing\n\tvar write_datas <-chan []byte\n\t... packing write ...\n\tfor {\n\t\tpkg := <-write_datas\n\t\tconn.Write(pkg)\n\t}\n}\n```\n\nBut, this is not necessary in [Netpoll][Netpoll], nocopy APIs supports operations on the original address of the buffer,\nand realizes automatic recycling and reuse of resources through reference counting.\n\nExamples are as follows(use function `Reader.Slice` and `Writer.Append`):\n\n```go\npackage main\n\nimport (\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc main() {\n\tvar conn netpoll.Connection\n\t\n\t// reading\n\treader := conn.Reader()\n\tfor {\n\t\t... unpacking & handling ...\n\t\tpkg, _ := reader.Slice(pkgsize)\n\t\tgo func() {\n\t\t\t... handling pkg ...\n\t\t\tpkg.Release()\n\t\t}\n\t}\n\t\n\t// writing\n\tvar write_datas <-chan netpoll.Writer\n\t... packing write ...\n\twriter := conn.Writer()\n\tfor {\n\t\tselect {\n\t\tcase pkg := <-write_datas:\n\t\t\twriter.Append(pkg)\n\t\tdefault:\n\t\t\tif writer.MallocLen() > 0 {\n\t\t\t\twriter.Flush()\n\t\t\t}\n\t\t}\n\t}\n}\n```\n\n# How To\n\n## 1. How to configure the number of pollers ?\n\n`NumLoops` represents the number of `epoll` created by [Netpoll][Netpoll], which has been automatically adjusted\naccording to the number of P (`runtime.GOMAXPROCS(0)`) by default, and users generally don't need to care.\n\nBut if your service has heavy I/O, you may need the following configuration:\n\n```go\npackage main\n\nimport (\n\t\"runtime\"\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc init() {\n\tnetpoll.SetNumLoops(runtime.GOMAXPROCS(0))\n}\n```\n\n## 2. How to configure poller's connection loadbalance ?\n\nWhen there are multiple pollers in [Netpoll][Netpoll], the connections in the service process will be loadbalanced to\neach poller.\n\nThe following strategies are supported now:\n\n1. Random\n    * The new connection will be assigned to a randomly picked poller.\n2. RoundRobin\n    * The new connection will be assigned to the poller in order.\n\n[Netpoll][Netpoll] uses `RoundRobin` by default, and users can change it in the following ways:\n\n```go\npackage main\n\nimport (\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc init() {\n\tnetpoll.SetLoadBalance(netpoll.Random)\n\t\n\t// or\n\tnetpoll.SetLoadBalance(netpoll.RoundRobin)\n}\n```\n\n## 3. How to configure [gopool][gopool] ?\n\n[Netpoll][Netpoll] uses [gopool][gopool] as the goroutine pool by default to optimize the `stack growth` problem that\ngenerally occurs in RPC services.\n\nIn the project [gopool][gopool], it explains how to change its configuration, so won't repeat it here.\n\nOf course, if your project does not have a `stack growth` problem, it is best to close [gopool][gopool] as follows:\n\n```go\npackage main\n\nimport (\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc init() {\n\tnetpoll.DisableGopool()\n}\n```\n\n## 4. How to prepare a new connection ?\n\nThere are different ways to prepare a new connection on the client and server.\n\n1. On the server side, `OnPrepare` is defined to prepare for the new connection, and it also supports returning\n   a `context`, which can be reused in subsequent business processing.\n   `WithOnPrepare` provides this registration. When the server accepts a new connection, it will automatically execute\n   the registered `OnPrepare` function to complete the preparation work. The example is as follows:\n\n```go\npackage main\n\nimport (\n\t\"context\"\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc main() {\n\t// register OnPrepare\n\tvar onPrepare netpoll.OnPrepare = prepare\n\tevl, _ := netpoll.NewEventLoop(handler, netpoll.WithOnPrepare(onPrepare))\n\t...\n}\n\nfunc prepare(connection netpoll.Connection) (ctx context.Context) {\n\t... prepare connection ...\n\treturn\n}\n```\n\n2. On the client side, the connection preparation needs to be completed by the user. Generally speaking, the connection\n   created by `Dialer` can be controlled by the user, which is different from passively accepting the connection on the\n   server side. Therefore, the user not relying on the trigger, just prepare a new connection like this:\n\n```go\npackage main\n\nimport (\n\t\"context\"\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc main() {\n\tconn, err := netpoll.DialConnection(network, address, timeout)\n\tif err != nil {\n\t\tpanic(\"dial netpoll connection failed\")\n\t}\n\t... prepare here directly ...\n\tprepare(conn)\n\t...\n}\n\nfunc prepare(connection netpoll.Connection) (ctx context.Context) {\n\t... prepare connection ...\n\treturn\n}\n```\n\n## 5. How to configure connection timeout ?\n\n[Netpoll][Netpoll] now supports two timeout configurations:\n\n1. `Read Timeout`\n    * In order to maintain the same operating style as `net.Conn`, `Connection.Reader` is also designed to block\n      reading. So provide `Read Timeout`.\n    * `Read Timeout` has no default value(wait infinitely), it can be configured via `Connection` or `EventLoop.Option`,\n      for example:\n\n```go\npackage main\n\nimport (\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc main() {\n\tvar conn netpoll.Connection\n\t\n\t// 1. setting by Connection\n\tconn.SetReadTimeout(timeout)\n\t\n\t// or\n\t\n\t// 2. setting with Option\n\tnetpoll.NewEventLoop(handler, netpoll.WithReadTimeout(timeout))\n\t...\n}\n```\n\n2. `Idle Timeout`\n    * `Idle Timeout` utilizes the `TCP KeepAlive` mechanism to kick out dead connections and reduce maintenance\n      overhead. When using [Netpoll][Netpoll], there is generally no need to create and close connections frequently,\n      and idle connections have little effect. When the connection is inactive for a long time, in order to prevent dead\n      connection caused by suspended animation, hang of the opposite end, abnormal disconnection, etc., the connection\n      will be actively closed after the `Idle Timeout`.\n    * The default minimum value of `Idle Timeout` is `10min`, which can be configured through `Connection` API\n      or `EventLoop.Option`, for example:\n\n```go\npackage main\n\nimport (\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc main() {\n\tvar conn netpoll.Connection\n\t\n\t// 1. setting by Connection\n\tconn.SetIdleTimeout(timeout)\n\t\n\t// or\n\t\n\t// 2. setting with Option\n\tnetpoll.NewEventLoop(handler, netpoll.WithIdleTimeout(timeout))\n\t...\n}\n```\n\n## 6. How to configure connection read event callback ?\n\n`OnRequest` refers to the callback triggered by [Netpoll][Netpoll] when a read event occurs on the connection. On the\nServer side, when creating the `EventLoop`, you can register an `OnRequest`, which will be triggered when each\nconnection data arrives and perform business processing. On the Client side, there is no `OnRequest` by default, and it\ncan be set via API when needed. E.g:\n\n```go\npackage main\n\nimport (\n\t\"context\"\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc main() {\n\tvar onRequest netpoll.OnRequest = handler\n\t\n\t// 1. on server side\n\tevl, _ := netpoll.NewEventLoop(onRequest, opts...)\n\t...\n\t\n\t// 2. on client side\n\tconn, _ := netpoll.DialConnection(network, address, timeout)\n\tconn.SetOnRequest(handler)\n\t...\n}\n\nfunc handler(ctx context.Context, connection netpoll.Connection) (err error) {\n\t... handling ...\n\treturn nil\n}\n```\n\n## 7. How to configure the connection close callback ?\n\n`CloseCallback` refers to the callback triggered by [Netpoll][Netpoll] when the connection is closed, which is used to\nperform additional processing after the connection is closed.\n[Netpoll][Netpoll] is able to perceive the connection status. When the connection is closed by peer or cleaned up by\nself, it will actively trigger `CloseCallback` instead of returning an error on the next `Read` or `Write`(the way\nof `net.Conn`).\n`Connection` provides API for adding `CloseCallback`, callbacks that have been added cannot be removed, and multiple\ncallbacks are supported.\n\n```go\npackage main\n\nimport (\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc main() {\n\tvar conn netpoll.Connection\n\t\n\t// add close callback\n\tvar cb netpoll.CloseCallback = callback\n\tconn.AddCloseCallback(cb)\n\t...\n}\n\nfunc callback(connection netpoll.Connection) error {\n\treturn nil\n}\n```\n\n# Attention\n\n## 1. Wrong setting of NumLoops\n\nIf your server is running on a physical machine, the number of P created by the Go process is equal to the number of\nCPUs of the machine. But the server may not use so many cores. In this case, too many pollers will cause performance\ndegradation.\n\nThere are several solutions:\n\n1. Use the `taskset` command to limit CPU usage, such as:\n\n```shell\ntaskset -c 0-3 $run_your_server\n```\n\n2. Actively set the number of P, for instance:\n\n```go\npackage main\n\nimport (\n\t\"runtime\"\n)\n\nfunc init() {\n\truntime.GOMAXPROCS(num_you_want)\n}\n```\n\n3. Actively set the number of pollers, e.g:\n\n```go\npackage main\n\nimport (\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc init() {\n\tnetpoll.SetNumLoops(num_you_want)\n}\n```\n\n[Netpoll]: https://github.com/cloudwego/netpoll\n\n[net]: https://github.com/golang/go/tree/master/src/net\n\n[gopool]: https://github.com/bytedance/gopkg/tree/develop/util/gopool\n\n[Examples]: https://github.com/cloudwego/netpoll-examples\n\n[server-example]: https://github.com/cloudwego/netpoll-examples/blob/main/server.go\n\n[client-example]: https://github.com/cloudwego/netpoll-examples/blob/main/client.go\n\n[netpoll.go]: https://github.com/cloudwego/netpoll/blob/main/netpoll.go\n\n[netpoll_options.go]: https://github.com/cloudwego/netpoll/blob/main/netpoll_options.go\n\n[nocopy.go]: https://github.com/cloudwego/netpoll/blob/main/nocopy.go\n"
  },
  {
    "path": "docs/reference/design_cn.md",
    "content": "# TODO"
  },
  {
    "path": "docs/reference/design_en.md",
    "content": "# TODO"
  },
  {
    "path": "docs/reference/explain.md",
    "content": "# DATA RACE EXPLAIN\n`Netpoll` declare different files by `//+build !race` and `//+build race` to avoid `DATA RACE` detection in some code.\n\nThe reason is that the `epoll` uses `unsafe.Pointer` to access the struct pointer, in order\n to improve performance. This operation is beyond the detection range of the `race detector`,\n so it is mistaken for data race, but not code bug actually.\n"
  },
  {
    "path": "eventloop.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage netpoll\n\nimport (\n\t\"context\"\n\t\"net\"\n)\n\n// A EventLoop is a network server.\ntype EventLoop interface {\n\t// Serve registers a listener and runs blockingly to provide services, including listening to ports,\n\t// accepting connections and processing trans data. When an exception occurs or Shutdown is invoked,\n\t// Serve will return an error which describes the specific reason.\n\tServe(ln net.Listener) error\n\n\t// Shutdown is used to graceful exit.\n\t// It will close all idle connections on the server, but will not change the underlying pollers.\n\t//\n\t// Argument: ctx set the waiting deadline, after which an error will be returned,\n\t// but will not force the closing of connections in progress.\n\tShutdown(ctx context.Context) error\n}\n\n/* The Connection Callback Sequence Diagram\n| Connection State                     | Callback Function | Notes\n|   Connected but not initialized      |    OnPrepare      | Conn is not registered into poller\n|   Connected and initialized          |    OnConnect      | Conn is ready for read or write\n|   Read first byte                    |    OnRequest      | Conn is ready for read or write\n|   Peer closed but conn is active     |    OnDisconnect   | Conn access will race with OnRequest function\n|   Self closed and conn is closed     |    CloseCallback  | Conn is destroyed\n\nExecution Order:\n  OnPrepare => OnConnect => OnRequest      => CloseCallback\n                            OnDisconnect\nNote: only OnRequest and OnDisconnect will be executed in parallel\n*/\n\n// OnPrepare is used to inject custom preparation at connection initialization,\n// which is optional but important in some scenarios. For example, a qps limiter\n// can be set by closing overloaded connections directly in OnPrepare.\n//\n// Return:\n// context will become the argument of OnRequest.\n// Usually, custom resources can be initialized in OnPrepare and used in OnRequest.\n//\n// PLEASE NOTE:\n// OnPrepare is executed without any data in the connection,\n// so Reader() or Writer() cannot be used here, but may be supported in the future.\ntype OnPrepare func(connection Connection) context.Context\n\n// OnConnect is called once connection created.\n// It supports read/write/close connection, and could return a ctx which will be passed to OnRequest.\n// OnConnect will not block the poller since it's executed asynchronously.\n// Only after OnConnect finished the OnRequest could be executed.\n//\n// An example usage in TCP Proxy scenario:\n//\n//\tfunc onConnect(ctx context.Context, upstream netpoll.Connection) context.Context {\n//\t\tdownstream, _ := netpoll.DialConnection(\"tcp\", downstreamAddr, time.Second)\n//\t\treturn context.WithValue(ctx, downstreamKey, downstream)\n//\t}\n//\n//\tfunc onRequest(ctx context.Context, upstream netpoll.Connection) error {\n//\t\tdownstream := ctx.Value(downstreamKey).(netpoll.Connection)\n//\t}\ntype OnConnect func(ctx context.Context, connection Connection) context.Context\n\n// OnDisconnect is called once connection is going to be closed.\n// OnDisconnect must return as quick as possible because it will block poller.\n// OnDisconnect is different from CloseCallback, you could check with \"The Connection Callback Sequence Diagram\" section.\ntype OnDisconnect func(ctx context.Context, connection Connection)\n\n// OnRequest defines the function for handling connection. When data is sent from the connection peer,\n// netpoll actively reads the data in LT mode and places it in the connection's input buffer.\n// Generally, OnRequest starts handling the data in the following way:\n//\n//\tfunc OnRequest(ctx context, connection Connection) error {\n//\t\tinput := connection.Reader().Next(n)\n//\t\thandling input data...\n//\t\tsend, _ := connection.Writer().Malloc(l)\n//\t\tcopy(send, output)\n//\t\tconnection.Flush()\n//\t\treturn nil\n//\t}\n//\n// OnRequest will run in a separate goroutine and\n// it is guaranteed that there is one and only one OnRequest running at the same time.\n// The underlying logic is similar to:\n//\n//\tgo func() {\n//\t\tfor !connection.Reader().IsEmpty() {\n//\t\t\tOnRequest(ctx, connection)\n//\t\t}\n//\t}()\n//\n// PLEASE NOTE:\n// OnRequest must either eventually read all the input data or actively Close the connection,\n// otherwise the goroutine will fall into a dead loop.\n//\n// Return: error is unused which will be ignored directly.\ntype OnRequest func(ctx context.Context, connection Connection) error\n"
  },
  {
    "path": "fd_operator.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage netpoll\n\nimport (\n\t\"runtime\"\n\t\"sync/atomic\"\n)\n\n// FDOperator is a collection of operations on file descriptors.\ntype FDOperator struct {\n\t// FD is file descriptor, poll will bind when register.\n\tFD int\n\n\t// The FDOperator provides three operations of reading, writing, and hanging.\n\t// The poll actively fire the FDOperator when fd changes, no check the return value of FDOperator.\n\tOnRead  func(p Poll) error\n\tOnWrite func(p Poll) error\n\tOnHup   func(p Poll) error\n\n\t// The following is the required fn, which must exist when used, or directly panic.\n\t// Fns are only called by the poll when handles connection events.\n\tInputs   func(vs [][]byte) (rs [][]byte)\n\tInputAck func(n int) (err error)\n\n\t// Outputs will locked if len(rs) > 0, which need unlocked by OutputAck.\n\t// supportZeroCopy is not implemented, and it will be ignored\n\tOutputs   func(vs [][]byte) (rs [][]byte, supportZeroCopy bool)\n\tOutputAck func(n int) (err error)\n\n\t// poll is the registered location of the file descriptor.\n\tpoll Poll\n\n\t// protect only detach once\n\tdetached int32\n\n\t// private, used by operatorCache\n\tnext  *FDOperator\n\tstate int32 // CAS: 0(unused) 1(inuse) 2(do-done)\n\tindex int32 // index in operatorCache\n}\n\nfunc (op *FDOperator) Control(event PollEvent) error {\n\tif event == PollDetach && atomic.AddInt32(&op.detached, 1) > 1 {\n\t\treturn nil\n\t}\n\treturn op.poll.Control(op, event)\n}\n\nfunc (op *FDOperator) Free() {\n\top.poll.Free(op)\n}\n\nfunc (op *FDOperator) do() (can bool) {\n\treturn atomic.CompareAndSwapInt32(&op.state, 1, 2)\n}\n\nfunc (op *FDOperator) done() {\n\tatomic.StoreInt32(&op.state, 1)\n}\n\nfunc (op *FDOperator) inuse() {\n\tfor !atomic.CompareAndSwapInt32(&op.state, 0, 1) {\n\t\tif atomic.LoadInt32(&op.state) == 1 {\n\t\t\treturn\n\t\t}\n\t\truntime.Gosched()\n\t}\n}\n\nfunc (op *FDOperator) unused() {\n\tfor !atomic.CompareAndSwapInt32(&op.state, 1, 0) {\n\t\tif atomic.LoadInt32(&op.state) == 0 {\n\t\t\treturn\n\t\t}\n\t\truntime.Gosched()\n\t}\n}\n\nfunc (op *FDOperator) isUnused() bool {\n\treturn atomic.LoadInt32(&op.state) == 0\n}\n\nfunc (op *FDOperator) reset() {\n\top.FD = 0\n\top.OnRead, op.OnWrite, op.OnHup = nil, nil, nil\n\top.Inputs, op.InputAck = nil, nil\n\top.Outputs, op.OutputAck = nil, nil\n\top.poll = nil\n\top.detached = 0\n}\n"
  },
  {
    "path": "fd_operator_cache.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage netpoll\n\nimport (\n\t\"runtime\"\n\t\"sync/atomic\"\n\t\"unsafe\"\n)\n\nfunc newOperatorCache() *operatorCache {\n\treturn &operatorCache{\n\t\tcache:    make([]*FDOperator, 0, 1024),\n\t\tfreelist: make([]int32, 0, 1024),\n\t}\n}\n\ntype operatorCache struct {\n\tfirst  *FDOperator\n\tcache  []*FDOperator\n\tlocked int32\n\t// freelist store the freeable operator\n\t// to reduce GC pressure, we only store op index here\n\tfreelocked int32\n\tfreelist   []int32\n}\n\nfunc (c *operatorCache) alloc() *FDOperator {\n\tlock(&c.locked)\n\tif c.first == nil {\n\t\tconst opSize = unsafe.Sizeof(FDOperator{})\n\t\tn := block4k / opSize\n\t\tif n == 0 {\n\t\t\tn = 1\n\t\t}\n\t\tindex := int32(len(c.cache))\n\t\tfor i := uintptr(0); i < n; i++ {\n\t\t\tpd := &FDOperator{index: index}\n\t\t\tc.cache = append(c.cache, pd)\n\t\t\tpd.next = c.first\n\t\t\tc.first = pd\n\t\t\tindex++\n\t\t}\n\t}\n\top := c.first\n\tc.first = op.next\n\tunlock(&c.locked)\n\treturn op\n}\n\n// freeable mark the operator that could be freed\n// only poller could do the real free action\nfunc (c *operatorCache) freeable(op *FDOperator) {\n\t// reset all state\n\top.unused()\n\top.reset()\n\tlock(&c.freelocked)\n\tc.freelist = append(c.freelist, op.index)\n\tunlock(&c.freelocked)\n}\n\nfunc (c *operatorCache) free() {\n\tlock(&c.freelocked)\n\tdefer unlock(&c.freelocked)\n\tif len(c.freelist) == 0 {\n\t\treturn\n\t}\n\n\tlock(&c.locked)\n\tfor _, idx := range c.freelist {\n\t\top := c.cache[idx]\n\t\top.next = c.first\n\t\tc.first = op\n\t}\n\tc.freelist = c.freelist[:0]\n\tunlock(&c.locked)\n}\n\nfunc lock(locked *int32) {\n\tfor !atomic.CompareAndSwapInt32(locked, 0, 1) {\n\t\truntime.Gosched()\n\t}\n}\n\nfunc unlock(locked *int32) {\n\tatomic.StoreInt32(locked, 0)\n}\n"
  },
  {
    "path": "fd_operator_cache_test.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n// +build !windows\n\npackage netpoll\n\nimport (\n\t\"runtime\"\n\t\"testing\"\n)\n\n// go test -v -gcflags=-d=checkptr -run=TestPersistFDOperator\nfunc TestPersistFDOperator(t *testing.T) {\n\topcache := newOperatorCache()\n\t// init\n\tsize := 2048\n\tops := make([]*FDOperator, size)\n\tfor i := 0; i < size; i++ {\n\t\top := opcache.alloc()\n\t\top.FD = i\n\t\tops[i] = op\n\t}\n\tEqual(t, len(opcache.freelist), 0)\n\t// gc\n\tfor i := 0; i < 4; i++ {\n\t\truntime.GC()\n\t}\n\t// check alloc\n\tfor i := range ops {\n\t\tEqual(t, ops[i].FD, i)\n\t\topcache.freeable(ops[i])\n\t\tEqual(t, len(opcache.freelist), i+1)\n\t}\n\tEqual(t, len(opcache.freelist), size)\n\topcache.free()\n\tEqual(t, len(opcache.freelist), 0)\n\tAssert(t, len(opcache.cache) >= size)\n}\n\nfunc BenchmarkPersistFDOperator1(b *testing.B) {\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\topcache := newOperatorCache()\n\tfor i := 0; i < b.N; i++ {\n\t\top := opcache.alloc()\n\t\topcache.freeable(op)\n\t\topcache.free()\n\t}\n}\n\nfunc BenchmarkPersistFDOperator2(b *testing.B) {\n\t// benchmark\n\tb.ReportAllocs()\n\tb.SetParallelism(128)\n\tb.ResetTimer()\n\topcache := newOperatorCache()\n\tb.RunParallel(func(pb *testing.PB) {\n\t\tfor pb.Next() {\n\t\t\top := opcache.alloc()\n\t\t\topcache.freeable(op)\n\t\t\topcache.free()\n\t\t}\n\t})\n}\n"
  },
  {
    "path": "go.mod",
    "content": "module github.com/cloudwego/netpoll\n\ngo 1.15\n\nrequire (\n\tgithub.com/bytedance/gopkg v0.1.1\n\tgithub.com/cloudwego/gopkg v0.1.4\n\tgolang.org/x/sys v0.19.0\n)\n"
  },
  {
    "path": "go.sum",
    "content": "github.com/bytedance/gopkg v0.1.1 h1:3azzgSkiaw79u24a+w9arfH8OfnQQ4MHUt9lJFREEaE=\ngithub.com/bytedance/gopkg v0.1.1/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM=\ngithub.com/cloudwego/gopkg v0.1.4 h1:EoQiCG4sTonTPHxOGE0VlQs+sQR+Hsi2uN0qqwu8O50=\ngithub.com/cloudwego/gopkg v0.1.4/go.mod h1:FQuXsRWRsSqJLsMVd5SYzp8/Z1y5gXKnVvRrWUOsCMI=\ngithub.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=\ngithub.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=\ngithub.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=\ngithub.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=\ngithub.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=\ngithub.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=\ngithub.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=\ngithub.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=\ngithub.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=\ngithub.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=\ngithub.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=\ngithub.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=\ngithub.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=\ngithub.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=\ngithub.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=\ngolang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=\ngolang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=\ngolang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=\ngolang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=\ngolang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=\ngolang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=\ngolang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=\ngolang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=\ngolang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=\ngolang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=\ngolang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=\ngolang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=\ngolang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=\ngolang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=\ngolang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=\ngolang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=\ngolang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=\ngolang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=\ngolang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=\ngolang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=\ngolang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=\ngolang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=\ngolang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=\ngolang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=\ngolang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=\ngolang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=\ngolang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=\ngolang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=\ngolang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=\ngopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\ngopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=\ngopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\n"
  },
  {
    "path": "internal/runner/runner.go",
    "content": "/*\n * Copyright 2025 CloudWeGo Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage runner\n\nimport (\n\t\"context\"\n\t\"os\"\n\t\"strconv\"\n\n\tbgopool \"github.com/bytedance/gopkg/util/gopool\"\n\tcgopool \"github.com/cloudwego/gopkg/concurrency/gopool\"\n)\n\n// RunTask runs the `f` in background, and `ctx` is optional.\n// `ctx` is used to pass to underlying implementation\nvar RunTask func(ctx context.Context, f func())\n\nfunc goRunTask(ctx context.Context, f func()) {\n\tgo f()\n}\n\nfunc init() {\n\t// netpoll uses github.com/bytedance/gopkg/util/gopool by default\n\t// if the env is set, change it to cloudwego/gopkg\n\t// for most users, using the 'go' keyword directly is more suitable.\n\tif yes, _ := strconv.ParseBool(os.Getenv(\"USE_CLOUDWEGO_GOPOOL\")); yes {\n\t\tRunTask = cgopool.CtxGo\n\t} else {\n\t\tRunTask = bgopool.CtxGo\n\t}\n}\n\n// UseGoRunTask updates RunTask with goRunTask which creates\n// a new goroutine for the given func, basically `go f()`\nfunc UseGoRunTask() {\n\tRunTask = goRunTask\n}\n\n// SetPanicHandler sets the panic handler for the global pool.\nfunc SetPanicHandler(f func(context.Context, interface{})) {\n\tbgopool.SetPanicHandler(f)\n\tcgopool.SetPanicHandler(f)\n}\n"
  },
  {
    "path": "internal/runner/runner_test.go",
    "content": "/*\n * Copyright 2025 CloudWeGo Authors\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *     http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\npackage runner\n\nimport (\n\t\"context\"\n\t\"sync\"\n\t\"testing\"\n)\n\nfunc TestRunTask(t *testing.T) {\n\tvar wg sync.WaitGroup\n\twg.Add(2)\n\tctx := context.Background()\n\tRunTask(ctx, func() {\n\t\twg.Done()\n\t})\n\tUseGoRunTask()\n\tRunTask(ctx, func() {\n\t\twg.Done()\n\t})\n\twg.Wait()\n}\n"
  },
  {
    "path": "lint.sh",
    "content": "#!/usr/bin/env bash\n\ngolangci-lint run\n"
  },
  {
    "path": "mux/mux_test.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n// +build !windows\n\npackage mux\n\nimport (\n\t\"testing\"\n)\n\nfunc MustNil(t *testing.T, val interface{}) {\n\tt.Helper()\n\tAssert(t, val == nil, val)\n\tif val != nil {\n\t\tt.Fatal(\"assertion nil failed, val=\", val)\n\t}\n}\n\nfunc MustTrue(t *testing.T, cond bool) {\n\tt.Helper()\n\tif !cond {\n\t\tt.Fatal(\"assertion true failed.\")\n\t}\n}\n\nfunc Equal(t *testing.T, got, expect interface{}) {\n\tt.Helper()\n\tif got != expect {\n\t\tt.Fatalf(\"assertion equal failed, got=[%v], expect=[%v]\", got, expect)\n\t}\n}\n\nfunc Assert(t *testing.T, cond bool, val ...interface{}) {\n\tt.Helper()\n\tif !cond {\n\t\tif len(val) > 0 {\n\t\t\tval = append([]interface{}{\"assertion failed:\"}, val...)\n\t\t\tt.Fatal(val...)\n\t\t} else {\n\t\t\tt.Fatal(\"assertion failed\")\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "mux/shard_queue.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage mux\n\nimport (\n\t\"fmt\"\n\t\"runtime\"\n\t\"sync\"\n\t\"sync/atomic\"\n\n\t\"github.com/cloudwego/netpoll\"\n\t\"github.com/cloudwego/netpoll/internal/runner\"\n)\n\n/* DOC:\n * ShardQueue uses the netpoll's nocopy API to merge and send data.\n * The Data Flush is passively triggered by ShardQueue.Add and does not require user operations.\n * If there is an error in the data transmission, the connection will be closed.\n *\n * ShardQueue.Add: add the data to be sent.\n * NewShardQueue: create a queue with netpoll.Connection.\n * ShardSize: the recommended number of shards is 32.\n */\nvar ShardSize int\n\nfunc init() {\n\tShardSize = runtime.GOMAXPROCS(0)\n}\n\n// NewShardQueue .\nfunc NewShardQueue(size int, conn netpoll.Connection) (queue *ShardQueue) {\n\tqueue = &ShardQueue{\n\t\tconn:    conn,\n\t\tsize:    int32(size),\n\t\tgetters: make([][]WriterGetter, size),\n\t\tswap:    make([]WriterGetter, 0, 64),\n\t\tlocks:   make([]int32, size),\n\t}\n\tfor i := range queue.getters {\n\t\tqueue.getters[i] = make([]WriterGetter, 0, 64)\n\t}\n\tqueue.list = make([]int32, size)\n\treturn queue\n}\n\n// WriterGetter is used to get a netpoll.Writer.\ntype WriterGetter func() (buf netpoll.Writer, isNil bool)\n\n// ShardQueue uses the netpoll's nocopy API to merge and send data.\n// The Data Flush is passively triggered by ShardQueue.Add and does not require user operations.\n// If there is an error in the data transmission, the connection will be closed.\n// ShardQueue.Add: add the data to be sent.\ntype ShardQueue struct {\n\tconn      netpoll.Connection\n\tidx, size int32\n\tgetters   [][]WriterGetter // len(getters) = size\n\tswap      []WriterGetter   // use for swap\n\tlocks     []int32          // len(locks) = size\n\tqueueTrigger\n}\n\nconst (\n\t// queueTrigger state\n\tactive  = 0\n\tclosing = 1\n\tclosed  = 2\n)\n\n// here for trigger\ntype queueTrigger struct {\n\ttrigger  int32\n\tstate    int32 // 0: active, 1: closing, 2: closed\n\trunNum   int32\n\tw, r     int32      // ptr of list\n\tlist     []int32    // record the triggered shard\n\tlistLock sync.Mutex // list total lock\n}\n\n// Add adds to q.getters[shard]\nfunc (q *ShardQueue) Add(gts ...WriterGetter) {\n\tif atomic.LoadInt32(&q.state) != active {\n\t\treturn\n\t}\n\tshard := atomic.AddInt32(&q.idx, 1) % q.size\n\tq.lock(shard)\n\ttrigger := len(q.getters[shard]) == 0\n\tq.getters[shard] = append(q.getters[shard], gts...)\n\tq.unlock(shard)\n\tif trigger {\n\t\tq.triggering(shard)\n\t}\n}\n\nfunc (q *ShardQueue) Close() error {\n\tif !atomic.CompareAndSwapInt32(&q.state, active, closing) {\n\t\treturn fmt.Errorf(\"shardQueue has been closed\")\n\t}\n\t// wait for all tasks finished\n\tfor atomic.LoadInt32(&q.state) != closed {\n\t\tif atomic.LoadInt32(&q.trigger) == 0 {\n\t\t\tatomic.StoreInt32(&q.state, closed)\n\t\t\treturn nil\n\t\t}\n\t\truntime.Gosched()\n\t}\n\treturn nil\n}\n\n// triggering shard.\nfunc (q *ShardQueue) triggering(shard int32) {\n\tq.listLock.Lock()\n\tq.w = (q.w + 1) % q.size\n\tq.list[q.w] = shard\n\tq.listLock.Unlock()\n\n\tif atomic.AddInt32(&q.trigger, 1) > 1 {\n\t\treturn\n\t}\n\tq.foreach()\n}\n\n// foreach swap r & w. It's not concurrency safe.\nfunc (q *ShardQueue) foreach() {\n\tif atomic.AddInt32(&q.runNum, 1) > 1 {\n\t\treturn\n\t}\n\trunner.RunTask(nil, func() {\n\t\tvar negNum int32 // is negative number of triggerNum\n\t\tfor triggerNum := atomic.LoadInt32(&q.trigger); triggerNum > 0; {\n\t\t\tq.r = (q.r + 1) % q.size\n\t\t\tshared := q.list[q.r]\n\n\t\t\t// lock & swap\n\t\t\tq.lock(shared)\n\t\t\ttmp := q.getters[shared]\n\t\t\tq.getters[shared] = q.swap[:0]\n\t\t\tq.swap = tmp\n\t\t\tq.unlock(shared)\n\n\t\t\t// deal\n\t\t\tq.deal(q.swap)\n\t\t\tnegNum--\n\t\t\tif triggerNum+negNum == 0 {\n\t\t\t\ttriggerNum = atomic.AddInt32(&q.trigger, negNum)\n\t\t\t\tnegNum = 0\n\t\t\t}\n\t\t}\n\t\tq.flush()\n\n\t\t// quit & check again\n\t\tatomic.StoreInt32(&q.runNum, 0)\n\t\tif atomic.LoadInt32(&q.trigger) > 0 {\n\t\t\tq.foreach()\n\t\t\treturn\n\t\t}\n\t\t// if state is closing, change it to closed\n\t\tatomic.CompareAndSwapInt32(&q.state, closing, closed)\n\t})\n}\n\n// deal is used to get deal of netpoll.Writer.\nfunc (q *ShardQueue) deal(gts []WriterGetter) {\n\tif !q.conn.IsActive() {\n\t\treturn\n\t}\n\twriter := q.conn.Writer()\n\tfor _, gt := range gts {\n\t\tbuf, isNil := gt()\n\t\tif !isNil {\n\t\t\terr := writer.Append(buf)\n\t\t\tif err != nil {\n\t\t\t\tq.conn.Close()\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n}\n\n// flush is used to flush netpoll.Writer.\nfunc (q *ShardQueue) flush() {\n\terr := q.conn.Writer().Flush()\n\tif err != nil {\n\t\tq.conn.Close()\n\t\treturn\n\t}\n}\n\n// lock shard.\nfunc (q *ShardQueue) lock(shard int32) {\n\tfor !atomic.CompareAndSwapInt32(&q.locks[shard], 0, 1) {\n\t\truntime.Gosched()\n\t}\n}\n\n// unlock shard.\nfunc (q *ShardQueue) unlock(shard int32) {\n\tatomic.StoreInt32(&q.locks[shard], 0)\n}\n"
  },
  {
    "path": "mux/shard_queue_test.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n// +build !windows\n\npackage mux\n\nimport (\n\t\"net\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/cloudwego/netpoll\"\n)\n\nfunc TestShardQueue(t *testing.T) {\n\tvar svrConn net.Conn\n\taccepted := make(chan struct{})\n\n\tnetwork, address := \"tcp\", \"localhost:12345\"\n\tln, err := net.Listen(\"tcp\", address)\n\tMustNil(t, err)\n\tstop := make(chan int, 1)\n\tdefer close(stop)\n\tgo func() {\n\t\tvar err error\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-stop:\n\t\t\t\terr = ln.Close()\n\t\t\t\tMustNil(t, err)\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\t\t\tsvrConn, err = ln.Accept()\n\t\t\tMustNil(t, err)\n\t\t\taccepted <- struct{}{}\n\t\t}\n\t}()\n\n\tconn, err := netpoll.DialConnection(network, address, time.Second)\n\tMustNil(t, err)\n\t<-accepted\n\n\t// test\n\tqueue := NewShardQueue(4, conn)\n\tcount, pkgsize := 16, 11\n\tfor i := 0; i < count; i++ {\n\t\tvar getter WriterGetter = func() (buf netpoll.Writer, isNil bool) {\n\t\t\tbuf = netpoll.NewLinkBuffer(pkgsize)\n\t\t\tbuf.Malloc(pkgsize)\n\t\t\treturn buf, false\n\t\t}\n\t\tqueue.Add(getter)\n\t}\n\n\terr = queue.Close()\n\tMustNil(t, err)\n\ttotal := count * pkgsize\n\trecv := make([]byte, total)\n\trn, err := svrConn.Read(recv)\n\tMustNil(t, err)\n\tEqual(t, rn, total)\n}\n\n// TODO: need mock flush\nfunc BenchmarkShardQueue(b *testing.B) {\n\tb.Skip()\n}\n"
  },
  {
    "path": "net_dialer.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n// +build !windows\n\npackage netpoll\n\nimport (\n\t\"context\"\n\t\"net\"\n\t\"time\"\n)\n\n// DialConnection is a default implementation of Dialer.\nfunc DialConnection(network, address string, timeout time.Duration) (connection Connection, err error) {\n\treturn defaultDialer.DialConnection(network, address, timeout)\n}\n\n// NewFDConnection create a Connection initialized by any fd\n// It's useful for writing unit tests for functions that have args with the type of netpoll.Connection\n// The typical usage is like:\n//\n//\trfd, wfd := netpoll.GetSysFdPairs()\n//\trconn, _ = netpoll.NewFDConnection(rfd)\n//\twconn, _ = netpoll.NewFDConnection(wfd)\nfunc NewFDConnection(fd int) (Connection, error) {\n\tconn := new(connection)\n\terr := conn.init(&netFD{fd: fd}, nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn conn, nil\n}\n\n// NewDialer only support TCP and unix socket now.\nfunc NewDialer() Dialer {\n\treturn &dialer{}\n}\n\nvar defaultDialer = NewDialer()\n\ntype dialer struct{}\n\n// DialTimeout implements Dialer.\nfunc (d *dialer) DialTimeout(network, address string, timeout time.Duration) (net.Conn, error) {\n\treturn d.DialConnection(network, address, timeout)\n}\n\n// DialConnection implements Dialer.\nfunc (d *dialer) DialConnection(network, address string, timeout time.Duration) (connection Connection, err error) {\n\tctx := context.Background()\n\tif timeout > 0 {\n\t\tsubCtx, cancel := context.WithTimeout(ctx, timeout)\n\t\tdefer cancel()\n\t\tctx = subCtx\n\t}\n\n\tswitch network {\n\tcase \"tcp\", \"tcp4\", \"tcp6\":\n\t\treturn d.dialTCP(ctx, network, address)\n\tcase \"unix\", \"unixgram\", \"unixpacket\":\n\t\traddr := &UnixAddr{\n\t\t\tUnixAddr: net.UnixAddr{Name: address, Net: network},\n\t\t}\n\t\treturn DialUnix(network, nil, raddr)\n\tdefault:\n\t\treturn nil, net.UnknownNetworkError(network)\n\t}\n}\n\nfunc (d *dialer) dialTCP(ctx context.Context, network, address string) (connection *TCPConnection, err error) {\n\thost, port, err := net.SplitHostPort(address)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tvar portnum int\n\tif portnum, err = net.DefaultResolver.LookupPort(ctx, network, port); err != nil {\n\t\treturn nil, err\n\t}\n\tvar ipaddrs []net.IPAddr\n\t// host maybe empty if address is :12345\n\tif host == \"\" {\n\t\tipaddrs = []net.IPAddr{{}}\n\t} else {\n\t\tipaddrs, err = net.DefaultResolver.LookupIPAddr(ctx, host)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif len(ipaddrs) == 0 {\n\t\t\treturn nil, &net.DNSError{Err: \"no such host\", Name: host, IsNotFound: true}\n\t\t}\n\t}\n\n\tvar firstErr error // The error from the first address is most relevant.\n\ttcpAddr := &TCPAddr{}\n\tfor _, ipaddr := range ipaddrs {\n\t\ttcpAddr.IP = ipaddr.IP\n\t\ttcpAddr.Port = portnum\n\t\ttcpAddr.Zone = ipaddr.Zone\n\t\tif ipaddr.IP != nil && ipaddr.IP.To4() == nil {\n\t\t\tconnection, err = DialTCP(ctx, \"tcp6\", nil, tcpAddr)\n\t\t} else {\n\t\t\tconnection, err = DialTCP(ctx, \"tcp\", nil, tcpAddr)\n\t\t}\n\t\tif err == nil {\n\t\t\treturn connection, nil\n\t\t}\n\t\tselect {\n\t\tcase <-ctx.Done(): // check timeout error\n\t\t\treturn nil, err\n\t\tdefault:\n\t\t}\n\t\tif firstErr == nil {\n\t\t\tfirstErr = err\n\t\t}\n\t}\n\n\tif firstErr == nil {\n\t\tfirstErr = &net.OpError{Op: \"dial\", Net: network, Source: nil, Addr: nil, Err: errMissingAddress}\n\t}\n\treturn nil, firstErr\n}\n\n// sysDialer contains a Dial's parameters and configuration.\ntype sysDialer struct {\n\tnet.Dialer\n\tnetwork, address string\n}\n"
  },
  {
    "path": "net_dialer_test.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n// +build !windows\n\npackage netpoll\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestDialerTCP(t *testing.T) {\n\tdialer := NewDialer()\n\taddress := getTestAddress()\n\tconn, err := dialer.DialTimeout(\"tcp\", address, time.Second)\n\tMustTrue(t, err != nil)\n\tMustTrue(t, conn.(*TCPConnection) == nil)\n\n\tln, err := CreateListener(\"tcp\", address)\n\tMustNil(t, err)\n\n\tstop := make(chan int, 1)\n\tdefer close(stop)\n\n\tgo func() {\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-stop:\n\t\t\t\terr := ln.Close()\n\t\t\t\tMustNil(t, err)\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\t\t\tconn, err := ln.Accept()\n\t\t\tif conn == nil && err == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t}()\n\n\tconn, err = dialer.DialTimeout(\"tcp\", address, time.Second)\n\tMustNil(t, err)\n\tMustTrue(t, strings.HasPrefix(conn.LocalAddr().String(), \"127.0.0.1:\"))\n\tEqual(t, conn.RemoteAddr().String(), address)\n}\n\nfunc TestDialerUnix(t *testing.T) {\n\tdialer := NewDialer()\n\tconn, err := dialer.DialTimeout(\"unix\", \"tmp.sock\", time.Second)\n\tMustTrue(t, err != nil)\n\tMustTrue(t, conn.(*UnixConnection) == nil)\n\n\tln, err := CreateListener(\"unix\", \"tmp.sock\")\n\tMustNil(t, err)\n\tdefer ln.Close()\n\n\tstop := make(chan int, 1)\n\tdefer func() {\n\t\tclose(stop)\n\t\ttime.Sleep(time.Millisecond)\n\t}()\n\n\tgo func() {\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-stop:\n\t\t\t\terr := ln.Close()\n\t\t\t\tMustNil(t, err)\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\t\t\tconn, err := ln.Accept()\n\t\t\tif conn == nil && err == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t}()\n\n\tconn, err = dialer.DialTimeout(\"unix\", \"tmp.sock\", time.Second)\n\tMustNil(t, err)\n\tif runtime.GOOS == \"linux\" {\n\t\tEqual(t, conn.LocalAddr().String(), \"@\")\n\t} else {\n\t\tEqual(t, conn.LocalAddr().String(), \"\")\n\t}\n\tEqual(t, conn.RemoteAddr().String(), \"tmp.sock\")\n}\n\nfunc TestDialerFdAlloc(t *testing.T) {\n\taddress := getTestAddress()\n\tln, err := CreateListener(\"tcp\", address)\n\tMustNil(t, err)\n\tdefer ln.Close()\n\tel1, _ := NewEventLoop(func(ctx context.Context, connection Connection) error {\n\t\tconnection.Close()\n\t\treturn nil\n\t})\n\tgo func() {\n\t\tel1.Serve(ln)\n\t}()\n\tctx1, cancel1 := context.WithTimeout(context.Background(), time.Second)\n\tdefer cancel1()\n\tdefer el1.Shutdown(ctx1)\n\n\tfor i := 0; i < 100; i++ {\n\t\tconn, err := DialConnection(\"tcp\", address, time.Second)\n\t\tMustNil(t, err)\n\t\tfd := conn.(*TCPConnection).fd\n\t\tconn.Write([]byte(\"hello world\"))\n\t\tfor conn.IsActive() {\n\t\t\truntime.Gosched()\n\t\t}\n\t\ttime.Sleep(time.Millisecond)\n\t\tsyscall.SetNonblock(fd, true)\n\t}\n}\n\nfunc TestFDClose(t *testing.T) {\n\taddress := getTestAddress()\n\tln, err := CreateListener(\"tcp\", address)\n\tMustNil(t, err)\n\tdefer ln.Close()\n\tel1, _ := NewEventLoop(func(ctx context.Context, connection Connection) error {\n\t\tconnection.Close()\n\t\treturn nil\n\t})\n\tgo func() {\n\t\tel1.Serve(ln)\n\t}()\n\tctx1, cancel1 := context.WithTimeout(context.Background(), time.Second)\n\tdefer cancel1()\n\tdefer el1.Shutdown(ctx1)\n\n\tvar fd int\n\tvar conn Connection\n\tconn, err = DialConnection(\"tcp\", address, time.Second)\n\tMustNil(t, err)\n\tfd = conn.(*TCPConnection).fd\n\tsyscall.SetNonblock(fd, true)\n\tconn.Close()\n\n\tconn, err = DialConnection(\"tcp\", address, time.Second)\n\tMustNil(t, err)\n\tfd = conn.(*TCPConnection).fd\n\tsyscall.SetNonblock(fd, true)\n\ttime.Sleep(time.Second)\n\tconn.Close()\n}\n\n// fd data package race test, use two servers and two dialers.\nfunc TestDialerThenClose(t *testing.T) {\n\taddress1 := getTestAddress()\n\taddress2 := getTestAddress()\n\t// server 1\n\tln1, _ := createTestListener(\"tcp\", address1)\n\tel1 := mockDialerEventLoop(1)\n\tgo func() {\n\t\tel1.Serve(ln1)\n\t}()\n\tctx1, cancel1 := context.WithTimeout(context.Background(), time.Second)\n\tdefer cancel1()\n\tdefer el1.Shutdown(ctx1)\n\n\t// server 2\n\tln2, _ := createTestListener(\"tcp\", address2)\n\tel2 := mockDialerEventLoop(2)\n\tgo func() {\n\t\tel2.Serve(ln2)\n\t}()\n\tctx2, cancel2 := context.WithTimeout(context.Background(), time.Second)\n\tdefer cancel2()\n\tdefer el2.Shutdown(ctx2)\n\n\tsize := 20\n\tvar wg sync.WaitGroup\n\twg.Add(size)\n\tfor i := 0; i < size; i++ {\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tfor i := 0; i < 50; i++ {\n\t\t\t\t// send server 1\n\t\t\t\tconn, err := DialConnection(\"tcp\", address1, time.Second)\n\t\t\t\tif err == nil {\n\t\t\t\t\tmockDialerSend(1, &conn.(*TCPConnection).connection)\n\t\t\t\t}\n\t\t\t\t// send server 2\n\t\t\t\tconn, err = DialConnection(\"tcp\", address2, time.Second)\n\t\t\t\tif err == nil {\n\t\t\t\t\tmockDialerSend(2, &conn.(*TCPConnection).connection)\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t}\n\twg.Wait()\n}\n\nfunc TestNewFDConnection(t *testing.T) {\n\tr, w := GetSysFdPairs()\n\trconn, err := NewFDConnection(r)\n\tMustNil(t, err)\n\twconn, err := NewFDConnection(w)\n\tMustNil(t, err)\n\t_, err = rconn.Writer().WriteString(\"hello\")\n\tMustNil(t, err)\n\terr = rconn.Writer().Flush()\n\tMustNil(t, err)\n\tbuf, err := wconn.Reader().Next(5)\n\tMustNil(t, err)\n\tEqual(t, string(buf), \"hello\")\n}\n\nfunc mockDialerEventLoop(idx int) EventLoop {\n\tel, _ := NewEventLoop(func(ctx context.Context, conn Connection) (err error) {\n\t\tdefer func() {\n\t\t\tif err != nil {\n\t\t\t\tfmt.Printf(\"Error: server%d conn closed: %s\", idx, err.Error())\n\t\t\t\tconn.Close()\n\t\t\t}\n\t\t}()\n\t\toperator := conn.(*connection)\n\t\tfd := operator.fd\n\t\tmsg := make([]byte, 15)\n\t\tn, err := operator.Read(msg)\n\t\tif err != nil {\n\t\t\tfmt.Printf(\"Error: conn[%d] server%d-read fail: %s\", operator.fd, idx, err.Error())\n\t\t\treturn err\n\t\t}\n\t\tif n < 1 {\n\t\t\treturn nil\n\t\t}\n\t\tif string(msg[0]) != strconv.Itoa(idx) {\n\t\t\tpanic(fmt.Sprintf(\"msg[%s] != [%d-xxx]\", msg, idx))\n\t\t}\n\n\t\tss := strings.Split(string(msg[:n]), \"-\")\n\t\trfd, _ := strconv.Atoi(ss[1])\n\t\t_, err = operator.Write([]byte(fmt.Sprintf(\"%d-%d\", idx, fd)))\n\t\tif err != nil {\n\t\t\tfmt.Printf(\"Error: conn[%d] rfd[%d] server%d-write fail: %s\", operator.fd, rfd, idx, err.Error())\n\t\t\treturn err\n\t\t}\n\t\treturn nil\n\t})\n\treturn el\n}\n\nfunc mockDialerSend(idx int, conn *connection) {\n\tdefer func() {\n\t\tconn.Close()\n\t}()\n\trandID1 := []byte(fmt.Sprintf(\"%d-%d\", idx, conn.fd))\n\t_, err := conn.Write(randID1)\n\tif err != nil {\n\t\tfmt.Printf(\"Error: conn[%d] client%d write fail: %s\", conn.fd, idx, err.Error())\n\t}\n\tmsg := make([]byte, 15)\n\t_, err = conn.Read(msg)\n\tif err != nil {\n\t\tfmt.Printf(\"Error: conn[%d] client%d Next fail: %s\", conn.fd, idx, err.Error())\n\t}\n}\n"
  },
  {
    "path": "net_io.go",
    "content": "// Copyright 2023 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build darwin || netbsd || freebsd || openbsd || dragonfly || linux\n// +build darwin netbsd freebsd openbsd dragonfly linux\n\npackage netpoll\n\nimport \"syscall\"\n\n// return value:\n// - n: n == 0 but err == nil, retry syscall\n// - err: if not nil, connection should be closed.\nfunc ioread(fd int, bs [][]byte, ivs []syscall.Iovec) (n int, err error) {\n\tn, err = readv(fd, bs, ivs)\n\tif n == 0 && err == nil { // means EOF\n\t\treturn 0, Exception(ErrEOF, \"\")\n\t}\n\tif err == syscall.EINTR || err == syscall.EAGAIN {\n\t\treturn 0, nil\n\t}\n\treturn n, err\n}\n\n// return value:\n// - n: n == 0 but err == nil, retry syscall\n// - err: if not nil, connection should be closed.\nfunc iosend(fd int, bs [][]byte, ivs []syscall.Iovec, zerocopy bool) (n int, err error) {\n\tn, err = sendmsg(fd, bs, ivs, zerocopy)\n\tif err == syscall.EAGAIN {\n\t\treturn 0, nil\n\t}\n\treturn n, err\n}\n"
  },
  {
    "path": "net_listener.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build darwin || netbsd || freebsd || openbsd || dragonfly || linux\n// +build darwin netbsd freebsd openbsd dragonfly linux\n\npackage netpoll\n\nimport (\n\t\"errors\"\n\t\"net\"\n\t\"os\"\n\t\"syscall\"\n)\n\n// CreateListener return a new Listener.\nfunc CreateListener(network, addr string) (l Listener, err error) {\n\tif network == \"udp\" || network == \"udp4\" || network == \"udp6\" {\n\t\treturn nil, Exception(ErrUnsupported, \"UDP\")\n\t}\n\t// tcp, tcp4, tcp6, unix\n\tln, err := net.Listen(network, addr)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn ConvertListener(ln)\n}\n\n// ConvertListener converts net.Listener to Listener\nfunc ConvertListener(l net.Listener) (nl Listener, err error) {\n\tif tmp, ok := l.(Listener); ok {\n\t\treturn tmp, nil\n\t}\n\tln := &listener{}\n\tln.ln = l\n\tln.addr = l.Addr()\n\terr = ln.parseFD()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn ln, syscall.SetNonblock(ln.fd, true)\n}\n\nvar _ net.Listener = &listener{}\n\ntype listener struct {\n\tfd   int\n\taddr net.Addr     // listener's local addr\n\tln   net.Listener // tcp|unix listener\n\tfile *os.File\n}\n\n// Accept implements Listener.\nfunc (ln *listener) Accept() (net.Conn, error) {\n\tfd, sa, err := syscall.Accept(ln.fd)\n\tif err != nil {\n\t\t/* https://man7.org/linux/man-pages/man2/accept.2.html\n\t\tEAGAIN or EWOULDBLOCK\n\t\t  The socket is marked nonblocking and no connections are\n\t\t  present to be accepted.  POSIX.1-2001 and POSIX.1-2008\n\t\t  allow either error to be returned for this case, and do\n\t\t  not require these constants to have the same value, so a\n\t\t  portable application should check for both possibilities.\n\t\t*/\n\t\tif err == syscall.EAGAIN || err == syscall.EWOULDBLOCK {\n\t\t\treturn nil, nil\n\t\t}\n\t\treturn nil, err\n\t}\n\tnfd := &netFD{}\n\tnfd.fd = fd\n\tnfd.localAddr = ln.addr\n\tnfd.network = ln.addr.Network()\n\tnfd.remoteAddr = sockaddrToAddr(sa)\n\treturn nfd, nil\n}\n\n// Close implements Listener.\nfunc (ln *listener) Close() error {\n\tif ln.fd != 0 {\n\t\tsyscall.Close(ln.fd)\n\t}\n\tif ln.file != nil {\n\t\tln.file.Close()\n\t}\n\tif ln.ln != nil {\n\t\tln.ln.Close()\n\t}\n\treturn nil\n}\n\n// Addr implements Listener.\nfunc (ln *listener) Addr() net.Addr {\n\treturn ln.addr\n}\n\n// Fd implements Listener.\nfunc (ln *listener) Fd() (fd int) {\n\treturn ln.fd\n}\n\nfunc (ln *listener) parseFD() (err error) {\n\tswitch netln := ln.ln.(type) {\n\tcase *net.TCPListener:\n\t\tln.file, err = netln.File()\n\tcase *net.UnixListener:\n\t\tln.file, err = netln.File()\n\tdefault:\n\t\treturn errors.New(\"listener type can't support\")\n\t}\n\tif err != nil {\n\t\treturn err\n\t}\n\tln.fd = int(ln.file.Fd())\n\treturn nil\n}\n"
  },
  {
    "path": "net_listener_test.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build darwin || netbsd || freebsd || openbsd || dragonfly || linux\n// +build darwin netbsd freebsd openbsd dragonfly linux\n\npackage netpoll\n\nimport (\n\t\"context\"\n\t\"net\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestListenerDialer(t *testing.T) {\n\tnetwork := \"tcp\"\n\taddr := getTestAddress()\n\tln, err := CreateListener(network, addr)\n\tMustNil(t, err)\n\tdefer ln.Close()\n\ttrigger := make(chan int)\n\tmsg := []byte(\"0123456789\")\n\n\tgo func() {\n\t\tfor {\n\t\t\tconn, err := ln.Accept()\n\t\t\tif conn == nil && err == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tgo func(conn net.Conn) {\n\t\t\t\t<-trigger\n\t\t\t\tbuf := make([]byte, 10)\n\t\t\t\tn, err := conn.Read(buf)\n\t\t\t\tMustNil(t, err)\n\t\t\t\tEqual(t, n, len(msg))\n\t\t\t\tEqual(t, string(buf[:n]), string(msg))\n\t\t\t\tn, err = conn.Write(buf)\n\t\t\t\tMustNil(t, err)\n\t\t\t\tEqual(t, n, len(msg))\n\t\t\t}(conn)\n\t\t}\n\t}()\n\n\t// trigger\n\tvar closed, read int32\n\n\tdialer := NewDialer()\n\tcallback := func(connection Connection) error {\n\t\tatomic.StoreInt32(&closed, 1)\n\t\treturn nil\n\t}\n\tonRequest := func(ctx context.Context, connection Connection) error {\n\t\tatomic.StoreInt32(&read, 1)\n\t\terr := connection.Close()\n\t\tMustNil(t, err)\n\t\treturn err\n\t}\n\tfor i := 0; i < 10; i++ {\n\t\tconn, err := dialer.DialConnection(network, addr, time.Second)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tconn.AddCloseCallback(callback)\n\t\tconn.SetOnRequest(onRequest)\n\n\t\tMustNil(t, err)\n\t\tn, err := conn.Write(msg)\n\t\tMustNil(t, err)\n\t\tEqual(t, n, len(msg))\n\t\ttime.Sleep(10 * time.Millisecond)\n\t\ttrigger <- 1\n\t\ttime.Sleep(10 * time.Millisecond)\n\t\tEqual(t, atomic.LoadInt32(&read), int32(1))\n\t\tEqual(t, atomic.LoadInt32(&closed), int32(1))\n\t}\n}\n\nfunc TestConvertListener(t *testing.T) {\n\tnetwork, address := \"unix\", \"mock.test.sock\"\n\tln, err := net.Listen(network, address)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tudsln, _ := ln.(*net.UnixListener)\n\t// udsln.SetUnlinkOnClose(false)\n\n\tnln, err := ConvertListener(udsln)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\terr = nln.Close()\n\tif err != nil {\n\t\tpanic(err)\n\t}\n}\n"
  },
  {
    "path": "net_netfd.go",
    "content": "// Copyright 2009 The Go Authors. All rights reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n//\n// This file may have been modified by CloudWeGo authors. (“CloudWeGo Modifications”).\n// All CloudWeGo Modifications are Copyright 2022 CloudWeGo authors.\n\n//go:build aix || darwin || dragonfly || freebsd || linux || nacl || netbsd || openbsd || solaris\n// +build aix darwin dragonfly freebsd linux nacl netbsd openbsd solaris\n\npackage netpoll\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"net\"\n\t\"os\"\n\t\"runtime\"\n\t\"syscall\"\n\t\"time\"\n)\n\n// nonDeadline and noCancel are just zero values for\n// readability with functions taking too many parameters.\nvar noDeadline = time.Time{}\n\ntype netFD struct {\n\t// file descriptor\n\tfd int\n\t// When calling netFD.dial(), fd will be registered into poll in some scenarios, such as dialing tcp socket,\n\t// but not in other scenarios, such as dialing unix socket.\n\t// This leads to a different behavior in register poller at after, so use this field to mark it.\n\tpd *pollDesc\n\t// closed marks whether fd has expired\n\tclosed uint32\n\t// Whether this is a streaming descriptor. Immutable.\n\tisStream bool\n\t// Whether a zero byte read indicates EOF. This is false for a\n\t// message based socket connection.\n\tzeroReadIsEOF bool\n\tfamily        int    // AF_INET, AF_INET6, syscall.AF_UNIX\n\tsotype        int    // syscall.SOCK_STREAM, syscall.SOCK_DGRAM, syscall.SOCK_RAW\n\tisConnected   bool   // handshake completed or use of association with peer\n\tnetwork       string // tcp, tcp4, tcp6, unix, unixgram, unixpacket\n\tlocalAddr     net.Addr\n\tremoteAddr    net.Addr\n\t// for detaching conn from poller\n\tdetaching bool\n}\n\nfunc newNetFD(fd, family, sotype int, net string) *netFD {\n\tret := &netFD{}\n\tret.fd = fd\n\tret.network = net\n\tret.family = family\n\tret.sotype = sotype\n\tret.isStream = sotype == syscall.SOCK_STREAM\n\tret.zeroReadIsEOF = sotype != syscall.SOCK_DGRAM && sotype != syscall.SOCK_RAW\n\treturn ret\n}\n\n// if dial connection error, you need exec netFD.Close actively\nfunc (c *netFD) dial(ctx context.Context, laddr, raddr sockaddr) (err error) {\n\tvar lsa syscall.Sockaddr\n\tif laddr != nil {\n\t\tif lsa, err = laddr.sockaddr(c.family); err != nil {\n\t\t\treturn err\n\t\t} else if lsa != nil {\n\t\t\t// bind local address\n\t\t\tif err = syscall.Bind(c.fd, lsa); err != nil {\n\t\t\t\treturn os.NewSyscallError(\"bind\", err)\n\t\t\t}\n\t\t}\n\t}\n\tvar rsa syscall.Sockaddr  // remote address from the user\n\tvar crsa syscall.Sockaddr // remote address we actually connected to\n\tif raddr != nil {\n\t\tif rsa, err = raddr.sockaddr(c.family); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\t// remote address we actually connected to\n\tif crsa, err = c.connect(ctx, lsa, rsa); err != nil {\n\t\treturn err\n\t}\n\tc.isConnected = true\n\n\t// Record the local and remote addresses from the actual socket.\n\t// Get the local address by calling Getsockname.\n\t// For the remote address, use\n\t// 1) the one returned by the connect method, if any; or\n\t// 2) the one from Getpeername, if it succeeds; or\n\t// 3) the one passed to us as the raddr parameter.\n\tlsa, _ = syscall.Getsockname(c.fd)\n\tc.localAddr = sockaddrToAddr(lsa)\n\tif crsa != nil {\n\t\tc.remoteAddr = sockaddrToAddr(crsa)\n\t} else if crsa, _ = syscall.Getpeername(c.fd); crsa != nil {\n\t\tc.remoteAddr = sockaddrToAddr(crsa)\n\t} else {\n\t\tc.remoteAddr = sockaddrToAddr(rsa)\n\t}\n\treturn nil\n}\n\nfunc (c *netFD) connect(ctx context.Context, la, ra syscall.Sockaddr) (rsa syscall.Sockaddr, retErr error) {\n\t// Do not need to call c.writing here,\n\t// because c is not yet accessible to user,\n\t// so no concurrent operations are possible.\n\tswitch err := syscall.Connect(c.fd, ra); err {\n\tcase syscall.EINPROGRESS, syscall.EALREADY, syscall.EINTR:\n\tcase nil, syscall.EISCONN:\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\treturn nil, mapErr(ctx.Err())\n\t\tdefault:\n\t\t}\n\t\treturn nil, nil\n\tcase syscall.EINVAL:\n\t\t// On Solaris we can see EINVAL if the socket has\n\t\t// already been accepted and closed by the server.\n\t\t// Treat this as a successful connection--writes to\n\t\t// the socket will see EOF.  For details and a test\n\t\t// case in C see https://golang.org/issue/6828.\n\t\tif runtime.GOOS == \"solaris\" {\n\t\t\treturn nil, nil\n\t\t}\n\t\tfallthrough\n\tdefault:\n\t\treturn nil, os.NewSyscallError(\"connect\", err)\n\t}\n\n\tc.pd = newPollDesc(c.fd)\n\tdefer func() {\n\t\t// free operator to avoid leak\n\t\tc.pd.operator.Free()\n\t\tc.pd = nil\n\t}()\n\tfor {\n\t\t// Performing multiple connect system calls on a\n\t\t// non-blocking socket under Unix variants does not\n\t\t// necessarily result in earlier errors being\n\t\t// returned. Instead, once runtime-integrated network\n\t\t// poller tells us that the socket is ready, get the\n\t\t// SO_ERROR socket option to see if the connection\n\t\t// succeeded or failed. See issue 7474 for further\n\t\t// details.\n\t\tif err := c.pd.WaitWrite(ctx); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tnerr, err := syscall.GetsockoptInt(c.fd, syscall.SOL_SOCKET, syscall.SO_ERROR)\n\t\tif err != nil {\n\t\t\treturn nil, os.NewSyscallError(\"getsockopt\", err)\n\t\t}\n\t\tswitch err := syscall.Errno(nerr); err {\n\t\tcase syscall.EINPROGRESS, syscall.EALREADY, syscall.EINTR:\n\t\tcase syscall.EISCONN:\n\t\t\treturn nil, nil\n\t\tcase syscall.Errno(0):\n\t\t\t// The runtime poller can wake us up spuriously;\n\t\t\t// see issues 14548 and 19289. Check that we are\n\t\t\t// really connected; if not, wait again.\n\t\t\tif rsa, err := syscall.Getpeername(c.fd); err == nil {\n\t\t\t\treturn rsa, nil\n\t\t\t}\n\t\tdefault:\n\t\t\treturn nil, os.NewSyscallError(\"connect\", err)\n\t\t}\n\t}\n}\n\n// Various errors contained in OpError.\nvar (\n\terrMissingAddress = errors.New(\"missing address\")\n\terrCanceled       = errors.New(\"operation was canceled\")\n\terrIOTimeout      = errors.New(\"i/o timeout\")\n)\n\n// mapErr maps from the context errors to the historical internal net\n// error values.\n//\n// TODO(bradfitz): get rid of this after adjusting tests and making\n// context.DeadlineExceeded implement net.Error?\nfunc mapErr(err error) error {\n\tswitch err {\n\tcase context.Canceled:\n\t\treturn errCanceled\n\tcase context.DeadlineExceeded:\n\t\treturn errIOTimeout\n\tdefault:\n\t\treturn err\n\t}\n}\n"
  },
  {
    "path": "net_netfd_conn.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build darwin || netbsd || freebsd || openbsd || dragonfly || linux\n// +build darwin netbsd freebsd openbsd dragonfly linux\n\npackage netpoll\n\nimport (\n\t\"net\"\n\t\"strings\"\n\t\"sync/atomic\"\n\t\"syscall\"\n\t\"time\"\n)\n\nvar _ Conn = &netFD{}\n\n// Fd implements Conn.\nfunc (c *netFD) Fd() (fd int) {\n\treturn c.fd\n}\n\n// Read implements Conn.\nfunc (c *netFD) Read(b []byte) (n int, err error) {\n\tn, err = syscall.Read(c.fd, b)\n\tif err != nil {\n\t\tif err == syscall.EAGAIN || err == syscall.EINTR {\n\t\t\treturn 0, nil\n\t\t}\n\t}\n\treturn n, err\n}\n\n// Write implements Conn.\nfunc (c *netFD) Write(b []byte) (n int, err error) {\n\tn, err = syscall.Write(c.fd, b)\n\tif err != nil {\n\t\tif err == syscall.EAGAIN {\n\t\t\treturn 0, nil\n\t\t}\n\t}\n\treturn n, err\n}\n\n// Close will be executed only once.\nfunc (c *netFD) Close() (err error) {\n\tif atomic.AddUint32(&c.closed, 1) != 1 {\n\t\treturn nil\n\t}\n\tif !c.detaching && c.fd > 2 {\n\t\terr = syscall.Close(c.fd)\n\t\tif err != nil {\n\t\t\tlogger.Printf(\"NETPOLL: netFD[%d] close error: %s\", c.fd, err.Error())\n\t\t}\n\t}\n\treturn err\n}\n\n// LocalAddr implements Conn.\nfunc (c *netFD) LocalAddr() (addr net.Addr) {\n\treturn c.localAddr\n}\n\n// RemoteAddr implements Conn.\nfunc (c *netFD) RemoteAddr() (addr net.Addr) {\n\treturn c.remoteAddr\n}\n\n// SetKeepAlive implements Conn.\n// TODO: only tcp conn is ok.\nfunc (c *netFD) SetKeepAlive(second int) error {\n\tif !strings.HasPrefix(c.network, \"tcp\") {\n\t\treturn nil\n\t}\n\tif second > 0 {\n\t\treturn SetKeepAlive(c.fd, second)\n\t}\n\treturn nil\n}\n\n// SetDeadline implements Conn.\nfunc (c *netFD) SetDeadline(t time.Time) error {\n\treturn Exception(ErrUnsupported, \"SetDeadline\")\n}\n\n// SetReadDeadline implements Conn.\nfunc (c *netFD) SetReadDeadline(t time.Time) error {\n\treturn Exception(ErrUnsupported, \"SetReadDeadline\")\n}\n\n// SetWriteDeadline implements Conn.\nfunc (c *netFD) SetWriteDeadline(t time.Time) error {\n\treturn Exception(ErrUnsupported, \"SetWriteDeadline\")\n}\n"
  },
  {
    "path": "net_polldesc.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n// +build !windows\n\npackage netpoll\n\nimport (\n\t\"context\"\n)\n\nfunc newPollDesc(fd int) *pollDesc {\n\tpd := &pollDesc{}\n\tpoll := pollmanager.Pick()\n\tpd.operator = poll.Alloc()\n\tpd.operator.poll = poll\n\tpd.operator.FD = fd\n\tpd.operator.OnWrite = pd.onwrite\n\tpd.operator.OnHup = pd.onhup\n\tpd.writeTrigger = make(chan struct{})\n\tpd.closeTrigger = make(chan struct{})\n\treturn pd\n}\n\ntype pollDesc struct {\n\toperator *FDOperator\n\t// The write event is OneShot, then mark the writable to skip duplicate calling.\n\twriteTrigger chan struct{}\n\tcloseTrigger chan struct{}\n}\n\n// WaitWrite .\nfunc (pd *pollDesc) WaitWrite(ctx context.Context) (err error) {\n\tif pd.operator.isUnused() {\n\t\t// add ET|Write|Hup\n\t\tif err = pd.operator.Control(PollWritable); err != nil {\n\t\t\tlogger.Printf(\"NETPOLL: pollDesc register operator failed: %v\", err)\n\t\t\treturn err\n\t\t}\n\t}\n\n\tselect {\n\tcase <-pd.writeTrigger: // triggered by poller\n\tcase <-pd.closeTrigger: // triggered by poller\n\t\t// no need to detach, since poller has done it in OnHup.\n\t\treturn Exception(ErrConnClosed, \"by peer\")\n\tcase <-ctx.Done(): // triggered by ctx\n\t\t// deregister from poller, upper caller function will close fd\n\t\tpd.detach()\n\t\treturn mapErr(ctx.Err())\n\t}\n\t// double check close trigger\n\tselect {\n\tcase <-pd.closeTrigger:\n\t\treturn Exception(ErrConnClosed, \"by peer\")\n\tdefault:\n\t\treturn nil\n\t}\n}\n\nfunc (pd *pollDesc) onwrite(p Poll) error {\n\tselect {\n\tcase <-pd.writeTrigger:\n\tdefault:\n\t\tpd.detach()\n\t\tclose(pd.writeTrigger)\n\t}\n\treturn nil\n}\n\nfunc (pd *pollDesc) onhup(p Poll) error {\n\tselect {\n\tcase <-pd.closeTrigger:\n\tdefault:\n\t\tclose(pd.closeTrigger)\n\t}\n\treturn nil\n}\n\nfunc (pd *pollDesc) detach() {\n\tif err := pd.operator.Control(PollDetach); err != nil {\n\t\tlogger.Printf(\"NETPOLL: pollDesc detach operator failed: %v\", err)\n\t}\n}\n"
  },
  {
    "path": "net_polldesc_test.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n// +build !windows\n\npackage netpoll\n\nimport (\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestZeroTimer(t *testing.T) {\n\tMustTrue(t, noDeadline.IsZero())\n}\n\nfunc TestRuntimePoll(t *testing.T) {\n\taddress := getTestAddress()\n\tln, err := CreateListener(\"tcp\", address)\n\tMustNil(t, err)\n\n\tstop := make(chan int, 1)\n\tdefer close(stop)\n\n\tgo func() {\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-stop:\n\t\t\t\terr := ln.Close()\n\t\t\t\tMustNil(t, err)\n\t\t\t\treturn\n\t\t\tdefault:\n\t\t\t}\n\t\t\tconn, err := ln.Accept()\n\t\t\tif conn == nil && err == nil {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t}()\n\n\tfor i := 0; i < 10; i++ {\n\t\tconn, err := DialConnection(\"tcp\", address, time.Second)\n\t\tMustNil(t, err)\n\t\tconn.Close()\n\t}\n}\n"
  },
  {
    "path": "net_sock.go",
    "content": "// Copyright 2009 The Go Authors. All rights reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n//\n// This file may have been modified by CloudWeGo authors. (“CloudWeGo Modifications”).\n// All CloudWeGo Modifications are Copyright 2022 CloudWeGo authors.\n\n//go:build !windows\n// +build !windows\n\npackage netpoll\n\nimport (\n\t\"context\"\n\t\"net\"\n\t\"runtime\"\n\t\"syscall\"\n)\n\n// A sockaddr represents a TCP, IP or Unix network endpoint\n// address that can be converted into a syscall.Sockaddr.\ntype sockaddr interface {\n\tnet.Addr\n\n\t// family returns the platform-dependent address family\n\t// identifier.\n\tfamily() int\n\n\t// isWildcard reports whether the address is a wildcard\n\t// address.\n\tisWildcard() bool\n\n\t// sockaddr returns the address converted into a syscall\n\t// sockaddr type that implements syscall.Sockaddr\n\t// interface. It returns a nil interface when the address is nil.\n\tsockaddr(family int) (syscall.Sockaddr, error)\n\n\t// toLocal maps the zero address to a local system address (127.0.0.1 or ::1)\n\ttoLocal(net string) sockaddr\n}\n\nfunc internetSocket(ctx context.Context, net string, laddr, raddr sockaddr, sotype, proto int, mode string) (conn *netFD, err error) {\n\tif (runtime.GOOS == \"aix\" || runtime.GOOS == \"openbsd\" || runtime.GOOS == \"nacl\") && raddr.isWildcard() {\n\t\traddr = raddr.toLocal(net)\n\t}\n\tfamily, ipv6only := favoriteAddrFamily(net, laddr, raddr)\n\treturn socket(ctx, net, family, sotype, proto, ipv6only, laddr, raddr)\n}\n\n// favoriteAddrFamily returns the appropriate address family for the\n// given network, laddr, raddr and mode.\n//\n// If mode indicates \"listen\" and laddr is a wildcard, we assume that\n// the user wants to make a passive-open connection with a wildcard\n// address family, both AF_INET and AF_INET6, and a wildcard address\n// like the following:\n//\n//   - A listen for a wildcard communication domain, \"tcp\",\n//     with a wildcard address: If the platform supports\n//     both IPv6 and IPv4-mapped IPv6 communication capabilities,\n//     or does not support IPv4, we use a dual stack, AF_INET6 and\n//     IPV6_V6ONLY=0, wildcard address listen. The dual stack\n//     wildcard address listen may fall back to an IPv6-only,\n//     AF_INET6 and IPV6_V6ONLY=1, wildcard address listen.\n//     Otherwise we prefer an IPv4-only, AF_INET, wildcard address\n//     listen.\n//\n//   - A listen for a wildcard communication domain, \"tcp\",\n//     with an IPv4 wildcard address: same as above.\n//\n//   - A listen for a wildcard communication domain, \"tcp\",\n//     with an IPv6 wildcard address: same as above.\n//\n//   - A listen for an IPv4 communication domain, \"tcp4\",\n//     with an IPv4 wildcard address: We use an IPv4-only, AF_INET,\n//     wildcard address listen.\n//\n//   - A listen for an IPv6 communication domain, \"tcp6\",\n//     with an IPv6 wildcard address: We use an IPv6-only, AF_INET6\n//     and IPV6_V6ONLY=1, wildcard address listen.\n//\n// Otherwise guess: If the addresses are IPv4 then returns AF_INET,\n// or else returns AF_INET6. It also returns a boolean value what\n// designates IPV6_V6ONLY option.\n//\n// Note that the latest DragonFly BSD and OpenBSD kernels allow\n// neither \"net.inet6.ip6.v6only=1\" change nor IPPROTO_IPV6 level\n// IPV6_V6ONLY socket option setting.\nfunc favoriteAddrFamily(network string, laddr, raddr sockaddr) (family int, ipv6only bool) {\n\tswitch network[len(network)-1] {\n\tcase '4':\n\t\treturn syscall.AF_INET, false\n\tcase '6':\n\t\treturn syscall.AF_INET6, true\n\t}\n\tif (laddr == nil || laddr.family() == syscall.AF_INET) &&\n\t\t(raddr == nil || raddr.family() == syscall.AF_INET) {\n\t\treturn syscall.AF_INET, false\n\t}\n\treturn syscall.AF_INET6, false\n}\n\n// socket returns a network file descriptor that is ready for\n// asynchronous I/O using the network poller.\nfunc socket(ctx context.Context, net string, family, sotype, proto int, ipv6only bool, laddr, raddr sockaddr) (netfd *netFD, err error) {\n\t// syscall.Socket & set socket options\n\tvar fd int\n\tfd, err = sysSocket(family, sotype, proto)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\terr = setDefaultSockopts(fd, family, sotype, ipv6only)\n\tif err != nil {\n\t\tsyscall.Close(fd)\n\t\treturn nil, err\n\t}\n\n\tnetfd = newNetFD(fd, family, sotype, net)\n\terr = netfd.dial(ctx, laddr, raddr)\n\tif err != nil {\n\t\tnetfd.Close()\n\t\treturn nil, err\n\t}\n\treturn netfd, nil\n}\n\n// sockaddrToAddr returns a go/net friendly address\nfunc sockaddrToAddr(sa syscall.Sockaddr) net.Addr {\n\tvar a net.Addr\n\tswitch sa := sa.(type) {\n\tcase *syscall.SockaddrInet4:\n\t\ta = &net.TCPAddr{\n\t\t\tIP:   sa.Addr[0:],\n\t\t\tPort: sa.Port,\n\t\t}\n\tcase *syscall.SockaddrInet6:\n\t\tvar zone string\n\t\tif sa.ZoneId != 0 {\n\t\t\tif ifi, err := net.InterfaceByIndex(int(sa.ZoneId)); err == nil {\n\t\t\t\tzone = ifi.Name\n\t\t\t}\n\t\t}\n\t\t// if zone == \"\" && sa.ZoneId != 0 {\n\t\t// }\n\t\ta = &net.TCPAddr{\n\t\t\tIP:   sa.Addr[0:],\n\t\t\tPort: sa.Port,\n\t\t\tZone: zone,\n\t\t}\n\tcase *syscall.SockaddrUnix:\n\t\ta = &net.UnixAddr{Net: \"unix\", Name: sa.Name}\n\t}\n\treturn a\n}\n"
  },
  {
    "path": "net_tcpsock.go",
    "content": "// Copyright 2009 The Go Authors. All rights reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n//\n// This file may have been modified by CloudWeGo authors. (“CloudWeGo Modifications”).\n// All CloudWeGo Modifications are Copyright 2022 CloudWeGo authors.\n\n//go:build !windows\n// +build !windows\n\npackage netpoll\n\nimport (\n\t\"context\"\n\t\"net\"\n\t\"os\"\n\t\"syscall\"\n)\n\n// TCPAddr represents the address of a TCP end point.\ntype TCPAddr struct {\n\tnet.TCPAddr\n}\n\nfunc (a *TCPAddr) isWildcard() bool {\n\tif a == nil || a.IP == nil {\n\t\treturn true\n\t}\n\treturn a.IP.IsUnspecified()\n}\n\nfunc (a *TCPAddr) opAddr() net.Addr {\n\tif a == nil {\n\t\treturn nil\n\t}\n\treturn a\n}\n\nfunc (a *TCPAddr) family() int {\n\tif a == nil || len(a.IP) <= net.IPv4len {\n\t\treturn syscall.AF_INET\n\t}\n\tif a.IP.To4() != nil {\n\t\treturn syscall.AF_INET\n\t}\n\treturn syscall.AF_INET6\n}\n\nfunc (a *TCPAddr) sockaddr(family int) (syscall.Sockaddr, error) {\n\tif a == nil {\n\t\treturn nil, nil\n\t}\n\treturn ipToSockaddr(family, a.IP, a.Port, a.Zone)\n}\n\nfunc (a *TCPAddr) toLocal(network string) sockaddr {\n\taddr := &TCPAddr{}\n\taddr.IP = loopbackIP(network)\n\taddr.Port = a.Port\n\taddr.Zone = a.Zone\n\treturn addr\n}\n\nfunc loopbackIP(network string) net.IP {\n\tif network != \"\" && network[len(network)-1] == '6' {\n\t\treturn net.IPv6loopback\n\t}\n\treturn net.IP{127, 0, 0, 1}\n}\n\nfunc ipToSockaddr(family int, ip net.IP, port int, zone string) (syscall.Sockaddr, error) {\n\tswitch family {\n\tcase syscall.AF_INET:\n\t\tif len(ip) == 0 {\n\t\t\tip = net.IPv4zero\n\t\t}\n\t\tip4 := ip.To4()\n\t\tif ip4 == nil {\n\t\t\treturn nil, &net.AddrError{Err: \"non-IPv4 address\", Addr: ip.String()}\n\t\t}\n\t\tsa := &syscall.SockaddrInet4{Port: port}\n\t\tcopy(sa.Addr[:], ip4)\n\t\treturn sa, nil\n\tcase syscall.AF_INET6:\n\t\t// In general, an IP wildcard address, which is either\n\t\t// \"0.0.0.0\" or \"::\", means the entire IP addressing\n\t\t// space. For some historical reason, it is used to\n\t\t// specify \"any available address\" on some operations\n\t\t// of IP node.\n\t\t//\n\t\t// When the IP node supports IPv4-mapped IPv6 address,\n\t\t// we allow an listener to listen to the wildcard\n\t\t// address of both IP addressing spaces by specifying\n\t\t// IPv6 wildcard address.\n\t\tif len(ip) == 0 || ip.Equal(net.IPv4zero) {\n\t\t\tip = net.IPv6zero\n\t\t}\n\t\t// We accept any IPv6 address including IPv4-mapped\n\t\t// IPv6 address.\n\t\tip6 := ip.To16()\n\t\tif ip6 == nil {\n\t\t\treturn nil, &net.AddrError{Err: \"non-IPv6 address\", Addr: ip.String()}\n\t\t}\n\t\t// TODO: sa := &syscall.SockaddrInet6{Port: port, ZoneId: uint32(zoneCache.index(zone))}\n\t\tsa := &syscall.SockaddrInet6{Port: port}\n\t\tcopy(sa.Addr[:], ip6)\n\t\treturn sa, nil\n\t}\n\treturn nil, &net.AddrError{Err: \"invalid address family\", Addr: ip.String()}\n}\n\n// ResolveTCPAddr returns an address of TCP end point.\n//\n// The network must be a TCP network name.\n//\n// If the host in the address parameter is not a literal IP address or\n// the port is not a literal port number, ResolveTCPAddr resolves the\n// address to an address of TCP end point.\n// Otherwise, it parses the address as a pair of literal IP address\n// and port number.\n// The address parameter can use a host name, but this is not\n// recommended, because it will return at most one of the host name's\n// IP addresses.\n//\n// See func Dial for a description of the network and address\n// parameters.\nfunc ResolveTCPAddr(network, address string) (*TCPAddr, error) {\n\taddr, err := net.ResolveTCPAddr(network, address)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &TCPAddr{*addr}, nil\n}\n\n// TCPConnection implements Connection.\ntype TCPConnection struct {\n\tconnection\n}\n\n// newTCPConnection wraps *TCPConnection.\nfunc newTCPConnection(conn Conn) (connection *TCPConnection, err error) {\n\tconnection = &TCPConnection{}\n\terr = connection.init(conn, nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn connection, nil\n}\n\n// DialTCP acts like Dial for TCP networks.\n//\n// The network must be a TCP network name; see func Dial for details.\n//\n// If laddr is nil, a local address is automatically chosen.\n// If the IP field of raddr is nil or an unspecified IP address, the\n// local system is assumed.\nfunc DialTCP(ctx context.Context, network string, laddr, raddr *TCPAddr) (*TCPConnection, error) {\n\tswitch network {\n\tcase \"tcp\", \"tcp4\", \"tcp6\":\n\tdefault:\n\t\treturn nil, &net.OpError{Op: \"dial\", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: net.UnknownNetworkError(network)}\n\t}\n\tif raddr == nil {\n\t\treturn nil, &net.OpError{Op: \"dial\", Net: network, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress}\n\t}\n\tif ctx == nil {\n\t\tctx = context.Background()\n\t}\n\tsd := &sysDialer{network: network, address: raddr.String()}\n\tc, err := sd.dialTCP(ctx, laddr, raddr)\n\tif err != nil {\n\t\treturn nil, &net.OpError{Op: \"dial\", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err}\n\t}\n\treturn c, nil\n}\n\nfunc (sd *sysDialer) dialTCP(ctx context.Context, laddr, raddr *TCPAddr) (*TCPConnection, error) {\n\tconn, err := internetSocket(ctx, sd.network, laddr, raddr, syscall.SOCK_STREAM, 0, \"dial\")\n\n\t// TCP has a rarely used mechanism called a 'simultaneous connection' in\n\t// which Dial(\"tcp\", addr1, addr2) run on the machine at addr1 can\n\t// connect to a simultaneous Dial(\"tcp\", addr2, addr1) run on the machine\n\t// at addr2, without either machine executing Listen. If laddr == nil,\n\t// it means we want the kernel to pick an appropriate originating local\n\t// address. Some Linux kernels cycle blindly through a fixed range of\n\t// local ports, regardless of destination port. If a kernel happens to\n\t// pick local port 50001 as the source for a Dial(\"tcp\", \"\", \"localhost:50001\"),\n\t// then the Dial will succeed, having simultaneously connected to itself.\n\t// This can only happen when we are letting the kernel pick a port (laddr == nil)\n\t// and when there is no listener for the destination address.\n\t// It's hard to argue this is anything other than a kernel bug. If we\n\t// see this happen, rather than expose the buggy effect to users, we\n\t// close the conn and try again. If it happens twice more, we relent and\n\t// use the result. See also:\n\t// \thttps://golang.org/issue/2690\n\t// \thttps://stackoverflow.com/questions/4949858/\n\t//\n\t// The opposite can also happen: if we ask the kernel to pick an appropriate\n\t// originating local address, sometimes it picks one that is already in use.\n\t// So if the error is EADDRNOTAVAIL, we have to try again too, just for\n\t// a different reason.\n\t//\n\t// The kernel socket code is no doubt enjoying watching us squirm.\n\tfor i := 0; i < 2 && (laddr == nil || laddr.Port == 0) && (selfConnect(conn, err) || spuriousENOTAVAIL(err)); i++ {\n\t\tif err == nil {\n\t\t\tconn.Close()\n\t\t}\n\t\tconn, err = internetSocket(ctx, sd.network, laddr, raddr, syscall.SOCK_STREAM, 0, \"dial\")\n\t}\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn newTCPConnection(conn)\n}\n\nfunc selfConnect(conn *netFD, err error) bool {\n\t// If the connect failed, we clearly didn't connect to ourselves.\n\tif err != nil {\n\t\treturn false\n\t}\n\n\t// The socket constructor can return an conn with raddr nil under certain\n\t// unknown conditions. The errors in the calls there to Getpeername\n\t// are discarded, but we can't catch the problem there because those\n\t// calls are sometimes legally erroneous with a \"socket not connected\".\n\t// Since this code (selfConnect) is already trying to work around\n\t// a problem, we make sure if this happens we recognize trouble and\n\t// ask the DialTCP routine to try again.\n\t// TODO: try to understand what's really going on.\n\tif conn.localAddr == nil || conn.remoteAddr == nil {\n\t\treturn true\n\t}\n\tl := conn.localAddr.(*net.TCPAddr)\n\tr := conn.remoteAddr.(*net.TCPAddr)\n\treturn l.Port == r.Port && l.IP.Equal(r.IP)\n}\n\nfunc spuriousENOTAVAIL(err error) bool {\n\tif op, ok := err.(*net.OpError); ok {\n\t\terr = op.Err\n\t}\n\tif sys, ok := err.(*os.SyscallError); ok {\n\t\terr = sys.Err\n\t}\n\treturn err == syscall.EADDRNOTAVAIL\n}\n"
  },
  {
    "path": "net_unixsock.go",
    "content": "// Copyright 2009 The Go Authors. All rights reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n//\n// This file may have been modified by CloudWeGo authors. (“CloudWeGo Modifications”).\n// All CloudWeGo Modifications are Copyright 2022 CloudWeGo authors.\n\n//go:build !windows\n// +build !windows\n\npackage netpoll\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"net\"\n\t\"syscall\"\n)\n\n// BUG(mikio): On JS, NaCl and Plan 9, methods and functions related\n// to UnixConn and UnixListener are not implemented.\n\n// BUG(mikio): On Windows, methods and functions related to UnixConn\n// and UnixListener don't work for \"unixgram\" and \"unixpacket\".\n\n// UnixAddr represents the address of a Unix domain socket end point.\ntype UnixAddr struct {\n\tnet.UnixAddr\n}\n\nfunc (a *UnixAddr) isWildcard() bool {\n\treturn a == nil || a.Name == \"\"\n}\n\nfunc (a *UnixAddr) opAddr() net.Addr {\n\tif a == nil {\n\t\treturn nil\n\t}\n\treturn a\n}\n\nfunc (a *UnixAddr) family() int {\n\treturn syscall.AF_UNIX\n}\n\nfunc (a *UnixAddr) sockaddr(family int) (syscall.Sockaddr, error) {\n\tif a == nil {\n\t\treturn nil, nil\n\t}\n\treturn &syscall.SockaddrUnix{Name: a.Name}, nil\n}\n\nfunc (a *UnixAddr) toLocal(net string) sockaddr {\n\treturn a\n}\n\n// ResolveUnixAddr returns an address of Unix domain socket end point.\n//\n// The network must be a Unix network name.\n//\n// See func Dial for a description of the network and address\n// parameters.\nfunc ResolveUnixAddr(network, address string) (*UnixAddr, error) {\n\taddr, err := net.ResolveUnixAddr(network, address)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &UnixAddr{*addr}, nil\n}\n\n// UnixConnection implements Connection.\ntype UnixConnection struct {\n\tconnection\n}\n\n// newUnixConnection wraps UnixConnection.\nfunc newUnixConnection(conn Conn) (connection *UnixConnection, err error) {\n\tconnection = &UnixConnection{}\n\terr = connection.init(conn, nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn connection, nil\n}\n\n// DialUnix acts like Dial for Unix networks.\n//\n// The network must be a Unix network name; see func Dial for details.\n//\n// If laddr is non-nil, it is used as the local address for the\n// connection.\nfunc DialUnix(network string, laddr, raddr *UnixAddr) (*UnixConnection, error) {\n\tswitch network {\n\tcase \"unix\", \"unixgram\", \"unixpacket\":\n\tdefault:\n\t\treturn nil, &net.OpError{Op: \"dial\", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: net.UnknownNetworkError(network)}\n\t}\n\tsd := &sysDialer{network: network, address: raddr.String()}\n\tc, err := sd.dialUnix(context.Background(), laddr, raddr)\n\tif err != nil {\n\t\treturn nil, &net.OpError{Op: \"dial\", Net: network, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err}\n\t}\n\treturn c, nil\n}\n\nfunc (sd *sysDialer) dialUnix(ctx context.Context, laddr, raddr *UnixAddr) (*UnixConnection, error) {\n\tconn, err := unixSocket(ctx, sd.network, laddr, raddr, \"dial\")\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn newUnixConnection(conn)\n}\n\nfunc unixSocket(ctx context.Context, network string, laddr, raddr sockaddr, mode string) (conn *netFD, err error) {\n\tvar sotype int\n\tswitch network {\n\tcase \"unix\":\n\t\tsotype = syscall.SOCK_STREAM\n\tcase \"unixgram\":\n\t\tsotype = syscall.SOCK_DGRAM\n\tcase \"unixpacket\":\n\t\tsotype = syscall.SOCK_SEQPACKET\n\tdefault:\n\t\treturn nil, net.UnknownNetworkError(network)\n\t}\n\n\tswitch mode {\n\tcase \"dial\":\n\t\tif laddr != nil && laddr.isWildcard() {\n\t\t\tladdr = nil\n\t\t}\n\t\tif raddr != nil && raddr.isWildcard() {\n\t\t\traddr = nil\n\t\t}\n\t\tif raddr == nil && (sotype != syscall.SOCK_DGRAM || laddr == nil) {\n\t\t\treturn nil, errMissingAddress\n\t\t}\n\tcase \"listen\":\n\tdefault:\n\t\treturn nil, errors.New(\"unknown mode: \" + mode)\n\t}\n\n\treturn socket(ctx, network, syscall.AF_UNIX, sotype, 0, false, laddr, raddr)\n}\n"
  },
  {
    "path": "netpoll_config.go",
    "content": "// Copyright 2024 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage netpoll\n\nimport (\n\t\"context\"\n\t\"io\"\n)\n\n// global config\nvar (\n\tdefaultLinkBufferSize = pagesize\n)\n\n// Config expose some tuning parameters to control the internal behaviors of netpoll.\n// Every parameter with the default zero value should keep the default behavior of netpoll.\ntype Config struct {\n\tPollerNum    int                                 // number of pollers\n\tBufferSize   int                                 // default size of a new connection's LinkBuffer\n\tRunner       func(ctx context.Context, f func()) // runner for event handler, most of the time use a goroutine pool.\n\tLoggerOutput io.Writer                           // logger output\n\tLoadBalance  LoadBalance                         // load balance for poller picker\n\tFeature                                          // define all features that not enable by default\n}\n\n// Feature expose some new features maybe promoted as a default behavior but not yet.\ntype Feature struct {\n\t// Deprecated: AlwaysNoCopyRead has no effect and will be removed in a future release.\n\tAlwaysNoCopyRead bool\n}\n"
  },
  {
    "path": "netpoll_options.go",
    "content": "// Copyright 2024 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage netpoll\n\nimport \"time\"\n\n// Option .\ntype Option struct {\n\tf func(*options)\n}\n\ntype options struct {\n\tonPrepare    OnPrepare\n\tonConnect    OnConnect\n\tonDisconnect OnDisconnect\n\tonRequest    OnRequest\n\treadTimeout  time.Duration\n\twriteTimeout time.Duration\n\tidleTimeout  time.Duration\n}\n\n// WithOnPrepare registers the OnPrepare method to EventLoop.\nfunc WithOnPrepare(onPrepare OnPrepare) Option {\n\treturn Option{func(op *options) {\n\t\top.onPrepare = onPrepare\n\t}}\n}\n\n// WithOnConnect registers the OnConnect method to EventLoop.\nfunc WithOnConnect(onConnect OnConnect) Option {\n\treturn Option{func(op *options) {\n\t\top.onConnect = onConnect\n\t}}\n}\n\n// WithOnDisconnect registers the OnDisconnect method to EventLoop.\nfunc WithOnDisconnect(onDisconnect OnDisconnect) Option {\n\treturn Option{func(op *options) {\n\t\top.onDisconnect = onDisconnect\n\t}}\n}\n\n// WithReadTimeout sets the read timeout of connections.\nfunc WithReadTimeout(timeout time.Duration) Option {\n\treturn Option{func(op *options) {\n\t\top.readTimeout = timeout\n\t}}\n}\n\n// WithWriteTimeout sets the write timeout of connections.\nfunc WithWriteTimeout(timeout time.Duration) Option {\n\treturn Option{func(op *options) {\n\t\top.writeTimeout = timeout\n\t}}\n}\n\n// WithIdleTimeout sets the idle timeout of connections.\nfunc WithIdleTimeout(timeout time.Duration) Option {\n\treturn Option{func(op *options) {\n\t\top.idleTimeout = timeout\n\t}}\n}\n"
  },
  {
    "path": "netpoll_server.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n// +build !windows\n\npackage netpoll\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n)\n\n// newServer wrap listener into server, quit will be invoked when server exit.\nfunc newServer(ln Listener, opts *options, onQuit func(err error)) *server {\n\treturn &server{\n\t\tln:     ln,\n\t\topts:   opts,\n\t\tonQuit: onQuit,\n\t}\n}\n\ntype server struct {\n\toperator    FDOperator\n\tln          Listener\n\topts        *options\n\tonQuit      func(err error)\n\tconnections sync.Map // key=fd, value=connection\n}\n\n// Run this server.\nfunc (s *server) Run() (err error) {\n\ts.operator = FDOperator{\n\t\tFD:     s.ln.Fd(),\n\t\tOnRead: s.OnRead,\n\t\tOnHup:  s.OnHup,\n\t}\n\ts.operator.poll = pollmanager.Pick()\n\terr = s.operator.Control(PollReadable)\n\tif err != nil {\n\t\ts.onQuit(err)\n\t}\n\treturn err\n}\n\n// Close this server with deadline.\nfunc (s *server) Close(ctx context.Context) error {\n\ts.operator.Control(PollDetach)\n\ts.ln.Close()\n\n\tfor {\n\t\tactiveConn := 0\n\t\ts.connections.Range(func(key, value interface{}) bool {\n\t\t\tconn, ok := value.(gracefulExit)\n\t\t\tif !ok || conn.isIdle() {\n\t\t\t\tvalue.(Connection).Close()\n\t\t\t} else {\n\t\t\t\tactiveConn++\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\t\tif activeConn == 0 { // all connections have been closed\n\t\t\treturn nil\n\t\t}\n\n\t\t// smart control graceful shutdown check internal\n\t\t// we should wait for more time if there are more active connections\n\t\twaitTime := time.Millisecond * time.Duration(activeConn)\n\t\tif waitTime > time.Second { // max wait time is 1000 ms\n\t\t\twaitTime = time.Millisecond * 1000\n\t\t} else if waitTime < time.Millisecond*50 { // min wait time is 50 ms\n\t\t\twaitTime = time.Millisecond * 50\n\t\t}\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\treturn ctx.Err()\n\t\tcase <-time.After(waitTime):\n\t\t\tcontinue\n\t\t}\n\t}\n}\n\n// OnRead implements FDOperator.\nfunc (s *server) OnRead(p Poll) error {\n\t// accept socket\n\tconn, err := s.ln.Accept()\n\tif err == nil {\n\t\tif conn != nil {\n\t\t\ts.onAccept(conn.(Conn))\n\t\t}\n\t\t// EAGAIN | EWOULDBLOCK if conn and err both nil\n\t\treturn nil\n\t}\n\tlogger.Printf(\"NETPOLL: accept conn failed: %v\", err)\n\n\t// delay accept when too many open files\n\tif isOutOfFdErr(err) {\n\t\t// since we use Epoll LT, we have to detach listener fd from epoll first\n\t\t// and re-register it when accept successfully or there is no available connection\n\t\tcerr := s.operator.Control(PollDetach)\n\t\tif cerr != nil {\n\t\t\tlogger.Printf(\"NETPOLL: detach listener fd failed: %v\", cerr)\n\t\t\treturn err\n\t\t}\n\t\tgo func() {\n\t\t\tretryTimes := []time.Duration{0, 10, 50, 100, 200, 500, 1000} // ms\n\t\t\tretryTimeIndex := 0\n\t\t\tfor {\n\t\t\t\tif retryTimeIndex > 0 {\n\t\t\t\t\ttime.Sleep(retryTimes[retryTimeIndex] * time.Millisecond)\n\t\t\t\t}\n\t\t\t\tconn, err := s.ln.Accept()\n\t\t\t\tif err == nil {\n\t\t\t\t\tif conn == nil {\n\t\t\t\t\t\t// recovery accept poll loop\n\t\t\t\t\t\ts.operator.Control(PollReadable)\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t\ts.onAccept(conn.(Conn))\n\t\t\t\t\tlogger.Println(\"NETPOLL: re-accept conn success:\", conn.RemoteAddr())\n\t\t\t\t\tretryTimeIndex = 0\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif retryTimeIndex+1 < len(retryTimes) {\n\t\t\t\t\tretryTimeIndex++\n\t\t\t\t}\n\t\t\t\tlogger.Printf(\"NETPOLL: re-accept conn failed, err=[%s] and next retrytime=%dms\", err.Error(), retryTimes[retryTimeIndex])\n\t\t\t}\n\t\t}()\n\t}\n\n\t// shut down\n\tif strings.Contains(err.Error(), \"closed\") {\n\t\ts.operator.Control(PollDetach)\n\t\ts.onQuit(err)\n\t\treturn err\n\t}\n\n\treturn err\n}\n\n// OnHup implements FDOperator.\nfunc (s *server) OnHup(p Poll) error {\n\ts.onQuit(errors.New(\"listener close\"))\n\treturn nil\n}\n\nfunc (s *server) onAccept(conn Conn) {\n\t// store & register connection\n\tnconn := new(connection)\n\tnconn.init(conn, s.opts)\n\tif !nconn.IsActive() {\n\t\treturn\n\t}\n\tfd := conn.Fd()\n\tnconn.AddCloseCallback(func(connection Connection) error {\n\t\ts.connections.Delete(fd)\n\t\treturn nil\n\t})\n\ts.connections.Store(fd, nconn)\n\n\t// trigger onConnect asynchronously\n\tnconn.onConnect()\n}\n\nfunc isOutOfFdErr(err error) bool {\n\tse, ok := err.(syscall.Errno)\n\treturn ok && (se == syscall.EMFILE || se == syscall.ENFILE)\n}\n"
  },
  {
    "path": "netpoll_unix.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build darwin || netbsd || freebsd || openbsd || dragonfly || linux\n// +build darwin netbsd freebsd openbsd dragonfly linux\n\npackage netpoll\n\nimport (\n\t\"context\"\n\t\"io\"\n\t\"log\"\n\t\"net\"\n\t\"os\"\n\t\"runtime\"\n\t\"sync\"\n\n\t\"github.com/cloudwego/netpoll/internal/runner\"\n)\n\nvar (\n\tpollmanager = newManager(runtime.GOMAXPROCS(0)/20 + 1) // pollmanager manage all pollers\n\tlogger      = log.New(os.Stderr, \"\", log.LstdFlags)\n)\n\n// Initialize the pollers actively. By default, it's lazy initialized.\n// It's safe to call it multi times.\nfunc Initialize() {\n\t// The first call of Pick() will init pollers\n\t_ = pollmanager.Pick()\n}\n\n// Configure the internal behaviors of netpoll.\n// Configure must called in init() function, because the poller will read some global variable after init() finished\nfunc Configure(config Config) (err error) {\n\tif config.PollerNum > 0 {\n\t\tif err = pollmanager.SetNumLoops(config.PollerNum); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tif config.BufferSize > 0 {\n\t\tdefaultLinkBufferSize = config.BufferSize\n\t}\n\n\tif config.Runner != nil {\n\t\trunner.RunTask = config.Runner\n\t}\n\tif config.LoggerOutput != nil {\n\t\tlogger = log.New(config.LoggerOutput, \"\", log.LstdFlags)\n\t}\n\tif config.LoadBalance >= 0 {\n\t\tif err = pollmanager.SetLoadBalance(config.LoadBalance); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// SetNumLoops is used to set the number of pollers, generally do not need to actively set.\n// By default, the number of pollers is equal to runtime.GOMAXPROCS(0)/20+1.\n// If the number of cores in your service process is less than 20c, theoretically only one poller is needed.\n// Otherwise, you may need to adjust the number of pollers to achieve the best results.\n// Experience recommends assigning a poller every 20c.\n//\n// You can only use SetNumLoops before any connection is created. An example usage:\n//\n//\tfunc init() {\n//\t    netpoll.SetNumLoops(...)\n//\t}\n//\n// Deprecated: use Configure instead.\nfunc SetNumLoops(numLoops int) error {\n\treturn pollmanager.SetNumLoops(numLoops)\n}\n\n// SetLoadBalance sets the load balancing method. Load balancing is always a best effort to attempt\n// to distribute the incoming connections between multiple polls.\n// This option only works when numLoops is set.\n// Deprecated: use Configure instead.\nfunc SetLoadBalance(lb LoadBalance) error {\n\treturn pollmanager.SetLoadBalance(lb)\n}\n\n// SetLoggerOutput sets the logger output target.\n// Deprecated: use Configure instead.\nfunc SetLoggerOutput(w io.Writer) {\n\tlogger = log.New(w, \"\", log.LstdFlags)\n}\n\n// SetRunner set the runner function for every OnRequest/OnConnect callback\n//\n// Deprecated: use Configure and specify config.Runner instead.\nfunc SetRunner(f func(ctx context.Context, f func())) {\n\trunner.RunTask = f\n}\n\n// DisableGopool will remove gopool(the goroutine pool used to run OnRequest),\n// which means that OnRequest will be run via `go OnRequest(...)`.\n// Usually, OnRequest will cause stack expansion, which can be solved by reusing goroutine.\n// But if you can confirm that the OnRequest will not cause stack expansion,\n// it is recommended to use DisableGopool to reduce redundancy and improve performance.\n//\n// Deprecated: use Configure() and specify config.Runner instead.\nfunc DisableGopool() error {\n\trunner.UseGoRunTask()\n\treturn nil\n}\n\n// NewEventLoop .\nfunc NewEventLoop(onRequest OnRequest, ops ...Option) (EventLoop, error) {\n\topts := &options{\n\t\tonRequest: onRequest,\n\t}\n\tfor _, do := range ops {\n\t\tdo.f(opts)\n\t}\n\treturn &eventLoop{\n\t\topts: opts,\n\t\tstop: make(chan error, 1),\n\t}, nil\n}\n\ntype eventLoop struct {\n\tsync.Mutex\n\topts *options\n\tsvr  *server\n\tstop chan error\n}\n\n// Serve implements EventLoop.\nfunc (evl *eventLoop) Serve(ln net.Listener) error {\n\tnpln, err := ConvertListener(ln)\n\tif err != nil {\n\t\treturn err\n\t}\n\tevl.Lock()\n\tevl.svr = newServer(npln, evl.opts, evl.quit)\n\tevl.svr.Run()\n\tevl.Unlock()\n\n\terr = evl.waitQuit()\n\t// ensure evl will not be finalized until Serve returns\n\truntime.SetFinalizer(evl, nil)\n\treturn err\n}\n\n// Shutdown signals a shutdown a begins server closing.\nfunc (evl *eventLoop) Shutdown(ctx context.Context) error {\n\tevl.Lock()\n\tsvr := evl.svr\n\tevl.svr = nil\n\tevl.Unlock()\n\n\tif svr == nil {\n\t\treturn nil\n\t}\n\tevl.quit(nil)\n\treturn svr.Close(ctx)\n}\n\n// waitQuit waits for a quit signal\nfunc (evl *eventLoop) waitQuit() error {\n\treturn <-evl.stop\n}\n\nfunc (evl *eventLoop) quit(err error) {\n\tselect {\n\tcase evl.stop <- err:\n\tdefault:\n\t}\n}\n"
  },
  {
    "path": "netpoll_unix_test.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n// +build !windows\n\npackage netpoll\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\t\"runtime\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"syscall\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/cloudwego/netpoll/internal/runner\"\n)\n\nfunc MustNil(t *testing.T, val interface{}) {\n\tt.Helper()\n\tAssert(t, val == nil, val)\n\tif val != nil {\n\t\tt.Fatal(\"assertion nil failed, val=\", val)\n\t}\n}\n\nfunc MustTrue(t *testing.T, cond bool) {\n\tt.Helper()\n\tif !cond {\n\t\tt.Fatal(\"assertion true failed.\")\n\t}\n}\n\nfunc Equal(t *testing.T, got, expect interface{}) {\n\tt.Helper()\n\tif got != expect {\n\t\tt.Fatalf(\"assertion equal failed, got=[%v], expect=[%v]\", got, expect)\n\t}\n}\n\nfunc Assert(t *testing.T, cond bool, val ...interface{}) {\n\tt.Helper()\n\tif !cond {\n\t\tif len(val) > 0 {\n\t\t\tval = append([]interface{}{\"assertion failed:\"}, val...)\n\t\t\tt.Fatal(val...)\n\t\t} else {\n\t\t\tt.Fatal(\"assertion failed\")\n\t\t}\n\t}\n}\n\nvar testPort int32 = 10000\n\n// getTestAddress return a unique port for every tests, so all tests will not share a same listener\nfunc getTestAddress() string {\n\treturn fmt.Sprintf(\"127.0.0.1:%d\", atomic.AddInt32(&testPort, 1))\n}\n\nfunc TestEqual(t *testing.T) {\n\tvar err error\n\tMustNil(t, err)\n\tMustTrue(t, err == nil)\n\tEqual(t, err, nil)\n\tAssert(t, err == nil, err)\n}\n\nfunc TestOnConnect(t *testing.T) {\n\tnetwork, address := \"tcp\", getTestAddress()\n\treq, resp := \"ping\", \"pong\"\n\tloop := newTestEventLoop(network, address,\n\t\tfunc(ctx context.Context, connection Connection) error {\n\t\t\treturn nil\n\t\t},\n\t\tWithOnConnect(func(ctx context.Context, conn Connection) context.Context {\n\t\t\tfor {\n\t\t\t\tinput, err := conn.Reader().Next(len(req))\n\t\t\t\tif errors.Is(err, ErrEOF) || errors.Is(err, ErrConnClosed) {\n\t\t\t\t\treturn ctx\n\t\t\t\t}\n\t\t\t\tMustNil(t, err)\n\t\t\t\tEqual(t, string(input), req)\n\n\t\t\t\t_, err = conn.Writer().WriteString(resp)\n\t\t\t\tMustNil(t, err)\n\t\t\t\terr = conn.Writer().Flush()\n\t\t\t\tMustNil(t, err)\n\t\t\t}\n\t\t}),\n\t)\n\tconn, err := DialConnection(network, address, time.Second)\n\tMustNil(t, err)\n\n\tfor i := 0; i < 1024; i++ {\n\t\t_, err = conn.Writer().WriteString(req)\n\t\tMustNil(t, err)\n\t\terr = conn.Writer().Flush()\n\t\tMustNil(t, err)\n\n\t\tinput, err := conn.Reader().Next(len(resp))\n\t\tMustNil(t, err)\n\t\tEqual(t, string(input), resp)\n\t}\n\n\terr = conn.Close()\n\tMustNil(t, err)\n\n\terr = loop.Shutdown(context.Background())\n\tMustNil(t, err)\n}\n\nfunc TestOnConnectWrite(t *testing.T) {\n\tnetwork, address := \"tcp\", getTestAddress()\n\tloop := newTestEventLoop(network, address,\n\t\tfunc(ctx context.Context, connection Connection) error {\n\t\t\treturn nil\n\t\t},\n\t\tWithOnConnect(func(ctx context.Context, connection Connection) context.Context {\n\t\t\t_, err := connection.Write([]byte(\"hello\"))\n\t\t\tMustNil(t, err)\n\t\t\treturn ctx\n\t\t}),\n\t)\n\tconn, err := DialConnection(network, address, time.Second)\n\tMustNil(t, err)\n\ts, err := conn.Reader().ReadString(5)\n\tMustNil(t, err)\n\tMustTrue(t, s == \"hello\")\n\n\terr = loop.Shutdown(context.Background())\n\tMustNil(t, err)\n}\n\nfunc TestOnDisconnect(t *testing.T) {\n\ttype ctxKey struct{}\n\tnetwork, address := \"tcp\", getTestAddress()\n\tvar canceled, closed int32\n\tvar conns int32 = 100\n\treq := \"ping\"\n\tloop := newTestEventLoop(network, address,\n\t\tfunc(ctx context.Context, connection Connection) error {\n\t\t\tcancelFunc, _ := ctx.Value(ctxKey{}).(context.CancelFunc)\n\t\t\tMustTrue(t, cancelFunc != nil)\n\t\t\tAssert(t, ctx.Done() != nil)\n\n\t\t\tbuf, err := connection.Reader().Next(4) // should consumed all data\n\t\t\tMustNil(t, err)\n\t\t\tEqual(t, string(buf), req)\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\tatomic.AddInt32(&canceled, 1)\n\t\t\tcase <-time.After(time.Second):\n\t\t\t}\n\t\t\treturn nil\n\t\t},\n\t\tWithOnConnect(func(ctx context.Context, conn Connection) context.Context {\n\t\t\tconn.AddCloseCallback(func(connection Connection) error {\n\t\t\t\tatomic.AddInt32(&closed, 1)\n\t\t\t\treturn nil\n\t\t\t})\n\t\t\tctx, cancel := context.WithCancel(ctx)\n\t\t\treturn context.WithValue(ctx, ctxKey{}, cancel)\n\t\t}),\n\t\tWithOnDisconnect(func(ctx context.Context, conn Connection) {\n\t\t\tcancelFunc, _ := ctx.Value(ctxKey{}).(context.CancelFunc)\n\t\t\tMustTrue(t, cancelFunc != nil)\n\t\t\tcancelFunc()\n\t\t}),\n\t)\n\n\tfor i := int32(0); i < conns; i++ {\n\t\tconn, err := DialConnection(network, address, time.Second)\n\t\tMustNil(t, err)\n\n\t\t_, err = conn.Writer().WriteString(req)\n\t\tMustNil(t, err)\n\t\terr = conn.Writer().Flush()\n\t\tMustNil(t, err)\n\n\t\terr = conn.Close()\n\t\tMustNil(t, err)\n\t}\n\tfor atomic.LoadInt32(&closed) < conns {\n\t\tt.Logf(\"closed: %d, canceled: %d\", atomic.LoadInt32(&closed), atomic.LoadInt32(&canceled))\n\t\truntime.Gosched()\n\t}\n\tEqual(t, atomic.LoadInt32(&closed), conns)\n\tEqual(t, atomic.LoadInt32(&canceled), conns)\n\n\terr := loop.Shutdown(context.Background())\n\tMustNil(t, err)\n}\n\nfunc TestOnDisconnectWhenOnConnect(t *testing.T) {\n\ttype ctxPrepareKey struct{}\n\ttype ctxConnectKey struct{}\n\tnetwork, address := \"tcp\", getTestAddress()\n\tvar conns int32 = 10\n\tvar wg sync.WaitGroup\n\twg.Add(int(conns) * 3)\n\tloop := newTestEventLoop(network, address,\n\t\tfunc(ctx context.Context, connection Connection) error {\n\t\t\t_, _ = connection.Reader().Next(connection.Reader().Len())\n\t\t\treturn nil\n\t\t},\n\t\tWithOnPrepare(func(connection Connection) context.Context {\n\t\t\tdefer wg.Done()\n\t\t\tvar counter int32\n\t\t\treturn context.WithValue(context.Background(), ctxPrepareKey{}, &counter)\n\t\t}),\n\t\tWithOnConnect(func(ctx context.Context, conn Connection) context.Context {\n\t\t\tdefer wg.Done()\n\t\t\tt.Logf(\"OnConnect: %v\", conn.RemoteAddr())\n\t\t\ttime.Sleep(time.Millisecond * 10) // wait for closed called\n\t\t\tcounter := ctx.Value(ctxPrepareKey{}).(*int32)\n\t\t\tok := atomic.CompareAndSwapInt32(counter, 0, 1)\n\t\t\tAssert(t, ok)\n\t\t\treturn context.WithValue(ctx, ctxConnectKey{}, \"123\")\n\t\t}),\n\t\tWithOnDisconnect(func(ctx context.Context, conn Connection) {\n\t\t\tdefer wg.Done()\n\t\t\tt.Logf(\"OnDisconnect: %v\", conn.RemoteAddr())\n\t\t\tcounter, _ := ctx.Value(ctxPrepareKey{}).(*int32)\n\t\t\tok := atomic.CompareAndSwapInt32(counter, 1, 2)\n\t\t\tAssert(t, ok)\n\t\t\tv := ctx.Value(ctxConnectKey{}).(string)\n\t\t\tEqual(t, v, \"123\")\n\t\t}),\n\t)\n\n\tfor i := int32(0); i < conns; i++ {\n\t\tconn, err := DialConnection(network, address, time.Second)\n\t\tMustNil(t, err)\n\t\terr = conn.Close()\n\t\tt.Logf(\"Close: %v\", conn.LocalAddr())\n\t\tMustNil(t, err)\n\t}\n\n\twg.Wait()\n\terr := loop.Shutdown(context.Background())\n\tMustNil(t, err)\n}\n\nfunc TestGracefulExit(t *testing.T) {\n\tnetwork, address := \"tcp\", getTestAddress()\n\n\t// exit without processing connections\n\teventLoop1 := newTestEventLoop(network, address,\n\t\tfunc(ctx context.Context, connection Connection) error {\n\t\t\treturn nil\n\t\t})\n\t_, err := DialConnection(network, address, time.Second)\n\tMustNil(t, err)\n\terr = eventLoop1.Shutdown(context.Background())\n\tMustNil(t, err)\n\n\t// exit with processing connections\n\ttrigger := make(chan struct{})\n\teventLoop2 := newTestEventLoop(network, address,\n\t\tfunc(ctx context.Context, conn Connection) error {\n\t\t\t<-trigger\n\t\t\trd := conn.Reader()\n\t\t\trd.Next(rd.Len()) // avoid dead loop\n\t\t\treturn errors.New(\"done\")\n\t\t})\n\tfor i := 0; i < 10; i++ {\n\t\t// connect success\n\t\tconn, err := DialConnection(network, address, time.Second)\n\t\tMustNil(t, err)\n\t\t_, err = conn.Write(make([]byte, 16))\n\t\tMustNil(t, err)\n\t}\n\t// shutdown timeout\n\tctx2, cancel2 := context.WithTimeout(context.Background(), time.Millisecond*100)\n\tdefer cancel2()\n\terr = eventLoop2.Shutdown(ctx2)\n\tMustTrue(t, err != nil)\n\tEqual(t, err.Error(), ctx2.Err().Error())\n\t// shutdown success\n\tclose(trigger)\n\terr = eventLoop2.Shutdown(ctx2)\n\tMustTrue(t, err == nil)\n\n\t// exit with read connections\n\tsize := 16\n\teventLoop3 := newTestEventLoop(network, address,\n\t\tfunc(ctx context.Context, connection Connection) error {\n\t\t\t_, err := connection.Reader().Next(size)\n\t\t\tMustNil(t, err)\n\t\t\treturn nil\n\t\t})\n\tfor i := 0; i < 10; i++ {\n\t\tconn, err := DialConnection(network, address, time.Second)\n\t\tMustNil(t, err)\n\t\tif i%2 == 0 {\n\t\t\t_, err := conn.Write(make([]byte, size))\n\t\t\tMustNil(t, err)\n\t\t}\n\t}\n\tctx3, cancel3 := context.WithTimeout(context.Background(), 5*time.Second)\n\tdefer cancel3()\n\terr = eventLoop3.Shutdown(ctx3)\n\tMustNil(t, err)\n}\n\nfunc TestCloseCallbackWhenOnRequest(t *testing.T) {\n\tnetwork, address := \"tcp\", getTestAddress()\n\trequested, closed := make(chan struct{}), make(chan struct{})\n\tloop := newTestEventLoop(network, address,\n\t\tfunc(ctx context.Context, connection Connection) error {\n\t\t\t_, err := connection.Reader().Next(connection.Reader().Len())\n\t\t\tMustNil(t, err)\n\t\t\terr = connection.AddCloseCallback(func(connection Connection) error {\n\t\t\t\tclosed <- struct{}{}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t\tMustNil(t, err)\n\t\t\trequested <- struct{}{}\n\t\t\treturn nil\n\t\t},\n\t)\n\tconn, err := DialConnection(network, address, time.Second)\n\tMustNil(t, err)\n\t_, err = conn.Writer().WriteString(\"hello\")\n\tMustNil(t, err)\n\terr = conn.Writer().Flush()\n\tMustNil(t, err)\n\t<-requested\n\terr = conn.Close()\n\tMustNil(t, err)\n\t<-closed\n\n\terr = loop.Shutdown(context.Background())\n\tMustNil(t, err)\n}\n\nfunc TestCloseCallbackWhenOnConnect(t *testing.T) {\n\tnetwork, address := \"tcp\", getTestAddress()\n\tconnected, closed := make(chan struct{}), make(chan struct{})\n\tloop := newTestEventLoop(network, address,\n\t\tnil,\n\t\tWithOnConnect(func(ctx context.Context, connection Connection) context.Context {\n\t\t\terr := connection.AddCloseCallback(func(connection Connection) error {\n\t\t\t\tclosed <- struct{}{}\n\t\t\t\treturn nil\n\t\t\t})\n\t\t\tMustNil(t, err)\n\t\t\tconnected <- struct{}{}\n\t\t\treturn ctx\n\t\t}),\n\t)\n\tconn, err := DialConnection(network, address, time.Second)\n\tMustNil(t, err)\n\terr = conn.Close()\n\tMustNil(t, err)\n\n\t<-connected\n\t<-closed\n\n\terr = loop.Shutdown(context.Background())\n\tMustNil(t, err)\n}\n\nfunc TestCloseConnWhenOnConnect(t *testing.T) {\n\tnetwork, address := \"tcp\", \"localhost:8888\"\n\tconns := 10\n\tvar wg sync.WaitGroup\n\twg.Add(conns)\n\tloop := newTestEventLoop(network, address,\n\t\tnil,\n\t\tWithOnConnect(func(ctx context.Context, connection Connection) context.Context {\n\t\t\tdefer wg.Done()\n\t\t\terr := connection.Close()\n\t\t\tMustNil(t, err)\n\t\t\treturn ctx\n\t\t}),\n\t)\n\n\tfor i := 0; i < conns; i++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tconn, err := DialConnection(network, address, time.Second)\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\t_, err = conn.Reader().Next(1)\n\t\t\tAssert(t, errors.Is(err, ErrEOF))\n\t\t\terr = conn.Close()\n\t\t\tMustNil(t, err)\n\t\t}()\n\t}\n\n\twg.Wait()\n\terr := loop.Shutdown(context.Background())\n\tMustNil(t, err)\n}\n\nfunc TestServerReadAndClose(t *testing.T) {\n\tnetwork, address := \"tcp\", getTestAddress()\n\tsendMsg := []byte(\"hello\")\n\tloop := newTestEventLoop(network, address,\n\t\tfunc(ctx context.Context, connection Connection) error {\n\t\t\t_, err := connection.Reader().Next(len(sendMsg))\n\t\t\tMustNil(t, err)\n\t\t\terr = connection.Close()\n\t\t\tMustNil(t, err)\n\t\t\treturn nil\n\t\t},\n\t)\n\n\tconn, err := DialConnection(network, address, time.Second)\n\tMustNil(t, err)\n\t_, err = conn.Writer().WriteBinary(sendMsg)\n\tMustNil(t, err)\n\terr = conn.Writer().Flush()\n\tMustNil(t, err)\n\n\tfor conn.IsActive() {\n\t\truntime.Gosched() // wait for poller close connection\n\t}\n\t_, err = conn.Writer().WriteBinary(sendMsg)\n\tAssert(t, errors.Is(err, ErrConnClosed), err)\n\n\terr = loop.Shutdown(context.Background())\n\tMustNil(t, err)\n}\n\nfunc TestServerPanicAndClose(t *testing.T) {\n\t// use custom RunTask to ignore panic log\n\trunfunc := runner.RunTask\n\tdefer func() { runner.RunTask = runfunc }()\n\trunner.RunTask = func(ctx context.Context, f func()) {\n\t\tgo func() {\n\t\t\tdefer func() { recover() }()\n\t\t\tf()\n\t\t}()\n\t}\n\n\tnetwork, address := \"tcp\", getTestAddress()\n\tsendMsg := []byte(\"hello\")\n\tvar panicked int32\n\tloop := newTestEventLoop(network, address,\n\t\tfunc(ctx context.Context, connection Connection) error {\n\t\t\t_, err := connection.Reader().Next(len(sendMsg))\n\t\t\tMustNil(t, err)\n\t\t\tatomic.StoreInt32(&panicked, 1)\n\t\t\tpanic(\"test\")\n\t\t},\n\t)\n\n\tconn, err := DialConnection(network, address, time.Second)\n\tMustNil(t, err)\n\t_, err = conn.Writer().WriteBinary(sendMsg)\n\tMustNil(t, err)\n\terr = conn.Writer().Flush()\n\tMustNil(t, err)\n\n\tfor atomic.LoadInt32(&panicked) == 0 {\n\t\truntime.Gosched() // wait for poller close connection\n\t}\n\tfor conn.IsActive() {\n\t\truntime.Gosched() // wait for poller close connection\n\t}\n\n\terr = loop.Shutdown(context.Background())\n\tMustNil(t, err)\n}\n\nfunc TestClientWriteAndClose(t *testing.T) {\n\tvar (\n\t\tnetwork, address            = \"tcp\", getTestAddress()\n\t\tconnnum                     = 10\n\t\tpacketsize, packetnum       = 1000 * 5, 1\n\t\trecvbytes             int32 = 0\n\t)\n\tloop := newTestEventLoop(network, address,\n\t\tfunc(ctx context.Context, connection Connection) error {\n\t\t\tbuf, err := connection.Reader().Next(connection.Reader().Len())\n\t\t\tif errors.Is(err, ErrConnClosed) {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tMustNil(t, err)\n\t\t\tatomic.AddInt32(&recvbytes, int32(len(buf)))\n\t\t\treturn nil\n\t\t},\n\t)\n\tvar wg sync.WaitGroup\n\tfor i := 0; i < connnum; i++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tdefer wg.Done()\n\t\t\tconn, err := DialConnection(network, address, time.Second)\n\t\t\tMustNil(t, err)\n\t\t\tsendMsg := make([]byte, packetsize)\n\t\t\tfor j := 0; j < packetnum; j++ {\n\t\t\t\t_, err = conn.Write(sendMsg)\n\t\t\t\tMustNil(t, err)\n\t\t\t}\n\t\t\terr = conn.Close()\n\t\t\tMustNil(t, err)\n\t\t}()\n\t}\n\twg.Wait()\n\texceptbytes := int32(packetsize * packetnum * connnum)\n\tfor atomic.LoadInt32(&recvbytes) != exceptbytes {\n\t\tt.Logf(\"left %d bytes not received\", exceptbytes-atomic.LoadInt32(&recvbytes))\n\t\truntime.Gosched()\n\t}\n\terr := loop.Shutdown(context.Background())\n\tMustNil(t, err)\n}\n\nfunc TestServerAcceptWhenTooManyOpenFiles(t *testing.T) {\n\tif os.Getenv(\"N_LOCAL\") == \"\" {\n\t\tt.Skip(\"Only test for debug purpose\")\n\t\treturn\n\t}\n\n\tvar originalRlimit syscall.Rlimit\n\terr := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &originalRlimit)\n\tMustNil(t, err)\n\tt.Logf(\"Original RLimit: %v\", originalRlimit)\n\n\trlimit := syscall.Rlimit{Cur: 32, Max: originalRlimit.Max}\n\terr = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &rlimit)\n\tMustNil(t, err)\n\terr = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit)\n\tMustNil(t, err)\n\tt.Logf(\"New RLimit: %v\", rlimit)\n\tdefer func() { // reset\n\t\terr = syscall.Setrlimit(syscall.RLIMIT_NOFILE, &originalRlimit)\n\t\tMustNil(t, err)\n\t}()\n\n\tnetwork, address := \"tcp\", getTestAddress()\n\tvar connected int32\n\tloop := newTestEventLoop(network, address,\n\t\tfunc(ctx context.Context, connection Connection) error {\n\t\t\tbuf, err := connection.Reader().Next(connection.Reader().Len())\n\t\t\tconnection.Writer().WriteBinary(buf)\n\t\t\tconnection.Writer().Flush()\n\t\t\treturn err\n\t\t},\n\t\tWithOnConnect(func(ctx context.Context, connection Connection) context.Context {\n\t\t\tatomic.AddInt32(&connected, 1)\n\t\t\tt.Logf(\"Conn[%s] accepted\", connection.RemoteAddr())\n\t\t\treturn ctx\n\t\t}),\n\t\tWithOnDisconnect(func(ctx context.Context, connection Connection) {\n\t\t\tt.Logf(\"Conn[%s] disconnected\", connection.RemoteAddr())\n\t\t}),\n\t)\n\ttime.Sleep(time.Millisecond * 10)\n\n\t// out of fds\n\tfiles := make([]*os.File, 0)\n\tfor {\n\t\tf, err := os.Open(\"/dev/null\")\n\t\tif err != nil {\n\t\t\tAssert(t, isOutOfFdErr(errors.Unwrap(err)), err)\n\t\t\tbreak\n\t\t}\n\t\tfiles = append(files, f)\n\t}\n\tgo func() {\n\t\ttime.Sleep(time.Second * 10)\n\t\tt.Logf(\"close all files\")\n\t\tfor _, f := range files {\n\t\t\tf.Close()\n\t\t}\n\t}()\n\n\t// we should use telnet manually\n\tconnections := 1\n\tfor atomic.LoadInt32(&connected) < int32(connections) {\n\t\tt.Logf(\"connected=%d\", atomic.LoadInt32(&connected))\n\t\ttime.Sleep(time.Second)\n\t}\n\ttime.Sleep(time.Second * 10)\n\n\terr = loop.Shutdown(context.Background())\n\tMustNil(t, err)\n}\n\nfunc createTestListener(network, address string) (Listener, error) {\n\tfor {\n\t\tln, err := CreateListener(network, address)\n\t\tif err == nil {\n\t\t\treturn ln, nil\n\t\t}\n\t\ttime.Sleep(time.Millisecond * 100)\n\t}\n}\n\nfunc newTestEventLoop(network, address string, onRequest OnRequest, opts ...Option) EventLoop {\n\tln, err := createTestListener(network, address)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\telp, err := NewEventLoop(onRequest, opts...)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tgo elp.Serve(ln)\n\treturn elp\n}\n"
  },
  {
    "path": "netpoll_windows.go",
    "content": "// Copyright 2024 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build windows\n// +build windows\n\n// The following methods would not be used, but are intended to compile on Windows.\npackage netpoll\n\nimport (\n\t\"net\"\n)\n\n// Configure the internal behaviors of netpoll.\nfunc Configure(config Config) (err error) {\n\treturn nil\n}\n\n// NewDialer only support TCP and unix socket now.\nfunc NewDialer() Dialer {\n\treturn nil\n}\n\n// NewEventLoop .\nfunc NewEventLoop(onRequest OnRequest, ops ...Option) (EventLoop, error) {\n\treturn nil, nil\n}\n\n// ConvertListener converts net.Listener to Listener\nfunc ConvertListener(l net.Listener) (nl Listener, err error) {\n\treturn nil, nil\n}\n\n// CreateListener return a new Listener.\nfunc CreateListener(network, addr string) (l Listener, err error) {\n\treturn nil, nil\n}\n"
  },
  {
    "path": "nocopy.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage netpoll\n\nimport (\n\t\"io\"\n\t\"reflect\"\n\t\"unsafe\"\n\n\t\"github.com/bytedance/gopkg/lang/dirtmake\"\n\t\"github.com/bytedance/gopkg/lang/mcache\"\n)\n\n// Reader is a collection of operations for nocopy reads.\n//\n// For ease of use, it is recommended to implement Reader as a blocking interface,\n// rather than simply fetching the buffer.\n// For example, the return of calling Next(n) should be blocked if there are fewer than n bytes, unless timeout.\n// The return value is guaranteed to meet the requirements or an error will be returned.\ntype Reader interface {\n\t// Next returns a slice containing the next n bytes from the buffer,\n\t// advancing the buffer as if the bytes had been returned by Read.\n\t//\n\t// If there are fewer than n bytes in the buffer, Next returns will be blocked\n\t// until data enough or an error occurs (such as a wait timeout).\n\t//\n\t// The slice p is only valid until the next call to the Release method.\n\t// Next is not globally optimal, and Skip, ReadString, ReadBinary methods\n\t// are recommended for specific scenarios.\n\t//\n\t// Return: len(p) must be n or 0, and p and error cannot be nil at the same time.\n\tNext(n int) (p []byte, err error)\n\n\t// Peek returns the next n bytes without advancing the reader.\n\t// Other behavior is the same as Next.\n\tPeek(n int) (buf []byte, err error)\n\n\t// Skip the next n bytes and advance the reader, which is\n\t// a faster implementation of Next when the next data is not used.\n\tSkip(n int) (err error)\n\n\t// Until reads until the first occurrence of delim in the input,\n\t// returning a slice stops with delim in the input buffer.\n\t// If Until encounters an error before finding a delimiter,\n\t// it returns all the data in the buffer and the error itself (often ErrEOF or ErrConnClosed).\n\t// Until returns err != nil only if line does not end in delim.\n\tUntil(delim byte) (line []byte, err error)\n\n\t// ReadString is a faster implementation of Next when a string needs to be returned.\n\t// It replaces:\n\t//\n\t//  var p, err = Next(n)\n\t//  return string(p), err\n\t//\n\tReadString(n int) (s string, err error)\n\n\t// ReadBinary is a faster implementation of Next when it needs to\n\t// return a copy of the slice that is not shared with the underlying layer.\n\t// It replaces:\n\t//\n\t//  var p, err = Next(n)\n\t//  var b = make([]byte, n)\n\t//  copy(b, p)\n\t//  return b, err\n\t//\n\tReadBinary(n int) (p []byte, err error)\n\n\t// ReadByte is a faster implementation of Next when a byte needs to be returned.\n\t// It replaces:\n\t//\n\t//  var p, err = Next(1)\n\t//  return p[0], err\n\t//\n\tReadByte() (b byte, err error)\n\n\t// Slice returns a new Reader containing the Next n bytes from this Reader.\n\t//\n\t// If you want to make a new Reader using the []byte returned by Next, Slice already does that,\n\t// and the operation is zero-copy. Besides, Slice would also Release this Reader.\n\t// The logic pseudocode is similar:\n\t//\n\t//  var p, err = this.Next(n)\n\t//  var reader = new Reader(p) // pseudocode\n\t//  this.Release()\n\t//  return reader, err\n\t//\n\tSlice(n int) (r Reader, err error)\n\n\t// Release the memory space occupied by all read slices. This method needs to be executed actively to\n\t// recycle the memory after confirming that the previously read data is no longer in use.\n\t// After invoking Release, the slices obtained by the method such as Next, Peek, Skip will\n\t// become an invalid address and cannot be used anymore.\n\tRelease() (err error)\n\n\t// Len returns the total length of the readable data in the reader.\n\tLen() (length int)\n}\n\n// Writer is a collection of operations for nocopy writes.\n//\n// The usage of the design is a two-step operation, first apply for a section of memory,\n// fill it and then submit. E.g:\n//\n//\tvar buf, _ = Malloc(n)\n//\tbuf = append(buf[:0], ...)\n//\tFlush()\n//\n// Note that it is not recommended to submit self-managed buffers to Writer.\n// Since the writer is processed asynchronously, if the self-managed buffer is used and recycled after submission,\n// it may cause inconsistent life cycle problems. Of course this is not within the scope of the design.\ntype Writer interface {\n\t// Malloc returns a slice containing the next n bytes from the buffer,\n\t// which will be written after submission(e.g. Flush).\n\t//\n\t// The slice p is only valid until the next submit(e.g. Flush).\n\t// Therefore, please make sure that all data has been written into the slice before submission.\n\tMalloc(n int) (buf []byte, err error)\n\n\t// WriteString is a faster implementation of Malloc when a string needs to be written.\n\t// It replaces:\n\t//\n\t//  var buf, err = Malloc(len(s))\n\t//  n = copy(buf, s)\n\t//  return n, err\n\t//\n\t// The argument string s will be referenced based on the original address and will not be copied,\n\t// so make sure that the string s will not be changed.\n\tWriteString(s string) (n int, err error)\n\n\t// WriteBinary is a faster implementation of Malloc when a slice needs to be written.\n\t// It replaces:\n\t//\n\t//  var buf, err = Malloc(len(b))\n\t//  n = copy(buf, b)\n\t//  return n, err\n\t//\n\t// The argument slice b will be referenced based on the original address and will not be copied,\n\t// so make sure that the slice b will not be changed.\n\tWriteBinary(b []byte) (n int, err error)\n\n\t// WriteByte is a faster implementation of Malloc when a byte needs to be written.\n\t// It replaces:\n\t//\n\t//  var buf, _ = Malloc(1)\n\t//  buf[0] = b\n\t//\n\tWriteByte(b byte) (err error)\n\n\t// WriteDirect is used to insert an additional slice of data on the current write stream.\n\t// For example, if you plan to execute:\n\t//\n\t//  var bufA, _ = Malloc(nA)\n\t//  WriteBinary(b)\n\t//  var bufB, _ = Malloc(nB)\n\t//\n\t// It can be replaced by:\n\t//\n\t//  var buf, _ = Malloc(nA+nB)\n\t//  WriteDirect(b, nB)\n\t//\n\t// where buf[:nA] = bufA, buf[nA:nA+nB] = bufB.\n\tWriteDirect(p []byte, remainCap int) error\n\n\t// MallocAck will keep the first n malloc bytes and discard the rest.\n\t// The following behavior:\n\t//\n\t//  var buf, _ = Malloc(8)\n\t//  buf = buf[:5]\n\t//  MallocAck(5)\n\t//\n\t// equivalent as\n\t//  var buf, _ = Malloc(5)\n\t//\n\tMallocAck(n int) (err error)\n\n\t// Append the argument writer to the tail of this writer and set the argument writer to nil,\n\t// the operation is zero-copy, similar to p = append(p, w.p).\n\tAppend(w Writer) (err error)\n\n\t// Flush will submit all malloc data and must confirm that the allocated bytes have been correctly assigned.\n\t// Its behavior is equivalent to the io.Writer hat already has parameters(slice b).\n\tFlush() (err error)\n\n\t// MallocLen returns the total length of the writable data that has not yet been submitted in the writer.\n\tMallocLen() (length int)\n}\n\n// ReadWriter is a combination of Reader and Writer.\ntype ReadWriter interface {\n\tReader\n\tWriter\n}\n\n// NewReader convert io.Reader to nocopy Reader\nfunc NewReader(r io.Reader) Reader {\n\treturn newZCReader(r)\n}\n\n// NewWriter convert io.Writer to nocopy Writer\nfunc NewWriter(w io.Writer) Writer {\n\treturn newZCWriter(w)\n}\n\n// NewReadWriter convert io.ReadWriter to nocopy ReadWriter\nfunc NewReadWriter(rw io.ReadWriter) ReadWriter {\n\treturn &zcReadWriter{\n\t\tzcReader: newZCReader(rw),\n\t\tzcWriter: newZCWriter(rw),\n\t}\n}\n\n// NewIOReader convert Reader to io.Reader\nfunc NewIOReader(r Reader) io.Reader {\n\tif reader, ok := r.(io.Reader); ok {\n\t\treturn reader\n\t}\n\treturn newIOReader(r)\n}\n\n// NewIOWriter convert Writer to io.Writer\nfunc NewIOWriter(w Writer) io.Writer {\n\tif writer, ok := w.(io.Writer); ok {\n\t\treturn writer\n\t}\n\treturn newIOWriter(w)\n}\n\n// NewIOReadWriter convert ReadWriter to io.ReadWriter\nfunc NewIOReadWriter(rw ReadWriter) io.ReadWriter {\n\tif rwer, ok := rw.(io.ReadWriter); ok {\n\t\treturn rwer\n\t}\n\treturn &ioReadWriter{\n\t\tReader: NewIOReader(rw),\n\t\tWriter: NewIOWriter(rw),\n\t}\n}\n\nconst (\n\tblock1k  = 1 * 1024\n\tblock2k  = 2 * 1024\n\tblock4k  = 4 * 1024\n\tblock8k  = 8 * 1024\n\tblock32k = 32 * 1024\n\n\tpagesize  = block8k\n\tmallocMax = block8k * block1k // mallocMax is 8MB\n\n\tdefaultLinkBufferMode = 0\n\t// flagUnmanaged marks a buffer node whose memory is not allocated by the LinkBuffer\n\t// (e.g. user-provided data via WriteDirect, or a zero-size node).\n\t// Unmanaged nodes are not reusable and are skipped during buffer growth.\n\tflagUnmanaged uint8 = 1 << 0 // 0000 0001\n\t// flagReadExposed marks a buffer node whose underlying memory has been returned\n\t// directly to user code via a zero-copy Reader method (Next, Peek, Slice, GetBytes).\n\t// The buffer may still be referenced by user code until Release is called.\n\tflagReadExposed uint8 = 1 << 1 // 0000 0010\n)\n\n// zero-copy slice convert to string\nfunc unsafeSliceToString(b []byte) string {\n\treturn *(*string)(unsafe.Pointer(&b))\n}\n\n// zero-copy string convert to slice\nfunc unsafeStringToSlice(s string) (b []byte) {\n\tp := unsafe.Pointer((*reflect.StringHeader)(unsafe.Pointer(&s)).Data)\n\thdr := (*reflect.SliceHeader)(unsafe.Pointer(&b))\n\thdr.Data = uintptr(p)\n\thdr.Cap = len(s)\n\thdr.Len = len(s)\n\treturn b\n}\n\n// malloc limits the cap of the buffer from mcache.\nfunc malloc(size, capacity int) []byte {\n\tif capacity > mallocMax {\n\t\treturn dirtmake.Bytes(size, capacity)\n\t}\n\treturn mcache.Malloc(size, capacity)\n}\n\n// free limits the cap of the buffer from mcache.\nfunc free(buf []byte) {\n\tif cap(buf) > mallocMax {\n\t\treturn\n\t}\n\tmcache.Free(buf)\n}\n"
  },
  {
    "path": "nocopy_linkbuffer.go",
    "content": "// Copyright 2024 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage netpoll\n\nimport (\n\t\"bytes\"\n\t\"errors\"\n\t\"fmt\"\n\t\"sync\"\n\t\"sync/atomic\"\n\n\t\"github.com/bytedance/gopkg/lang/dirtmake\"\n)\n\n// BinaryInplaceThreshold marks the minimum value of the nocopy slice length,\n// which is the threshold to use copy to minimize overhead.\nconst BinaryInplaceThreshold = block4k\n\n// LinkBufferCap that can be modified marks the minimum value of each node of LinkBuffer.\nvar LinkBufferCap = block4k\n\nvar untilErr = errors.New(\"link buffer read slice cannot find delim\")\n\nvar (\n\t_ Reader = &LinkBuffer{}\n\t_ Writer = &LinkBuffer{}\n)\n\n// NewLinkBuffer size defines the initial capacity, but there is no readable data.\nfunc NewLinkBuffer(size ...int) *LinkBuffer {\n\tbuf := &LinkBuffer{}\n\tvar l int\n\tif len(size) > 0 {\n\t\tl = size[0]\n\t}\n\tnode := newLinkBufferNode(l)\n\tbuf.head, buf.read, buf.flush, buf.write = node, node, node, node\n\treturn buf\n}\n\n// UnsafeLinkBuffer implements ReadWriter.\ntype UnsafeLinkBuffer struct {\n\tlength     int64\n\tmallocSize int\n\n\thead  *linkBufferNode // release head\n\tread  *linkBufferNode // read head\n\tflush *linkBufferNode // malloc head\n\twrite *linkBufferNode // malloc tail\n\n\t// buf allocated by Next when cross-package, which should be freed when release\n\tcaches [][]byte\n\n\t// for `Peek` only, avoid creating too many []byte in `caches`\n\t// fix the issue when we have a large buffer and we call `Peek` multiple times\n\tcachePeek []byte\n}\n\n// Len implements Reader.\nfunc (b *UnsafeLinkBuffer) Len() int {\n\tl := atomic.LoadInt64(&b.length)\n\treturn int(l)\n}\n\n// IsEmpty check if this LinkBuffer is empty.\nfunc (b *UnsafeLinkBuffer) IsEmpty() (ok bool) {\n\treturn b.Len() == 0\n}\n\n// ------------------------------------------ implement copy reader ------------------------------------------\n\n// readCopy copies up to len(p) bytes from the buffer into p without exposing\n// the underlying buffer to user code (flagReadExposed is not set).\n// After copying, it releases consumed nodes where readExposed is false.\n// Nodes with readExposed are left for the next Release call.\nfunc (b *UnsafeLinkBuffer) readCopy(p []byte) (n int) {\n\tl := len(p)\n\tif l == 0 || b.Len() == 0 {\n\t\treturn 0\n\t}\n\tif has := b.Len(); has < l {\n\t\tl = has\n\t}\n\tb.recalLen(-l)\n\n\t// copy from nodes\n\tfor ack := l; ack > 0; {\n\t\tif b.read.Len() == 0 {\n\t\t\tb.read = b.read.next\n\t\t\tcontinue\n\t\t}\n\t\trd := b.read.Len()\n\t\tif rd >= ack {\n\t\t\tn += copy(p[n:], b.read.buf[b.read.off:b.read.off+ack])\n\t\t\tb.read.off += ack\n\t\t\tbreak\n\t\t}\n\t\tn += copy(p[n:], b.read.buf[b.read.off:])\n\t\tack -= rd\n\t\tb.read = b.read.next\n\t}\n\n\t// advance read past empty nodes\n\tfor b.read != b.flush && b.read.Len() == 0 {\n\t\tb.read = b.read.next\n\t}\n\t// release consumed nodes that are not readExposed.\n\t// exposed nodes stay in the chain so Release() can free them later.\n\t//\n\t// Example: [exposed/consumed] → [not-exposed/consumed] → [read/partial]\n\t// After:   head → [exposed] → [read/partial]\n\t//          the middle node is detached and released.\n\tvar prev *linkBufferNode\n\tnewHead := b.read\n\tfor cur := b.head; cur != b.read; {\n\t\tnext := cur.next\n\t\tif cur.readExposed() {\n\t\t\tif prev == nil {\n\t\t\t\tnewHead = cur\n\t\t\t}\n\t\t\tprev = cur\n\t\t} else {\n\t\t\tcur.Release()\n\t\t\tif prev != nil {\n\t\t\t\tprev.next = next\n\t\t\t}\n\t\t}\n\t\tcur = next\n\t}\n\tb.head = newHead\n\treturn n\n}\n\n// ------------------------------------------ implement zero-copy reader ------------------------------------------\n\n// Next implements Reader.\nfunc (b *UnsafeLinkBuffer) Next(n int) (p []byte, err error) {\n\tif n <= 0 {\n\t\treturn\n\t}\n\t// check whether enough or not.\n\tif b.Len() < n {\n\t\treturn p, fmt.Errorf(\"link buffer next[%d] not enough\", n)\n\t}\n\tb.recalLen(-n) // re-cal length\n\n\t// single node\n\tif b.isSingleNode(n) {\n\t\tb.read.setFlag(flagReadExposed)\n\t\treturn b.read.Next(n), nil\n\t}\n\t// multiple nodes\n\tvar pIdx int\n\tif block1k < n && n <= mallocMax {\n\t\tp = malloc(n, n)\n\t\tb.caches = append(b.caches, p)\n\t} else {\n\t\tp = dirtmake.Bytes(n, n)\n\t}\n\tvar l int\n\tfor ack := n; ack > 0; ack = ack - l {\n\t\tl = b.read.Len()\n\t\tif l >= ack {\n\t\t\tpIdx += copy(p[pIdx:], b.read.Next(ack))\n\t\t\tbreak\n\t\t} else if l > 0 {\n\t\t\tpIdx += copy(p[pIdx:], b.read.Next(l))\n\t\t}\n\t\tb.read = b.read.next\n\t}\n\t_ = pIdx\n\treturn p, nil\n}\n\n// Peek does not have an independent lifecycle, and there is no signal to\n// indicate that Peek content can be released, so Peek will not introduce mcache for now.\nfunc (b *UnsafeLinkBuffer) Peek(n int) (p []byte, err error) {\n\tif n <= 0 {\n\t\treturn\n\t}\n\t// check whether enough or not.\n\tif b.Len() < n {\n\t\treturn p, fmt.Errorf(\"link buffer peek[%d] not enough\", n)\n\t}\n\t// single node\n\tif b.isSingleNode(n) {\n\t\tb.read.setFlag(flagReadExposed)\n\t\treturn b.read.Peek(n), nil\n\t}\n\n\t// multiple nodes\n\n\t// try to make use of the cap of b.cachePeek, if can't, free it.\n\tif b.cachePeek != nil && cap(b.cachePeek) < n {\n\t\tfree(b.cachePeek)\n\t\tb.cachePeek = nil\n\t}\n\tif b.cachePeek == nil {\n\t\tb.cachePeek = malloc(0, n) // init with zero len, will append later\n\t}\n\tp = b.cachePeek\n\tif len(p) >= n {\n\t\t// in case we peek smaller than last time,\n\t\t// we can return cache data directly.\n\t\t// we will reset cachePeek when Next or Skip, no worries about stale data\n\t\treturn p[:n], nil\n\t}\n\n\t// How it works >>>>>>\n\t// [ -------- node0 -------- ][ --------- node1 --------- ]  <- b.read\n\t// [ --------------- p --------------- ]\n\t//                                     ^ len(p)     ^ n here\n\t//                           ^ scanned\n\t// `scanned` var is the len of last nodes which we scanned and already copied to p\n\t// `len(p) - scanned` is the start pos of current node for p to copy from\n\t// `n - len(p)` is the len of bytes we're going to append to p\n\t// \t\twe copy `len(node1)` - `len(p) - scanned` bytes in case node1 doesn't have enough data\n\tfor scanned, node := 0, b.read; len(p) < n; node = node.next {\n\t\tl := node.Len()\n\t\tif scanned+l <= len(p) { // already copied in p, skip\n\t\t\tscanned += l\n\t\t\tcontinue\n\t\t}\n\t\tstart := len(p) - scanned // `start` must be smaller than l coz `scanned+l <= len(p)` is false\n\t\tcopyn := n - len(p)\n\t\tif nodeLeftN := l - start; copyn > nodeLeftN {\n\t\t\tcopyn = nodeLeftN\n\t\t}\n\t\tp = append(p, node.Peek(l)[start:start+copyn]...)\n\t\tscanned += l\n\t}\n\tb.cachePeek = p\n\treturn p[:n], nil\n}\n\n// Skip implements Reader.\nfunc (b *UnsafeLinkBuffer) Skip(n int) (err error) {\n\tif n <= 0 {\n\t\treturn\n\t}\n\t// check whether enough or not.\n\tif b.Len() < n {\n\t\treturn fmt.Errorf(\"link buffer skip[%d] not enough\", n)\n\t}\n\tb.recalLen(-n) // re-cal length\n\n\tvar l int\n\tfor ack := n; ack > 0; ack = ack - l {\n\t\tl = b.read.Len()\n\t\tif l >= ack {\n\t\t\tb.read.off += ack\n\t\t\tbreak\n\t\t}\n\t\tb.read = b.read.next\n\t}\n\treturn nil\n}\n\n// Release the node that has been read.\n// b.flush == nil indicates that this LinkBuffer is created by LinkBuffer.Slice\nfunc (b *UnsafeLinkBuffer) Release() (err error) {\n\tfor b.read != b.flush && b.read.Len() == 0 {\n\t\tb.read = b.read.next\n\t}\n\tfor b.head != b.read {\n\t\tnode := b.head\n\t\tb.head = b.head.next\n\t\tnode.Release()\n\t}\n\tfor i := range b.caches {\n\t\tfree(b.caches[i])\n\t\tb.caches[i] = nil\n\t}\n\tb.caches = b.caches[:0]\n\tif b.cachePeek != nil {\n\t\tfree(b.cachePeek)\n\t\tb.cachePeek = nil\n\t}\n\treturn nil\n}\n\n// ReadString implements Reader.\nfunc (b *UnsafeLinkBuffer) ReadString(n int) (s string, err error) {\n\tif n <= 0 {\n\t\treturn\n\t}\n\t// check whether enough or not.\n\tif b.Len() < n {\n\t\treturn s, fmt.Errorf(\"link buffer read string[%d] not enough\", n)\n\t}\n\treturn unsafeSliceToString(b.readBinary(n)), nil\n}\n\n// ReadBinary implements Reader.\nfunc (b *UnsafeLinkBuffer) ReadBinary(n int) (p []byte, err error) {\n\tif n <= 0 {\n\t\treturn\n\t}\n\t// check whether enough or not.\n\tif b.Len() < n {\n\t\treturn p, fmt.Errorf(\"link buffer read binary[%d] not enough\", n)\n\t}\n\treturn b.readBinary(n), nil\n}\n\n// readBinary cannot use mcache, because the memory allocated by readBinary will not be recycled.\nfunc (b *UnsafeLinkBuffer) readBinary(n int) (p []byte) {\n\tb.recalLen(-n) // re-cal length\n\n\t// single node\n\tif b.isSingleNode(n) {\n\t\tp = dirtmake.Bytes(n, n)\n\t\tcopy(p, b.read.Next(n))\n\t\treturn p\n\t}\n\tp = dirtmake.Bytes(n, n)\n\t// multiple nodes\n\tvar pIdx int\n\tvar l int\n\tfor ack := n; ack > 0; ack = ack - l {\n\t\tl = b.read.Len()\n\t\tif l >= ack {\n\t\t\tpIdx += copy(p[pIdx:], b.read.Next(ack))\n\t\t\tbreak\n\t\t} else if l > 0 {\n\t\t\tpIdx += copy(p[pIdx:], b.read.Next(l))\n\t\t}\n\t\tb.read = b.read.next\n\t}\n\t_ = pIdx\n\treturn p\n}\n\n// ReadByte implements Reader.\nfunc (b *UnsafeLinkBuffer) ReadByte() (p byte, err error) {\n\t// check whether enough or not.\n\tif b.Len() < 1 {\n\t\treturn p, errors.New(\"link buffer read byte is empty\")\n\t}\n\tb.recalLen(-1) // re-cal length\n\tfor {\n\t\tif b.read.Len() >= 1 {\n\t\t\treturn b.read.Next(1)[0], nil\n\t\t}\n\t\tb.read = b.read.next\n\t}\n}\n\n// Until returns a slice ends with the delim in the buffer.\nfunc (b *UnsafeLinkBuffer) Until(delim byte) (line []byte, err error) {\n\tn := b.indexByte(delim, 0)\n\tif n < 0 {\n\t\treturn nil, untilErr\n\t}\n\treturn b.Next(n + 1)\n}\n\n// Slice returns a new LinkBuffer, which is a zero-copy slice of this LinkBuffer,\n// and only holds the ability of Reader.\n//\n// Slice will automatically execute a Release.\nfunc (b *UnsafeLinkBuffer) Slice(n int) (r Reader, err error) {\n\tif n <= 0 {\n\t\treturn NewLinkBuffer(0), nil\n\t}\n\t// check whether enough or not.\n\tif b.Len() < n {\n\t\treturn r, fmt.Errorf(\"link buffer readv[%d] not enough\", n)\n\t}\n\tb.recalLen(-n) // re-cal length\n\n\t// just use for range\n\tp := new(LinkBuffer)\n\tp.length = int64(n)\n\n\tdefer func() {\n\t\t// set to read-only\n\t\tp.flush = p.flush.next\n\t\tp.write = p.flush\n\t}()\n\n\t// single node\n\tif b.isSingleNode(n) {\n\t\tb.read.setFlag(flagReadExposed)\n\t\tnode := b.read.Refer(n)\n\t\tp.head, p.read, p.flush = node, node, node\n\t\treturn p, nil\n\t}\n\t// multiple nodes\n\tl := b.read.Len()\n\tb.read.setFlag(flagReadExposed)\n\tnode := b.read.Refer(l)\n\tb.read = b.read.next\n\n\tp.head, p.read, p.flush = node, node, node\n\tfor ack := n - l; ack > 0; ack = ack - l {\n\t\tl = b.read.Len()\n\t\tif l >= ack {\n\t\t\tb.read.setFlag(flagReadExposed)\n\t\t\tp.flush.next = b.read.Refer(ack)\n\t\t\tp.flush = p.flush.next\n\t\t\tbreak\n\t\t} else if l > 0 {\n\t\t\tb.read.setFlag(flagReadExposed)\n\t\t\tp.flush.next = b.read.Refer(l)\n\t\t\tp.flush = p.flush.next\n\t\t}\n\t\tb.read = b.read.next\n\t}\n\treturn p, b.Release()\n}\n\n// ------------------------------------------ implement zero-copy writer ------------------------------------------\n\n// Malloc pre-allocates memory, which is not readable, and becomes readable data after submission(e.g. Flush).\nfunc (b *UnsafeLinkBuffer) Malloc(n int) (buf []byte, err error) {\n\tif n <= 0 {\n\t\treturn\n\t}\n\tb.mallocSize += n\n\tb.growth(n)\n\treturn b.write.Malloc(n), nil\n}\n\n// MallocLen implements Writer.\nfunc (b *UnsafeLinkBuffer) MallocLen() (length int) {\n\treturn b.mallocSize\n}\n\n// MallocAck will keep the first n malloc bytes and discard the rest.\nfunc (b *UnsafeLinkBuffer) MallocAck(n int) (err error) {\n\tif n < 0 {\n\t\treturn fmt.Errorf(\"link buffer malloc ack[%d] invalid\", n)\n\t}\n\tb.mallocSize = n\n\tb.write = b.flush\n\n\tvar l int\n\tfor ack := n; ack > 0; ack = ack - l {\n\t\tl = b.write.malloc - len(b.write.buf)\n\t\tif l >= ack {\n\t\t\tb.write.malloc = ack + len(b.write.buf)\n\t\t\tbreak\n\t\t}\n\t\tb.write = b.write.next\n\t}\n\t// discard the rest\n\tfor node := b.write.next; node != nil; node = node.next {\n\t\tnode.malloc, node.refer, node.buf = node.off, 1, node.buf[:node.off]\n\t}\n\treturn nil\n}\n\n// Flush will submit all malloc data and must confirm that the allocated bytes have been correctly assigned.\nfunc (b *UnsafeLinkBuffer) Flush() (err error) {\n\tb.mallocSize = 0\n\t// FIXME: The tail node must not be larger than 8KB to prevent Out Of Memory.\n\tif cap(b.write.buf) > pagesize {\n\t\tb.write.next = newLinkBufferNode(0)\n\t\tb.write = b.write.next\n\t}\n\tvar n int\n\tfor node := b.flush; node != b.write.next; node = node.next {\n\t\tdelta := node.malloc - len(node.buf)\n\t\tif delta > 0 {\n\t\t\tn += delta\n\t\t\tnode.buf = node.buf[:node.malloc]\n\t\t}\n\t}\n\tb.flush = b.write\n\t// re-cal length\n\tb.recalLen(n)\n\treturn nil\n}\n\n// Append implements Writer.\nfunc (b *UnsafeLinkBuffer) Append(w Writer) (err error) {\n\tbuf, ok := w.(*LinkBuffer)\n\tif !ok {\n\t\treturn errors.New(\"unsupported writer which is not LinkBuffer\")\n\t}\n\treturn b.WriteBuffer(buf)\n}\n\n// WriteBuffer will not submit(e.g. Flush) data to ensure normal use of MallocLen.\n// you must actively submit before read the data.\n// The argument buf can't be used after calling WriteBuffer. (set it to nil)\nfunc (b *UnsafeLinkBuffer) WriteBuffer(buf *LinkBuffer) (err error) {\n\tif buf == nil {\n\t\treturn\n\t}\n\tbufLen, bufMallocLen := buf.Len(), buf.MallocLen()\n\tif bufLen+bufMallocLen <= 0 {\n\t\treturn nil\n\t}\n\tb.write.next = buf.read\n\tb.write = buf.write\n\n\t// close buf, prevents reuse.\n\tfor buf.head != buf.read {\n\t\tnd := buf.head\n\t\tbuf.head = buf.head.next\n\t\tnd.Release()\n\t}\n\tfor buf.write = buf.write.next; buf.write != nil; {\n\t\tnd := buf.write\n\t\tbuf.write = buf.write.next\n\t\tnd.Release()\n\t}\n\tbuf.length, buf.mallocSize, buf.head, buf.read, buf.flush, buf.write = 0, 0, nil, nil, nil, nil\n\n\t// DON'T MODIFY THE CODE BELOW UNLESS YOU KNOW WHAT YOU ARE DOING !\n\t//\n\t// You may encounter a chain of bugs and not be able to\n\t// find out within a week that they are caused by modifications here.\n\t//\n\t// After release buf, continue to adjust b.\n\tb.write.next = nil\n\tif bufLen > 0 {\n\t\tb.recalLen(bufLen)\n\t}\n\tb.mallocSize += bufMallocLen\n\treturn nil\n}\n\n// WriteString implements Writer.\nfunc (b *UnsafeLinkBuffer) WriteString(s string) (n int, err error) {\n\tif len(s) == 0 {\n\t\treturn\n\t}\n\tbuf := unsafeStringToSlice(s)\n\treturn b.WriteBinary(buf)\n}\n\n// WriteBinary implements Writer.\nfunc (b *UnsafeLinkBuffer) WriteBinary(p []byte) (n int, err error) {\n\tn = len(p)\n\tif n == 0 {\n\t\treturn\n\t}\n\tb.mallocSize += n\n\n\t// TODO: Verify that all nocopy is possible under mcache.\n\tif n > BinaryInplaceThreshold {\n\t\t// expand buffer directly with nocopy\n\t\tb.write.next = newLinkBufferNode(0)\n\t\tb.write = b.write.next\n\t\tb.write.buf, b.write.malloc = p[:0], n\n\t\treturn n, nil\n\t}\n\t// here will copy\n\tb.growth(n)\n\tbuf := b.write.Malloc(n)\n\treturn copy(buf, p), nil\n}\n\n// WriteDirect cannot be mixed with WriteString or WriteBinary functions.\nfunc (b *UnsafeLinkBuffer) WriteDirect(extra []byte, remainLen int) error {\n\tn := len(extra)\n\tif n == 0 || remainLen < 0 {\n\t\treturn nil\n\t}\n\t// find origin\n\torigin := b.flush\n\tmalloc := b.mallocSize - remainLen // calculate the remaining malloc length\n\tfor t := origin.malloc - len(origin.buf); t < malloc; t = origin.malloc - len(origin.buf) {\n\t\tmalloc -= t\n\t\torigin = origin.next\n\t}\n\t// Add the buf length of the original node\n\t// `malloc` is the origin buffer offset that already malloced, the extra buffer should be inserted after that offset.\n\tmalloc += len(origin.buf)\n\n\t// Create dataNode and newNode and insert them into the chain\n\t// dataNode wrap the user buffer extra, and newNode wrap the origin left netpoll buffer\n\t// - originNode{buf=origin, off=0, malloc=malloc, readonly=true} : non-reusable\n\t// - dataNode{buf=extra, off=0, malloc=len(extra), readonly=true} : non-reusable\n\t// - newNode{buf=origin, off=malloc, malloc=origin.malloc, readonly=false} : reusable\n\tdataNode := newLinkBufferNode(0) // zero node will be set by readonly mode\n\tdataNode.buf, dataNode.malloc = extra[:0], n\n\n\tif remainLen > 0 {\n\t\t// split a single buffer node to originNode and newNode\n\t\tnewNode := newLinkBufferNode(0)\n\t\tnewNode.off = malloc\n\t\tnewNode.buf = origin.buf[:malloc]\n\t\tnewNode.malloc = origin.malloc\n\t\tnewNode.unsetFlag(flagUnmanaged)\n\t\torigin.malloc = malloc\n\t\torigin.setFlag(flagUnmanaged)\n\n\t\t// link nodes\n\t\tdataNode.next = newNode\n\t\tnewNode.next = origin.next\n\t\torigin.next = dataNode\n\t} else {\n\t\t// link nodes\n\t\tdataNode.next = origin.next\n\t\torigin.next = dataNode\n\t}\n\n\t// adjust b.write\n\tfor b.write.next != nil {\n\t\tb.write = b.write.next\n\t}\n\n\tb.mallocSize += n\n\treturn nil\n}\n\n// WriteByte implements Writer.\nfunc (b *UnsafeLinkBuffer) WriteByte(p byte) (err error) {\n\tdst, err := b.Malloc(1)\n\tif len(dst) == 1 {\n\t\tdst[0] = p\n\t}\n\treturn err\n}\n\n// Close will recycle all buffer.\nfunc (b *UnsafeLinkBuffer) Close() (err error) {\n\tatomic.StoreInt64(&b.length, 0)\n\tb.mallocSize = 0\n\t// just release all\n\tb.Release()\n\tfor node := b.head; node != nil; {\n\t\tnd := node\n\t\tnode = node.next\n\t\tnd.Release()\n\t}\n\tb.head, b.read, b.flush, b.write = nil, nil, nil, nil\n\treturn nil\n}\n\n// ------------------------------------------ implement connection interface ------------------------------------------\n\n// Bytes returns all the readable bytes of this LinkBuffer.\nfunc (b *UnsafeLinkBuffer) Bytes() []byte {\n\tnode, flush := b.read, b.flush\n\tif node == flush {\n\t\treturn node.buf[node.off:]\n\t}\n\tn := 0\n\tp := dirtmake.Bytes(b.Len(), b.Len())\n\tfor ; node != flush; node = node.next {\n\t\tif node.Len() > 0 {\n\t\t\tn += copy(p[n:], node.buf[node.off:])\n\t\t}\n\t}\n\tn += copy(p[n:], flush.buf[flush.off:])\n\treturn p[:n]\n}\n\n// GetBytes will read and fill the slice p as much as possible.\n// If p is not passed, return all readable bytes.\nfunc (b *UnsafeLinkBuffer) GetBytes(p [][]byte) (vs [][]byte) {\n\tnode, flush := b.read, b.flush\n\tif len(p) == 0 {\n\t\tn := 0\n\t\tfor ; node != flush; node = node.next {\n\t\t\tn++\n\t\t}\n\t\tnode = b.read\n\t\tp = make([][]byte, n)\n\t}\n\tvar i int\n\tfor i = 0; node != flush && i < len(p); node = node.next {\n\t\tif node.Len() > 0 {\n\t\t\tnode.setFlag(flagReadExposed)\n\t\t\tp[i] = node.buf[node.off:]\n\t\t\ti++\n\t\t}\n\t}\n\tif i < len(p) {\n\t\tflush.setFlag(flagReadExposed)\n\t\tp[i] = flush.buf[flush.off:]\n\t\ti++\n\t}\n\treturn p[:i]\n}\n\n// book will grow and malloc buffer to hold data.\n//\n// bookSize: The size of data that can be read at once.\n// maxSize: The maximum size of data between two Release(). In some cases, this can\n//\n//\tguarantee all data allocated in one node to reduce copy.\nfunc (b *UnsafeLinkBuffer) book(bookSize, maxSize int) (p []byte) {\n\tl := cap(b.write.buf) - b.write.malloc\n\t// grow linkBuffer\n\tif l == 0 {\n\t\tl = maxSize\n\t\tb.write.next = newLinkBufferNode(maxSize)\n\t\tb.write = b.write.next\n\t}\n\tif l > bookSize {\n\t\tl = bookSize\n\t}\n\treturn b.write.Malloc(l)\n}\n\n// bookAck will ack the first n malloc bytes and discard the rest.\n//\n// length: The size of data in inputBuffer. It is used to calculate the maxSize\nfunc (b *UnsafeLinkBuffer) bookAck(n int) (length int, err error) {\n\tb.write.malloc = n + len(b.write.buf)\n\tb.write.buf = b.write.buf[:b.write.malloc]\n\tb.flush = b.write\n\n\t// re-cal length\n\tlength = b.recalLen(n)\n\treturn length, nil\n}\n\n// calcMaxSize will calculate the data size between two Release()\nfunc (b *UnsafeLinkBuffer) calcMaxSize() (sum int) {\n\tfor node := b.head; node != b.read; node = node.next {\n\t\tsum += len(node.buf)\n\t}\n\tsum += len(b.read.buf)\n\treturn sum\n}\n\n// resetTail will reset tail node or add an empty tail node to\n// guarantee the tail node is not larger than 8KB\nfunc (b *UnsafeLinkBuffer) resetTail(maxSize int) {\n\tif maxSize <= pagesize {\n\t\t// no need to reset a small buffer tail node\n\t\treturn\n\t}\n\t// set nil tail\n\tb.write.next = newLinkBufferNode(0)\n\tb.write = b.write.next\n\tb.flush = b.write\n}\n\n// indexByte returns the index of the first instance of c in buffer, or -1 if c is not present in buffer.\nfunc (b *UnsafeLinkBuffer) indexByte(c byte, skip int) int {\n\tsize := b.Len()\n\tif skip >= size {\n\t\treturn -1\n\t}\n\tvar unread, n, l int\n\tnode := b.read\n\tfor unread = size; unread > 0; unread -= n {\n\t\tl = node.Len()\n\t\tif l >= unread { // last node\n\t\t\tn = unread\n\t\t} else { // read full node\n\t\t\tn = l\n\t\t}\n\n\t\t// skip current node\n\t\tif skip >= n {\n\t\t\tskip -= n\n\t\t\tnode = node.next\n\t\t\tcontinue\n\t\t}\n\t\ti := bytes.IndexByte(node.Peek(n)[skip:], c)\n\t\tif i >= 0 {\n\t\t\treturn (size - unread) + skip + i // past_read + skip_read + index\n\t\t}\n\t\tskip = 0 // no skip bytes\n\t\tnode = node.next\n\t}\n\treturn -1\n}\n\n// ------------------------------------------ private function ------------------------------------------\n\n// recalLen re-calculate the length\nfunc (b *UnsafeLinkBuffer) recalLen(delta int) (length int) {\n\tif delta < 0 && len(b.cachePeek) > 0 {\n\t\t// b.cachePeek will contain stale data if we read out even a single byte from buffer,\n\t\t// so we need to reset it or the next Peek call will return invalid bytes.\n\t\tb.cachePeek = b.cachePeek[:0]\n\t}\n\treturn int(atomic.AddInt64(&b.length, int64(delta)))\n}\n\n// growth directly create the next node, when b.write is not enough.\nfunc (b *UnsafeLinkBuffer) growth(n int) {\n\tif n <= 0 {\n\t\treturn\n\t}\n\t// the memory of readonly node if not malloc by us so should skip them\n\tfor b.write.getFlag(flagUnmanaged) || cap(b.write.buf)-b.write.malloc < n {\n\t\tif b.write.next == nil {\n\t\t\tb.write.next = newLinkBufferNode(n)\n\t\t\tb.write = b.write.next\n\t\t\treturn\n\t\t}\n\t\tb.write = b.write.next\n\t}\n}\n\n// isSingleNode determines whether reading needs to cross nodes.\n// isSingleNode will move b.read to latest non-empty node if there is a zero-size node\n// Must require b.Len() > 0\nfunc (b *UnsafeLinkBuffer) isSingleNode(readN int) (single bool) {\n\tif readN <= 0 {\n\t\treturn true\n\t}\n\tl := b.read.Len()\n\tfor l == 0 && b.read != b.flush {\n\t\tb.read = b.read.next\n\t\tl = b.read.Len()\n\t}\n\treturn l >= readN\n}\n\n// memorySize return the real memory size in bytes the LinkBuffer occupied\nfunc (b *LinkBuffer) memorySize() (bytes int) {\n\tfor node := b.head; node != nil; node = node.next {\n\t\tbytes += cap(node.buf)\n\t}\n\tfor _, c := range b.caches {\n\t\tbytes += cap(c)\n\t}\n\tbytes += cap(b.cachePeek)\n\treturn bytes\n}\n\n// ------------------------------------------ implement link node ------------------------------------------\n\n// newLinkBufferNode create or reuse linkBufferNode.\n// Nodes with size <= 0 are marked as readonly, which means the node.buf is not allocated by this mcache.\nfunc newLinkBufferNode(size int) *linkBufferNode {\n\tnode := linkedPool.Get().(*linkBufferNode)\n\t// reset node offset\n\tnode.off, node.malloc, node.refer, node.mode = 0, 0, 1, defaultLinkBufferMode\n\tif size <= 0 {\n\t\tnode.setFlag(flagUnmanaged)\n\t\treturn node\n\t}\n\tif size < LinkBufferCap {\n\t\tsize = LinkBufferCap\n\t}\n\tnode.buf = malloc(0, size)\n\treturn node\n}\n\nvar linkedPool = sync.Pool{\n\tNew: func() interface{} {\n\t\treturn &linkBufferNode{\n\t\t\trefer: 1, // comes with 1 reference\n\t\t}\n\t},\n}\n\ntype linkBufferNode struct {\n\tbuf    []byte          // buffer\n\toff    int             // read-offset\n\tmalloc int             // write-offset\n\trefer  int32           // reference count\n\tmode   uint8           // mode store all bool bit status\n\torigin *linkBufferNode // the root node of the extends\n\tnext   *linkBufferNode // the next node of the linked buffer\n}\n\nfunc (node *linkBufferNode) Len() (l int) {\n\treturn len(node.buf) - node.off\n}\n\nfunc (node *linkBufferNode) IsEmpty() (ok bool) {\n\treturn node.off == len(node.buf)\n}\n\nfunc (node *linkBufferNode) Reset() {\n\tif node.origin != nil || atomic.LoadInt32(&node.refer) != 1 {\n\t\treturn\n\t}\n\tnode.off, node.malloc = 0, 0\n\tnode.buf = node.buf[:0]\n}\n\nfunc (node *linkBufferNode) Next(n int) (p []byte) {\n\toff := node.off\n\tnode.off += n\n\treturn node.buf[off:node.off:node.off]\n}\n\nfunc (node *linkBufferNode) Peek(n int) (p []byte) {\n\treturn node.buf[node.off : node.off+n : node.off+n]\n}\n\nfunc (node *linkBufferNode) Malloc(n int) (buf []byte) {\n\tmalloc := node.malloc\n\tnode.malloc += n\n\treturn node.buf[malloc:node.malloc:node.malloc]\n}\n\n// Refer holds a reference count at the same time as Next, and releases the real buffer after Release.\n// The node obtained by Refer is read-only.\nfunc (node *linkBufferNode) Refer(n int) (p *linkBufferNode) {\n\tp = newLinkBufferNode(0)\n\tp.buf = node.Next(n)\n\n\tif node.origin != nil {\n\t\tp.origin = node.origin\n\t} else {\n\t\tp.origin = node\n\t}\n\tatomic.AddInt32(&p.origin.refer, 1)\n\treturn p\n}\n\n// Release consists of two parts:\n// 1. reduce the reference count of itself and origin.\n// 2. recycle the buf when the reference count is 0.\nfunc (node *linkBufferNode) Release() (err error) {\n\tif node.origin != nil {\n\t\tnode.origin.Release()\n\t}\n\t// release self\n\tif atomic.AddInt32(&node.refer, -1) == 0 {\n\t\t// readonly nodes cannot recycle node.buf, other node.buf are recycled to mcache.\n\t\tif node.reusable() {\n\t\t\tfree(node.buf)\n\t\t}\n\t\tnode.buf, node.origin, node.next = nil, nil, nil\n\t\tlinkedPool.Put(node)\n\t}\n\treturn nil\n}\n\nfunc (node *linkBufferNode) getFlag(flag uint8) bool {\n\treturn node.mode&flag > 0\n}\n\nfunc (node *linkBufferNode) setFlag(flag uint8) {\n\tnode.mode |= flag\n}\n\nfunc (node *linkBufferNode) unsetFlag(flag uint8) {\n\tnode.mode &^= flag\n}\n\n// reusable reports whether the node's buffer memory is owned by the LinkBuffer and can be recycled.\n// Called during Release to decide if node.buf should be returned to mcache via free.\nfunc (node *linkBufferNode) reusable() bool {\n\treturn node.mode&flagUnmanaged == 0\n}\n\n// readExposed reports whether the node's buffer has been returned directly to user code\n// via a zero-copy Reader method and may still be referenced externally.\nfunc (node *linkBufferNode) readExposed() bool {\n\treturn node.mode&flagReadExposed > 0\n}\n"
  },
  {
    "path": "nocopy_linkbuffer_norace.go",
    "content": "// Copyright 2024 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !race\n// +build !race\n\npackage netpoll\n\ntype LinkBuffer = UnsafeLinkBuffer\n"
  },
  {
    "path": "nocopy_linkbuffer_race.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build race\n// +build race\n\npackage netpoll\n\nimport (\n\t\"sync\"\n)\n\ntype LinkBuffer = SafeLinkBuffer\n\n// SafeLinkBuffer only used to in go tests with -race\ntype SafeLinkBuffer struct {\n\tsync.Mutex\n\tUnsafeLinkBuffer\n}\n\n// ------------------------------------------ implement copy reader ------------------------------------------\n\nfunc (b *SafeLinkBuffer) readCopy(p []byte) int {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.readCopy(p)\n}\n\n// ------------------------------------------ implement zero-copy reader ------------------------------------------\n\n// Next implements Reader.\nfunc (b *SafeLinkBuffer) Next(n int) (p []byte, err error) {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.Next(n)\n}\n\n// Peek implements Reader.\nfunc (b *SafeLinkBuffer) Peek(n int) (p []byte, err error) {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.Peek(n)\n}\n\n// Skip implements Reader.\nfunc (b *SafeLinkBuffer) Skip(n int) (err error) {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.Skip(n)\n}\n\n// Until implements Reader.\nfunc (b *SafeLinkBuffer) Until(delim byte) (line []byte, err error) {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.Until(delim)\n}\n\n// Release implements Reader.\nfunc (b *SafeLinkBuffer) Release() (err error) {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.Release()\n}\n\n// ReadString implements Reader.\nfunc (b *SafeLinkBuffer) ReadString(n int) (s string, err error) {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.ReadString(n)\n}\n\n// ReadBinary implements Reader.\nfunc (b *SafeLinkBuffer) ReadBinary(n int) (p []byte, err error) {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.ReadBinary(n)\n}\n\n// ReadByte implements Reader.\nfunc (b *SafeLinkBuffer) ReadByte() (p byte, err error) {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.ReadByte()\n}\n\n// Slice implements Reader.\nfunc (b *SafeLinkBuffer) Slice(n int) (r Reader, err error) {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.Slice(n)\n}\n\n// ------------------------------------------ implement zero-copy writer ------------------------------------------\n\n// Malloc implements Writer.\nfunc (b *SafeLinkBuffer) Malloc(n int) (buf []byte, err error) {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.Malloc(n)\n}\n\n// MallocLen implements Writer.\nfunc (b *SafeLinkBuffer) MallocLen() (length int) {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.MallocLen()\n}\n\n// MallocAck implements Writer.\nfunc (b *SafeLinkBuffer) MallocAck(n int) (err error) {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.MallocAck(n)\n}\n\n// Flush implements Writer.\nfunc (b *SafeLinkBuffer) Flush() (err error) {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.Flush()\n}\n\n// Append implements Writer.\nfunc (b *SafeLinkBuffer) Append(w Writer) (err error) {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.Append(w)\n}\n\n// WriteBuffer implements Writer.\nfunc (b *SafeLinkBuffer) WriteBuffer(buf *LinkBuffer) (err error) {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.WriteBuffer(buf)\n}\n\n// WriteString implements Writer.\nfunc (b *SafeLinkBuffer) WriteString(s string) (n int, err error) {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.WriteString(s)\n}\n\n// WriteBinary implements Writer.\nfunc (b *SafeLinkBuffer) WriteBinary(p []byte) (n int, err error) {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.WriteBinary(p)\n}\n\n// WriteDirect cannot be mixed with WriteString or WriteBinary functions.\nfunc (b *SafeLinkBuffer) WriteDirect(p []byte, remainLen int) error {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.WriteDirect(p, remainLen)\n}\n\n// WriteByte implements Writer.\nfunc (b *SafeLinkBuffer) WriteByte(p byte) (err error) {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.WriteByte(p)\n}\n\n// Close will recycle all buffer.\nfunc (b *SafeLinkBuffer) Close() (err error) {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.Close()\n}\n\n// ------------------------------------------ implement connection interface ------------------------------------------\n\n// Bytes returns all the readable bytes of this SafeLinkBuffer.\nfunc (b *SafeLinkBuffer) Bytes() []byte {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.Bytes()\n}\n\n// GetBytes will read and fill the slice p as much as possible.\nfunc (b *SafeLinkBuffer) GetBytes(p [][]byte) (vs [][]byte) {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.GetBytes(p)\n}\n\n// book will grow and malloc buffer to hold data.\n//\n// bookSize: The size of data that can be read at once.\n// maxSize: The maximum size of data between two Release(). In some cases, this can\n//\n//\tguarantee all data allocated in one node to reduce copy.\nfunc (b *SafeLinkBuffer) book(bookSize, maxSize int) (p []byte) {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.book(bookSize, maxSize)\n}\n\n// bookAck will ack the first n malloc bytes and discard the rest.\n//\n// length: The size of data in inputBuffer. It is used to calculate the maxSize\nfunc (b *SafeLinkBuffer) bookAck(n int) (length int, err error) {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.bookAck(n)\n}\n\n// calcMaxSize will calculate the data size between two Release()\nfunc (b *SafeLinkBuffer) calcMaxSize() (sum int) {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.calcMaxSize()\n}\n\nfunc (b *SafeLinkBuffer) resetTail(maxSize int) {\n\tb.Lock()\n\tdefer b.Unlock()\n\tb.UnsafeLinkBuffer.resetTail(maxSize)\n}\n\nfunc (b *SafeLinkBuffer) indexByte(c byte, skip int) int {\n\tb.Lock()\n\tdefer b.Unlock()\n\treturn b.UnsafeLinkBuffer.indexByte(c, skip)\n}\n"
  },
  {
    "path": "nocopy_linkbuffer_test.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n// +build !windows\n\npackage netpoll\n\nimport (\n\t\"bytes\"\n\t\"encoding/binary\"\n\t\"fmt\"\n\t\"reflect\"\n\t\"sync/atomic\"\n\t\"testing\"\n)\n\nfunc TestLinkBuffer(t *testing.T) {\n\t// clean & new\n\tLinkBufferCap = 128\n\n\tbuf := NewLinkBuffer()\n\tEqual(t, buf.Len(), 0)\n\tMustTrue(t, buf.IsEmpty())\n\n\thead := buf.head\n\n\tp, err := buf.Next(10)\n\tEqual(t, len(p), 0)\n\tMustTrue(t, err != nil)\n\n\tbuf.Malloc(128)\n\tMustTrue(t, buf.IsEmpty())\n\n\tp, err = buf.Peek(10)\n\tEqual(t, len(p), 0)\n\tMustTrue(t, err != nil)\n\n\tbuf.Flush()\n\tEqual(t, buf.Len(), 128)\n\tMustTrue(t, !buf.IsEmpty())\n\n\tp, err = buf.Next(28)\n\tEqual(t, len(p), 28)\n\tEqual(t, buf.Len(), 100)\n\tMustNil(t, err)\n\tMustTrue(t, buf.read.readExposed()) // single-node Next exposes buffer\n\n\tp, err = buf.Peek(90)\n\tEqual(t, len(p), 90)\n\tEqual(t, buf.Len(), 100)\n\tMustNil(t, err)\n\tMustTrue(t, buf.read.readExposed()) // single-node Peek exposes buffer\n\n\tread := buf.read\n\tEqual(t, buf.head, head)\n\terr = buf.Release()\n\tMustNil(t, err)\n\tEqual(t, buf.head, read)\n\n\tinputs := buf.book(block1k, block8k)\n\tEqual(t, len(inputs), block1k)\n\tEqual(t, buf.Len(), 100)\n\n\tbuf.MallocAck(block1k)\n\tEqual(t, buf.Len(), 100)\n\tEqual(t, buf.MallocLen(), block1k)\n\tbuf.Flush()\n\tEqual(t, buf.Len(), 100+block1k)\n\tEqual(t, buf.MallocLen(), 0)\n\n\toutputs := buf.GetBytes(make([][]byte, 16))\n\tEqual(t, len(outputs), 2)\n\n\terr = buf.Skip(block1k)\n\tMustNil(t, err)\n\tEqual(t, buf.Len(), 100)\n}\n\nfunc TestLinkBufferGetBytes(t *testing.T) {\n\tbuf := NewLinkBuffer()\n\tvar (\n\t\tnum         = 10\n\t\tb           = 1\n\t\texpectedLen = 0\n\t)\n\tfor i := 0; i < num; i++ {\n\t\texpectedLen += b\n\t\tn, err := buf.WriteBinary(make([]byte, b))\n\t\tMustNil(t, err)\n\t\tEqual(t, n, b)\n\t\tb *= 10\n\t}\n\tbuf.Flush()\n\tEqual(t, int(buf.length), expectedLen)\n\tbs := buf.GetBytes(nil)\n\tactualLen := 0\n\tfor i := 0; i < len(bs); i++ {\n\t\tactualLen += len(bs[i])\n\t}\n\tEqual(t, actualLen, expectedLen)\n}\n\n// TestLinkBufferWithZero test more case with n is invalid.\nfunc TestLinkBufferWithInvalid(t *testing.T) {\n\t// clean & new\n\tLinkBufferCap = 128\n\n\tbuf := NewLinkBuffer()\n\tEqual(t, buf.Len(), 0)\n\tMustTrue(t, buf.IsEmpty())\n\n\tfor n := 0; n > -5; n-- {\n\t\t// test writer\n\t\tp, err := buf.Malloc(n)\n\t\tEqual(t, len(p), 0)\n\t\tEqual(t, buf.MallocLen(), 0)\n\t\tEqual(t, buf.Len(), 0)\n\t\tMustNil(t, err)\n\n\t\tvar wn int\n\t\twn, err = buf.WriteString(\"\")\n\t\tEqual(t, wn, 0)\n\t\tEqual(t, buf.MallocLen(), 0)\n\t\tEqual(t, buf.Len(), 0)\n\t\tMustNil(t, err)\n\n\t\twn, err = buf.WriteBinary(nil)\n\t\tEqual(t, wn, 0)\n\t\tEqual(t, buf.MallocLen(), 0)\n\t\tEqual(t, buf.Len(), 0)\n\t\tMustNil(t, err)\n\n\t\terr = buf.WriteDirect(nil, n)\n\t\tEqual(t, buf.MallocLen(), 0)\n\t\tEqual(t, buf.Len(), 0)\n\t\tMustNil(t, err)\n\n\t\tvar w *LinkBuffer\n\t\terr = buf.Append(w)\n\t\tEqual(t, buf.MallocLen(), 0)\n\t\tEqual(t, buf.Len(), 0)\n\t\tMustNil(t, err)\n\n\t\terr = buf.MallocAck(n)\n\t\tEqual(t, buf.MallocLen(), 0)\n\t\tEqual(t, buf.Len(), 0)\n\t\tif n == 0 {\n\t\t\tMustNil(t, err)\n\t\t} else {\n\t\t\tMustTrue(t, err != nil)\n\t\t}\n\n\t\terr = buf.Flush()\n\t\tMustNil(t, err)\n\n\t\t// test reader\n\t\tp, err = buf.Next(n)\n\t\tEqual(t, len(p), 0)\n\t\tMustNil(t, err)\n\n\t\tp, err = buf.Peek(n)\n\t\tEqual(t, len(p), 0)\n\t\tMustNil(t, err)\n\n\t\terr = buf.Skip(n)\n\t\tEqual(t, len(p), 0)\n\t\tMustNil(t, err)\n\n\t\tvar s string\n\t\ts, err = buf.ReadString(n)\n\t\tEqual(t, len(s), 0)\n\t\tMustNil(t, err)\n\n\t\tp, err = buf.ReadBinary(n)\n\t\tEqual(t, len(p), 0)\n\t\tMustNil(t, err)\n\n\t\tvar r Reader\n\t\tr, err = buf.Slice(n)\n\t\tEqual(t, r.Len(), 0)\n\t\tMustNil(t, err)\n\n\t\terr = buf.Release()\n\t\tMustNil(t, err)\n\t}\n}\n\nfunc TestLinkBufferMultiNode(t *testing.T) {\n\t// clean & new\n\tLinkBufferCap = 8\n\n\tbuf := NewLinkBuffer()\n\tEqual(t, buf.Len(), 0)\n\tMustTrue(t, buf.IsEmpty())\n\tvar p []byte\n\n\tp, _ = buf.Malloc(15)\n\tfor i := 0; i < len(p); i++ { // updates p[0] - p[14] to 0 - 14\n\t\tp[i] = byte(i)\n\t}\n\tEqual(t, len(p), 15)\n\tMustTrue(t, buf.read == buf.flush)\n\tEqual(t, buf.read.off, 0)\n\tEqual(t, buf.read.malloc, 0)\n\tEqual(t, buf.write.off, 0)\n\tEqual(t, buf.write.malloc, 15)\n\tEqual(t, cap(buf.write.buf), 16) // mcache up-aligned to the power of 2\n\n\tp, _ = buf.Malloc(7)\n\tfor i := 0; i < len(p); i++ { // updates p[0] - p[6] to 15 - 21\n\t\tp[i] = byte(i + 15)\n\t}\n\tEqual(t, len(p), 7)\n\tMustTrue(t, buf.read == buf.flush)\n\tEqual(t, buf.read.off, 0)\n\tEqual(t, buf.read.malloc, 0)\n\tEqual(t, buf.write.off, 0)\n\tEqual(t, buf.write.malloc, 7)\n\tEqual(t, cap(buf.write.buf), LinkBufferCap)\n\n\tbuf.Flush()\n\tMustTrue(t, buf.read != buf.flush)\n\tMustTrue(t, buf.flush == buf.write)\n\tEqual(t, buf.read.off, 0)\n\tEqual(t, len(buf.read.buf), 0)\n\tEqual(t, buf.read.next.off, 0)\n\tEqual(t, len(buf.read.next.buf), 15)\n\tEqual(t, buf.flush.off, 0)\n\tEqual(t, buf.flush.malloc, 7)\n\tEqual(t, len(buf.flush.buf), 7)\n\n\tp, _ = buf.Next(13)\n\tEqual(t, len(p), 13)\n\tEqual(t, p[0], byte(0))\n\tEqual(t, p[12], byte(12))\n\tMustTrue(t, buf.read != buf.flush)\n\tEqual(t, buf.read.off, 13)\n\tEqual(t, buf.read.Len(), 2)\n\tEqual(t, buf.read.next.Len(), 7)\n\tEqual(t, buf.flush.off, 0)\n\tEqual(t, buf.flush.malloc, 7)\n\tMustTrue(t, buf.read.readExposed())   // single-node Next\n\tMustTrue(t, !buf.flush.readExposed()) // not touched yet\n\n\t// Peek\n\tp, _ = buf.Peek(4)\n\tEqual(t, len(p), 4)\n\tEqual(t, p[0], byte(13))\n\tEqual(t, p[1], byte(14))\n\tEqual(t, p[2], byte(15))\n\tEqual(t, p[3], byte(16))\n\tEqual(t, len(buf.cachePeek), 4)\n\tp, _ = buf.Peek(3) // case: smaller than the last call\n\tEqual(t, len(p), 3)\n\tEqual(t, p[0], byte(13))\n\tEqual(t, p[2], byte(15))\n\tEqual(t, len(buf.cachePeek), 4)\n\tp, _ = buf.Peek(5) // case: Peek than the max call, and cap(buf.cachePeek) < n\n\tEqual(t, len(p), 5)\n\tEqual(t, p[0], byte(13))\n\tEqual(t, p[4], byte(17))\n\tEqual(t, len(buf.cachePeek), 5)\n\tp, _ = buf.Peek(6) // case: Peek than the last call, and cap(buf.cachePeek) > n\n\tEqual(t, len(p), 6)\n\tEqual(t, p[0], byte(13))\n\tEqual(t, p[5], byte(18))\n\tEqual(t, len(buf.cachePeek), 6)\n\tMustTrue(t, buf.read != buf.flush)\n\tEqual(t, buf.read.off, 13)\n\tEqual(t, buf.read.Len(), 2)\n\tEqual(t, buf.flush.off, 0)\n\tEqual(t, buf.flush.malloc, 7)\n\tMustTrue(t, !buf.flush.readExposed()) // multi-node Peek copies, doesn't expose\n\t// Peek ends\n\n\tbuf.book(block8k, block8k)\n\tMustTrue(t, buf.flush == buf.write)\n\tEqual(t, buf.flush.off, 0)\n\tEqual(t, buf.flush.malloc, 8)\n\tEqual(t, buf.flush.Len(), 7)\n\tEqual(t, buf.write.off, 0)\n\tEqual(t, buf.write.malloc, 8)\n\tEqual(t, buf.write.Len(), 7)\n\n\tbuf.book(block8k, block8k)\n\tMustTrue(t, buf.flush != buf.write)\n\tEqual(t, buf.flush.off, 0)\n\tEqual(t, buf.flush.malloc, 8)\n\tEqual(t, buf.flush.Len(), 7)\n\tEqual(t, buf.write.off, 0)\n\tEqual(t, buf.write.malloc, 8192)\n\tEqual(t, buf.write.Len(), 0)\n\n\tbuf.MallocAck(5)\n\tMustTrue(t, buf.flush != buf.write)\n\tEqual(t, buf.write.off, 0)\n\tEqual(t, buf.write.malloc, 4)\n\tEqual(t, buf.write.Len(), 0)\n\tMustTrue(t, buf.write.next == nil)\n\tbuf.Flush()\n\n\tp, _ = buf.Next(8)\n\tEqual(t, len(p), 8)\n\tMustTrue(t, buf.read != buf.flush)\n\tEqual(t, buf.read.off, 6)\n\tEqual(t, buf.read.Len(), 2)\n\tEqual(t, buf.flush.off, 0)\n\tEqual(t, buf.flush.malloc, 4)\n\tEqual(t, buf.flush.Len(), 4)\n\n\terr := buf.Skip(3)\n\tMustNil(t, err)\n\tMustTrue(t, buf.read == buf.flush)\n\tEqual(t, buf.read.off, 1)\n\tEqual(t, buf.read.Len(), 3)\n\tEqual(t, buf.flush.malloc, 4)\n}\n\nfunc TestLinkBufferRefer(t *testing.T) {\n\t// clean & new\n\tLinkBufferCap = 8\n\n\twbuf := NewLinkBuffer()\n\twbuf.book(block8k, block8k)\n\twbuf.Malloc(7)\n\twbuf.Flush()\n\tEqual(t, wbuf.Len(), block8k+7)\n\n\tbuf := NewLinkBuffer()\n\tvar p []byte\n\n\t// writev\n\tbuf.WriteBuffer(wbuf)\n\tbuf.Flush()\n\tEqual(t, buf.Len(), block8k+7)\n\n\tp, _ = buf.Next(5)\n\tEqual(t, len(p), 5)\n\tMustTrue(t, buf.read != buf.flush)\n\tEqual(t, buf.read.off, 5)\n\tEqual(t, buf.read.Len(), block8k-5)\n\tEqual(t, buf.flush.off, 0)\n\tEqual(t, buf.flush.malloc, 7)\n\tEqual(t, cap(buf.flush.buf), 8)\n\tMustTrue(t, buf.read.readExposed()) // single-node Next\n\n\t// readv\n\t_rbuf, err := buf.Slice(4)\n\trbuf, ok := _rbuf.(*LinkBuffer)\n\tMustNil(t, err)\n\tMustTrue(t, ok)\n\tEqual(t, rbuf.Len(), 4)\n\tMustTrue(t, rbuf.read != rbuf.flush)\n\tEqual(t, rbuf.read.off, 0)\n\tEqual(t, rbuf.read.Len(), 4)\n\n\tMustTrue(t, buf.head != buf.read) // Slice will Release\n\tMustTrue(t, rbuf.read != buf.read)\n\tEqual(t, buf.Len(), block8k-2)\n\tMustTrue(t, buf.read != buf.flush)\n\tEqual(t, buf.read.off, 9)\n\tEqual(t, buf.read.malloc, block8k)\n\n\t// release\n\tnode1 := rbuf.head\n\tnode2 := buf.head\n\trbuf.Skip(rbuf.Len())\n\terr = rbuf.Release()\n\tMustNil(t, err)\n\tMustTrue(t, rbuf.head != node1)\n\tMustTrue(t, buf.head == node2)\n\n\terr = buf.Release()\n\tMustNil(t, err)\n\tMustTrue(t, buf.head != node2)\n\tMustTrue(t, buf.head == buf.read)\n\tEqual(t, buf.read.off, 9)\n\tEqual(t, buf.read.malloc, block8k)\n\tEqual(t, buf.read.refer, int32(1))\n\tEqual(t, buf.read.Len(), block8k-9)\n}\n\nfunc TestLinkBufferResetTail(t *testing.T) {\n\texcept := byte(1)\n\n\tLinkBufferCap = 8\n\tbuf := NewLinkBuffer()\n\n\t// 1. slice reader\n\tbuf.WriteByte(except)\n\tbuf.Flush()\n\tr1, _ := buf.Slice(1)\n\tt.Logf(\"1: %x\\n\", buf.flush.buf)\n\t// 2. release & reset tail\n\tbuf.resetTail(LinkBufferCap)\n\tbuf.WriteByte(byte(2))\n\tt.Logf(\"2: %x\\n\", buf.flush.buf)\n\n\t// check slice reader\n\tgot, _ := r1.ReadByte()\n\tEqual(t, got, except)\n}\n\nfunc TestLinkBufferWriteBuffer(t *testing.T) {\n\tbuf1 := NewLinkBuffer()\n\tbuf2 := NewLinkBuffer()\n\tb2, _ := buf2.Malloc(1)\n\tb2[0] = 2\n\tbuf2.Flush()\n\tbuf3 := NewLinkBuffer()\n\tb3, _ := buf3.Malloc(1)\n\tb3[0] = 3\n\tbuf3.Flush()\n\tbuf1.WriteBuffer(buf2)\n\tbuf1.WriteBuffer(buf3)\n\tbuf1.Flush()\n\tMustTrue(t, bytes.Equal(buf1.Bytes(), []byte{2, 3}))\n}\n\nfunc TestLinkBufferCheckSingleNode(t *testing.T) {\n\tbuf := NewLinkBuffer(block4k)\n\t_, err := buf.Malloc(block8k)\n\tMustNil(t, err)\n\tbuf.Flush()\n\tMustTrue(t, buf.read.Len() == 0)\n\tis := buf.isSingleNode(block8k)\n\tMustTrue(t, is)\n\tMustTrue(t, buf.read.Len() == block8k)\n\tis = buf.isSingleNode(block8k + 1)\n\tMustTrue(t, !is)\n\n\t// cross node malloc, but b.read.Len() still == 0\n\tbuf = NewLinkBuffer(block4k)\n\t_, err = buf.Malloc(block8k)\n\tMustNil(t, err)\n\t// not malloc ack yet\n\t// read function will call isSingleNode inside\n\tbuf.isSingleNode(1)\n}\n\nfunc TestLinkBufferWriteMultiFlush(t *testing.T) {\n\tbuf := NewLinkBuffer()\n\tb1, _ := buf.Malloc(4)\n\tb1[0] = 1\n\tb1[2] = 2\n\terr := buf.Flush()\n\tMustNil(t, err)\n\terr = buf.Flush()\n\tMustNil(t, err)\n\tMustTrue(t, buf.Bytes()[0] == 1)\n\tMustTrue(t, len(buf.Bytes()) == 4)\n\n\terr = buf.Skip(2)\n\tMustNil(t, err)\n\tMustTrue(t, buf.Bytes()[0] == 2)\n\tMustTrue(t, len(buf.Bytes()) == 2)\n\terr = buf.Flush()\n\tMustNil(t, err)\n\tMustTrue(t, buf.Bytes()[0] == 2)\n\tMustTrue(t, len(buf.Bytes()) == 2)\n\n\tb2, _ := buf.Malloc(2)\n\tb2[0] = 3\n\terr = buf.Flush()\n\tMustNil(t, err)\n\tMustTrue(t, buf.Bytes()[0] == 2)\n\tMustTrue(t, buf.Bytes()[2] == 3)\n\tMustTrue(t, len(buf.Bytes()) == 4)\n}\n\nfunc TestLinkBufferWriteBinary(t *testing.T) {\n\t// clean & new\n\tLinkBufferCap = 8\n\n\t// new b: cap=16, len=9\n\tb := make([]byte, 16)\n\tbuf := NewLinkBuffer()\n\tbuf.WriteBinary(b[:9])\n\tbuf.Flush()\n\n\t// Currently, b[9:] should no longer be held.\n\t// WriteBinary/Malloc etc. cannot start from b[9:]\n\tbuf.WriteBinary([]byte{1})\n\tEqual(t, b[9], byte(0))\n\tbs, err := buf.Malloc(1)\n\tMustNil(t, err)\n\tbs[0] = 2\n\tbuf.Flush()\n\tEqual(t, b[9], byte(0))\n}\n\nfunc TestLinkBufferWriteDirect(t *testing.T) {\n\t// clean & new\n\tLinkBufferCap = 32\n\n\tbuf := NewLinkBuffer()\n\tbt, _ := buf.Malloc(32)\n\tbt[0] = 'a'\n\tbt[1] = 'b'\n\tbuf.WriteDirect([]byte(\"cdef\"), 30)\n\tbt[2] = 'g'\n\tbuf.WriteDirect([]byte(\"hijkl\"), 29)\n\tbt[3] = 'm'\n\tbuf.WriteDirect([]byte(\"nopqrst\"), 28)\n\tbt[4] = 'u'\n\tbuf.WriteDirect([]byte(\"vwxyz\"), 27)\n\tcopy(bt[5:], \"abcdefghijklmnopqrstuvwxyza\")\n\tbuf.WriteDirect([]byte(\"abcdefghijklmnopqrstuvwxyz\"), 0)\n\tbuf.Flush()\n\tbs := buf.Bytes()\n\tstr := \"abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzaabcdefghijklmnopqrstuvwxyz\"\n\tfor i := 0; i < len(str); i++ {\n\t\tif bs[i] != str[i] {\n\t\t\tt.Error(\"not equal!\")\n\t\t}\n\t}\n}\n\nfunc TestLinkBufferBufferMode(t *testing.T) {\n\tbufnode := newLinkBufferNode(0)\n\tMustTrue(t, bufnode.getFlag(flagUnmanaged))\n\tMustTrue(t, !bufnode.reusable())\n\tMustTrue(t, !bufnode.readExposed())\n\n\tbufnode = newLinkBufferNode(1)\n\tMustTrue(t, !bufnode.getFlag(flagUnmanaged))\n\tMustTrue(t, bufnode.reusable())\n\tMustTrue(t, !bufnode.readExposed())\n}\n\nfunc TestLinkBufferReadCopy(t *testing.T) {\n\tt.Run(\"SingleNode\", func(t *testing.T) {\n\t\tLinkBufferCap = 128\n\t\tbuf := NewLinkBuffer(128)\n\t\tp, _ := buf.Malloc(16)\n\t\tfor i := range p {\n\t\t\tp[i] = byte(i)\n\t\t}\n\t\tbuf.Flush()\n\n\t\tdst := make([]byte, 10)\n\t\tn := buf.readCopy(dst)\n\t\tEqual(t, n, 10)\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tEqual(t, dst[i], byte(i))\n\t\t}\n\t\tEqual(t, buf.Len(), 6)\n\t\t// readCopy must not set readExposed\n\t\tMustTrue(t, !buf.read.readExposed())\n\t})\n\n\tt.Run(\"MultiNode\", func(t *testing.T) {\n\t\tLinkBufferCap = 8\n\t\tbuf := NewLinkBuffer(8)\n\t\tp, _ := buf.Malloc(8)\n\t\tfor i := range p {\n\t\t\tp[i] = byte(i)\n\t\t}\n\t\tbuf.Flush()\n\t\tp, _ = buf.Malloc(8)\n\t\tfor i := range p {\n\t\t\tp[i] = byte(i + 8)\n\t\t}\n\t\tbuf.Flush()\n\n\t\tdst := make([]byte, 16)\n\t\tn := buf.readCopy(dst)\n\t\tEqual(t, n, 16)\n\t\tfor i := 0; i < 16; i++ {\n\t\t\tEqual(t, dst[i], byte(i))\n\t\t}\n\t\tEqual(t, buf.Len(), 0)\n\t})\n\n\tt.Run(\"PartialRead\", func(t *testing.T) {\n\t\tLinkBufferCap = 128\n\t\tbuf := NewLinkBuffer(128)\n\t\tp, _ := buf.Malloc(4)\n\t\tfor i := range p {\n\t\t\tp[i] = byte(i + 1)\n\t\t}\n\t\tbuf.Flush()\n\n\t\t// read more than available\n\t\tdst := make([]byte, 16)\n\t\tn := buf.readCopy(dst)\n\t\tEqual(t, n, 4)\n\t\tEqual(t, dst[0], byte(1))\n\t\tEqual(t, dst[3], byte(4))\n\t\tEqual(t, buf.Len(), 0)\n\t})\n\n\tt.Run(\"ReleasesNonExposedNodes\", func(t *testing.T) {\n\t\tLinkBufferCap = 8\n\t\tbuf := NewLinkBuffer(8)\n\t\tbuf.Malloc(8)\n\t\tbuf.Flush()\n\t\tbuf.Malloc(8)\n\t\tbuf.Flush()\n\t\tnode1 := buf.read\n\n\t\tdst := make([]byte, 16)\n\t\tbuf.readCopy(dst)\n\t\t// node1 was not exposed, should be released (head advanced past it)\n\t\tMustTrue(t, buf.head != node1)\n\t})\n\n\tt.Run(\"SkipsExposedNodes\", func(t *testing.T) {\n\t\tLinkBufferCap = 8\n\t\tbuf := NewLinkBuffer(8)\n\t\tp, _ := buf.Malloc(8)\n\t\tfor i := range p {\n\t\t\tp[i] = byte(i)\n\t\t}\n\t\tbuf.Flush()\n\t\tbuf.Malloc(8)\n\t\tbuf.Flush()\n\n\t\t// expose node1 via Peek\n\t\tbuf.Peek(4)\n\t\tnode1 := buf.read\n\t\tMustTrue(t, node1.readExposed())\n\n\t\t// readCopy past both nodes\n\t\tdst := make([]byte, 16)\n\t\tn := buf.readCopy(dst)\n\t\tEqual(t, n, 16)\n\t\tEqual(t, dst[0], byte(0))\n\t\t// head should stay at exposed node1\n\t\tEqual(t, buf.head, node1)\n\n\t\t// subsequent Release frees the exposed node\n\t\tbuf.Release()\n\t\tMustTrue(t, buf.head != node1)\n\t})\n\n\t// [exposed/consumed] → [not-exposed/consumed] → [partial-consumed/read]\n\tt.Run(\"ExposedThenNonExposedThenPartial\", func(t *testing.T) {\n\t\tLinkBufferCap = 8\n\t\tbuf := NewLinkBuffer(8)\n\t\t// node1: 8 bytes\n\t\tp, _ := buf.Malloc(8)\n\t\tfor i := range p {\n\t\t\tp[i] = byte(i)\n\t\t}\n\t\tbuf.Flush()\n\t\t// node2: 8 bytes\n\t\tp, _ = buf.Malloc(8)\n\t\tfor i := range p {\n\t\t\tp[i] = byte(i + 8)\n\t\t}\n\t\tbuf.Flush()\n\t\t// node3: 8 bytes\n\t\tp, _ = buf.Malloc(8)\n\t\tfor i := range p {\n\t\t\tp[i] = byte(i + 16)\n\t\t}\n\t\tbuf.Flush()\n\n\t\t// expose node1 via Peek\n\t\tbuf.Peek(4)\n\t\tnode1 := buf.read\n\t\tnode2 := node1.next\n\t\tMustTrue(t, node1.readExposed())\n\t\tMustTrue(t, !node2.readExposed())\n\n\t\t// readCopy 20 bytes: consumes node1(8) + node2(8) + 4 from node3\n\t\tdst := make([]byte, 20)\n\t\tn := buf.readCopy(dst)\n\t\tEqual(t, n, 20)\n\t\tfor i := 0; i < 20; i++ {\n\t\t\tEqual(t, dst[i], byte(i))\n\t\t}\n\t\tEqual(t, buf.Len(), 4)\n\n\t\t// head should be node1 (exposed, kept in chain)\n\t\tEqual(t, buf.head, node1)\n\t\t// node2 was released, node1.next should skip to read (node3)\n\t\tEqual(t, node1.next, buf.read)\n\n\t\t// subsequent Release frees the exposed node\n\t\tbuf.Release()\n\t\tMustTrue(t, buf.head == buf.read)\n\t})\n}\n\nfunc BenchmarkLinkBufferConcurrentReadWrite(b *testing.B) {\n\tb.StopTimer()\n\n\tbuf := NewLinkBuffer()\n\tvar rwTag uint32\n\treadMsg := []string{\n\t\t\"0123456\",\n\t\t\"7890123\",\n\t\t\"4567890\",\n\t\t\"1234567\",\n\t\t\"8901234\",\n\t\t\"5678901\",\n\t\t\"2345678\",\n\t\t\"9012345\",\n\t\t\"6789012\",\n\t\t\"3456789\",\n\t}\n\twriteMsg := []byte(\"0123456789\")\n\n\t// benchmark\n\tb.ReportAllocs()\n\tb.StartTimer()\n\tb.SetParallelism(2) // one read one write\n\tb.RunParallel(func(pb *testing.PB) {\n\t\tswitch atomic.AddUint32(&rwTag, 1) {\n\t\tcase 1:\n\t\t\t// 1 is write\n\t\t\tfor pb.Next() {\n\t\t\t\tp, err := buf.Malloc(80)\n\t\t\t\tif err != nil {\n\t\t\t\t\tpanic(fmt.Sprintf(\"malloc error %s\", err.Error()))\n\t\t\t\t}\n\t\t\t\tfor i := 0; i < 7; i++ {\n\t\t\t\t\tcopy(p[i*10:i*10+10], writeMsg)\n\t\t\t\t}\n\t\t\t\tbuf.MallocAck(70)\n\t\t\t\tbuf.Flush()\n\t\t\t}\n\t\tcase 2:\n\t\t\t// 2 is read\n\t\t\tfor pb.Next() {\n\t\t\t\tfor i := 0; i < 10; {\n\t\t\t\t\tp, err := buf.Next(7)\n\t\t\t\t\tif err == nil {\n\t\t\t\t\t\tif string(p) != readMsg[i] {\n\t\t\t\t\t\t\tpanic(fmt.Sprintf(\"NEXT p[%s] != msg[%s]\", p, readMsg[i]))\n\t\t\t\t\t\t}\n\t\t\t\t\t} else {\n\t\t\t\t\t\t// No read data, wait for write\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\ti++\n\t\t\t\t}\n\t\t\t\tbuf.Release()\n\t\t\t}\n\t\t}\n\t})\n}\n\nfunc TestUnsafeStringToSlice(t *testing.T) {\n\ts := \"hello world\"\n\tbs := unsafeStringToSlice(s)\n\ts = \"hi, boy\"\n\t_ = s\n\tEqual(t, string(bs), \"hello world\")\n}\n\nfunc TestLinkBufferIndexByte(t *testing.T) {\n\t// clean & new\n\tLinkBufferCap = 128\n\tloopSize := 1000\n\ttrigger := make(chan struct{}, 16)\n\n\tlb := NewLinkBuffer()\n\tempty := make([]byte, 1002)\n\tgo func() {\n\t\tfor i := 0; i < loopSize; i++ {\n\t\t\tbuf, err := lb.Malloc(1002)\n\t\t\t// need clear buffer\n\t\t\tcopy(buf, empty)\n\t\t\tbuf[500] = '\\n'\n\t\t\tbuf[1001] = '\\n'\n\t\t\tMustNil(t, err)\n\t\t\tlb.Flush()\n\t\t\ttrigger <- struct{}{}\n\t\t}\n\t}()\n\n\tfor i := 0; i < loopSize; i++ {\n\t\t<-trigger\n\t\tlast := i * 1002\n\t\tn := lb.indexByte('\\n', 0+last)\n\t\tEqual(t, n, 500+last)\n\t\tn = lb.indexByte('\\n', 500+last)\n\t\tEqual(t, n, 500+last)\n\t\tn = lb.indexByte('\\n', 501+last)\n\t\tEqual(t, n, 1001+last)\n\t}\n}\n\nfunc TestLinkBufferPeekOutOfMemory(t *testing.T) {\n\tbufCap := 1024 * 8\n\tbufNodes := 100\n\tmagicN := uint64(2024)\n\tbuf := NewLinkBuffer(bufCap)\n\tMustTrue(t, buf.IsEmpty())\n\tEqual(t, cap(buf.write.buf), bufCap)\n\tEqual(t, buf.memorySize(), bufCap)\n\n\tvar p []byte\n\tvar err error\n\t// write data that cross multi nodes\n\tfor n := 0; n < bufNodes; n++ {\n\t\tp, err = buf.Malloc(bufCap)\n\t\tMustNil(t, err)\n\t\tEqual(t, len(p), bufCap)\n\t\tbinary.BigEndian.PutUint64(p, magicN)\n\t}\n\tEqual(t, buf.MallocLen(), bufCap*bufNodes)\n\tbuf.Flush()\n\tEqual(t, buf.MallocLen(), 0)\n\n\t// peak data that in single node\n\tfor i := 0; i < 10; i++ {\n\t\tp, err = buf.Peek(bufCap)\n\t\tEqual(t, binary.BigEndian.Uint64(p), magicN)\n\t\tMustNil(t, err)\n\t\tEqual(t, len(p), bufCap)\n\t\tEqual(t, buf.memorySize(), bufCap*bufNodes)\n\t}\n\n\t// peak data that cross nodes\n\tmemorySize := 0\n\tfor i := 0; i < 1024; i++ {\n\t\tp, err = buf.Peek(bufCap + 1)\n\t\tMustNil(t, err)\n\t\tEqual(t, binary.BigEndian.Uint64(p), magicN)\n\t\tEqual(t, len(p), bufCap+1)\n\t\tif memorySize == 0 {\n\t\t\tmemorySize = buf.memorySize()\n\t\t\tt.Logf(\"after Peek: memorySize=%d\", memorySize)\n\t\t} else {\n\t\t\tEqual(t, buf.memorySize(), memorySize)\n\t\t}\n\t}\n}\n\nfunc TestMallocAck(t *testing.T) {\n\tsLen := 1024 * 7\n\tbuf1 := []byte{1, 2, 3, 4}\n\tbuf2 := []byte{5, 6, 7, 8}\n\tlb := NewLinkBuffer(0)\n\n\tbuf, err := lb.Malloc(4 + sLen)\n\tMustNil(t, err)\n\tcopy(buf[:4], buf1)\n\ts := make([]byte, sLen)\n\terr = lb.WriteDirect(s, sLen)\n\tMustNil(t, err)\n\n\terr = lb.MallocAck(4 + sLen)\n\tMustNil(t, err)\n\tlb.Flush()\n\n\tbuf, err = lb.Malloc(4)\n\tMustNil(t, err)\n\tcopy(buf[:4], buf2)\n\tlb.Flush()\n\n\tbuf, err = lb.Next(8 + sLen)\n\tMustNil(t, err)\n\n\tMustTrue(t, reflect.DeepEqual(buf, append(append(buf1, s...), buf2...)))\n}\n\nfunc BenchmarkStringToSliceByte(b *testing.B) {\n\tb.StopTimer()\n\ts := \"hello world\"\n\tvar bs []byte\n\tif false {\n\t\tb.Logf(\"bs = %s\", bs)\n\t}\n\n\t// benchmark\n\tb.ReportAllocs()\n\tb.StartTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tbs = unsafeStringToSlice(s)\n\t}\n\t_ = bs\n}\n\nfunc BenchmarkStringToCopy(b *testing.B) {\n\tb.StopTimer()\n\ts := \"hello world\"\n\tvar bs []byte\n\tb.Logf(\"bs = %s\", bs)\n\n\t// benchmark\n\tb.ReportAllocs()\n\tb.StartTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tbs = []byte(s)\n\t}\n\t_ = bs\n}\n\nfunc BenchmarkLinkBufferPoolGet(b *testing.B) {\n\tvar v *linkBufferNode\n\tif false {\n\t\tb.Logf(\"bs = %v\", v)\n\t}\n\n\t// benchmark\n\tb.ReportAllocs()\n\tb.SetParallelism(100)\n\tb.ResetTimer()\n\tb.RunParallel(func(pb *testing.PB) {\n\t\tfor pb.Next() {\n\t\t\tv = newLinkBufferNode(0)\n\t\t\tv.Release()\n\t\t}\n\t})\n}\n\nfunc BenchmarkCopyString(b *testing.B) {\n\ts := make([]byte, 128*1024)\n\n\t// benchmark\n\tb.ReportAllocs()\n\tb.SetParallelism(100)\n\tb.ResetTimer()\n\tb.RunParallel(func(pb *testing.PB) {\n\t\tv := make([]byte, 1024)\n\t\tfor pb.Next() {\n\t\t\tcopy(v, s)\n\t\t}\n\t})\n}\n\nfunc BenchmarkLinkBufferNoCopyRead(b *testing.B) {\n\ttotalSize := 0\n\tminSize := 32\n\tmaxSize := minSize << 9\n\tfor size := minSize; size <= maxSize; size = size << 1 {\n\t\ttotalSize += size\n\t}\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\tb.RunParallel(func(pb *testing.PB) {\n\t\tbuffer := NewLinkBuffer(pagesize)\n\t\tfor pb.Next() {\n\t\t\tbuf, err := buffer.Malloc(totalSize)\n\t\t\tif len(buf) != totalSize || err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\t\t\terr = buffer.MallocAck(totalSize)\n\t\t\tif err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\t\t\terr = buffer.Flush()\n\t\t\tif err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\n\t\t\tfor size := minSize; size <= maxSize; size = size << 1 {\n\t\t\t\tbuf, err = buffer.ReadBinary(size)\n\t\t\t\tif len(buf) != size || err != nil {\n\t\t\t\t\tb.Fatal(err)\n\t\t\t\t}\n\t\t\t}\n\t\t\t// buffer.Release will not reuse memory since we use no copy mode here\n\t\t\terr = buffer.Release()\n\t\t\tif err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\t\t}\n\t})\n}\n"
  },
  {
    "path": "nocopy_readwriter.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage netpoll\n\nimport (\n\t\"fmt\"\n\t\"io\"\n)\n\nconst maxReadCycle = 16\n\nfunc newZCReader(r io.Reader) *zcReader {\n\treturn &zcReader{\n\t\tr:   r,\n\t\tbuf: NewLinkBuffer(),\n\t}\n}\n\nvar _ Reader = &zcReader{}\n\n// zcReader implements Reader.\ntype zcReader struct {\n\tr   io.Reader\n\tbuf *LinkBuffer\n}\n\n// Next implements Reader.\nfunc (r *zcReader) Next(n int) (p []byte, err error) {\n\tif err = r.waitRead(n); err != nil {\n\t\treturn p, err\n\t}\n\treturn r.buf.Next(n)\n}\n\n// Peek implements Reader.\nfunc (r *zcReader) Peek(n int) (buf []byte, err error) {\n\tif err = r.waitRead(n); err != nil {\n\t\treturn buf, err\n\t}\n\treturn r.buf.Peek(n)\n}\n\n// Skip implements Reader.\nfunc (r *zcReader) Skip(n int) (err error) {\n\tif err = r.waitRead(n); err != nil {\n\t\treturn err\n\t}\n\treturn r.buf.Skip(n)\n}\n\n// Release implements Reader.\nfunc (r *zcReader) Release() (err error) {\n\treturn r.buf.Release()\n}\n\n// Slice implements Reader.\nfunc (r *zcReader) Slice(n int) (reader Reader, err error) {\n\tif err = r.waitRead(n); err != nil {\n\t\treturn nil, err\n\t}\n\treturn r.buf.Slice(n)\n}\n\n// Len implements Reader.\nfunc (r *zcReader) Len() (length int) {\n\treturn r.buf.Len()\n}\n\n// ReadString implements Reader.\nfunc (r *zcReader) ReadString(n int) (s string, err error) {\n\tif err = r.waitRead(n); err != nil {\n\t\treturn s, err\n\t}\n\treturn r.buf.ReadString(n)\n}\n\n// ReadBinary implements Reader.\nfunc (r *zcReader) ReadBinary(n int) (p []byte, err error) {\n\tif err = r.waitRead(n); err != nil {\n\t\treturn p, err\n\t}\n\treturn r.buf.ReadBinary(n)\n}\n\n// ReadByte implements Reader.\nfunc (r *zcReader) ReadByte() (b byte, err error) {\n\tif err = r.waitRead(1); err != nil {\n\t\treturn b, err\n\t}\n\treturn r.buf.ReadByte()\n}\n\nfunc (r *zcReader) Until(delim byte) (line []byte, err error) {\n\treturn r.buf.Until(delim)\n}\n\nfunc (r *zcReader) waitRead(n int) (err error) {\n\tfor r.buf.Len() < n {\n\t\terr = r.fill(n)\n\t\tif err != nil {\n\t\t\tif err == io.EOF {\n\t\t\t\terr = Exception(ErrEOF, \"\")\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// fill buffer to greater than n, range no more than 16 times.\nfunc (r *zcReader) fill(n int) (err error) {\n\tvar buf []byte\n\tvar num int\n\tfor i := 0; i < maxReadCycle && r.buf.Len() < n && err == nil; i++ {\n\t\tbuf, err = r.buf.Malloc(block4k)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tnum, err = r.r.Read(buf)\n\t\tif num < 0 {\n\t\t\tif err == nil {\n\t\t\t\terr = fmt.Errorf(\"zcReader fill negative count[%d]\", num)\n\t\t\t}\n\t\t\tnum = 0\n\t\t}\n\t\tr.buf.MallocAck(num)\n\t\tr.buf.Flush()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn err\n}\n\nfunc newZCWriter(w io.Writer) *zcWriter {\n\treturn &zcWriter{\n\t\tw:   w,\n\t\tbuf: NewLinkBuffer(),\n\t}\n}\n\nvar _ Writer = &zcWriter{}\n\n// zcWriter implements Writer.\ntype zcWriter struct {\n\tw   io.Writer\n\tbuf *LinkBuffer\n}\n\n// Malloc implements Writer.\nfunc (w *zcWriter) Malloc(n int) (buf []byte, err error) {\n\treturn w.buf.Malloc(n)\n}\n\n// MallocLen implements Writer.\nfunc (w *zcWriter) MallocLen() (length int) {\n\treturn w.buf.MallocLen()\n}\n\n// Flush implements Writer.\nfunc (w *zcWriter) Flush() (err error) {\n\tw.buf.Flush()\n\tn, err := w.w.Write(w.buf.Bytes())\n\tif n > 0 {\n\t\tw.buf.Skip(n)\n\t\tw.buf.Release()\n\t}\n\treturn err\n}\n\n// MallocAck implements Writer.\nfunc (w *zcWriter) MallocAck(n int) (err error) {\n\treturn w.buf.MallocAck(n)\n}\n\n// Append implements Writer.\nfunc (w *zcWriter) Append(w2 Writer) (err error) {\n\treturn w.buf.Append(w2)\n}\n\n// WriteString implements Writer.\nfunc (w *zcWriter) WriteString(s string) (n int, err error) {\n\treturn w.buf.WriteString(s)\n}\n\n// WriteBinary implements Writer.\nfunc (w *zcWriter) WriteBinary(b []byte) (n int, err error) {\n\treturn w.buf.WriteBinary(b)\n}\n\n// WriteDirect implements Writer.\nfunc (w *zcWriter) WriteDirect(p []byte, remainCap int) error {\n\treturn w.buf.WriteDirect(p, remainCap)\n}\n\n// WriteByte implements Writer.\nfunc (w *zcWriter) WriteByte(b byte) (err error) {\n\treturn w.buf.WriteByte(b)\n}\n\n// zcWriter implements ReadWriter.\ntype zcReadWriter struct {\n\t*zcReader\n\t*zcWriter\n}\n\nfunc newIOReader(r Reader) *ioReader {\n\treturn &ioReader{\n\t\tr: r,\n\t}\n}\n\nvar _ io.Reader = &ioReader{}\n\n// ioReader implements io.Reader.\n//\n// Deprecated: connection already implements Read directly with optimized buffer access.\n// This wrapper exists only for external Reader implementations.\ntype ioReader struct {\n\tr Reader\n}\n\n// Read implements io.Reader.\n//\n// BUG: Read calls Release which invalidates any slices previously returned by Next or Peek\n// on the same Reader. Do not mix Next/Peek and Read on the same Reader without first\n// calling Release.\nfunc (r *ioReader) Read(p []byte) (n int, err error) {\n\tl := len(p)\n\tif l == 0 {\n\t\treturn 0, nil\n\t}\n\t// read min(len(p), buffer.Len)\n\tif has := r.r.Len(); has < l {\n\t\tl = has\n\t}\n\tif l == 0 {\n\t\treturn 0, io.EOF\n\t}\n\tsrc, err := r.r.Next(l)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tn = copy(p, src)\n\terr = r.r.Release()\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\treturn n, nil\n}\n\nfunc newIOWriter(w Writer) *ioWriter {\n\treturn &ioWriter{\n\t\tw: w,\n\t}\n}\n\nvar _ io.Writer = &ioWriter{}\n\n// ioWriter implements io.Writer.\ntype ioWriter struct {\n\tw Writer\n}\n\n// Write implements io.Writer.\nfunc (w *ioWriter) Write(p []byte) (n int, err error) {\n\tdst, err := w.w.Malloc(len(p))\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tn = copy(dst, p)\n\terr = w.w.Flush()\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\treturn n, nil\n}\n\n// ioReadWriter implements io.ReadWriter.\ntype ioReadWriter struct {\n\tio.Reader\n\tio.Writer\n}\n"
  },
  {
    "path": "nocopy_readwriter_test.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n// +build !windows\n\npackage netpoll\n\nimport (\n\t\"errors\"\n\t\"io\"\n\t\"io/ioutil\"\n\t\"testing\"\n)\n\nfunc TestZCReader(t *testing.T) {\n\treader := &MockIOReadWriter{\n\t\tread: func(p []byte) (n int, err error) {\n\t\t\treturn len(p), nil\n\t\t},\n\t}\n\tr := newZCReader(reader)\n\n\tp, err := r.Next(block8k)\n\tMustNil(t, err)\n\tEqual(t, len(p), block8k)\n\tEqual(t, r.buf.Len(), 0)\n\n\tp, err = r.Peek(block4k)\n\tMustNil(t, err)\n\tEqual(t, len(p), block4k)\n\tEqual(t, r.buf.Len(), block4k)\n\n\terr = r.Skip(block4k)\n\tMustNil(t, err)\n\tEqual(t, r.buf.Len(), 0)\n\n\terr = r.Release()\n\tMustNil(t, err)\n}\n\nfunc TestZCWriter(t *testing.T) {\n\twriter := &MockIOReadWriter{\n\t\twrite: func(p []byte) (n int, err error) {\n\t\t\treturn len(p), nil\n\t\t},\n\t}\n\tw := newZCWriter(writer)\n\n\tp, err := w.Malloc(block1k)\n\tMustNil(t, err)\n\tEqual(t, len(p), block1k)\n\tEqual(t, w.buf.Len(), 0)\n\n\terr = w.Flush()\n\tMustNil(t, err)\n\tEqual(t, w.buf.Len(), 0)\n\n\tp, err = w.Malloc(block2k)\n\tMustNil(t, err)\n\tEqual(t, len(p), block2k)\n\tEqual(t, w.buf.Len(), 0)\n\n\terr = w.buf.Flush()\n\tMustNil(t, err)\n\tEqual(t, w.buf.Len(), block2k)\n\n\terr = w.Flush()\n\tMustNil(t, err)\n\tEqual(t, w.buf.Len(), 0)\n}\n\nfunc TestZCEOF(t *testing.T) {\n\treader := &MockIOReadWriter{\n\t\tread: func(p []byte) (n int, err error) {\n\t\t\treturn 0, io.EOF\n\t\t},\n\t}\n\tr := newZCReader(reader)\n\n\t_, err := r.Next(block8k)\n\tMustTrue(t, errors.Is(err, ErrEOF))\n}\n\ntype MockIOReadWriter struct {\n\tread  func(p []byte) (n int, err error)\n\twrite func(p []byte) (n int, err error)\n}\n\nfunc (rw *MockIOReadWriter) Read(p []byte) (n int, err error) {\n\tif rw.read != nil {\n\t\treturn rw.read(p)\n\t}\n\treturn\n}\n\nfunc (rw *MockIOReadWriter) Write(p []byte) (n int, err error) {\n\tif rw.write != nil {\n\t\treturn rw.write(p)\n\t}\n\treturn\n}\n\nfunc TestIOReadWriter(t *testing.T) {\n\tbuf := NewLinkBuffer(block1k)\n\treader, writer := newIOReader(buf), newIOWriter(buf)\n\tmsg := []byte(\"hello world\")\n\tn, err := writer.Write(msg)\n\tMustNil(t, err)\n\tEqual(t, n, len(msg))\n\n\tp := make([]byte, block1k)\n\tn, err = reader.Read(p)\n\tMustNil(t, err)\n\tEqual(t, n, len(msg))\n}\n\nfunc TestIOReadWriter2(t *testing.T) {\n\tbuf := NewLinkBuffer(block1k)\n\treader, writer := newIOReader(buf), newIOWriter(buf)\n\tmsg := []byte(\"hello world\")\n\tn, err := writer.Write(msg)\n\tMustNil(t, err)\n\tEqual(t, n, len(msg))\n\n\tp, err := ioutil.ReadAll(reader)\n\tMustNil(t, err)\n\tEqual(t, len(p), len(msg))\n}\n"
  },
  {
    "path": "poll.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage netpoll\n\n// Poll monitors fd(file descriptor), calls the FDOperator to perform specific actions,\n// and shields underlying differences. On linux systems, poll uses epoll by default,\n// and kevent by default on bsd systems.\ntype Poll interface {\n\t// Wait will poll all registered fds, and schedule processing based on the triggered event.\n\t// The call will block, so the usage can be like:\n\t//\n\t//  go wait()\n\t//\n\tWait() error\n\n\t// Close the poll and shutdown Wait().\n\tClose() error\n\n\t// Trigger can be used to actively refresh the loop where Wait is located when no event is triggered.\n\t// On linux systems, eventfd is used by default, and kevent by default on bsd systems.\n\tTrigger() error\n\n\t// Control the event of file descriptor and the operations is defined by PollEvent.\n\tControl(operator *FDOperator, event PollEvent) error\n\n\t// Alloc the operator from cache.\n\tAlloc() (operator *FDOperator)\n\n\t// Free the operator from cache.\n\tFree(operator *FDOperator)\n}\n\n// PollEvent defines the operation of poll.Control.\ntype PollEvent int\n\nconst (\n\t// PollReadable is used to monitor whether the FDOperator registered by\n\t// listener and connection is readable or closed.\n\tPollReadable PollEvent = 0x1\n\n\t// PollWritable is used to monitor whether the FDOperator created by the dialer is writable or closed.\n\t// ET mode must be used (still need to poll hup after being writable)\n\tPollWritable PollEvent = 0x2\n\n\t// PollDetach is used to remove the FDOperator from poll.\n\tPollDetach PollEvent = 0x3\n\n\t// PollR2RW is used to monitor writable for FDOperator,\n\t// which is only called when the socket write buffer is full.\n\tPollR2RW PollEvent = 0x5\n\n\t// PollRW2R is used to remove the writable monitor of FDOperator, generally used with PollR2RW.\n\tPollRW2R PollEvent = 0x6\n)\n"
  },
  {
    "path": "poll_default.go",
    "content": "// Copyright 2023 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build darwin || netbsd || freebsd || openbsd || dragonfly || linux\n// +build darwin netbsd freebsd openbsd dragonfly linux\n\npackage netpoll\n\nfunc (p *defaultPoll) Alloc() (operator *FDOperator) {\n\top := p.opcache.alloc()\n\top.poll = p\n\treturn op\n}\n\nfunc (p *defaultPoll) Free(operator *FDOperator) {\n\tp.opcache.freeable(operator)\n}\n\nfunc (p *defaultPoll) appendHup(operator *FDOperator) {\n\tp.hups = append(p.hups, operator.OnHup)\n\tp.detach(operator)\n\toperator.done()\n}\n\nfunc (p *defaultPoll) detach(operator *FDOperator) {\n\tif err := operator.Control(PollDetach); err != nil {\n\t\tlogger.Printf(\"NETPOLL: poller detach operator failed: %v\", err)\n\t}\n}\n\nfunc (p *defaultPoll) onhups() {\n\tif len(p.hups) == 0 {\n\t\treturn\n\t}\n\thups := p.hups\n\tp.hups = nil\n\tgo func(onhups []func(p Poll) error) {\n\t\tfor i := range onhups {\n\t\t\tif onhups[i] != nil {\n\t\t\t\tonhups[i](p)\n\t\t\t}\n\t\t}\n\t}(hups)\n}\n\n// readall read all left data before close connection\nfunc readall(op *FDOperator, br barrier) (total int, err error) {\n\tivs := br.ivs\n\tvar n int\n\tfor {\n\t\tbs := op.Inputs(br.bs)\n\t\tif len(bs) == 0 {\n\t\t\treturn total, nil\n\t\t}\n\n\tTryRead:\n\t\tn, err = ioread(op.FD, bs, ivs)\n\t\top.InputAck(n)\n\t\ttotal += n\n\t\tif err != nil {\n\t\t\treturn total, err\n\t\t}\n\t\tif n == 0 {\n\t\t\tgoto TryRead\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "poll_default_bsd.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build darwin || netbsd || freebsd || openbsd || dragonfly\n// +build darwin netbsd freebsd openbsd dragonfly\n\npackage netpoll\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"syscall\"\n\t\"unsafe\"\n)\n\nfunc openPoll() (Poll, error) {\n\treturn openDefaultPoll()\n}\n\nfunc openDefaultPoll() (*defaultPoll, error) {\n\tl := new(defaultPoll)\n\tp, err := syscall.Kqueue()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tl.fd = p\n\t_, err = syscall.Kevent(l.fd, []syscall.Kevent_t{{\n\t\tIdent:  0,\n\t\tFilter: syscall.EVFILT_USER,\n\t\tFlags:  syscall.EV_ADD | syscall.EV_CLEAR,\n\t}}, nil, nil)\n\tif err != nil {\n\t\tsyscall.Close(l.fd)\n\t\treturn nil, err\n\t}\n\tl.opcache = newOperatorCache()\n\treturn l, nil\n}\n\ntype defaultPoll struct {\n\tfd      int\n\ttrigger uint32\n\tm       sync.Map       //nolint:unused // only used in go:race\n\topcache *operatorCache // operator cache\n\thups    []func(p Poll) error\n}\n\n// Wait implements Poll.\nfunc (p *defaultPoll) Wait() error {\n\t// init\n\tsize, caps := 1024, barriercap\n\tevents, barriers := make([]syscall.Kevent_t, size), make([]barrier, size)\n\tfor i := range barriers {\n\t\tbarriers[i].bs = make([][]byte, caps)\n\t\tbarriers[i].ivs = make([]syscall.Iovec, caps)\n\t}\n\t// wait\n\tvar triggerRead, triggerWrite, triggerHup bool\n\tfor {\n\t\tn, err := syscall.Kevent(p.fd, nil, events, nil)\n\t\tif err != nil && err != syscall.EINTR {\n\t\t\t// exit gracefully\n\t\t\tif err == syscall.EBADF {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\t\tfor i := 0; i < n; i++ {\n\t\t\tfd := int(events[i].Ident)\n\t\t\t// trigger\n\t\t\tif fd == 0 {\n\t\t\t\t// clean trigger\n\t\t\t\tatomic.StoreUint32(&p.trigger, 0)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\toperator := p.getOperator(fd, unsafe.Pointer(&events[i].Udata))\n\t\t\tif operator == nil || !operator.do() {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tvar totalRead int\n\t\t\tevt := events[i]\n\t\t\ttriggerRead = evt.Filter == syscall.EVFILT_READ && evt.Flags&syscall.EV_ENABLE != 0\n\t\t\ttriggerWrite = evt.Filter == syscall.EVFILT_WRITE && evt.Flags&syscall.EV_ENABLE != 0\n\t\t\ttriggerHup = evt.Flags&syscall.EV_EOF != 0\n\n\t\t\tif triggerRead {\n\t\t\t\tif operator.OnRead != nil {\n\t\t\t\t\t// for non-connection\n\t\t\t\t\toperator.OnRead(p)\n\t\t\t\t} else {\n\t\t\t\t\t// only for connection\n\t\t\t\t\tbs := operator.Inputs(barriers[i].bs)\n\t\t\t\t\tif len(bs) > 0 {\n\t\t\t\t\t\tn, err := ioread(operator.FD, bs, barriers[i].ivs)\n\t\t\t\t\t\toperator.InputAck(n)\n\t\t\t\t\t\ttotalRead += n\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\tp.appendHup(operator)\n\t\t\t\t\t\t\tcontinue\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\tif triggerHup {\n\t\t\t\tif triggerRead && operator.Inputs != nil {\n\t\t\t\t\tvar leftRead int\n\t\t\t\t\t// read all left data if peer send and close\n\t\t\t\t\tif leftRead, err = readall(operator, barriers[i]); err != nil && !errors.Is(err, ErrEOF) {\n\t\t\t\t\t\tlogger.Printf(\"NETPOLL: readall(fd=%d)=%d before close: %s\", operator.FD, total, err.Error())\n\t\t\t\t\t}\n\t\t\t\t\ttotalRead += leftRead\n\t\t\t\t}\n\t\t\t\t// only close connection if no further read bytes\n\t\t\t\tif totalRead == 0 {\n\t\t\t\t\tp.appendHup(operator)\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\t\t\tif triggerWrite {\n\t\t\t\tif operator.OnWrite != nil {\n\t\t\t\t\t// for non-connection\n\t\t\t\t\toperator.OnWrite(p)\n\t\t\t\t} else {\n\t\t\t\t\t// only for connection\n\t\t\t\t\tbs, supportZeroCopy := operator.Outputs(barriers[i].bs)\n\t\t\t\t\tif len(bs) > 0 {\n\t\t\t\t\t\t// TODO: Let the upper layer pass in whether to use ZeroCopy.\n\t\t\t\t\t\tn, err := iosend(operator.FD, bs, barriers[i].ivs, false && supportZeroCopy)\n\t\t\t\t\t\toperator.OutputAck(n)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\tp.appendHup(operator)\n\t\t\t\t\t\t\tcontinue\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t\toperator.done()\n\t\t}\n\t\t// hup conns together to avoid blocking the poll.\n\t\tp.onhups()\n\t\tp.opcache.free()\n\t}\n}\n\n// TODO: Close will bad file descriptor here\nfunc (p *defaultPoll) Close() error {\n\terr := syscall.Close(p.fd)\n\treturn err\n}\n\n// Trigger implements Poll.\nfunc (p *defaultPoll) Trigger() error {\n\tif atomic.AddUint32(&p.trigger, 1) > 1 {\n\t\treturn nil\n\t}\n\t_, err := syscall.Kevent(p.fd, []syscall.Kevent_t{{\n\t\tIdent:  0,\n\t\tFilter: syscall.EVFILT_USER,\n\t\tFflags: syscall.NOTE_TRIGGER,\n\t}}, nil, nil)\n\treturn err\n}\n\n// Control implements Poll.\nfunc (p *defaultPoll) Control(operator *FDOperator, event PollEvent) error {\n\tevs := make([]syscall.Kevent_t, 1)\n\tevs[0].Ident = uint64(operator.FD)\n\tp.setOperator(unsafe.Pointer(&evs[0].Udata), operator)\n\tswitch event {\n\tcase PollReadable:\n\t\toperator.inuse()\n\t\tevs[0].Filter, evs[0].Flags = syscall.EVFILT_READ, syscall.EV_ADD|syscall.EV_ENABLE\n\tcase PollWritable:\n\t\toperator.inuse()\n\t\tevs[0].Filter, evs[0].Flags = syscall.EVFILT_WRITE, syscall.EV_ADD|syscall.EV_ENABLE\n\tcase PollDetach:\n\t\tif operator.OnWrite != nil { // means WaitWrite finished\n\t\t\tevs[0].Filter, evs[0].Flags = syscall.EVFILT_WRITE, syscall.EV_DELETE\n\t\t} else {\n\t\t\tevs[0].Filter, evs[0].Flags = syscall.EVFILT_READ, syscall.EV_DELETE\n\t\t}\n\t\tp.delOperator(operator)\n\tcase PollR2RW:\n\t\tevs[0].Filter, evs[0].Flags = syscall.EVFILT_WRITE, syscall.EV_ADD|syscall.EV_ENABLE\n\tcase PollRW2R:\n\t\tevs[0].Filter, evs[0].Flags = syscall.EVFILT_WRITE, syscall.EV_DELETE\n\t}\n\t_, err := syscall.Kevent(p.fd, evs, nil, nil)\n\treturn err\n}\n"
  },
  {
    "path": "poll_default_bsd_norace.go",
    "content": "// Copyright 2023 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build (darwin || netbsd || freebsd || openbsd || dragonfly) && !race\n// +build darwin netbsd freebsd openbsd dragonfly\n// +build !race\n\npackage netpoll\n\nimport \"unsafe\"\n\nfunc (p *defaultPoll) getOperator(fd int, ptr unsafe.Pointer) *FDOperator {\n\treturn *(**FDOperator)(ptr)\n}\n\nfunc (p *defaultPoll) setOperator(ptr unsafe.Pointer, operator *FDOperator) {\n\t*(**FDOperator)(ptr) = operator\n}\n\nfunc (p *defaultPoll) delOperator(operator *FDOperator) {\n}\n"
  },
  {
    "path": "poll_default_bsd_race.go",
    "content": "// Copyright 2023 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build (darwin || netbsd || freebsd || openbsd || dragonfly) && race\n// +build darwin netbsd freebsd openbsd dragonfly\n// +build race\n\npackage netpoll\n\nimport \"unsafe\"\n\nfunc (p *defaultPoll) getOperator(fd int, ptr unsafe.Pointer) *FDOperator {\n\ttmp, _ := p.m.Load(fd)\n\tif tmp == nil {\n\t\treturn nil\n\t}\n\treturn tmp.(*FDOperator)\n}\n\nfunc (p *defaultPoll) setOperator(ptr unsafe.Pointer, operator *FDOperator) {\n\tp.m.Store(operator.FD, operator)\n}\n\nfunc (p *defaultPoll) delOperator(operator *FDOperator) {\n\tp.m.Delete(operator.FD)\n}\n"
  },
  {
    "path": "poll_default_linux.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage netpoll\n\nimport (\n\t\"errors\"\n\t\"runtime\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"syscall\"\n\t\"unsafe\"\n)\n\nfunc openPoll() (Poll, error) {\n\treturn openDefaultPoll()\n}\n\nfunc openDefaultPoll() (*defaultPoll, error) {\n\tpoll := new(defaultPoll)\n\n\tpoll.buf = make([]byte, 8)\n\tp, err := EpollCreate(0)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tpoll.fd = p\n\n\tr0, _, e0 := syscall.Syscall(syscall.SYS_EVENTFD2, 0, 0, 0)\n\tif e0 != 0 {\n\t\t_ = syscall.Close(poll.fd)\n\t\treturn nil, e0\n\t}\n\n\tpoll.Reset = poll.reset\n\tpoll.Handler = poll.handler\n\tpoll.wop = &FDOperator{FD: int(r0)}\n\n\tif err = poll.Control(poll.wop, PollReadable); err != nil {\n\t\t_ = syscall.Close(poll.wop.FD)\n\t\t_ = syscall.Close(poll.fd)\n\t\treturn nil, err\n\t}\n\n\tpoll.opcache = newOperatorCache()\n\treturn poll, nil\n}\n\ntype defaultPoll struct {\n\tpollArgs\n\tfd      int            // epoll fd\n\twop     *FDOperator    // eventfd, wake epoll_wait\n\tbuf     []byte         // read wfd trigger msg\n\ttrigger uint32         // trigger flag\n\tm       sync.Map       //nolint:unused // only used in go:race\n\topcache *operatorCache // operator cache\n\t// fns for handle events\n\tReset   func(size, caps int)\n\tHandler func(events []epollevent) (closed bool)\n}\n\ntype pollArgs struct {\n\tsize     int\n\tcaps     int\n\tevents   []epollevent\n\tbarriers []barrier\n\thups     []func(p Poll) error\n}\n\nfunc (a *pollArgs) reset(size, caps int) {\n\ta.size, a.caps = size, caps\n\ta.events, a.barriers = make([]epollevent, size), make([]barrier, size)\n\tfor i := range a.barriers {\n\t\ta.barriers[i].bs = make([][]byte, a.caps)\n\t\ta.barriers[i].ivs = make([]syscall.Iovec, a.caps)\n\t}\n}\n\n// Wait implements Poll.\nfunc (p *defaultPoll) Wait() (err error) {\n\t// init\n\tcaps, msec, n := barriercap, -1, 0\n\tp.Reset(128, caps)\n\t// wait\n\tfor {\n\t\tif n == p.size && p.size < 128*1024 {\n\t\t\tp.Reset(p.size<<1, caps)\n\t\t}\n\t\tn, err = EpollWait(p.fd, p.events, msec)\n\t\tif err != nil && err != syscall.EINTR {\n\t\t\treturn err\n\t\t}\n\t\tif n <= 0 {\n\t\t\tmsec = -1\n\t\t\truntime.Gosched()\n\t\t\tcontinue\n\t\t}\n\t\tmsec = 0\n\t\tif p.Handler(p.events[:n]) {\n\t\t\treturn nil\n\t\t}\n\t\t// we can make sure that there is no op remaining if Handler finished\n\t\tp.opcache.free()\n\t}\n}\n\nfunc (p *defaultPoll) handler(events []epollevent) (closed bool) {\n\tvar triggerRead, triggerWrite, triggerHup, triggerError bool\n\tvar err error\n\tfor i := range events {\n\t\toperator := p.getOperator(0, unsafe.Pointer(&events[i].data))\n\t\tif operator == nil || !operator.do() {\n\t\t\tcontinue\n\t\t}\n\n\t\tvar totalRead int\n\t\tevt := events[i].events\n\t\ttriggerRead = evt&syscall.EPOLLIN != 0\n\t\ttriggerWrite = evt&syscall.EPOLLOUT != 0\n\t\ttriggerHup = evt&(syscall.EPOLLHUP|syscall.EPOLLRDHUP) != 0\n\t\ttriggerError = evt&syscall.EPOLLERR != 0\n\n\t\t// trigger or exit gracefully\n\t\tif operator.FD == p.wop.FD {\n\t\t\t// must clean trigger first\n\t\t\tsyscall.Read(p.wop.FD, p.buf)\n\t\t\tatomic.StoreUint32(&p.trigger, 0)\n\t\t\t// if closed & exit\n\t\t\tif p.buf[0] > 0 {\n\t\t\t\tsyscall.Close(p.wop.FD)\n\t\t\t\tsyscall.Close(p.fd)\n\t\t\t\toperator.done()\n\t\t\t\treturn true\n\t\t\t}\n\t\t\toperator.done()\n\t\t\tcontinue\n\t\t}\n\n\t\tif triggerRead {\n\t\t\tif operator.OnRead != nil {\n\t\t\t\t// for non-connection\n\t\t\t\toperator.OnRead(p)\n\t\t\t} else if operator.Inputs != nil {\n\t\t\t\t// for connection\n\t\t\t\tbs := operator.Inputs(p.barriers[i].bs)\n\t\t\t\tif len(bs) > 0 {\n\t\t\t\t\tn, err := ioread(operator.FD, bs, p.barriers[i].ivs)\n\t\t\t\t\toperator.InputAck(n)\n\t\t\t\t\ttotalRead += n\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tp.appendHup(operator)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tlogger.Printf(\"NETPOLL: operator has critical problem! event=%d operator=%v\", evt, operator)\n\t\t\t}\n\t\t}\n\t\tif triggerHup {\n\t\t\tif triggerRead && operator.Inputs != nil {\n\t\t\t\t// read all left data if peer send and close\n\t\t\t\tvar leftRead int\n\t\t\t\t// read all left data if peer send and close\n\t\t\t\tif leftRead, err = readall(operator, p.barriers[i]); err != nil && !errors.Is(err, ErrEOF) {\n\t\t\t\t\tlogger.Printf(\"NETPOLL: readall(fd=%d)=%d before close: %s\", operator.FD, totalRead, err.Error())\n\t\t\t\t}\n\t\t\t\ttotalRead += leftRead\n\t\t\t}\n\t\t\t// only close connection if no further read bytes\n\t\t\tif totalRead == 0 {\n\t\t\t\tp.appendHup(operator)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\tif triggerError {\n\t\t\t// Under block-zerocopy, the kernel may give an error callback, which is not a real error, just an EAGAIN.\n\t\t\t// So here we need to check this error, if it is EAGAIN then do nothing, otherwise still mark as hup.\n\t\t\tif _, _, _, _, err := syscall.Recvmsg(operator.FD, nil, nil, syscall.MSG_ERRQUEUE); err != syscall.EAGAIN {\n\t\t\t\tp.appendHup(operator)\n\t\t\t} else {\n\t\t\t\toperator.done()\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tif triggerWrite {\n\t\t\tif operator.OnWrite != nil {\n\t\t\t\t// for non-connection\n\t\t\t\toperator.OnWrite(p)\n\t\t\t} else if operator.Outputs != nil {\n\t\t\t\t// for connection\n\t\t\t\tbs, _ := operator.Outputs(p.barriers[i].bs)\n\t\t\t\tif len(bs) > 0 {\n\t\t\t\t\tn, err := iosend(operator.FD, bs, p.barriers[i].ivs, false)\n\t\t\t\t\toperator.OutputAck(n)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tp.appendHup(operator)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tlogger.Printf(\"NETPOLL: operator has critical problem! event=%d operator=%v\", evt, operator)\n\t\t\t}\n\t\t}\n\t\toperator.done()\n\t}\n\t// hup conns together to avoid blocking the poll.\n\tp.onhups()\n\treturn false\n}\n\n// Close will write 10000000\nfunc (p *defaultPoll) Close() error {\n\t_, err := syscall.Write(p.wop.FD, []byte{1, 0, 0, 0, 0, 0, 0, 0})\n\treturn err\n}\n\n// Trigger implements Poll.\nfunc (p *defaultPoll) Trigger() error {\n\tif atomic.AddUint32(&p.trigger, 1) > 1 {\n\t\treturn nil\n\t}\n\t// MAX(eventfd) = 0xfffffffffffffffe\n\t_, err := syscall.Write(p.wop.FD, []byte{0, 0, 0, 0, 0, 0, 0, 1})\n\treturn err\n}\n\n// Control implements Poll.\nfunc (p *defaultPoll) Control(operator *FDOperator, event PollEvent) error {\n\t// DON'T move `fd=operator.FD` behind inuse() call, we can only access operator before op.inuse() for avoid race\n\t// G1:              G2:\n\t// op.inuse()       op.unused()\n\t// op.FD  -- T1     op.FD = 0  -- T2\n\t// T1 and T2 may happen together\n\tfd := operator.FD\n\tvar op int\n\tvar evt epollevent\n\tp.setOperator(unsafe.Pointer(&evt.data), operator)\n\tswitch event {\n\tcase PollReadable: // server accept a new connection and wait read\n\t\toperator.inuse()\n\t\top, evt.events = syscall.EPOLL_CTL_ADD, syscall.EPOLLIN|syscall.EPOLLRDHUP|syscall.EPOLLERR\n\tcase PollWritable: // client create a new connection and wait connect finished\n\t\toperator.inuse()\n\t\top, evt.events = syscall.EPOLL_CTL_ADD, EPOLLET|syscall.EPOLLOUT|syscall.EPOLLRDHUP|syscall.EPOLLERR\n\tcase PollDetach: // deregister\n\t\tp.delOperator(operator)\n\t\top, evt.events = syscall.EPOLL_CTL_DEL, syscall.EPOLLIN|syscall.EPOLLOUT|syscall.EPOLLRDHUP|syscall.EPOLLERR\n\tcase PollR2RW: // connection wait read/write\n\t\top, evt.events = syscall.EPOLL_CTL_MOD, syscall.EPOLLIN|syscall.EPOLLOUT|syscall.EPOLLRDHUP|syscall.EPOLLERR\n\tcase PollRW2R: // connection wait read\n\t\top, evt.events = syscall.EPOLL_CTL_MOD, syscall.EPOLLIN|syscall.EPOLLRDHUP|syscall.EPOLLERR\n\t}\n\treturn EpollCtl(p.fd, op, fd, &evt)\n}\n"
  },
  {
    "path": "poll_default_linux_norace.go",
    "content": "// Copyright 2023 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build linux && !race\n// +build linux,!race\n\npackage netpoll\n\nimport \"unsafe\"\n\nfunc (p *defaultPoll) getOperator(fd int, ptr unsafe.Pointer) *FDOperator {\n\treturn *(**FDOperator)(ptr)\n}\n\nfunc (p *defaultPoll) setOperator(ptr unsafe.Pointer, operator *FDOperator) {\n\t*(**FDOperator)(ptr) = operator\n}\n\nfunc (p *defaultPoll) delOperator(operator *FDOperator) {\n}\n"
  },
  {
    "path": "poll_default_linux_race.go",
    "content": "// Copyright 2023 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build linux && race\n// +build linux,race\n\npackage netpoll\n\nimport \"unsafe\"\n\ntype eventdata struct {\n\tfd  int32\n\tpad int32\n}\n\nfunc (p *defaultPoll) getOperator(fd int, ptr unsafe.Pointer) *FDOperator {\n\tdata := *(*eventdata)(ptr)\n\ttmp, _ := p.m.Load(int(data.fd))\n\tif tmp == nil {\n\t\treturn nil\n\t}\n\treturn tmp.(*FDOperator)\n}\n\nfunc (p *defaultPoll) setOperator(ptr unsafe.Pointer, operator *FDOperator) {\n\t*(*eventdata)(ptr) = eventdata{fd: int32(operator.FD)}\n\tp.m.Store(operator.FD, operator)\n}\n\nfunc (p *defaultPoll) delOperator(operator *FDOperator) {\n\tp.m.Delete(operator.FD)\n}\n"
  },
  {
    "path": "poll_default_linux_test.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build linux\n// +build linux\n\npackage netpoll\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"syscall\"\n\t\"testing\"\n\n\t\"golang.org/x/sys/unix\"\n)\n\nfunc TestEpollEvent(t *testing.T) {\n\tepollfd, err := EpollCreate(0)\n\tMustNil(t, err)\n\tdefer syscall.Close(epollfd)\n\n\trfd, wfd := GetSysFdPairs()\n\tdefer syscall.Close(rfd)\n\tdefer syscall.Close(wfd)\n\n\tsend := []byte(\"hello\")\n\trecv := make([]byte, 5)\n\tevents := make([]epollevent, 128)\n\teventdata1 := [8]byte{0, 0, 0, 0, 0, 0, 0, 1}\n\teventdata2 := [8]byte{0, 0, 0, 0, 0, 0, 0, 2}\n\teventdata3 := [8]byte{0, 0, 0, 0, 0, 0, 0, 3}\n\tevent1 := &epollevent{\n\t\tevents: syscall.EPOLLIN,\n\t\tdata:   eventdata1,\n\t}\n\tevent2 := &epollevent{\n\t\tevents: syscall.EPOLLIN,\n\t\tdata:   eventdata2,\n\t}\n\tevent3 := &epollevent{\n\t\tevents: syscall.EPOLLIN | syscall.EPOLLOUT,\n\t\tdata:   eventdata3,\n\t}\n\n\t// EPOLL: add ,del and add\n\terr = EpollCtl(epollfd, unix.EPOLL_CTL_ADD, rfd, event1)\n\tMustNil(t, err)\n\terr = EpollCtl(epollfd, unix.EPOLL_CTL_DEL, rfd, event1)\n\tMustNil(t, err)\n\terr = EpollCtl(epollfd, unix.EPOLL_CTL_ADD, rfd, event2)\n\tMustNil(t, err)\n\t_, err = syscall.Write(wfd, send)\n\tMustNil(t, err)\n\tn, err := epollWaitUntil(epollfd, events, -1)\n\tMustNil(t, err)\n\tEqual(t, n, 1)\n\tEqual(t, events[0].data, eventdata2)\n\t_, err = syscall.Read(rfd, recv)\n\tMustTrue(t, err == nil && string(recv) == string(send))\n\terr = EpollCtl(epollfd, unix.EPOLL_CTL_DEL, rfd, event2)\n\tMustNil(t, err)\n\n\t// EPOLL: add ,mod and mod\n\terr = EpollCtl(epollfd, unix.EPOLL_CTL_ADD, rfd, event1)\n\tMustNil(t, err)\n\terr = EpollCtl(epollfd, unix.EPOLL_CTL_MOD, rfd, event2)\n\tMustNil(t, err)\n\terr = EpollCtl(epollfd, unix.EPOLL_CTL_MOD, rfd, event3)\n\tMustNil(t, err)\n\t_, err = syscall.Write(wfd, send)\n\tMustNil(t, err)\n\tn, err = epollWaitUntil(epollfd, events, -1)\n\tMustNil(t, err)\n\tEqual(t, n, 1)\n\tEqual(t, events[0].data, eventdata3)\n\t_, err = syscall.Read(rfd, recv)\n\tMustTrue(t, err == nil && string(recv) == string(send))\n\tAssert(t, events[0].events&syscall.EPOLLIN != 0)\n\tAssert(t, events[0].events&syscall.EPOLLOUT != 0)\n\n\terr = EpollCtl(epollfd, unix.EPOLL_CTL_DEL, rfd, event2)\n\tMustNil(t, err)\n}\n\nfunc TestEpollWait(t *testing.T) {\n\tepollfd, err := EpollCreate(0)\n\tMustNil(t, err)\n\tdefer syscall.Close(epollfd)\n\n\trfd, wfd := GetSysFdPairs()\n\tdefer syscall.Close(wfd)\n\n\tsend := []byte(\"hello\")\n\trecv := make([]byte, 5)\n\tevents := make([]epollevent, 128)\n\teventdata := [8]byte{0, 0, 0, 0, 0, 0, 0, 1}\n\n\t// EPOLL: init state\n\tevent := &epollevent{\n\t\tevents: syscall.EPOLLIN | syscall.EPOLLOUT | syscall.EPOLLRDHUP | syscall.EPOLLERR,\n\t\tdata:   eventdata,\n\t}\n\terr = EpollCtl(epollfd, unix.EPOLL_CTL_ADD, rfd, event)\n\tMustNil(t, err)\n\t_, err = epollWaitUntil(epollfd, events, -1)\n\tMustNil(t, err)\n\tAssert(t, events[0].events&syscall.EPOLLIN == 0)\n\tAssert(t, events[0].events&syscall.EPOLLOUT != 0)\n\n\t// EPOLL: readable\n\t_, err = syscall.Write(wfd, send)\n\tMustNil(t, err)\n\t_, err = epollWaitUntil(epollfd, events, -1)\n\tMustNil(t, err)\n\tAssert(t, events[0].events&syscall.EPOLLIN != 0)\n\tAssert(t, events[0].events&syscall.EPOLLOUT != 0)\n\t_, err = syscall.Read(rfd, recv)\n\tMustTrue(t, err == nil && string(recv) == string(send))\n\n\t// EPOLL: read finished\n\t_, err = epollWaitUntil(epollfd, events, -1)\n\tMustNil(t, err)\n\tAssert(t, events[0].events&syscall.EPOLLIN == 0)\n\tAssert(t, events[0].events&syscall.EPOLLOUT != 0)\n\n\t// EPOLL: close peer fd\n\terr = syscall.Close(wfd)\n\tMustNil(t, err)\n\t_, err = epollWaitUntil(epollfd, events, -1)\n\tMustNil(t, err)\n\tAssert(t, events[0].events&syscall.EPOLLIN != 0)\n\tAssert(t, events[0].events&syscall.EPOLLOUT != 0)\n\tAssert(t, events[0].events&syscall.EPOLLRDHUP != 0)\n\tAssert(t, events[0].events&syscall.EPOLLERR == 0)\n\n\t// EPOLL: close current fd\n\trfd2, wfd2 := GetSysFdPairs()\n\tdefer syscall.Close(wfd2)\n\terr = EpollCtl(epollfd, unix.EPOLL_CTL_ADD, rfd2, event)\n\tMustNil(t, err)\n\terr = syscall.Close(rfd2)\n\tMustNil(t, err)\n\t_, err = epollWaitUntil(epollfd, events, -1)\n\tMustNil(t, err)\n\tAssert(t, events[0].events&syscall.EPOLLIN != 0)\n\tAssert(t, events[0].events&syscall.EPOLLOUT != 0)\n\tAssert(t, events[0].events&syscall.EPOLLRDHUP != 0)\n\tAssert(t, events[0].events&syscall.EPOLLERR == 0)\n\n\terr = EpollCtl(epollfd, unix.EPOLL_CTL_DEL, rfd, event)\n\tMustNil(t, err)\n}\n\nfunc TestEpollETClose(t *testing.T) {\n\tepollfd, err := EpollCreate(0)\n\tMustNil(t, err)\n\tdefer syscall.Close(epollfd)\n\trfd, wfd := GetSysFdPairs()\n\tevents := make([]epollevent, 128)\n\teventdata := [8]byte{0, 0, 0, 0, 0, 0, 0, 1}\n\tevent := &epollevent{\n\t\tevents: EPOLLET | syscall.EPOLLIN | syscall.EPOLLOUT | syscall.EPOLLRDHUP | syscall.EPOLLERR,\n\t\tdata:   eventdata,\n\t}\n\n\t// EPOLL: init state\n\terr = EpollCtl(epollfd, unix.EPOLL_CTL_ADD, rfd, event)\n\tMustNil(t, err)\n\t_, err = epollWaitUntil(epollfd, events, -1)\n\tMustNil(t, err)\n\tAssert(t, events[0].events&syscall.EPOLLIN == 0)\n\tAssert(t, events[0].events&syscall.EPOLLOUT != 0)\n\tAssert(t, events[0].events&syscall.EPOLLRDHUP == 0)\n\tAssert(t, events[0].events&syscall.EPOLLERR == 0)\n\n\t// EPOLL: close current fd\n\t// nothing will happen\n\terr = syscall.Close(rfd)\n\tMustNil(t, err)\n\tn, err := epollWaitUntil(epollfd, events, 100)\n\tMustNil(t, err)\n\tAssert(t, n == 0, n)\n\terr = syscall.Close(wfd)\n\tMustNil(t, err)\n\n\t// EPOLL: close peer fd\n\t// EPOLLIN and EPOLLOUT\n\trfd, wfd = GetSysFdPairs()\n\terr = EpollCtl(epollfd, unix.EPOLL_CTL_ADD, rfd, event)\n\tMustNil(t, err)\n\terr = syscall.Close(wfd)\n\tMustNil(t, err)\n\tn, err = epollWaitUntil(epollfd, events, 100)\n\tMustNil(t, err)\n\tAssert(t, n == 1, n)\n\tAssert(t, events[0].events&syscall.EPOLLIN != 0)\n\tAssert(t, events[0].events&syscall.EPOLLOUT != 0)\n\tAssert(t, events[0].events&syscall.EPOLLRDHUP != 0)\n\tAssert(t, events[0].events&syscall.EPOLLERR == 0)\n\tbuf := make([]byte, 1024)\n\tivs := make([]syscall.Iovec, 1)\n\tn, err = ioread(rfd, [][]byte{buf}, ivs) // EOF\n\tAssert(t, n == 0 && errors.Is(err, ErrEOF), n, err)\n}\n\nfunc TestEpollETDel(t *testing.T) {\n\tepollfd, err := EpollCreate(0)\n\tMustNil(t, err)\n\tdefer syscall.Close(epollfd)\n\trfd, wfd := GetSysFdPairs()\n\tsend := []byte(\"hello\")\n\tevents := make([]epollevent, 128)\n\teventdata := [8]byte{0, 0, 0, 0, 0, 0, 0, 1}\n\tevent := &epollevent{\n\t\tevents: EPOLLET | syscall.EPOLLIN | syscall.EPOLLRDHUP | syscall.EPOLLERR,\n\t\tdata:   eventdata,\n\t}\n\n\t// EPOLL: del partly\n\terr = EpollCtl(epollfd, unix.EPOLL_CTL_ADD, rfd, event)\n\tMustNil(t, err)\n\tevent.events = syscall.EPOLLIN | syscall.EPOLLOUT | syscall.EPOLLRDHUP | syscall.EPOLLERR\n\terr = EpollCtl(epollfd, unix.EPOLL_CTL_DEL, rfd, event)\n\tMustNil(t, err)\n\t_, err = syscall.Write(wfd, send)\n\tMustNil(t, err)\n\t_, err = epollWaitUntil(epollfd, events, 100)\n\tMustNil(t, err)\n\tAssert(t, events[0].events&syscall.EPOLLIN == 0)\n\tAssert(t, events[0].events&syscall.EPOLLRDHUP == 0)\n\tAssert(t, events[0].events&syscall.EPOLLERR == 0)\n}\n\nfunc TestEpollConnectSameFD(t *testing.T) {\n\taddr := syscall.SockaddrInet4{\n\t\tPort: 12345,\n\t\tAddr: [4]byte{127, 0, 0, 1},\n\t}\n\tloop := newTestEventLoop(\"tcp\", \"127.0.0.1:12345\",\n\t\tfunc(ctx context.Context, connection Connection) error {\n\t\t\t_, err := connection.Reader().Next(connection.Reader().Len())\n\t\t\treturn err\n\t\t},\n\t)\n\tdefer loop.Shutdown(context.Background())\n\n\tepollfd, err := EpollCreate(0)\n\tMustNil(t, err)\n\tdefer syscall.Close(epollfd)\n\tevents := make([]epollevent, 128)\n\teventdata1 := [8]byte{0, 0, 0, 0, 0, 0, 0, 1}\n\teventdata2 := [8]byte{0, 0, 0, 0, 0, 0, 0, 2}\n\tevent1 := &epollevent{\n\t\tevents: EPOLLET | syscall.EPOLLOUT | syscall.EPOLLRDHUP | syscall.EPOLLERR,\n\t\tdata:   eventdata1,\n\t}\n\tevent2 := &epollevent{\n\t\tevents: EPOLLET | syscall.EPOLLOUT | syscall.EPOLLRDHUP | syscall.EPOLLERR,\n\t\tdata:   eventdata2,\n\t}\n\teventin := &epollevent{\n\t\tevents: syscall.EPOLLIN | syscall.EPOLLRDHUP | syscall.EPOLLERR,\n\t\tdata:   eventdata1,\n\t}\n\n\t// connect non-block socket\n\tfd1, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_TCP)\n\tMustNil(t, err)\n\tt.Logf(\"create fd: %d\", fd1)\n\terr = syscall.SetNonblock(fd1, true)\n\tMustNil(t, err)\n\terr = EpollCtl(epollfd, unix.EPOLL_CTL_ADD, fd1, event1)\n\tMustNil(t, err)\n\terr = syscall.Connect(fd1, &addr)\n\tt.Log(err) // EINPROGRESS\n\t_, err = epollWaitUntil(epollfd, events, -1)\n\tMustNil(t, err)\n\tAssert(t, events[0].events&syscall.EPOLLOUT != 0)\n\tAssert(t, events[0].events&syscall.EPOLLRDHUP == 0)\n\tAssert(t, events[0].events&syscall.EPOLLERR == 0)\n\t// forget to del fd\n\t// err = EpollCtl(epollfd, unix.EPOLL_CTL_DEL, fd1, event1)\n\t// MustNil(t, err)\n\terr = syscall.Close(fd1) // close fd1\n\tMustNil(t, err)\n\n\t// connect non-block socket with same fd\n\tfd2, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_TCP)\n\tMustNil(t, err)\n\tt.Logf(\"create fd: %d\", fd2)\n\terr = syscall.SetNonblock(fd2, true)\n\tMustNil(t, err)\n\terr = EpollCtl(epollfd, unix.EPOLL_CTL_ADD, fd2, event2)\n\tMustNil(t, err)\n\terr = syscall.Connect(fd2, &addr)\n\tt.Log(err) // EINPROGRESS\n\t_, err = epollWaitUntil(epollfd, events, -1)\n\tMustNil(t, err)\n\tAssert(t, events[0].events&syscall.EPOLLOUT != 0)\n\tAssert(t, events[0].events&syscall.EPOLLRDHUP == 0)\n\tAssert(t, events[0].events&syscall.EPOLLERR == 0)\n\terr = EpollCtl(epollfd, unix.EPOLL_CTL_DEL, fd2, event2)\n\tMustNil(t, err)\n\terr = syscall.Close(fd2) // close fd2\n\tMustNil(t, err)\n\tEqual(t, events[0].data, eventdata2)\n\n\t// no event after close fd\n\tfd3, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_TCP)\n\tMustNil(t, err)\n\tt.Logf(\"create fd: %d\", fd3)\n\terr = syscall.SetNonblock(fd3, true)\n\tMustNil(t, err)\n\terr = EpollCtl(epollfd, unix.EPOLL_CTL_ADD, fd3, event1)\n\tMustNil(t, err)\n\terr = syscall.Connect(fd3, &addr)\n\tt.Log(err) // EINPROGRESS\n\t_, err = epollWaitUntil(epollfd, events, -1)\n\tMustNil(t, err)\n\tAssert(t, events[0].events&syscall.EPOLLOUT != 0)\n\tAssert(t, events[0].events&syscall.EPOLLRDHUP == 0)\n\tAssert(t, events[0].events&syscall.EPOLLERR == 0)\n\tMustNil(t, err)\n\terr = EpollCtl(epollfd, unix.EPOLL_CTL_MOD, fd3, eventin)\n\tMustNil(t, err)\n\terr = syscall.Close(fd3) // close fd3\n\tMustNil(t, err)\n\tn, err := epollWaitUntil(epollfd, events, 100)\n\tMustNil(t, err)\n\tAssert(t, n == 0)\n}\n\nfunc epollWaitUntil(epfd int, events []epollevent, msec int) (n int, err error) {\nWAIT:\n\tn, err = EpollWait(epfd, events, msec)\n\tif err == syscall.EINTR {\n\t\tgoto WAIT\n\t}\n\treturn n, err\n}\n"
  },
  {
    "path": "poll_loadbalance.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage netpoll\n\nimport (\n\t\"sync/atomic\"\n\n\t\"github.com/bytedance/gopkg/lang/fastrand\"\n)\n\n// LoadBalance sets the load balancing method.\ntype LoadBalance int\n\nconst (\n\t// RoundRobin requests that connections are distributed to a Poll\n\t// in a round-robin fashion.\n\tRoundRobin LoadBalance = iota\n\t// Random requests that connections are randomly distributed.\n\tRandom\n)\n\n// loadbalance sets the load balancing method for []*polls\ntype loadbalance interface {\n\tLoadBalance() LoadBalance\n\t// Pick choose the most qualified Poll\n\tPick() (poll Poll)\n\n\tRebalance(polls []Poll)\n}\n\nfunc newLoadbalance(lb LoadBalance, polls []Poll) loadbalance {\n\tswitch lb {\n\tcase RoundRobin:\n\t\treturn newRoundRobinLB(polls)\n\tcase Random:\n\t\treturn newRandomLB(polls)\n\t}\n\treturn newRoundRobinLB(polls)\n}\n\nfunc newRandomLB(polls []Poll) loadbalance {\n\treturn &randomLB{polls: polls, pollSize: len(polls)}\n}\n\ntype randomLB struct {\n\tpolls    []Poll\n\tpollSize int\n}\n\nfunc (b *randomLB) LoadBalance() LoadBalance {\n\treturn Random\n}\n\nfunc (b *randomLB) Pick() (poll Poll) {\n\tidx := fastrand.Intn(b.pollSize)\n\treturn b.polls[idx]\n}\n\nfunc (b *randomLB) Rebalance(polls []Poll) {\n\tb.polls, b.pollSize = polls, len(polls)\n}\n\nfunc newRoundRobinLB(polls []Poll) loadbalance {\n\treturn &roundRobinLB{polls: polls, pollSize: len(polls)}\n}\n\ntype roundRobinLB struct {\n\tpolls    []Poll\n\taccepted uintptr // accept counter\n\tpollSize int\n}\n\nfunc (b *roundRobinLB) LoadBalance() LoadBalance {\n\treturn RoundRobin\n}\n\nfunc (b *roundRobinLB) Pick() (poll Poll) {\n\tidx := int(atomic.AddUintptr(&b.accepted, 1)) % b.pollSize\n\treturn b.polls[idx]\n}\n\nfunc (b *roundRobinLB) Rebalance(polls []Poll) {\n\tb.polls, b.pollSize = polls, len(polls)\n}\n"
  },
  {
    "path": "poll_manager.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n// +build !windows\n\npackage netpoll\n\nimport (\n\t\"fmt\"\n\t\"runtime\"\n\t\"sync/atomic\"\n)\n\nconst (\n\tmanagerUninitialized = iota\n\tmanagerInitializing\n\tmanagerInitialized\n)\n\nfunc newManager(numLoops int) *manager {\n\tm := new(manager)\n\tm.SetLoadBalance(RoundRobin)\n\tm.SetNumLoops(numLoops)\n\treturn m\n}\n\n// LoadBalance is used to do load balancing among multiple pollers.\n// a single poller may not be optimal if the number of cores is large (40C+).\ntype manager struct {\n\tnumLoops int32\n\tstatus   int32       // 0: uninitialized, 1: initializing, 2: initialized\n\tbalance  loadbalance // load balancing method\n\tpolls    []Poll      // all the polls\n}\n\n// SetNumLoops will return error when set numLoops < 1\nfunc (m *manager) SetNumLoops(numLoops int) (err error) {\n\tif numLoops < 1 {\n\t\treturn fmt.Errorf(\"set invalid numLoops[%d]\", numLoops)\n\t}\n\t// note: set new numLoops first and then change the status\n\tatomic.StoreInt32(&m.numLoops, int32(numLoops))\n\tatomic.StoreInt32(&m.status, managerUninitialized)\n\treturn nil\n}\n\n// SetLoadBalance set load balance.\nfunc (m *manager) SetLoadBalance(lb LoadBalance) error {\n\tif m.balance != nil && m.balance.LoadBalance() == lb {\n\t\treturn nil\n\t}\n\tm.balance = newLoadbalance(lb, m.polls)\n\treturn nil\n}\n\n// Close release all resources.\nfunc (m *manager) Close() (err error) {\n\tfor _, poll := range m.polls {\n\t\terr = poll.Close()\n\t}\n\tm.numLoops = 0\n\tm.balance = nil\n\tm.polls = nil\n\treturn err\n}\n\n// Run all pollers.\nfunc (m *manager) Run() (err error) {\n\tdefer func() {\n\t\tif err != nil {\n\t\t\t_ = m.Close()\n\t\t}\n\t}()\n\n\tnumLoops := int(atomic.LoadInt32(&m.numLoops))\n\tif numLoops == len(m.polls) {\n\t\treturn nil\n\t}\n\tpolls := make([]Poll, numLoops)\n\tif numLoops < len(m.polls) {\n\t\t// shrink polls\n\t\tcopy(polls, m.polls[:numLoops])\n\t\tfor idx := numLoops; idx < len(m.polls); idx++ {\n\t\t\t// close redundant polls\n\t\t\tif err = m.polls[idx].Close(); err != nil {\n\t\t\t\tlogger.Printf(\"NETPOLL: poller close failed: %v\\n\", err)\n\t\t\t}\n\t\t}\n\t} else {\n\t\t// growth polls\n\t\tcopy(polls, m.polls)\n\t\tfor idx := len(m.polls); idx < numLoops; idx++ {\n\t\t\tvar poll Poll\n\t\t\tpoll, err = openPoll()\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tpolls[idx] = poll\n\t\t\tgo poll.Wait()\n\t\t}\n\t}\n\tm.polls = polls\n\n\t// LoadBalance must be set before calling Run, otherwise it will panic.\n\tm.balance.Rebalance(m.polls)\n\treturn nil\n}\n\n// Reset pollers, this operation is very dangerous, please make sure to do this when calling !\nfunc (m *manager) Reset() error {\n\tfor _, poll := range m.polls {\n\t\tpoll.Close()\n\t}\n\tm.polls = nil\n\treturn m.Run()\n}\n\n// Pick will select the poller for use each time based on the LoadBalance.\nfunc (m *manager) Pick() Poll {\nSTART:\n\t// fast path\n\tif atomic.LoadInt32(&m.status) == managerInitialized {\n\t\treturn m.balance.Pick()\n\t}\n\t// slow path\n\t// try to get initializing lock failed, wait others finished the init work, and try again\n\tif !atomic.CompareAndSwapInt32(&m.status, managerUninitialized, managerInitializing) {\n\t\truntime.Gosched()\n\t\tgoto START\n\t}\n\t// adjust polls\n\t// m.Run() will finish very quickly, so will not many goroutines block on Pick.\n\t_ = m.Run()\n\n\t//nolint:staticcheck // SA9003: empty branch\n\tif !atomic.CompareAndSwapInt32(&m.status, managerInitializing, managerInitialized) {\n\t\t// SetNumLoops called during m.Run() which cause CAS failed\n\t\t// The polls will be adjusted next Pick\n\t}\n\treturn m.balance.Pick()\n}\n"
  },
  {
    "path": "poll_manager_test.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n// +build !windows\n\npackage netpoll\n\nimport (\n\t\"runtime\"\n\t\"sync\"\n\t\"testing\"\n)\n\nfunc TestPollManager(t *testing.T) {\n\tr, w := GetSysFdPairs()\n\trconn, wconn := &connection{}, &connection{}\n\terr := rconn.init(&netFD{fd: r}, nil)\n\tMustNil(t, err)\n\terr = wconn.init(&netFD{fd: w}, nil)\n\tMustNil(t, err)\n\n\tmsg := []byte(\"hello world\")\n\tn, err := wconn.Write(msg)\n\tMustNil(t, err)\n\tEqual(t, n, len(msg))\n\n\tp, err := rconn.Reader().Next(n)\n\tMustNil(t, err)\n\tEqual(t, string(p), string(msg))\n\n\terr = wconn.Close()\n\tMustNil(t, err)\n\tfor rconn.IsActive() || wconn.IsActive() {\n\t\truntime.Gosched()\n\t}\n}\n\nfunc TestPollManagerReset(t *testing.T) {\n\tn := pollmanager.numLoops\n\terr := pollmanager.Reset()\n\tMustNil(t, err)\n\tEqual(t, len(pollmanager.polls), int(n))\n}\n\nfunc TestPollManagerSetNumLoops(t *testing.T) {\n\tpm := newManager(1)\n\n\tstartGs := runtime.NumGoroutine()\n\tpoll := pm.Pick()\n\tnewGs := runtime.NumGoroutine()\n\tAssert(t, poll != nil)\n\tt.Logf(\"old=%d, new=%d\", startGs, newGs)\n\t// FIXME: it's unstable due to background goroutines created by other tests\n\t// Assert(t, newGs-startGs == 1)\n\n\t// change pollers\n\toldGs := newGs\n\terr := pm.SetNumLoops(100)\n\tMustNil(t, err)\n\tnewGs = runtime.NumGoroutine()\n\tt.Logf(\"old=%d, new=%d\", oldGs, newGs)\n\t// Assert(t, newGs == oldGs)\n\n\t// trigger polls adjustment\n\tvar wg sync.WaitGroup\n\tfinish := make(chan struct{})\n\tfor i := 0; i < 32; i++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tpoll := pm.Pick()\n\t\t\tAssert(t, poll != nil)\n\t\t\tAssert(t, len(pm.polls) == 100)\n\t\t\twg.Done()\n\t\t\t<-finish // hold goroutines\n\t\t}()\n\t}\n\twg.Wait()\n\tclose(finish)\n}\n"
  },
  {
    "path": "poll_test.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n// +build !windows\n\npackage netpoll\n\nimport (\n\t\"runtime\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"syscall\"\n\t\"testing\"\n\t\"time\"\n)\n\n// Trigger has been validated, but no usage for now.\nfunc TestPollTrigger(t *testing.T) {\n\tt.Skip()\n\tvar trigger int\n\tstop := make(chan error)\n\tp, err := openDefaultPoll()\n\tMustNil(t, err)\n\n\tgo func() {\n\t\tstop <- p.Wait()\n\t}()\n\n\ttime.Sleep(time.Millisecond)\n\tEqual(t, trigger, 0)\n\tp.Trigger()\n\ttime.Sleep(time.Millisecond)\n\tEqual(t, trigger, 1)\n\tp.Trigger()\n\ttime.Sleep(time.Millisecond)\n\tEqual(t, trigger, 2)\n\n\tp.Close()\n\terr = <-stop\n\tMustNil(t, err)\n}\n\nfunc TestPollMod(t *testing.T) {\n\tvar rn, wn, hn int32\n\tread := func(p Poll) error {\n\t\tatomic.AddInt32(&rn, 1)\n\t\treturn nil\n\t}\n\twrite := func(p Poll) error {\n\t\tatomic.AddInt32(&wn, 1)\n\t\treturn nil\n\t}\n\thup := func(p Poll) error {\n\t\tatomic.AddInt32(&hn, 1)\n\t\treturn nil\n\t}\n\tstop := make(chan error)\n\tp, err := openDefaultPoll()\n\tMustNil(t, err)\n\tgo func() {\n\t\tstop <- p.Wait()\n\t}()\n\n\trfd, wfd := GetSysFdPairs()\n\trop := &FDOperator{FD: rfd, OnRead: read, OnWrite: write, OnHup: hup, poll: p}\n\twop := &FDOperator{FD: wfd, OnRead: read, OnWrite: write, OnHup: hup, poll: p}\n\tvar r, w, h int32\n\tr, w, h = atomic.LoadInt32(&rn), atomic.LoadInt32(&wn), atomic.LoadInt32(&hn)\n\tAssert(t, r == 0 && w == 0 && h == 0, r, w, h)\n\terr = p.Control(rop, PollReadable)\n\tMustNil(t, err)\n\tr, w, h = atomic.LoadInt32(&rn), atomic.LoadInt32(&wn), atomic.LoadInt32(&hn)\n\tAssert(t, r == 0 && w == 0 && h == 0, r, w, h)\n\n\terr = p.Control(wop, PollWritable) // trigger one shot\n\tMustNil(t, err)\n\tfor atomic.LoadInt32(&wn) == 0 {\n\t\truntime.Gosched()\n\t}\n\tr, w, h = atomic.LoadInt32(&rn), atomic.LoadInt32(&wn), atomic.LoadInt32(&hn)\n\tAssert(t, r == 0 && w >= 1 && h == 0, r, w, h)\n\n\terr = p.Control(rop, PollR2RW) // trigger write\n\tMustNil(t, err)\n\tfor atomic.LoadInt32(&wn) <= 1 {\n\t\truntime.Gosched()\n\t}\n\tr, w, h = atomic.LoadInt32(&rn), atomic.LoadInt32(&wn), atomic.LoadInt32(&hn)\n\tAssert(t, r == 0 && w >= 2 && h == 0, r, w, h)\n\n\t// close wfd, then trigger hup rfd\n\terr = syscall.Close(wfd) // trigger hup\n\tMustNil(t, err)\n\tfor atomic.LoadInt32(&hn) == 0 {\n\t\truntime.Gosched()\n\t}\n\tw, h = atomic.LoadInt32(&wn), atomic.LoadInt32(&hn)\n\tAssert(t, w >= 2 && h >= 1, r, w, h)\n\n\tp.Close()\n\terr = <-stop\n\tMustNil(t, err)\n}\n\nfunc TestPollClose(t *testing.T) {\n\tp, err := openDefaultPoll()\n\tMustNil(t, err)\n\tvar wg sync.WaitGroup\n\twg.Add(1)\n\tgo func() {\n\t\tp.Wait()\n\t\twg.Done()\n\t}()\n\tp.Close()\n\twg.Wait()\n}\n\nfunc BenchmarkPollMod(b *testing.B) {\n\tb.StopTimer()\n\tp, _ := openDefaultPoll()\n\tr, _ := GetSysFdPairs()\n\toperator := &FDOperator{FD: r}\n\tp.Control(operator, PollReadable)\n\n\t// benchmark\n\tb.ReportAllocs()\n\tb.StartTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tp.Control(operator, PollR2RW)\n\t}\n}\n"
  },
  {
    "path": "sys_epoll_linux.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !arm64 && !loong64\n// +build !arm64,!loong64\n\npackage netpoll\n\nimport (\n\t\"syscall\"\n\t\"unsafe\"\n)\n\nconst EPOLLET = -syscall.EPOLLET\n\ntype epollevent struct {\n\tevents uint32\n\tdata   [8]byte // unaligned uintptr\n}\n\n// EpollCreate implements epoll_create1.\nfunc EpollCreate(flag int) (fd int, err error) {\n\tvar r0 uintptr\n\tr0, _, err = syscall.RawSyscall(syscall.SYS_EPOLL_CREATE1, uintptr(flag), 0, 0)\n\tif err == syscall.Errno(0) {\n\t\terr = nil\n\t}\n\treturn int(r0), err\n}\n\n// EpollCtl implements epoll_ctl.\nfunc EpollCtl(epfd, op, fd int, event *epollevent) (err error) {\n\t_, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_CTL, uintptr(epfd), uintptr(op), uintptr(fd), uintptr(unsafe.Pointer(event)), 0, 0)\n\tif err == syscall.Errno(0) {\n\t\terr = nil\n\t}\n\treturn err\n}\n\n// EpollWait implements epoll_wait.\nfunc EpollWait(epfd int, events []epollevent, msec int) (n int, err error) {\n\tvar r0 uintptr\n\t_p0 := unsafe.Pointer(&events[0])\n\tif msec == 0 {\n\t\tr0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_WAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), 0, 0, 0)\n\t} else {\n\t\tr0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_WAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0)\n\t}\n\tif err == syscall.Errno(0) {\n\t\terr = nil\n\t}\n\treturn int(r0), err\n}\n"
  },
  {
    "path": "sys_epoll_linux_arm64.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage netpoll\n\nimport (\n\t\"syscall\"\n\t\"unsafe\"\n)\n\nconst EPOLLET = syscall.EPOLLET\n\ntype epollevent struct {\n\tevents uint32\n\t_      int32\n\tdata   [8]byte // unaligned uintptr\n}\n\n// EpollCreate implements epoll_create1.\nfunc EpollCreate(flag int) (fd int, err error) {\n\tvar r0 uintptr\n\tr0, _, err = syscall.RawSyscall(syscall.SYS_EPOLL_CREATE1, uintptr(flag), 0, 0)\n\tif err == syscall.Errno(0) {\n\t\terr = nil\n\t}\n\treturn int(r0), err\n}\n\n// EpollCtl implements epoll_ctl.\nfunc EpollCtl(epfd int, op int, fd int, event *epollevent) (err error) {\n\t_, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_CTL, uintptr(epfd), uintptr(op), uintptr(fd), uintptr(unsafe.Pointer(event)), 0, 0)\n\tif err == syscall.Errno(0) {\n\t\terr = nil\n\t}\n\treturn err\n}\n\n// EpollWait implements epoll_wait.\nfunc EpollWait(epfd int, events []epollevent, msec int) (n int, err error) {\n\tvar r0 uintptr\n\t_p0 := unsafe.Pointer(&events[0])\n\tif msec == 0 {\n\t\tr0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), 0, 0, 0)\n\t} else {\n\t\tr0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0)\n\t}\n\tif err == syscall.Errno(0) {\n\t\terr = nil\n\t}\n\treturn int(r0), err\n}\n"
  },
  {
    "path": "sys_epoll_linux_loong64.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build linux && loong64\n// +build linux,loong64\n\npackage netpoll\n\nimport (\n\t\"syscall\"\n\t\"unsafe\"\n)\n\nconst EPOLLET = syscall.EPOLLET\n\ntype epollevent struct {\n\tevents uint32\n\t_      int32\n\tdata   [8]byte // unaligned uintptr\n}\n\n// EpollCreate implements epoll_create1.\nfunc EpollCreate(flag int) (fd int, err error) {\n\tvar r0 uintptr\n\tr0, _, err = syscall.RawSyscall(syscall.SYS_EPOLL_CREATE1, uintptr(flag), 0, 0)\n\tif err == syscall.Errno(0) {\n\t\terr = nil\n\t}\n\treturn int(r0), err\n}\n\n// EpollCtl implements epoll_ctl.\nfunc EpollCtl(epfd int, op int, fd int, event *epollevent) (err error) {\n\t_, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_CTL, uintptr(epfd), uintptr(op), uintptr(fd), uintptr(unsafe.Pointer(event)), 0, 0)\n\tif err == syscall.Errno(0) {\n\t\terr = nil\n\t}\n\treturn err\n}\n\n// EpollWait implements epoll_wait.\nfunc EpollWait(epfd int, events []epollevent, msec int) (n int, err error) {\n\tvar r0 uintptr\n\t_p0 := unsafe.Pointer(&events[0])\n\tif msec == 0 {\n\t\tr0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), 0, 0, 0)\n\t} else {\n\t\tr0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0)\n\t}\n\tif err == syscall.Errno(0) {\n\t\terr = nil\n\t}\n\treturn int(r0), err\n}\n"
  },
  {
    "path": "sys_exec.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n// +build !windows\n\npackage netpoll\n\nimport (\n\t\"math\"\n\t\"os\"\n\t\"syscall\"\n\t\"unsafe\"\n)\n\n// GetSysFdPairs creates and returns the fds of a pair of sockets.\nfunc GetSysFdPairs() (r, w int) {\n\tfds, _ := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_STREAM, 0)\n\treturn fds[0], fds[1]\n}\n\n// setTCPNoDelay set the TCP_NODELAY flag on socket\nfunc setTCPNoDelay(fd int, b bool) (err error) {\n\treturn syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_NODELAY, boolint(b))\n}\n\n// Wrapper around the socket system call that marks the returned file\n// descriptor as nonblocking and close-on-exec.\nfunc sysSocket(family, sotype, proto int) (int, error) {\n\t// See ../syscall/exec_unix.go for description of ForkLock.\n\tsyscall.ForkLock.RLock()\n\ts, err := syscall.Socket(family, sotype, proto)\n\tif err == nil {\n\t\tsyscall.CloseOnExec(s)\n\t}\n\tsyscall.ForkLock.RUnlock()\n\tif err != nil {\n\t\treturn -1, os.NewSyscallError(\"socket\", err)\n\t}\n\tif err = syscall.SetNonblock(s, true); err != nil {\n\t\tsyscall.Close(s)\n\t\treturn -1, os.NewSyscallError(\"setnonblock\", err)\n\t}\n\treturn s, nil\n}\n\nconst barriercap = 32\n\ntype barrier struct {\n\tbs  [][]byte\n\tivs []syscall.Iovec\n}\n\n// writev wraps the writev system call.\nfunc writev(fd int, bs [][]byte, ivs []syscall.Iovec) (n int, err error) {\n\tiovLen := iovecs(bs, ivs)\n\tif iovLen == 0 {\n\t\treturn 0, nil\n\t}\n\t// syscall\n\tr, _, e := syscall.RawSyscall(syscall.SYS_WRITEV, uintptr(fd), uintptr(unsafe.Pointer(&ivs[0])), uintptr(iovLen))\n\tresetIovecs(bs, ivs[:iovLen])\n\tif e != 0 {\n\t\treturn int(r), e\n\t}\n\treturn int(r), nil\n}\n\n// readv wraps the readv system call.\n// return 0, nil means EOF.\nfunc readv(fd int, bs [][]byte, ivs []syscall.Iovec) (n int, err error) {\n\tiovLen := iovecs(bs, ivs)\n\tif iovLen == 0 {\n\t\treturn 0, nil\n\t}\n\t// syscall\n\tr, _, e := syscall.RawSyscall(syscall.SYS_READV, uintptr(fd), uintptr(unsafe.Pointer(&ivs[0])), uintptr(iovLen))\n\tresetIovecs(bs, ivs[:iovLen])\n\tif e != 0 {\n\t\treturn int(r), e\n\t}\n\treturn int(r), nil\n}\n\n// TODO: read from sysconf(_SC_IOV_MAX)? The Linux default is\n//\n//\t1024 and this seems conservative enough for now. Darwin's\n//\tUIO_MAXIOV also seems to be 1024.\n//\n// iovecs limit length to 2GB(2^31)\nfunc iovecs(bs [][]byte, ivs []syscall.Iovec) (iovLen int) {\n\ttotalLen := 0\n\tfor i := 0; i < len(bs); i++ {\n\t\tchunk := bs[i]\n\t\tl := len(chunk)\n\t\tif l == 0 {\n\t\t\tcontinue\n\t\t}\n\t\tivs[iovLen].Base = &chunk[0]\n\t\ttotalLen += l\n\t\tif totalLen < math.MaxInt32 {\n\t\t\tivs[iovLen].SetLen(l)\n\t\t\tiovLen++\n\t\t} else {\n\t\t\tnewLen := math.MaxInt32 - totalLen + l\n\t\t\tivs[iovLen].SetLen(newLen)\n\t\t\tiovLen++\n\t\t\treturn iovLen\n\t\t}\n\t}\n\n\treturn iovLen\n}\n\nfunc resetIovecs(bs [][]byte, ivs []syscall.Iovec) {\n\tfor i := 0; i < len(bs); i++ {\n\t\tbs[i] = nil\n\t}\n\tfor i := 0; i < len(ivs); i++ {\n\t\tivs[i].Base = nil\n\t}\n}\n\n// Boolean to int.\nfunc boolint(b bool) int {\n\tif b {\n\t\treturn 1\n\t}\n\treturn 0\n}\n"
  },
  {
    "path": "sys_exec_test.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build !windows\n// +build !windows\n\npackage netpoll\n\nimport (\n\t\"math\"\n\t\"syscall\"\n\t\"testing\"\n)\n\nfunc TestIovecs(t *testing.T) {\n\tvar got int\n\tvar bs [][]byte\n\tivs := make([]syscall.Iovec, 4)\n\n\t// case 1\n\tbs = [][]byte{\n\t\tmake([]byte, 10),\n\t\tmake([]byte, 20),\n\t\tmake([]byte, 30),\n\t\tmake([]byte, 40),\n\t}\n\tgot = iovecs(bs, ivs)\n\tEqual(t, got, 4)\n\tEqual(t, int(ivs[0].Len), 10)\n\tEqual(t, int(ivs[1].Len), 20)\n\tEqual(t, int(ivs[2].Len), 30)\n\tEqual(t, int(ivs[3].Len), 40)\n\n\t// case 2\n\tresetIovecs(bs, ivs)\n\tbs = [][]byte{\n\t\tmake([]byte, math.MaxInt32+100),\n\t\tmake([]byte, 20),\n\t\tmake([]byte, 30),\n\t\tmake([]byte, 40),\n\t}\n\tgot = iovecs(bs, ivs)\n\tEqual(t, got, 1)\n\tEqual(t, int(ivs[0].Len), math.MaxInt32)\n\tAssert(t, ivs[1].Base == nil)\n\tAssert(t, ivs[2].Base == nil)\n\tAssert(t, ivs[3].Base == nil)\n\n\t// case 3\n\tresetIovecs(bs, ivs)\n\tbs = [][]byte{\n\t\tmake([]byte, 10),\n\t\tmake([]byte, 20),\n\t\tmake([]byte, math.MaxInt32+100),\n\t\tmake([]byte, 40),\n\t}\n\tgot = iovecs(bs, ivs)\n\tEqual(t, got, 3)\n\tEqual(t, int(ivs[0].Len), 10)\n\tEqual(t, int(ivs[1].Len), 20)\n\tEqual(t, int(ivs[2].Len), math.MaxInt32-30)\n\tAssert(t, ivs[3].Base == nil)\n}\n\nfunc TestWritev(t *testing.T) {\n\tr, w := GetSysFdPairs()\n\tbarrier := barrier{}\n\tbarrier.bs = [][]byte{\n\t\t[]byte(\"\"),            // len=0\n\t\t[]byte(\"first line\"),  // len=10\n\t\t[]byte(\"second line\"), // len=11\n\t\t[]byte(\"third line\"),  // len=10\n\t}\n\tbarrier.ivs = make([]syscall.Iovec, len(barrier.bs))\n\twn, err := writev(w, barrier.bs, barrier.ivs)\n\tMustNil(t, err)\n\tEqual(t, wn, 31)\n\tp := make([]byte, 50)\n\trn, err := syscall.Read(r, p)\n\tMustNil(t, err)\n\tEqual(t, rn, 31)\n\tt.Logf(\"READ %s\", p[:rn])\n}\n\nfunc TestReadv(t *testing.T) {\n\tr, w := GetSysFdPairs()\n\tvs := [][]byte{\n\t\t[]byte(\"first line\"),  // len=10\n\t\t[]byte(\"second line\"), // len=11\n\t\t[]byte(\"third line\"),  // len=10\n\t}\n\tw1, _ := syscall.Write(w, vs[0])\n\tw2, _ := syscall.Write(w, vs[1])\n\tw3, _ := syscall.Write(w, vs[2])\n\tEqual(t, w1+w2+w3, 31)\n\n\tbarrier := barrier{\n\t\tbs: make([][]byte, 4),\n\t}\n\tres := [][]byte{\n\t\tmake([]byte, 0),\n\t\tmake([]byte, 10),\n\t\tmake([]byte, 11),\n\t\tmake([]byte, 10),\n\t}\n\tcopy(barrier.bs, res)\n\tbarrier.ivs = make([]syscall.Iovec, len(barrier.bs))\n\trn, err := readv(r, barrier.bs, barrier.ivs)\n\tMustNil(t, err)\n\tEqual(t, rn, 31)\n\tfor i, v := range res {\n\t\tt.Logf(\"READ [%d] %s\", i, v)\n\t}\n}\n\nfunc TestSendmsg(t *testing.T) {\n\tr, w := GetSysFdPairs()\n\tbarrier := barrier{}\n\tbarrier.bs = [][]byte{\n\t\t[]byte(\"\"),            // len=0\n\t\t[]byte(\"first line\"),  // len=10\n\t\t[]byte(\"second line\"), // len=11\n\t\t[]byte(\"third line\"),  // len=10\n\t}\n\tbarrier.ivs = make([]syscall.Iovec, len(barrier.bs))\n\twn, err := sendmsg(w, barrier.bs, barrier.ivs, false)\n\tMustNil(t, err)\n\tEqual(t, wn, 31)\n\tp := make([]byte, 50)\n\trn, err := syscall.Read(r, p)\n\tMustNil(t, err)\n\tEqual(t, rn, 31)\n\tt.Logf(\"READ %s\", p[:rn])\n}\n\nfunc BenchmarkWrite(b *testing.B) {\n\tb.StopTimer()\n\tr, w := GetSysFdPairs()\n\tmessage := \"hello, world!\"\n\tsize := 5\n\n\tgo func() {\n\t\tbuffer := make([]byte, 13)\n\t\tfor {\n\t\t\tsyscall.Read(r, buffer)\n\t\t}\n\t}()\n\n\t// benchmark\n\tb.ReportAllocs()\n\tb.StartTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\twmsg := make([]byte, len(message)*5)\n\t\tvar n int\n\t\tfor j := 0; j < size; j++ {\n\t\t\tn += copy(wmsg[n:], message)\n\t\t}\n\t\tsyscall.Write(w, wmsg)\n\t}\n}\n\nfunc BenchmarkWritev(b *testing.B) {\n\tb.StopTimer()\n\tr, w := GetSysFdPairs()\n\tmessage := \"hello, world!\"\n\tsize := 5\n\tbarrier := barrier{}\n\tbarrier.bs = make([][]byte, size)\n\tbarrier.ivs = make([]syscall.Iovec, len(barrier.bs))\n\tfor i := range barrier.bs {\n\t\tbarrier.bs[i] = make([]byte, len(message))\n\t}\n\n\tgo func() {\n\t\tbuffer := make([]byte, 13)\n\t\tfor {\n\t\t\tsyscall.Read(r, buffer)\n\t\t}\n\t}()\n\n\t// benchmark\n\tb.ReportAllocs()\n\tb.StartTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\twritev(w, barrier.bs, barrier.ivs)\n\t}\n}\n\nfunc BenchmarkSendmsg(b *testing.B) {\n\tb.StopTimer()\n\tr, w := GetSysFdPairs()\n\tmessage := \"hello, world!\"\n\tsize := 5\n\tbarrier := barrier{}\n\tbarrier.bs = make([][]byte, size)\n\tbarrier.ivs = make([]syscall.Iovec, len(barrier.bs))\n\tfor i := range barrier.bs {\n\t\tbarrier.bs[i] = make([]byte, len(message))\n\t}\n\n\tgo func() {\n\t\tbuffer := make([]byte, 13)\n\t\tfor {\n\t\t\tsyscall.Read(r, buffer)\n\t\t}\n\t}()\n\n\t// benchmark\n\tb.ReportAllocs()\n\tb.StartTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tsendmsg(w, barrier.bs, barrier.ivs, false)\n\t}\n}\n\nfunc BenchmarkRead(b *testing.B) {\n\tb.StopTimer()\n\tr, w := GetSysFdPairs()\n\tmessage := \"hello, world!\"\n\tsize := 5\n\twmsg := make([]byte, size*len(message))\n\tvar n int\n\tfor j := 0; j < size; j++ {\n\t\tn += copy(wmsg[n:], message)\n\t}\n\n\tgo func() {\n\t\tfor {\n\t\t\tsyscall.Write(w, wmsg)\n\t\t}\n\t}()\n\n\t// benchmark\n\tb.ReportAllocs()\n\tb.StartTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tbuffer := make([]byte, size*len(message))\n\t\tsyscall.Read(r, buffer)\n\t}\n}\n\nfunc BenchmarkReadv(b *testing.B) {\n\tb.StopTimer()\n\tr, w := GetSysFdPairs()\n\tmessage := \"hello, world!\"\n\tsize := 5\n\tbarrier := barrier{}\n\tbarrier.bs = make([][]byte, size)\n\tbarrier.ivs = make([]syscall.Iovec, len(barrier.bs))\n\tfor i := range barrier.bs {\n\t\tbarrier.bs[i] = make([]byte, len(message))\n\t}\n\n\tgo func() {\n\t\tfor {\n\t\t\twriteAll(w, []byte(message))\n\t\t}\n\t}()\n\n\t// benchmark\n\tb.ReportAllocs()\n\tb.StartTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\treadv(r, barrier.bs, barrier.ivs)\n\t}\n}\n"
  },
  {
    "path": "sys_keepalive_darwin.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage netpoll\n\nimport \"syscall\"\n\n// SetKeepAlive sets the keepalive for the connection\nfunc SetKeepAlive(fd, secs int) error {\n\tif err := syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_KEEPALIVE, 1); err != nil {\n\t\treturn err\n\t}\n\tswitch err := syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, 0x101, secs); err {\n\tcase nil, syscall.ENOPROTOOPT: // OS X 10.7 and earlier don't support this option\n\tdefault:\n\t\treturn err\n\t}\n\treturn syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_KEEPALIVE, secs)\n}\n"
  },
  {
    "path": "sys_keepalive_openbsd.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage netpoll\n\n// SetKeepAlive sets the keepalive for the connection\nfunc SetKeepAlive(fd, secs int) error {\n\t// OpenBSD has no user-settable per-socket TCP keepalive options.\n\treturn nil\n}\n"
  },
  {
    "path": "sys_keepalive_unix.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build netbsd || freebsd || dragonfly || linux\n// +build netbsd freebsd dragonfly linux\n\npackage netpoll\n\nimport \"syscall\"\n\n// just support ipv4\nfunc SetKeepAlive(fd, secs int) error {\n\t// open keep-alive\n\tif err := syscall.SetsockoptInt(fd, syscall.SOL_SOCKET, syscall.SO_KEEPALIVE, 1); err != nil {\n\t\treturn err\n\t}\n\t// tcp_keepalive_intvl\n\tif err := syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_KEEPINTVL, secs); err != nil {\n\t\treturn err\n\t}\n\t// tcp_keepalive_probes\n\t// if err := syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_KEEPCNT, 1); err != nil {\n\t// \treturn err\n\t// }\n\t// tcp_keepalive_time\n\treturn syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_KEEPIDLE, secs)\n}\n"
  },
  {
    "path": "sys_sendmsg_bsd.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\n//go:build darwin || dragonfly || freebsd || netbsd || openbsd\n// +build darwin dragonfly freebsd netbsd openbsd\n\npackage netpoll\n\nimport (\n\t\"syscall\"\n\t\"unsafe\"\n)\n\n// sendmsg wraps the sendmsg system call.\n// Must len(iovs) >= len(vs)\nfunc sendmsg(fd int, bs [][]byte, ivs []syscall.Iovec, zerocopy bool) (n int, err error) {\n\tiovLen := iovecs(bs, ivs)\n\tif iovLen == 0 {\n\t\treturn 0, nil\n\t}\n\tmsghdr := syscall.Msghdr{\n\t\tIov:    &ivs[0],\n\t\tIovlen: int32(iovLen),\n\t}\n\t// flags = syscall.MSG_DONTWAIT\n\tr, _, e := syscall.RawSyscall(syscall.SYS_SENDMSG, uintptr(fd), uintptr(unsafe.Pointer(&msghdr)), uintptr(0))\n\tresetIovecs(bs, ivs[:iovLen])\n\tif e != 0 {\n\t\treturn int(r), e\n\t}\n\treturn int(r), nil\n}\n"
  },
  {
    "path": "sys_sendmsg_linux.go",
    "content": "// Copyright 2022 CloudWeGo Authors\n//\n// Licensed under the Apache License, Version 2.0 (the \"License\");\n// you may not use this file except in compliance with the License.\n// You may obtain a copy of the License at\n//\n//    http://www.apache.org/licenses/LICENSE-2.0\n//\n// Unless required by applicable law or agreed to in writing, software\n// distributed under the License is distributed on an \"AS IS\" BASIS,\n// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n// See the License for the specific language governing permissions and\n// limitations under the License.\n\npackage netpoll\n\nimport (\n\t\"syscall\"\n\t\"unsafe\"\n)\n\n//func init() {\n//\terr := syscall.Setrlimit(8, &syscall.Rlimit{\n//\t\tCur: 0xffffffff,\n//\t\tMax: 0xffffffff,\n//\t})\n//\tif err != nil {\n//\t\tpanic(err)\n//\t}\n//}\n\n// sendmsg wraps the sendmsg system call.\n// Must len(iovs) >= len(vs)\nfunc sendmsg(fd int, bs [][]byte, ivs []syscall.Iovec, zerocopy bool) (n int, err error) {\n\tiovLen := iovecs(bs, ivs)\n\tif iovLen == 0 {\n\t\treturn 0, nil\n\t}\n\tmsghdr := syscall.Msghdr{\n\t\tIov:    &ivs[0],\n\t\tIovlen: uint64(iovLen),\n\t}\n\tr, _, e := syscall.RawSyscall(syscall.SYS_SENDMSG, uintptr(fd), uintptr(unsafe.Pointer(&msghdr)), 0)\n\tresetIovecs(bs, ivs[:iovLen])\n\tif e != 0 {\n\t\treturn int(r), e\n\t}\n\treturn int(r), nil\n}\n"
  },
  {
    "path": "sys_sockopt_bsd.go",
    "content": "// Copyright 2009 The Go Authors. All rights reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n//\n// This file may have been modified by CloudWeGo authors. (“CloudWeGo Modifications”).\n// All CloudWeGo Modifications are Copyright 2022 CloudWeGo authors.\n\n//go:build darwin || dragonfly || freebsd || netbsd || openbsd\n// +build darwin dragonfly freebsd netbsd openbsd\n\npackage netpoll\n\nimport (\n\t\"os\"\n\t\"runtime\"\n\t\"syscall\"\n)\n\nfunc setDefaultSockopts(s, family, sotype int, ipv6only bool) error {\n\tif runtime.GOOS == \"dragonfly\" && sotype != syscall.SOCK_RAW {\n\t\t// On DragonFly BSD, we adjust the ephemeral port\n\t\t// range because unlike other BSD systems its default\n\t\t// port range doesn't conform to IANA recommendation\n\t\t// as described in RFC 6056 and is pretty narrow.\n\t\tswitch family {\n\t\tcase syscall.AF_INET:\n\t\t\tsyscall.SetsockoptInt(s, syscall.IPPROTO_IP, syscall.IP_PORTRANGE, syscall.IP_PORTRANGE_HIGH)\n\t\tcase syscall.AF_INET6:\n\t\t\tsyscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_PORTRANGE, syscall.IPV6_PORTRANGE_HIGH)\n\t\t}\n\t}\n\t// Allow broadcast.\n\treturn os.NewSyscallError(\"setsockopt\", syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1))\n}\n"
  },
  {
    "path": "sys_sockopt_linux.go",
    "content": "// Copyright 2009 The Go Authors. All rights reserved.\n// Use of this source code is governed by a BSD-style\n// license that can be found in the LICENSE file.\n//\n// This file may have been modified by CloudWeGo authors. (“CloudWeGo Modifications”).\n// All CloudWeGo Modifications are Copyright 2022 CloudWeGo authors.\n\npackage netpoll\n\nimport (\n\t\"os\"\n\t\"syscall\"\n)\n\nfunc setDefaultSockopts(s, family, sotype int, ipv6only bool) error {\n\tif family == syscall.AF_INET6 && sotype != syscall.SOCK_RAW {\n\t\t// Allow both IP versions even if the OS default\n\t\t// is otherwise. Note that some operating systems\n\t\t// never admit this option.\n\t\tsyscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, boolint(ipv6only))\n\t}\n\n\t// Allow broadcast.\n\treturn os.NewSyscallError(\"setsockopt\", syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1))\n}\n"
  },
  {
    "path": "test_conns.sh",
    "content": "#!/usr/bin/env bash\n\nip=\"$1\"\nport=\"$2\"\nconns=\"$3\"\ntimeout=\"$4\"\n\nfor i in $(seq 1 $conns);\ndo\n  nc -v -w $timeout $ip $port < /dev/null &\ndone\n\nwait\n"
  }
]