[ { "path": ".circleci/config.yml", "content": "version: 2\njobs:\n lint:\n docker:\n - image: golangci/golangci-lint:v1.45-alpine\n steps:\n - checkout\n - run: golangci-lint run\n\n kafka-270:\n working_directory: &working_directory /go/src/github.com/segmentio/kafka-go\n environment:\n KAFKA_VERSION: \"2.7.0\"\n\n # Need to skip nettest to avoid these kinds of errors:\n # --- FAIL: TestConn/nettest (17.56s)\n # --- FAIL: TestConn/nettest/PingPong (7.40s)\n # conntest.go:112: unexpected Read error: [7] Request Timed Out: the request exceeded the user-specified time limit in the request\n # conntest.go:118: mismatching value: got 77, want 78\n # conntest.go:118: mismatching value: got 78, want 79\n # ...\n #\n # TODO: Figure out why these are happening and fix them (they don't appear to be new).\n KAFKA_SKIP_NETTEST: \"1\"\n docker:\n - image: circleci/golang\n - image: bitnamilegacy/zookeeper:latest\n ports:\n - 2181:2181\n environment:\n ALLOW_ANONYMOUS_LOGIN: yes\n - image: bitnamilegacy/kafka:2.7.0\n ports:\n - 9092:9092\n - 9093:9093\n environment: &environment\n KAFKA_CFG_BROKER_ID: 1\n KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'\n KAFKA_CFG_ADVERTISED_HOST_NAME: 'localhost'\n KAFKA_CFG_ADVERTISED_PORT: '9092'\n KAFKA_CFG_ZOOKEEPER_CONNECT: localhost:2181\n KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'true'\n KAFKA_CFG_MESSAGE_MAX_BYTES: '200000000'\n KAFKA_CFG_LISTENERS: 'PLAINTEXT://:9092,SASL_PLAINTEXT://:9093'\n KAFKA_CFG_ADVERTISED_LISTENERS: 'PLAINTEXT://localhost:9092,SASL_PLAINTEXT://localhost:9093'\n KAFKA_CFG_SASL_ENABLED_MECHANISMS: 'PLAIN,SCRAM-SHA-256,SCRAM-SHA-512'\n KAFKA_CFG_AUTHORIZER_CLASS_NAME: 'kafka.security.auth.SimpleAclAuthorizer'\n KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true'\n KAFKA_OPTS: \"-Djava.security.auth.login.config=/opt/bitnami/kafka/config/kafka_jaas.conf\"\n ALLOW_PLAINTEXT_LISTENER: yes\n entrypoint: &entrypoint\n - \"/bin/bash\"\n - \"-c\"\n - echo -e 'KafkaServer {\\norg.apache.kafka.common.security.scram.ScramLoginModule required\\n username=\"adminscram\"\\n password=\"admin-secret\";\\n org.apache.kafka.common.security.plain.PlainLoginModule required\\n username=\"adminplain\"\\n password=\"admin-secret\"\\n user_adminplain=\"admin-secret\";\\n };' > /opt/bitnami/kafka/config/kafka_jaas.conf; /opt/bitnami/kafka/bin/kafka-configs.sh --zookeeper localhost:2181 --alter --add-config \"SCRAM-SHA-256=[password=admin-secret-256],SCRAM-SHA-512=[password=admin-secret-512]\" --entity-type users --entity-name adminscram; exec /entrypoint.sh /run.sh\n steps: &steps\n - checkout\n - restore_cache:\n key: kafka-go-mod-{{ checksum \"go.sum\" }}-1\n - run:\n name: Download dependencies\n command: go mod download\n - save_cache:\n key: kafka-go-mod-{{ checksum \"go.sum\" }}-1\n paths:\n - /go/pkg/mod\n - run:\n name: Wait for kafka\n command: ./scripts/wait-for-kafka.sh\n - run:\n name: Test kafka-go\n command: go test -race -cover ./...\n - run:\n name: Test kafka-go unsafe\n command: go test -tags=unsafe -race -cover ./...\n - run:\n name: Test kafka-go/sasl/aws_msk_iam\n working_directory: ./sasl/aws_msk_iam\n command: go test -race -cover ./...\n\n kafka-281:\n working_directory: *working_directory\n environment:\n KAFKA_VERSION: \"2.8.1\"\n\n # Need to skip nettest to avoid these kinds of errors:\n # --- FAIL: TestConn/nettest (17.56s)\n # --- FAIL: TestConn/nettest/PingPong (7.40s)\n # conntest.go:112: unexpected Read error: [7] Request Timed Out: the request exceeded the user-specified time limit in the request\n # conntest.go:118: mismatching value: got 77, want 78\n # conntest.go:118: mismatching value: got 78, want 79\n # ...\n #\n # TODO: Figure out why these are happening and fix them (they don't appear to be new).\n KAFKA_SKIP_NETTEST: \"1\"\n docker:\n - image: circleci/golang\n - image: bitnamilegacy/zookeeper:latest\n ports:\n - 2181:2181\n environment:\n ALLOW_ANONYMOUS_LOGIN: yes\n - image: bitnamilegacy/kafka:2.8.1\n ports:\n - 9092:9092\n - 9093:9093\n environment: *environment\n entrypoint: *entrypoint\n steps: *steps\n\n kafka-370:\n working_directory: *working_directory\n environment:\n KAFKA_VERSION: \"3.7.0\"\n\n # Need to skip nettest to avoid these kinds of errors:\n # --- FAIL: TestConn/nettest (17.56s)\n # --- FAIL: TestConn/nettest/PingPong (7.40s)\n # conntest.go:112: unexpected Read error: [7] Request Timed Out: the request exceeded the user-specified time limit in the request\n # conntest.go:118: mismatching value: got 77, want 78\n # conntest.go:118: mismatching value: got 78, want 79\n # ...\n #\n # TODO: Figure out why these are happening and fix them (they don't appear to be new).\n KAFKA_SKIP_NETTEST: \"1\"\n docker:\n - image: circleci/golang\n - image: bitnamilegacy/zookeeper:latest\n ports:\n - 2181:2181\n environment:\n ALLOW_ANONYMOUS_LOGIN: yes\n - image: bitnamilegacy/kafka:3.7.0\n ports:\n - 9092:9092\n - 9093:9093\n environment:\n <<: *environment\n KAFKA_CFG_AUTHORIZER_CLASS_NAME: 'kafka.security.authorizer.AclAuthorizer'\n entrypoint: *entrypoint\n steps: *steps\n\n # NOTE: this fails quite often due to Java heap errors from Kafka.\n # Once we figure out how to fix that, we can re-enable this.\n # https://github.com/segmentio/kafka-go/issues/1360#issuecomment-2858935900\n # kafka-400:\n # working_directory: *working_directory\n # environment:\n # KAFKA_VERSION: \"4.0.0\"\n\n # # Need to skip nettest to avoid these kinds of errors:\n # # --- FAIL: TestConn/nettest (17.56s)\n # # --- FAIL: TestConn/nettest/PingPong (7.40s)\n # # conntest.go:112: unexpected Read error: [7] Request Timed Out: the request exceeded the user-specified time limit in the request\n # # conntest.go:118: mismatching value: got 77, want 78\n # # conntest.go:118: mismatching value: got 78, want 79\n # # ...\n # #\n # # TODO: Figure out why these are happening and fix them (they don't appear to be new).\n # KAFKA_SKIP_NETTEST: \"1\"\n # docker:\n # - image: circleci/golang\n # - image: bitnamilegacy/kafka:4.0.0\n # ports:\n # - 9092:9092\n # - 9093:9093\n # environment:\n # KAFKA_CFG_NODE_ID: 1\n # KAFKA_CFG_BROKER_ID: 1\n # KAFKA_CFG_PROCESS_ROLES: broker,controller\n # KAFKA_CFG_ADVERTISED_HOST_NAME: 'localhost'\n # KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER\n # KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAIN:PLAINTEXT,SASL:SASL_PLAINTEXT\n # KAFKA_CFG_LISTENERS: CONTROLLER://:9094,PLAIN://:9092,SASL://:9093\n # KAFKA_CFG_ADVERTISED_LISTENERS: PLAIN://localhost:9092,SASL://localhost:9093\n # KAFKA_CFG_INTER_BROKER_LISTENER_NAME: PLAIN\n # KAFKA_CFG_SASL_ENABLED_MECHANISMS: 'PLAIN,SCRAM-SHA-256,SCRAM-SHA-512'\n # KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 1@localhost:9094\n # ALLOW_PLAINTEXT_LISTENER: yes\n # KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true'\n # KAFKA_OPTS: \"-Djava.security.auth.login.config=/opt/bitnami/kafka/config/kafka_jaas.conf\"\n # KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'true'\n # KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'\n # KAFKA_CFG_MESSAGE_MAX_BYTES: '200000000'\n # KAFKA_CFG_AUTHORIZER_CLASS_NAME: 'org.apache.kafka.metadata.authorizer.StandardAuthorizer'\n # KAFKA_CFG_SUPER_USERS: User:adminscram256;User:adminscram512;User:adminplain\n # KAFKA_CLIENT_USERS: adminscram256,adminscram512,adminplain\n # KAFKA_CLIENT_PASSWORDS: admin-secret-256,admin-secret-512,admin-secret\n # KAFKA_CLIENT_SASL_MECHANISMS: SCRAM-SHA-256,SCRAM-SHA-512,PLAIN\n # KAFKA_INTER_BROKER_USER: adminscram512\n # KAFKA_INTER_BROKER_PASSWORD: admin-secret-512\n # KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL: SCRAM-SHA-512\n # steps: *steps\n\nworkflows:\n version: 2\n run:\n jobs:\n - lint\n - kafka-270\n - kafka-281\n - kafka-370 \n #- kafka-400\n" }, { "path": ".gitattributes", "content": "fixtures/*.hex binary\n" }, { "path": ".github/ISSUE_TEMPLATE/bug_report.md", "content": "---\nname: Bug report\nabout: Create a report to help us improve\ntitle: ''\nlabels: bug\nassignees: ''\n\n---\n\n**Describe the bug**\n\n> A clear and concise description of what the bug is.\n\n**Kafka Version**\n\n> * What version(s) of Kafka are you testing against?\n> * What version of kafka-go are you using?\n\n**To Reproduce**\n\n> Resources to reproduce the behavior:\n\n```yaml\n---\n# docker-compose.yaml\n#\n# Adding a docker-compose file will help the maintainers setup the environment\n# to reproduce the issue.\n#\n# If one the docker-compose files available in the repository may be used,\n# mentioning it is also a useful alternative.\n...\n```\n\n```go\npackage main\n\nimport (\n \"github.com/segmentio/kafka-go\"\n)\n\nfunc main() {\n // Adding a fully reproducible example will help maintainers provide\n // assistance to debug the issues.\n ...\n}\n```\n\n**Expected Behavior**\n\n> A clear and concise description of what you expected to happen.\n\n**Observed Behavior**\n\n> A clear and concise description of the behavior you observed.\n\n```\nOften times, pasting the logging output from a kafka.Reader or kafka.Writer will\nprovide useful details to help maintainers investigate the issue and provide a\nfix. If possible, providing stack traces or CPU/memory profiles may also contain\nvaluable information to understand the conditions that triggered the issue.\n```\n\n**Additional Context**\n\n> Add any other context about the problem here.\n" }, { "path": ".github/ISSUE_TEMPLATE/feature_request.md", "content": "---\nname: Feature request\nabout: Suggest an idea for this project\ntitle: ''\nlabels: enhancement\nassignees: ''\n\n---\n\n**Describe the solution you would like**\n\n> A clear and concise description of what you want to happen.\n\n**Supporting documentation**\n\n> Please provides links to relevant Kafka protocol docs and/or KIPs.\n" }, { "path": ".gitignore", "content": "# Compiled Object files, Static and Dynamic libs (Shared Objects)\n*.o\n*.a\n*.so\n\n# Folders\n_obj\n_test\n\n# Architecture specific extensions/prefixes\n*.[568vq]\n[568vq].out\n\n*.cgo1.go\n*.cgo2.c\n_cgo_defun.c\n_cgo_gotypes.go\n_cgo_export.*\n\n_testmain.go\n\n*.exe\n*.test\n*.prof\n/kafkacli\n\n# Emacs\n*~\n\n# VIM\n*.swp\n\n# Goland\n.idea\n\n#IntelliJ\n*.iml\n\n# govendor\n/vendor/*/\n" }, { "path": ".golangci.yml", "content": "linters:\n enable:\n - bodyclose\n - errorlint\n - goconst\n - godot\n - gofmt\n - goimports\n - prealloc\n\n disable:\n # Temporarily disabling so it can be addressed in a dedicated PR.\n - errcheck\n - goerr113\n\nlinters-settings:\n goconst:\n ignore-tests: true\n" }, { "path": "CODE_OF_CONDUCT.md", "content": "# Contributor Covenant Code of Conduct\n\n## Our Pledge\n\nIn the interest of fostering an open and welcoming environment, we as\ncontributors and maintainers pledge to making participation in our project and\nour community a harassment-free experience for everyone, regardless of age, body\nsize, disability, ethnicity, sex characteristics, gender identity and expression,\nlevel of experience, education, socio-economic status, nationality, personal\nappearance, race, religion, or sexual identity and orientation.\n\n## Our Standards\n\nExamples of behavior that contributes to creating a positive environment\ninclude:\n\n- Using welcoming and inclusive language\n- Being respectful of differing viewpoints and experiences\n- Gracefully accepting constructive criticism\n- Focusing on what is best for the community\n- Showing empathy towards other community members\n\nExamples of unacceptable behavior by participants include:\n\n- The use of sexualized language or imagery and unwelcome sexual attention or\n advances\n- Trolling, insulting/derogatory comments, and personal or political attacks\n- Public or private harassment\n- Publishing others' private information, such as a physical or electronic\n address, without explicit permission\n- Other conduct which could reasonably be considered inappropriate in a\n professional setting\n\n## Our Responsibilities\n\nProject maintainers are responsible for clarifying the standards of acceptable\nbehavior and are expected to take appropriate and fair corrective action in\nresponse to any instances of unacceptable behavior.\n\nProject maintainers have the right and responsibility to remove, edit, or\nreject comments, commits, code, wiki edits, issues, and other contributions\nthat are not aligned to this Code of Conduct, or to ban temporarily or\npermanently any contributor for other behaviors that they deem inappropriate,\nthreatening, offensive, or harmful.\n\nProject maintainers are available at [#kafka-go](https://gophers.slack.com/archives/CG4H0N9PX) channel inside the [Gophers Slack](https://gophers.slack.com)\n\n## Scope\n\nThis Code of Conduct applies both within project spaces and in public spaces\nwhen an individual is representing the project or its community. Examples of\nrepresenting a project or community include using an official project e-mail\naddress, posting via an official social media account, or acting as an appointed\nrepresentative at an online or offline event. Representation of a project may be\nfurther defined and clarified by project maintainers.\n\n## Enforcement\n\nInstances of abusive, harassing, or otherwise unacceptable behavior may be\nreported by contacting the project team at open-source@twilio.com. All\ncomplaints will be reviewed and investigated and will result in a response that\nis deemed necessary and appropriate to the circumstances. The project team is\nobligated to maintain confidentiality with regard to the reporter of an incident.\nFurther details of specific enforcement policies may be posted separately.\n\nProject maintainers who do not follow or enforce the Code of Conduct in good\nfaith may face temporary or permanent repercussions as determined by other\nmembers of the project's leadership.\n\n## Attribution\n\nThis Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,\navailable at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html\n\n[homepage]: https://www.contributor-covenant.org\n" }, { "path": "CONTRIBUTING.md", "content": "# Contributing to kafka-go\n\nkafka-go is an open source project. We welcome contributions to kafka-go of any kind including documentation,\norganization, tutorials, bug reports, issues, feature requests, feature implementations, pull requests, etc.\n\n## Table of Contents\n\n* [Reporting Issues](#reporting-issues)\n* [Submitting Patches](#submitting-patches)\n * [Code Contribution Guidelines](#code-contribution-guidelines)\n * [Git Commit Message Guidelines](#git-commit-message-guidelines)\n * [Fetching the Source From GitHub](#fetching-the-sources-from-github)\n * [Building kafka-go with Your Changes](#building-kakfa-go-with-your-changes)\n\n## Reporting Issues\n\nIf you believe you have found a defect in kafka-go, use the GitHub issue tracker to report\nthe problem to the maintainers. \nWhen reporting the issue, please provide the version of kafka-go, what version(s) of Kafka \nare you testing against, and your operating system.\n\n - [kafka-go Issues segmentio/kafka-go](https://github.com/segmentio/kafka-go/issues)\n\n## Submitting Patches\n\nkafka-go project welcomes all contributors and contributions regardless of skill or experience levels. If you are\ninterested in helping with the project, we will help you with your contribution.\n\n### Code Contribution\n\nTo make contributions as seamless as possible, we ask the following:\n\n* Go ahead and fork the project and make your changes. We encourage pull requests to allow for review and discussion of code changes.\n* When you’re ready to create a pull request, be sure to:\n * Have test cases for the new code. If you have questions about how to do this, please ask in your pull request.\n * Run `go fmt`.\n * Squash your commits into a single commit. `git rebase -i`. It’s okay to force update your pull request with `git push -f`.\n * Follow the **Git Commit Message Guidelines** below.\n\n### Git Commit Message Guidelines\n\nThis [blog article](http://chris.beams.io/posts/git-commit/) is a good resource for learning how to write good commit messages,\nthe most important part being that each commit message should have a title/subject in imperative mood starting with a capital letter and no trailing period:\n*\"Return error on wrong use of the Reader\"*, **NOT** *\"returning some error.\"*\n\nAlso, if your commit references one or more GitHub issues, always end your commit message body with *See #1234* or *Fixes #1234*.\nReplace *1234* with the GitHub issue ID. The last example will close the issue when the commit is merged into *master*.\n\nPlease use a short and descriptive branch name, e.g. NOT \"patch-1\". It's very common but creates a naming conflict each\ntime when a submission is pulled for a review.\n\nAn example:\n\n```text\nAdd Code of Conduct and Code Contribution Guidelines\n\nAdd a full Code of Conduct and Code Contribution Guidelines document. \nProvide description on how best to retrieve code, fork, checkout, and commit changes.\n\nFixes #688\n```\n\n### Fetching the Sources From GitHub\n\nWe use Go Modules support built into Go 1.11 to build. The easiest way is to clone kafka-go into a directory outside of\n`GOPATH`, as in the following example:\n\n```bash\nmkdir $HOME/src\ncd $HOME/src\ngit clone https://github.com/segmentio/kafka-go.git\ncd kafka-go\ngo build ./...\n```\n\nTo make changes to kafka-go's source:\n\n1. Create a new branch for your changes (the branch name is arbitrary):\n\n ```bash\n git checkout -b branch1234\n ```\n\n1. After making your changes, commit them to your new branch:\n\n ```bash\n git commit -a -v\n ```\n\n1. Fork kafka-go in GitHub\n\n1. Add your fork as a new remote (the remote name, \"upstream\" in this example, is arbitrary):\n\n ```bash\n git remote add upstream git@github.com:USERNAME/kafka-go.git\n ```\n\n1. Push your branch (the remote name, \"upstream\" in this example, is arbitrary):\n\n ```bash\n git push upstream \n ```\n\n1. You are now ready to submit a PR based upon the new branch in your forked repository.\n\n### Using the forked library\n\nTo replace the original version of kafka-go library with a forked version is accomplished this way.\n\n1. Make sure your application already has a go.mod entry depending on kafka-go\n\n ```bash\n module github.com/myusername/myapp\n\n require (\n ...\n github.com/segmentio/kafka-go v1.2.3\n ...\n )\n ```\n\n1. Add the following entry to the beginning of the modules file.\n\n ```bash\n module github.com/myusername/myapp\n\n replace github.com/segmentio/kafka-go v1.2.3 => ../local/directory\n\n require (\n ...\n github.com/segmentio/kafka-go v1.2.3\n ...\n )\n ```\n1. Depending on if you are using `vendor`ing or not you might need to run the following command to pull in the new bits.\n\n ```bash\n > go mod vendor\n ```\n" }, { "path": "LICENSE", "content": "MIT License\n\nCopyright (c) 2017 Segment\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "Makefile", "content": "test:\n\tKAFKA_SKIP_NETTEST=1 \\\n\tKAFKA_VERSION=2.3.1 \\\n\tgo test -race -cover ./...\n\ndocker:\n\tdocker compose up -d\n" }, { "path": "README.md", "content": "# kafka-go [![CircleCI](https://circleci.com/gh/segmentio/kafka-go.svg?style=shield)](https://circleci.com/gh/segmentio/kafka-go) [![Go Report Card](https://goreportcard.com/badge/github.com/segmentio/kafka-go)](https://goreportcard.com/report/github.com/segmentio/kafka-go) [![GoDoc](https://godoc.org/github.com/segmentio/kafka-go?status.svg)](https://godoc.org/github.com/segmentio/kafka-go)\n\n## Motivations\n\nWe rely on both Go and Kafka a lot at Segment. Unfortunately, the state of the Go\nclient libraries for Kafka at the time of this writing was not ideal. The available\noptions were:\n\n- [sarama](https://github.com/Shopify/sarama), which is by far the most popular\nbut is quite difficult to work with. It is poorly documented, the API exposes\nlow level concepts of the Kafka protocol, and it doesn't support recent Go features\nlike [contexts](https://golang.org/pkg/context/). It also passes all values as\npointers which causes large numbers of dynamic memory allocations, more frequent\ngarbage collections, and higher memory usage.\n\n- [confluent-kafka-go](https://github.com/confluentinc/confluent-kafka-go) is a\ncgo based wrapper around [librdkafka](https://github.com/edenhill/librdkafka),\nwhich means it introduces a dependency to a C library on all Go code that uses\nthe package. It has much better documentation than sarama but still lacks support\nfor Go contexts.\n\n- [goka](https://github.com/lovoo/goka) is a more recent Kafka client for Go\nwhich focuses on a specific usage pattern. It provides abstractions for using Kafka\nas a message passing bus between services rather than an ordered log of events, but\nthis is not the typical use case of Kafka for us at Segment. The package also\ndepends on sarama for all interactions with Kafka.\n\nThis is where `kafka-go` comes into play. It provides both low and high level\nAPIs for interacting with Kafka, mirroring concepts and implementing interfaces of\nthe Go standard library to make it easy to use and integrate with existing\nsoftware.\n\n#### Note:\n\nIn order to better align with our newly adopted Code of Conduct, the kafka-go\nproject has renamed our default branch to `main`. For the full details of our\nCode Of Conduct see [this](./CODE_OF_CONDUCT.md) document.\n\n## Kafka versions\n\n`kafka-go` is currently tested with Kafka versions 0.10.1.0 to 2.7.1.\nWhile it should also be compatible with later versions, newer features available\nin the Kafka API may not yet be implemented in the client.\n\n## Go versions\n\n`kafka-go` requires Go version 1.15 or later.\n\n## Connection [![GoDoc](https://godoc.org/github.com/segmentio/kafka-go?status.svg)](https://godoc.org/github.com/segmentio/kafka-go#Conn)\n\nThe `Conn` type is the core of the `kafka-go` package. It wraps around a raw\nnetwork connection to expose a low-level API to a Kafka server.\n\nHere are some examples showing typical use of a connection object:\n```go\n// to produce messages\ntopic := \"my-topic\"\npartition := 0\n\nconn, err := kafka.DialLeader(context.Background(), \"tcp\", \"localhost:9092\", topic, partition)\nif err != nil {\n log.Fatal(\"failed to dial leader:\", err)\n}\n\nconn.SetWriteDeadline(time.Now().Add(10*time.Second))\n_, err = conn.WriteMessages(\n kafka.Message{Value: []byte(\"one!\")},\n kafka.Message{Value: []byte(\"two!\")},\n kafka.Message{Value: []byte(\"three!\")},\n)\nif err != nil {\n log.Fatal(\"failed to write messages:\", err)\n}\n\nif err := conn.Close(); err != nil {\n log.Fatal(\"failed to close writer:\", err)\n}\n```\n```go\n// to consume messages\ntopic := \"my-topic\"\npartition := 0\n\nconn, err := kafka.DialLeader(context.Background(), \"tcp\", \"localhost:9092\", topic, partition)\nif err != nil {\n log.Fatal(\"failed to dial leader:\", err)\n}\n\nconn.SetReadDeadline(time.Now().Add(10*time.Second))\nbatch := conn.ReadBatch(10e3, 1e6) // fetch 10KB min, 1MB max\n\nb := make([]byte, 10e3) // 10KB max per message\nfor {\n n, err := batch.Read(b)\n if err != nil {\n break\n }\n fmt.Println(string(b[:n]))\n}\n\nif err := batch.Close(); err != nil {\n log.Fatal(\"failed to close batch:\", err)\n}\n\nif err := conn.Close(); err != nil {\n log.Fatal(\"failed to close connection:\", err)\n}\n```\n\n### To Create Topics\nBy default kafka has the `auto.create.topics.enable='true'` (`KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE='true'` in the bitnami/kafka kafka docker image). If this value is set to `'true'` then topics will be created as a side effect of `kafka.DialLeader` like so:\n```go\n// to create topics when auto.create.topics.enable='true'\nconn, err := kafka.DialLeader(context.Background(), \"tcp\", \"localhost:9092\", \"my-topic\", 0)\nif err != nil {\n panic(err.Error())\n}\n```\n\nIf `auto.create.topics.enable='false'` then you will need to create topics explicitly like so:\n```go\n// to create topics when auto.create.topics.enable='false'\ntopic := \"my-topic\"\n\nconn, err := kafka.Dial(\"tcp\", \"localhost:9092\")\nif err != nil {\n panic(err.Error())\n}\ndefer conn.Close()\n\ncontroller, err := conn.Controller()\nif err != nil {\n panic(err.Error())\n}\nvar controllerConn *kafka.Conn\ncontrollerConn, err = kafka.Dial(\"tcp\", net.JoinHostPort(controller.Host, strconv.Itoa(controller.Port)))\nif err != nil {\n panic(err.Error())\n}\ndefer controllerConn.Close()\n\n\ntopicConfigs := []kafka.TopicConfig{\n {\n Topic: topic,\n NumPartitions: 1,\n ReplicationFactor: 1,\n },\n}\n\nerr = controllerConn.CreateTopics(topicConfigs...)\nif err != nil {\n panic(err.Error())\n}\n```\n\n### To Connect To Leader Via a Non-leader Connection\n```go\n// to connect to the kafka leader via an existing non-leader connection rather than using DialLeader\nconn, err := kafka.Dial(\"tcp\", \"localhost:9092\")\nif err != nil {\n panic(err.Error())\n}\ndefer conn.Close()\ncontroller, err := conn.Controller()\nif err != nil {\n panic(err.Error())\n}\nvar connLeader *kafka.Conn\nconnLeader, err = kafka.Dial(\"tcp\", net.JoinHostPort(controller.Host, strconv.Itoa(controller.Port)))\nif err != nil {\n panic(err.Error())\n}\ndefer connLeader.Close()\n```\n\n### To list topics\n```go\nconn, err := kafka.Dial(\"tcp\", \"localhost:9092\")\nif err != nil {\n panic(err.Error())\n}\ndefer conn.Close()\n\npartitions, err := conn.ReadPartitions()\nif err != nil {\n panic(err.Error())\n}\n\nm := map[string]struct{}{}\n\nfor _, p := range partitions {\n m[p.Topic] = struct{}{}\n}\nfor k := range m {\n fmt.Println(k)\n}\n```\n\n\nBecause it is low level, the `Conn` type turns out to be a great building block\nfor higher level abstractions, like the `Reader` for example.\n\n## Reader [![GoDoc](https://godoc.org/github.com/segmentio/kafka-go?status.svg)](https://godoc.org/github.com/segmentio/kafka-go#Reader)\n\nA `Reader` is another concept exposed by the `kafka-go` package, which intends\nto make it simpler to implement the typical use case of consuming from a single\ntopic-partition pair.\nA `Reader` also automatically handles reconnections and offset management, and\nexposes an API that supports asynchronous cancellations and timeouts using Go\ncontexts.\n\nNote that it is important to call `Close()` on a `Reader` when a process exits.\nThe kafka server needs a graceful disconnect to stop it from continuing to\nattempt to send messages to the connected clients. The given example will not\ncall `Close()` if the process is terminated with SIGINT (ctrl-c at the shell) or\nSIGTERM (as docker stop or a kubernetes restart does). This can result in a\ndelay when a new reader on the same topic connects (e.g. new process started\nor new container running). Use a `signal.Notify` handler to close the reader on\nprocess shutdown.\n\n```go\n// make a new reader that consumes from topic-A, partition 0, at offset 42\nr := kafka.NewReader(kafka.ReaderConfig{\n Brokers: []string{\"localhost:9092\",\"localhost:9093\", \"localhost:9094\"},\n Topic: \"topic-A\",\n Partition: 0,\n MaxBytes: 10e6, // 10MB\n})\nr.SetOffset(42)\n\nfor {\n m, err := r.ReadMessage(context.Background())\n if err != nil {\n break\n }\n fmt.Printf(\"message at offset %d: %s = %s\\n\", m.Offset, string(m.Key), string(m.Value))\n}\n\nif err := r.Close(); err != nil {\n log.Fatal(\"failed to close reader:\", err)\n}\n```\n\n### Consumer Groups\n\n```kafka-go``` also supports Kafka consumer groups including broker managed offsets.\nTo enable consumer groups, simply specify the GroupID in the ReaderConfig.\n\nReadMessage automatically commits offsets when using consumer groups.\n\n```go\n// make a new reader that consumes from topic-A\nr := kafka.NewReader(kafka.ReaderConfig{\n Brokers: []string{\"localhost:9092\", \"localhost:9093\", \"localhost:9094\"},\n GroupID: \"consumer-group-id\",\n Topic: \"topic-A\",\n MaxBytes: 10e6, // 10MB\n})\n\nfor {\n m, err := r.ReadMessage(context.Background())\n if err != nil {\n break\n }\n fmt.Printf(\"message at topic/partition/offset %v/%v/%v: %s = %s\\n\", m.Topic, m.Partition, m.Offset, string(m.Key), string(m.Value))\n}\n\nif err := r.Close(); err != nil {\n log.Fatal(\"failed to close reader:\", err)\n}\n```\n\nThere are a number of limitations when using consumer groups:\n\n* ```(*Reader).SetOffset``` will return an error when GroupID is set\n* ```(*Reader).Offset``` will always return ```-1``` when GroupID is set\n* ```(*Reader).Lag``` will always return ```-1``` when GroupID is set\n* ```(*Reader).ReadLag``` will return an error when GroupID is set\n* ```(*Reader).Stats``` will return a partition of ```-1``` when GroupID is set\n\n### Explicit Commits\n\n```kafka-go``` also supports explicit commits. Instead of calling ```ReadMessage```,\ncall ```FetchMessage``` followed by ```CommitMessages```.\n\n```go\nctx := context.Background()\nfor {\n m, err := r.FetchMessage(ctx)\n if err != nil {\n break\n }\n fmt.Printf(\"message at topic/partition/offset %v/%v/%v: %s = %s\\n\", m.Topic, m.Partition, m.Offset, string(m.Key), string(m.Value))\n if err := r.CommitMessages(ctx, m); err != nil {\n log.Fatal(\"failed to commit messages:\", err)\n }\n}\n```\n\nWhen committing messages in consumer groups, the message with the highest offset\nfor a given topic/partition determines the value of the committed offset for\nthat partition. For example, if messages at offset 1, 2, and 3 of a single\npartition were retrieved by call to `FetchMessage`, calling `CommitMessages`\nwith message offset 3 will also result in committing the messages at offsets 1\nand 2 for that partition.\n\n### Managing Commits\n\nBy default, CommitMessages will synchronously commit offsets to Kafka. For\nimproved performance, you can instead periodically commit offsets to Kafka\nby setting CommitInterval on the ReaderConfig.\n\n\n```go\n// make a new reader that consumes from topic-A\nr := kafka.NewReader(kafka.ReaderConfig{\n Brokers: []string{\"localhost:9092\", \"localhost:9093\", \"localhost:9094\"},\n GroupID: \"consumer-group-id\",\n Topic: \"topic-A\",\n MaxBytes: 10e6, // 10MB\n CommitInterval: time.Second, // flushes commits to Kafka every second\n})\n```\n\n## Writer [![GoDoc](https://godoc.org/github.com/segmentio/kafka-go?status.svg)](https://godoc.org/github.com/segmentio/kafka-go#Writer)\n\nTo produce messages to Kafka, a program may use the low-level `Conn` API, but\nthe package also provides a higher level `Writer` type which is more appropriate\nto use in most cases as it provides additional features:\n\n- Automatic retries and reconnections on errors.\n- Configurable distribution of messages across available partitions.\n- Synchronous or asynchronous writes of messages to Kafka.\n- Asynchronous cancellation using contexts.\n- Flushing of pending messages on close to support graceful shutdowns.\n- Creation of a missing topic before publishing a message. *Note!* it was the default behaviour up to the version `v0.4.30`.\n\n```go\n// make a writer that produces to topic-A, using the least-bytes distribution\nw := &kafka.Writer{\n\tAddr: kafka.TCP(\"localhost:9092\", \"localhost:9093\", \"localhost:9094\"),\n\tTopic: \"topic-A\",\n\tBalancer: &kafka.LeastBytes{},\n}\n\nerr := w.WriteMessages(context.Background(),\n\tkafka.Message{\n\t\tKey: []byte(\"Key-A\"),\n\t\tValue: []byte(\"Hello World!\"),\n\t},\n\tkafka.Message{\n\t\tKey: []byte(\"Key-B\"),\n\t\tValue: []byte(\"One!\"),\n\t},\n\tkafka.Message{\n\t\tKey: []byte(\"Key-C\"),\n\t\tValue: []byte(\"Two!\"),\n\t},\n)\nif err != nil {\n log.Fatal(\"failed to write messages:\", err)\n}\n\nif err := w.Close(); err != nil {\n log.Fatal(\"failed to close writer:\", err)\n}\n```\n\n### Missing topic creation before publication\n\n```go\n// Make a writer that publishes messages to topic-A.\n// The topic will be created if it is missing.\nw := &Writer{\n Addr: kafka.TCP(\"localhost:9092\", \"localhost:9093\", \"localhost:9094\"),\n Topic: \"topic-A\",\n AllowAutoTopicCreation: true,\n}\n\nmessages := []kafka.Message{\n {\n Key: []byte(\"Key-A\"),\n Value: []byte(\"Hello World!\"),\n },\n {\n Key: []byte(\"Key-B\"),\n Value: []byte(\"One!\"),\n },\n {\n Key: []byte(\"Key-C\"),\n Value: []byte(\"Two!\"),\n },\n}\n\nvar err error\nconst retries = 3\nfor i := 0; i < retries; i++ {\n ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)\n defer cancel()\n \n // attempt to create topic prior to publishing the message\n err = w.WriteMessages(ctx, messages...)\n if errors.Is(err, kafka.LeaderNotAvailable) || errors.Is(err, context.DeadlineExceeded) {\n time.Sleep(time.Millisecond * 250)\n continue\n }\n\n if err != nil {\n log.Fatalf(\"unexpected error %v\", err)\n }\n break\n}\n\nif err := w.Close(); err != nil {\n log.Fatal(\"failed to close writer:\", err)\n}\n```\n\n### Writing to multiple topics\n\nNormally, the `WriterConfig.Topic` is used to initialize a single-topic writer.\nBy excluding that particular configuration, you are given the ability to define\nthe topic on a per-message basis by setting `Message.Topic`.\n\n```go\nw := &kafka.Writer{\n\tAddr: kafka.TCP(\"localhost:9092\", \"localhost:9093\", \"localhost:9094\"),\n // NOTE: When Topic is not defined here, each Message must define it instead.\n\tBalancer: &kafka.LeastBytes{},\n}\n\nerr := w.WriteMessages(context.Background(),\n // NOTE: Each Message has Topic defined, otherwise an error is returned.\n\tkafka.Message{\n Topic: \"topic-A\",\n\t\tKey: []byte(\"Key-A\"),\n\t\tValue: []byte(\"Hello World!\"),\n\t},\n\tkafka.Message{\n Topic: \"topic-B\",\n\t\tKey: []byte(\"Key-B\"),\n\t\tValue: []byte(\"One!\"),\n\t},\n\tkafka.Message{\n Topic: \"topic-C\",\n\t\tKey: []byte(\"Key-C\"),\n\t\tValue: []byte(\"Two!\"),\n\t},\n)\nif err != nil {\n log.Fatal(\"failed to write messages:\", err)\n}\n\nif err := w.Close(); err != nil {\n log.Fatal(\"failed to close writer:\", err)\n}\n```\n\n**NOTE:** These 2 patterns are mutually exclusive, if you set `Writer.Topic`,\nyou must not also explicitly define `Message.Topic` on the messages you are\nwriting. The opposite applies when you do not define a topic for the writer.\nThe `Writer` will return an error if it detects this ambiguity.\n\n### Compatibility with other clients\n\n#### Sarama\n\nIf you're switching from Sarama and need/want to use the same algorithm for message partitioning, you can either use \nthe `kafka.Hash` balancer or the `kafka.ReferenceHash` balancer:\n* `kafka.Hash` = `sarama.NewHashPartitioner`\n* `kafka.ReferenceHash` = `sarama.NewReferenceHashPartitioner`\n\nThe `kafka.Hash` and `kafka.ReferenceHash` balancers would route messages to the same partitions that the two \naforementioned Sarama partitioners would route them to.\n\n```go\nw := &kafka.Writer{\n\tAddr: kafka.TCP(\"localhost:9092\", \"localhost:9093\", \"localhost:9094\"),\n\tTopic: \"topic-A\",\n\tBalancer: &kafka.Hash{},\n}\n```\n\n#### librdkafka and confluent-kafka-go\n\nUse the ```kafka.CRC32Balancer``` balancer to get the same behaviour as librdkafka's\ndefault ```consistent_random``` partition strategy.\n\n```go\nw := &kafka.Writer{\n\tAddr: kafka.TCP(\"localhost:9092\", \"localhost:9093\", \"localhost:9094\"),\n\tTopic: \"topic-A\",\n\tBalancer: kafka.CRC32Balancer{},\n}\n```\n\n#### Java\n\nUse the ```kafka.Murmur2Balancer``` balancer to get the same behaviour as the canonical\nJava client's default partitioner. Note: the Java class allows you to directly specify\nthe partition which is not permitted.\n\n```go\nw := &kafka.Writer{\n\tAddr: kafka.TCP(\"localhost:9092\", \"localhost:9093\", \"localhost:9094\"),\n\tTopic: \"topic-A\",\n\tBalancer: kafka.Murmur2Balancer{},\n}\n```\n\n### Compression\n\nCompression can be enabled on the `Writer` by setting the `Compression` field:\n\n```go\nw := &kafka.Writer{\n\tAddr: kafka.TCP(\"localhost:9092\", \"localhost:9093\", \"localhost:9094\"),\n\tTopic: \"topic-A\",\n\tCompression: kafka.Snappy,\n}\n```\n\nThe `Reader` will by determine if the consumed messages are compressed by\nexamining the message attributes. However, the package(s) for all expected\ncodecs must be imported so that they get loaded correctly.\n\n_Note: in versions prior to 0.4 programs had to import compression packages to\ninstall codecs and support reading compressed messages from kafka. This is no\nlonger the case and import of the compression packages are now no-ops._\n\n## TLS Support\n\nFor a bare bones Conn type or in the Reader/Writer configs you can specify a dialer option for TLS support. If the TLS field is nil, it will not connect with TLS.\n*Note:* Connecting to a Kafka cluster with TLS enabled without configuring TLS on the Conn/Reader/Writer can manifest in opaque io.ErrUnexpectedEOF errors.\n\n\n### Connection\n\n```go\ndialer := &kafka.Dialer{\n Timeout: 10 * time.Second,\n DualStack: true,\n TLS: &tls.Config{...tls config...},\n}\n\nconn, err := dialer.DialContext(ctx, \"tcp\", \"localhost:9093\")\n```\n\n### Reader\n\n```go\ndialer := &kafka.Dialer{\n Timeout: 10 * time.Second,\n DualStack: true,\n TLS: &tls.Config{...tls config...},\n}\n\nr := kafka.NewReader(kafka.ReaderConfig{\n Brokers: []string{\"localhost:9092\", \"localhost:9093\", \"localhost:9094\"},\n GroupID: \"consumer-group-id\",\n Topic: \"topic-A\",\n Dialer: dialer,\n})\n```\n\n### Writer\n\n\nDirect Writer creation\n\n```go\nw := kafka.Writer{\n Addr: kafka.TCP(\"localhost:9092\", \"localhost:9093\", \"localhost:9094\"), \n Topic: \"topic-A\",\n Balancer: &kafka.Hash{},\n Transport: &kafka.Transport{\n TLS: &tls.Config{},\n },\n }\n```\n\nUsing `kafka.NewWriter`\n\n```go\ndialer := &kafka.Dialer{\n Timeout: 10 * time.Second,\n DualStack: true,\n TLS: &tls.Config{...tls config...},\n}\n\nw := kafka.NewWriter(kafka.WriterConfig{\n\tBrokers: []string{\"localhost:9092\", \"localhost:9093\", \"localhost:9094\"},\n\tTopic: \"topic-A\",\n\tBalancer: &kafka.Hash{},\n\tDialer: dialer,\n})\n```\nNote that `kafka.NewWriter` and `kafka.WriterConfig` are deprecated and will be removed in a future release.\n\n## SASL Support\n\nYou can specify an option on the `Dialer` to use SASL authentication. The `Dialer` can be used directly to open a `Conn` or it can be passed to a `Reader` or `Writer` via their respective configs. If the `SASLMechanism` field is `nil`, it will not authenticate with SASL.\n\n### SASL Authentication Types\n\n#### [Plain](https://godoc.org/github.com/segmentio/kafka-go/sasl/plain#Mechanism)\n```go\nmechanism := plain.Mechanism{\n Username: \"username\",\n Password: \"password\",\n}\n```\n\n#### [SCRAM](https://godoc.org/github.com/segmentio/kafka-go/sasl/scram#Mechanism)\n```go\nmechanism, err := scram.Mechanism(scram.SHA512, \"username\", \"password\")\nif err != nil {\n panic(err)\n}\n```\n\n### Connection\n\n```go\nmechanism, err := scram.Mechanism(scram.SHA512, \"username\", \"password\")\nif err != nil {\n panic(err)\n}\n\ndialer := &kafka.Dialer{\n Timeout: 10 * time.Second,\n DualStack: true,\n SASLMechanism: mechanism,\n}\n\nconn, err := dialer.DialContext(ctx, \"tcp\", \"localhost:9093\")\n```\n\n\n### Reader\n\n```go\nmechanism, err := scram.Mechanism(scram.SHA512, \"username\", \"password\")\nif err != nil {\n panic(err)\n}\n\ndialer := &kafka.Dialer{\n Timeout: 10 * time.Second,\n DualStack: true,\n SASLMechanism: mechanism,\n}\n\nr := kafka.NewReader(kafka.ReaderConfig{\n Brokers: []string{\"localhost:9092\",\"localhost:9093\", \"localhost:9094\"},\n GroupID: \"consumer-group-id\",\n Topic: \"topic-A\",\n Dialer: dialer,\n})\n```\n\n### Writer\n\n```go\nmechanism, err := scram.Mechanism(scram.SHA512, \"username\", \"password\")\nif err != nil {\n panic(err)\n}\n\n// Transports are responsible for managing connection pools and other resources,\n// it's generally best to create a few of these and share them across your\n// application.\nsharedTransport := &kafka.Transport{\n SASL: mechanism,\n}\n\nw := kafka.Writer{\n\tAddr: kafka.TCP(\"localhost:9092\", \"localhost:9093\", \"localhost:9094\"),\n\tTopic: \"topic-A\",\n\tBalancer: &kafka.Hash{},\n\tTransport: sharedTransport,\n}\n```\n\n### Client\n\n```go\nmechanism, err := scram.Mechanism(scram.SHA512, \"username\", \"password\")\nif err != nil {\n panic(err)\n}\n\n// Transports are responsible for managing connection pools and other resources,\n// it's generally best to create a few of these and share them across your\n// application.\nsharedTransport := &kafka.Transport{\n SASL: mechanism,\n}\n\nclient := &kafka.Client{\n Addr: kafka.TCP(\"localhost:9092\", \"localhost:9093\", \"localhost:9094\"),\n Timeout: 10 * time.Second,\n Transport: sharedTransport,\n}\n```\n\n#### Reading all messages within a time range\n\n```go\nstartTime := time.Now().Add(-time.Hour)\nendTime := time.Now()\nbatchSize := int(10e6) // 10MB\n\nr := kafka.NewReader(kafka.ReaderConfig{\n Brokers: []string{\"localhost:9092\", \"localhost:9093\", \"localhost:9094\"},\n Topic: \"my-topic1\",\n Partition: 0,\n MaxBytes: batchSize,\n})\n\nr.SetOffsetAt(context.Background(), startTime)\n\nfor {\n m, err := r.ReadMessage(context.Background())\n\n if err != nil {\n break\n }\n if m.Time.After(endTime) {\n break\n }\n // TODO: process message\n fmt.Printf(\"message at offset %d: %s = %s\\n\", m.Offset, string(m.Key), string(m.Value))\n}\n\nif err := r.Close(); err != nil {\n log.Fatal(\"failed to close reader:\", err)\n}\n```\n\n\n## Logging\n\nFor visiblity into the operations of the Reader/Writer types, configure a logger on creation.\n\n\n### Reader\n\n```go\nfunc logf(msg string, a ...interface{}) {\n\tfmt.Printf(msg, a...)\n\tfmt.Println()\n}\n\nr := kafka.NewReader(kafka.ReaderConfig{\n\tBrokers: []string{\"localhost:9092\", \"localhost:9093\", \"localhost:9094\"},\n\tTopic: \"my-topic1\",\n\tPartition: 0,\n\tLogger: kafka.LoggerFunc(logf),\n\tErrorLogger: kafka.LoggerFunc(logf),\n})\n```\n\n### Writer\n\n```go\nfunc logf(msg string, a ...interface{}) {\n\tfmt.Printf(msg, a...)\n\tfmt.Println()\n}\n\nw := &kafka.Writer{\n\tAddr: kafka.TCP(\"localhost:9092\"),\n\tTopic: \"topic\",\n\tLogger: kafka.LoggerFunc(logf),\n\tErrorLogger: kafka.LoggerFunc(logf),\n}\n```\n\n\n\n## Testing\n\nSubtle behavior changes in later Kafka versions have caused some historical tests to break, if you are running against Kafka 2.3.1 or later, exporting the `KAFKA_SKIP_NETTEST=1` environment variables will skip those tests.\n\nRun Kafka locally in docker\n\n```bash\ndocker-compose up -d\n```\n\nRun tests\n\n```bash\nKAFKA_VERSION=2.3.1 \\\n KAFKA_SKIP_NETTEST=1 \\\n go test -race ./...\n```\n\n(or) to clean up the cached test results and run tests:\n```\ngo clean -cache && make test\n```\n" }, { "path": "addoffsetstotxn.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/addoffsetstotxn\"\n)\n\n// AddOffsetsToTxnRequest is the request structure for the AddOffsetsToTxn function.\ntype AddOffsetsToTxnRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// The transactional id key\n\tTransactionalID string\n\n\t// The Producer ID (PID) for the current producer session;\n\t// received from an InitProducerID request.\n\tProducerID int\n\n\t// The epoch associated with the current producer session for the given PID\n\tProducerEpoch int\n\n\t// The unique group identifier.\n\tGroupID string\n}\n\n// AddOffsetsToTxnResponse is the response structure for the AddOffsetsToTxn function.\ntype AddOffsetsToTxnResponse struct {\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// An error that may have occurred when attempting to add the offsets\n\t// to a transaction.\n\t//\n\t// The errors contain the kafka error code. Programs may use the standard\n\t// errors.Is function to test the error against kafka error codes.\n\tError error\n}\n\n// AddOffsetsToTnx sends an add offsets to txn request to a kafka broker and returns the response.\nfunc (c *Client) AddOffsetsToTxn(\n\tctx context.Context,\n\treq *AddOffsetsToTxnRequest,\n) (*AddOffsetsToTxnResponse, error) {\n\tm, err := c.roundTrip(ctx, req.Addr, &addoffsetstotxn.Request{\n\t\tTransactionalID: req.TransactionalID,\n\t\tProducerID: int64(req.ProducerID),\n\t\tProducerEpoch: int16(req.ProducerEpoch),\n\t\tGroupID: req.GroupID,\n\t})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).AddOffsetsToTxn: %w\", err)\n\t}\n\n\tr := m.(*addoffsetstotxn.Response)\n\n\tres := &AddOffsetsToTxnResponse{\n\t\tThrottle: makeDuration(r.ThrottleTimeMs),\n\t\tError: makeError(r.ErrorCode, \"\"),\n\t}\n\n\treturn res, nil\n}\n" }, { "path": "addoffsetstotxn_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"log\"\n\t\"os\"\n\t\"testing\"\n\t\"time\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc TestClientAddOffsetsToTxn(t *testing.T) {\n\tif !ktesting.KafkaIsAtLeast(\"0.11.0\") {\n\t\tt.Skip(\"Skipping test because kafka version is not high enough.\")\n\t}\n\n\t// TODO: look into why this test fails on Kafka 3.0.0 and higher when transactional support\n\t// work is revisited.\n\tif ktesting.KafkaIsAtLeast(\"3.0.0\") {\n\t\tt.Skip(\"Skipping test because it fails on Kafka version 3.0.0 or higher.\")\n\t}\n\n\ttopic := makeTopic()\n\ttransactionalID := makeTransactionalID()\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\terr := clientCreateTopic(client, topic, 3)\n\tdefer deleteTopic(t, topic)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tctx, cancel := context.WithTimeout(context.Background(), time.Second*30)\n\twaitForTopic(ctx, t, topic)\n\tdefer cancel()\n\trespc, err := waitForCoordinatorIndefinitely(ctx, client, &FindCoordinatorRequest{\n\t\tAddr: client.Addr,\n\t\tKey: transactionalID,\n\t\tKeyType: CoordinatorKeyTypeConsumer,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif respc.Error != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tgroupID := makeGroupID()\n\n\tgroup, err := NewConsumerGroup(ConsumerGroupConfig{\n\t\tID: groupID,\n\t\tTopics: []string{topic},\n\t\tBrokers: []string{\"localhost:9092\"},\n\t\tHeartbeatInterval: 2 * time.Second,\n\t\tRebalanceTimeout: 2 * time.Second,\n\t\tRetentionTime: time.Hour,\n\t\tLogger: log.New(os.Stdout, \"cg-test: \", 0),\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer group.Close()\n\n\tctx, cancel = context.WithTimeout(context.Background(), time.Second*30)\n\tdefer cancel()\n\t_, err = group.Next(ctx)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tctx, cancel = context.WithTimeout(context.Background(), time.Second*30)\n\tdefer cancel()\n\trespc, err = waitForCoordinatorIndefinitely(ctx, client, &FindCoordinatorRequest{\n\t\tAddr: client.Addr,\n\t\tKey: transactionalID,\n\t\tKeyType: CoordinatorKeyTypeTransaction,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif respc.Error != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tipResp, err := client.InitProducerID(ctx, &InitProducerIDRequest{\n\t\tTransactionalID: transactionalID,\n\t\tTransactionTimeoutMs: 10000,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif ipResp.Error != nil {\n\t\tt.Fatal(ipResp.Error)\n\t}\n\n\tdefer func() {\n\t\terr := clientEndTxn(client, &EndTxnRequest{\n\t\t\tTransactionalID: transactionalID,\n\t\t\tProducerID: ipResp.Producer.ProducerID,\n\t\t\tProducerEpoch: ipResp.Producer.ProducerEpoch,\n\t\t\tCommitted: false,\n\t\t})\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tctx, cancel = context.WithTimeout(context.Background(), time.Second*30)\n\tdefer cancel()\n\n\tresp, err := client.AddOffsetsToTxn(ctx, &AddOffsetsToTxnRequest{\n\t\tTransactionalID: transactionalID,\n\t\tProducerID: ipResp.Producer.ProducerID,\n\t\tProducerEpoch: ipResp.Producer.ProducerEpoch,\n\t\tGroupID: groupID,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif resp.Error != nil {\n\t\tt.Fatal(err)\n\t}\n}\n" }, { "path": "addpartitionstotxn.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/addpartitionstotxn\"\n)\n\n// AddPartitionToTxn represents a partition to be added\n// to a transaction.\ntype AddPartitionToTxn struct {\n\t// Partition is the ID of a partition to add to the transaction.\n\tPartition int\n}\n\n// AddPartitionsToTxnRequest is the request structure fo the AddPartitionsToTxn function.\ntype AddPartitionsToTxnRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// The transactional id key\n\tTransactionalID string\n\n\t// The Producer ID (PID) for the current producer session;\n\t// received from an InitProducerID request.\n\tProducerID int\n\n\t// The epoch associated with the current producer session for the given PID\n\tProducerEpoch int\n\n\t// Mappings of topic names to lists of partitions.\n\tTopics map[string][]AddPartitionToTxn\n}\n\n// AddPartitionsToTxnResponse is the response structure for the AddPartitionsToTxn function.\ntype AddPartitionsToTxnResponse struct {\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// Mappings of topic names to partitions being added to a transactions.\n\tTopics map[string][]AddPartitionToTxnPartition\n}\n\n// AddPartitionToTxnPartition represents the state of a single partition\n// in response to adding to a transaction.\ntype AddPartitionToTxnPartition struct {\n\t// The ID of the partition.\n\tPartition int\n\n\t// An error that may have occurred when attempting to add the partition\n\t// to a transaction.\n\t//\n\t// The errors contain the kafka error code. Programs may use the standard\n\t// errors.Is function to test the error against kafka error codes.\n\tError error\n}\n\n// AddPartitionsToTnx sends an add partitions to txn request to a kafka broker and returns the response.\nfunc (c *Client) AddPartitionsToTxn(\n\tctx context.Context,\n\treq *AddPartitionsToTxnRequest,\n) (*AddPartitionsToTxnResponse, error) {\n\tprotoReq := &addpartitionstotxn.Request{\n\t\tTransactionalID: req.TransactionalID,\n\t\tProducerID: int64(req.ProducerID),\n\t\tProducerEpoch: int16(req.ProducerEpoch),\n\t}\n\tprotoReq.Topics = make([]addpartitionstotxn.RequestTopic, 0, len(req.Topics))\n\n\tfor topic, partitions := range req.Topics {\n\t\treqTopic := addpartitionstotxn.RequestTopic{\n\t\t\tName: topic,\n\t\t\tPartitions: make([]int32, len(partitions)),\n\t\t}\n\t\tfor i, partition := range partitions {\n\t\t\treqTopic.Partitions[i] = int32(partition.Partition)\n\t\t}\n\t\tprotoReq.Topics = append(protoReq.Topics, reqTopic)\n\t}\n\n\tm, err := c.roundTrip(ctx, req.Addr, protoReq)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).AddPartitionsToTxn: %w\", err)\n\t}\n\n\tr := m.(*addpartitionstotxn.Response)\n\n\tres := &AddPartitionsToTxnResponse{\n\t\tThrottle: makeDuration(r.ThrottleTimeMs),\n\t\tTopics: make(map[string][]AddPartitionToTxnPartition, len(r.Results)),\n\t}\n\n\tfor _, result := range r.Results {\n\t\tpartitions := make([]AddPartitionToTxnPartition, 0, len(result.Results))\n\t\tfor _, rp := range result.Results {\n\t\t\tpartitions = append(partitions, AddPartitionToTxnPartition{\n\t\t\t\tPartition: int(rp.PartitionIndex),\n\t\t\t\tError: makeError(rp.ErrorCode, \"\"),\n\t\t\t})\n\t\t}\n\t\tres.Topics[result.Name] = partitions\n\t}\n\n\treturn res, nil\n}\n" }, { "path": "addpartitionstotxn_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"net\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc TestClientAddPartitionsToTxn(t *testing.T) {\n\tif !ktesting.KafkaIsAtLeast(\"0.11.0\") {\n\t\tt.Skip(\"Skipping test because kafka version is not high enough.\")\n\t}\n\n\t// TODO: look into why this test fails on Kafka 3.0.0 and higher when transactional support\n\t// work is revisited.\n\tif ktesting.KafkaIsAtLeast(\"3.0.0\") {\n\t\tt.Skip(\"Skipping test because it fails on Kafka version 3.0.0 or higher.\")\n\t}\n\n\ttopic1 := makeTopic()\n\ttopic2 := makeTopic()\n\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\terr := clientCreateTopic(client, topic1, 3)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = clientCreateTopic(client, topic2, 3)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttransactionalID := makeTransactionalID()\n\n\tctx, cancel := context.WithTimeout(context.Background(), time.Second*30)\n\tdefer cancel()\n\trespc, err := waitForCoordinatorIndefinitely(ctx, client, &FindCoordinatorRequest{\n\t\tAddr: client.Addr,\n\t\tKey: transactionalID,\n\t\tKeyType: CoordinatorKeyTypeTransaction,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttransactionCoordinator := TCP(net.JoinHostPort(respc.Coordinator.Host, strconv.Itoa(int(respc.Coordinator.Port))))\n\tclient, shutdown = newClient(transactionCoordinator)\n\tdefer shutdown()\n\n\tipResp, err := client.InitProducerID(ctx, &InitProducerIDRequest{\n\t\tTransactionalID: transactionalID,\n\t\tTransactionTimeoutMs: 10000,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif ipResp.Error != nil {\n\t\tt.Fatal(ipResp.Error)\n\t}\n\n\tdefer func() {\n\t\terr := clientEndTxn(client, &EndTxnRequest{\n\t\t\tTransactionalID: transactionalID,\n\t\t\tProducerID: ipResp.Producer.ProducerID,\n\t\t\tProducerEpoch: ipResp.Producer.ProducerEpoch,\n\t\t\tCommitted: false,\n\t\t})\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}()\n\n\tctx, cancel = context.WithTimeout(context.Background(), time.Second*30)\n\tdefer cancel()\n\tresp, err := client.AddPartitionsToTxn(ctx, &AddPartitionsToTxnRequest{\n\t\tTransactionalID: transactionalID,\n\t\tProducerID: ipResp.Producer.ProducerID,\n\t\tProducerEpoch: ipResp.Producer.ProducerEpoch,\n\t\tTopics: map[string][]AddPartitionToTxn{\n\t\t\ttopic1: {\n\t\t\t\t{\n\t\t\t\t\tPartition: 0,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tPartition: 1,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tPartition: 2,\n\t\t\t\t},\n\t\t\t},\n\t\t\ttopic2: {\n\t\t\t\t{\n\t\t\t\t\tPartition: 0,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tPartition: 2,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(resp.Topics) != 2 {\n\t\tt.Errorf(\"expected responses for 2 topics; got: %d\", len(resp.Topics))\n\t}\n\tfor topic, partitions := range resp.Topics {\n\t\tif topic == topic1 {\n\t\t\tif len(partitions) != 3 {\n\t\t\t\tt.Errorf(\"expected 3 partitions in response for topic %s; got: %d\", topic, len(partitions))\n\t\t\t}\n\t\t}\n\t\tif topic == topic2 {\n\t\t\tif len(partitions) != 2 {\n\t\t\t\tt.Errorf(\"expected 2 partitions in response for topic %s; got: %d\", topic, len(partitions))\n\t\t\t}\n\t\t}\n\t\tfor _, partition := range partitions {\n\t\t\tif partition.Error != nil {\n\t\t\t\tt.Error(partition.Error)\n\t\t\t}\n\t\t}\n\t}\n}\n" }, { "path": "address.go", "content": "package kafka\n\nimport (\n\t\"net\"\n\t\"strings\"\n)\n\n// TCP constructs an address with the network set to \"tcp\".\nfunc TCP(address ...string) net.Addr { return makeNetAddr(\"tcp\", address) }\n\nfunc makeNetAddr(network string, addresses []string) net.Addr {\n\tswitch len(addresses) {\n\tcase 0:\n\t\treturn nil // maybe panic instead?\n\tcase 1:\n\t\treturn makeAddr(network, addresses[0])\n\tdefault:\n\t\treturn makeMultiAddr(network, addresses)\n\t}\n}\n\nfunc makeAddr(network, address string) net.Addr {\n\treturn &networkAddress{\n\t\tnetwork: network,\n\t\taddress: canonicalAddress(address),\n\t}\n}\n\nfunc makeMultiAddr(network string, addresses []string) net.Addr {\n\tmulti := make(multiAddr, len(addresses))\n\tfor i, address := range addresses {\n\t\tmulti[i] = makeAddr(network, address)\n\t}\n\treturn multi\n}\n\ntype networkAddress struct {\n\tnetwork string\n\taddress string\n}\n\nfunc (a *networkAddress) Network() string { return a.network }\n\nfunc (a *networkAddress) String() string { return a.address }\n\ntype multiAddr []net.Addr\n\nfunc (m multiAddr) Network() string { return m.join(net.Addr.Network) }\n\nfunc (m multiAddr) String() string { return m.join(net.Addr.String) }\n\nfunc (m multiAddr) join(f func(net.Addr) string) string {\n\tswitch len(m) {\n\tcase 0:\n\t\treturn \"\"\n\tcase 1:\n\t\treturn f(m[0])\n\t}\n\ts := make([]string, len(m))\n\tfor i, a := range m {\n\t\ts[i] = f(a)\n\t}\n\treturn strings.Join(s, \",\")\n}\n" }, { "path": "address_test.go", "content": "package kafka\n\nimport (\n\t\"net\"\n\t\"testing\"\n)\n\nfunc TestNetworkAddress(t *testing.T) {\n\ttests := []struct {\n\t\taddr net.Addr\n\t\tnetwork string\n\t\taddress string\n\t}{\n\t\t{\n\t\t\taddr: TCP(\"127.0.0.1\"),\n\t\t\tnetwork: \"tcp\",\n\t\t\taddress: \"127.0.0.1:9092\",\n\t\t},\n\n\t\t{\n\t\t\taddr: TCP(\"::1\"),\n\t\t\tnetwork: \"tcp\",\n\t\t\taddress: \"[::1]:9092\",\n\t\t},\n\n\t\t{\n\t\t\taddr: TCP(\"localhost\"),\n\t\t\tnetwork: \"tcp\",\n\t\t\taddress: \"localhost:9092\",\n\t\t},\n\n\t\t{\n\t\t\taddr: TCP(\"localhost:9092\"),\n\t\t\tnetwork: \"tcp\",\n\t\t\taddress: \"localhost:9092\",\n\t\t},\n\n\t\t{\n\t\t\taddr: TCP(\"localhost\", \"localhost:9093\", \"localhost:9094\"),\n\t\t\tnetwork: \"tcp,tcp,tcp\",\n\t\t\taddress: \"localhost:9092,localhost:9093,localhost:9094\",\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.network+\"+\"+test.address, func(t *testing.T) {\n\t\t\tif s := test.addr.Network(); s != test.network {\n\t\t\t\tt.Errorf(\"network mismatch: want %q but got %q\", test.network, s)\n\t\t\t}\n\t\t\tif s := test.addr.String(); s != test.address {\n\t\t\t\tt.Errorf(\"network mismatch: want %q but got %q\", test.address, s)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "alterclientquotas.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/alterclientquotas\"\n)\n\n// AlterClientQuotasRequest represents a request sent to a kafka broker to\n// alter client quotas.\ntype AlterClientQuotasRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// List of client quotas entries to alter.\n\tEntries []AlterClientQuotaEntry\n\n\t// Whether the alteration should be validated, but not performed.\n\tValidateOnly bool\n}\n\ntype AlterClientQuotaEntry struct {\n\t// The quota entities to alter.\n\tEntities []AlterClientQuotaEntity\n\n\t// An individual quota configuration entry to alter.\n\tOps []AlterClientQuotaOps\n}\n\ntype AlterClientQuotaEntity struct {\n\t// The quota entity type.\n\tEntityType string\n\n\t// The name of the quota entity, or null if the default.\n\tEntityName string\n}\n\ntype AlterClientQuotaOps struct {\n\t// The quota configuration key.\n\tKey string\n\n\t// The quota configuration value to set, otherwise ignored if the value is to be removed.\n\tValue float64\n\n\t// Whether the quota configuration value should be removed, otherwise set.\n\tRemove bool\n}\n\ntype AlterClientQuotaResponseQuotas struct {\n\t// Error is set to a non-nil value including the code and message if a top-level\n\t// error was encountered when doing the update.\n\tError error\n\n\t// The altered quota entities.\n\tEntities []AlterClientQuotaEntity\n}\n\n// AlterClientQuotasResponse represents a response from a kafka broker to an alter client\n// quotas request.\ntype AlterClientQuotasResponse struct {\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// List of altered client quotas responses.\n\tEntries []AlterClientQuotaResponseQuotas\n}\n\n// AlterClientQuotas sends client quotas alteration request to a kafka broker and returns\n// the response.\nfunc (c *Client) AlterClientQuotas(ctx context.Context, req *AlterClientQuotasRequest) (*AlterClientQuotasResponse, error) {\n\tentries := make([]alterclientquotas.Entry, len(req.Entries))\n\n\tfor entryIdx, entry := range req.Entries {\n\t\tentities := make([]alterclientquotas.Entity, len(entry.Entities))\n\t\tfor entityIdx, entity := range entry.Entities {\n\t\t\tentities[entityIdx] = alterclientquotas.Entity{\n\t\t\t\tEntityType: entity.EntityType,\n\t\t\t\tEntityName: entity.EntityName,\n\t\t\t}\n\t\t}\n\n\t\tops := make([]alterclientquotas.Ops, len(entry.Ops))\n\t\tfor opsIdx, op := range entry.Ops {\n\t\t\tops[opsIdx] = alterclientquotas.Ops{\n\t\t\t\tKey: op.Key,\n\t\t\t\tValue: op.Value,\n\t\t\t\tRemove: op.Remove,\n\t\t\t}\n\t\t}\n\n\t\tentries[entryIdx] = alterclientquotas.Entry{\n\t\t\tEntities: entities,\n\t\t\tOps: ops,\n\t\t}\n\t}\n\n\tm, err := c.roundTrip(ctx, req.Addr, &alterclientquotas.Request{\n\t\tEntries: entries,\n\t\tValidateOnly: req.ValidateOnly,\n\t})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).AlterClientQuotas: %w\", err)\n\t}\n\n\tres := m.(*alterclientquotas.Response)\n\tresponseEntries := make([]AlterClientQuotaResponseQuotas, len(res.Results))\n\n\tfor responseEntryIdx, responseEntry := range res.Results {\n\t\tresponseEntities := make([]AlterClientQuotaEntity, len(responseEntry.Entities))\n\t\tfor responseEntityIdx, responseEntity := range responseEntry.Entities {\n\t\t\tresponseEntities[responseEntityIdx] = AlterClientQuotaEntity{\n\t\t\t\tEntityType: responseEntity.EntityType,\n\t\t\t\tEntityName: responseEntity.EntityName,\n\t\t\t}\n\t\t}\n\n\t\tresponseEntries[responseEntryIdx] = AlterClientQuotaResponseQuotas{\n\t\t\tError: makeError(responseEntry.ErrorCode, responseEntry.ErrorMessage),\n\t\t\tEntities: responseEntities,\n\t\t}\n\t}\n\tret := &AlterClientQuotasResponse{\n\t\tThrottle: makeDuration(res.ThrottleTimeMs),\n\t\tEntries: responseEntries,\n\t}\n\n\treturn ret, nil\n}\n" }, { "path": "alterclientquotas_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n\t\"time\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestClientAlterClientQuotas(t *testing.T) {\n\t// Added in Version 2.6.0 https://issues.apache.org/jira/browse/KAFKA-7740\n\tif !ktesting.KafkaIsAtLeast(\"2.6.0\") {\n\t\treturn\n\t}\n\n\tconst (\n\t\tentityType = \"client-id\"\n\t\tentityName = \"my-client-id\"\n\t\tkey = \"producer_byte_rate\"\n\t\tvalue = 500000.0\n\t)\n\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\talterResp, err := client.AlterClientQuotas(context.Background(), &AlterClientQuotasRequest{\n\t\tEntries: []AlterClientQuotaEntry{\n\t\t\t{\n\t\t\t\tEntities: []AlterClientQuotaEntity{\n\t\t\t\t\t{\n\t\t\t\t\t\tEntityType: entityType,\n\t\t\t\t\t\tEntityName: entityName,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tOps: []AlterClientQuotaOps{\n\t\t\t\t\t{\n\t\t\t\t\t\tKey: key,\n\t\t\t\t\t\tValue: value,\n\t\t\t\t\t\tRemove: false,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\texpectedAlterResp := AlterClientQuotasResponse{\n\t\tThrottle: 0,\n\t\tEntries: []AlterClientQuotaResponseQuotas{\n\t\t\t{\n\t\t\t\tError: makeError(0, \"\"),\n\t\t\t\tEntities: []AlterClientQuotaEntity{\n\t\t\t\t\t{\n\t\t\t\t\t\tEntityName: entityName,\n\t\t\t\t\t\tEntityType: entityType,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tassert.Equal(t, expectedAlterResp, *alterResp)\n\n\t// kraft mode is slow\n\tif ktesting.KafkaIsAtLeast(\"3.7.0\") {\n\t\ttime.Sleep(3 * time.Second)\n\t}\n\n\tdescribeResp, err := client.DescribeClientQuotas(context.Background(), &DescribeClientQuotasRequest{\n\t\tComponents: []DescribeClientQuotasRequestComponent{\n\t\t\t{\n\t\t\t\tEntityType: entityType,\n\t\t\t\tMatchType: 0,\n\t\t\t\tMatch: entityName,\n\t\t\t},\n\t\t},\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\texpectedDescribeResp := DescribeClientQuotasResponse{\n\t\tThrottle: 0,\n\t\tError: makeError(0, \"\"),\n\t\tEntries: []DescribeClientQuotasResponseQuotas{\n\t\t\t{\n\t\t\t\tEntities: []DescribeClientQuotasEntity{\n\t\t\t\t\t{\n\t\t\t\t\t\tEntityType: entityType,\n\t\t\t\t\t\tEntityName: entityName,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tValues: []DescribeClientQuotasValue{\n\t\t\t\t\t{\n\t\t\t\t\t\tKey: key,\n\t\t\t\t\t\tValue: value,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tassert.Equal(t, expectedDescribeResp, *describeResp)\n}\n" }, { "path": "alterconfigs.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/alterconfigs\"\n)\n\n// AlterConfigsRequest represents a request sent to a kafka broker to alter configs.\ntype AlterConfigsRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// List of resources to update.\n\tResources []AlterConfigRequestResource\n\n\t// When set to true, topics are not created but the configuration is\n\t// validated as if they were.\n\tValidateOnly bool\n}\n\ntype AlterConfigRequestResource struct {\n\t// Resource Type\n\tResourceType ResourceType\n\n\t// Resource Name\n\tResourceName string\n\n\t// Configs is a list of configuration updates.\n\tConfigs []AlterConfigRequestConfig\n}\n\ntype AlterConfigRequestConfig struct {\n\t// Configuration key name\n\tName string\n\n\t// The value to set for the configuration key.\n\tValue string\n}\n\n// AlterConfigsResponse represents a response from a kafka broker to an alter config request.\ntype AlterConfigsResponse struct {\n\t// Duration for which the request was throttled due to a quota violation.\n\tThrottle time.Duration\n\n\t// Mapping of topic names to errors that occurred while attempting to create\n\t// the topics.\n\t//\n\t// The errors contain the kafka error code. Programs may use the standard\n\t// errors.Is function to test the error against kafka error codes.\n\tErrors map[AlterConfigsResponseResource]error\n}\n\n// AlterConfigsResponseResource helps map errors to specific resources in an\n// alter config response.\ntype AlterConfigsResponseResource struct {\n\tType int8\n\tName string\n}\n\n// AlterConfigs sends a config altering request to a kafka broker and returns the\n// response.\nfunc (c *Client) AlterConfigs(ctx context.Context, req *AlterConfigsRequest) (*AlterConfigsResponse, error) {\n\tresources := make([]alterconfigs.RequestResources, len(req.Resources))\n\n\tfor i, t := range req.Resources {\n\t\tconfigs := make([]alterconfigs.RequestConfig, len(t.Configs))\n\t\tfor j, v := range t.Configs {\n\t\t\tconfigs[j] = alterconfigs.RequestConfig{\n\t\t\t\tName: v.Name,\n\t\t\t\tValue: v.Value,\n\t\t\t}\n\t\t}\n\t\tresources[i] = alterconfigs.RequestResources{\n\t\t\tResourceType: int8(t.ResourceType),\n\t\t\tResourceName: t.ResourceName,\n\t\t\tConfigs: configs,\n\t\t}\n\t}\n\n\tm, err := c.roundTrip(ctx, req.Addr, &alterconfigs.Request{\n\t\tResources: resources,\n\t\tValidateOnly: req.ValidateOnly,\n\t})\n\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).AlterConfigs: %w\", err)\n\t}\n\n\tres := m.(*alterconfigs.Response)\n\tret := &AlterConfigsResponse{\n\t\tThrottle: makeDuration(res.ThrottleTimeMs),\n\t\tErrors: make(map[AlterConfigsResponseResource]error, len(res.Responses)),\n\t}\n\n\tfor _, t := range res.Responses {\n\t\tret.Errors[AlterConfigsResponseResource{\n\t\t\tType: t.ResourceType,\n\t\t\tName: t.ResourceName,\n\t\t}] = makeError(t.ErrorCode, t.ErrorMessage)\n\t}\n\n\treturn ret, nil\n}\n" }, { "path": "alterconfigs_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestClientAlterConfigs(t *testing.T) {\n\tif !ktesting.KafkaIsAtLeast(\"0.11.0\") {\n\t\treturn\n\t}\n\n\tconst (\n\t\tMaxMessageBytes = \"max.message.bytes\"\n\t\tMaxMessageBytesValue = \"200000\"\n\t)\n\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\ttopic := makeTopic()\n\tcreateTopic(t, topic, 1)\n\tdefer deleteTopic(t, topic)\n\n\t_, err := client.AlterConfigs(context.Background(), &AlterConfigsRequest{\n\t\tResources: []AlterConfigRequestResource{{\n\t\t\tResourceType: ResourceTypeTopic,\n\t\t\tResourceName: topic,\n\t\t\tConfigs: []AlterConfigRequestConfig{{\n\t\t\t\tName: MaxMessageBytes,\n\t\t\t\tValue: MaxMessageBytesValue,\n\t\t\t},\n\t\t\t},\n\t\t}},\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdescribeResp, err := client.DescribeConfigs(context.Background(), &DescribeConfigsRequest{\n\t\tResources: []DescribeConfigRequestResource{{\n\t\t\tResourceType: ResourceTypeTopic,\n\t\t\tResourceName: topic,\n\t\t\tConfigNames: []string{MaxMessageBytes},\n\t\t}},\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmaxMessageBytesValue := \"0\"\n\tfor _, resource := range describeResp.Resources {\n\t\tif resource.ResourceType == int8(ResourceTypeTopic) && resource.ResourceName == topic {\n\t\t\tfor _, entry := range resource.ConfigEntries {\n\t\t\t\tif entry.ConfigName == MaxMessageBytes {\n\t\t\t\t\tmaxMessageBytesValue = entry.ConfigValue\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tassert.Equal(t, maxMessageBytesValue, MaxMessageBytesValue)\n}\n" }, { "path": "alterpartitionreassignments.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/alterpartitionreassignments\"\n)\n\n// AlterPartitionReassignmentsRequest is a request to the AlterPartitionReassignments API.\ntype AlterPartitionReassignmentsRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// Topic is the name of the topic to alter partitions in. Keep this field empty and use Topic in AlterPartitionReassignmentsRequestAssignment to\n\t// reassign to multiple topics.\n\tTopic string\n\n\t// Assignments is the list of partition reassignments to submit to the API.\n\tAssignments []AlterPartitionReassignmentsRequestAssignment\n\n\t// Timeout is the amount of time to wait for the request to complete.\n\tTimeout time.Duration\n}\n\n// AlterPartitionReassignmentsRequestAssignment contains the requested reassignments for a single\n// partition.\ntype AlterPartitionReassignmentsRequestAssignment struct {\n\t// Topic is the name of the topic to alter partitions in. If empty, the value of Topic in AlterPartitionReassignmentsRequest is used.\n\tTopic string\n\n\t// PartitionID is the ID of the partition to make the reassignments in.\n\tPartitionID int\n\n\t// BrokerIDs is a slice of brokers to set the partition replicas to, or null to cancel a pending reassignment for this partition.\n\tBrokerIDs []int\n}\n\n// AlterPartitionReassignmentsResponse is a response from the AlterPartitionReassignments API.\ntype AlterPartitionReassignmentsResponse struct {\n\t// Error is set to a non-nil value including the code and message if a top-level\n\t// error was encountered when doing the update.\n\tError error\n\n\t// PartitionResults contains the specific results for each partition.\n\tPartitionResults []AlterPartitionReassignmentsResponsePartitionResult\n}\n\n// AlterPartitionReassignmentsResponsePartitionResult contains the detailed result of\n// doing reassignments for a single partition.\ntype AlterPartitionReassignmentsResponsePartitionResult struct {\n\t// Topic is the topic name.\n\tTopic string\n\n\t// PartitionID is the ID of the partition that was altered.\n\tPartitionID int\n\n\t// Error is set to a non-nil value including the code and message if an error was encountered\n\t// during the update for this partition.\n\tError error\n}\n\nfunc (c *Client) AlterPartitionReassignments(\n\tctx context.Context,\n\treq *AlterPartitionReassignmentsRequest,\n) (*AlterPartitionReassignmentsResponse, error) {\n\tapiTopicMap := make(map[string]*alterpartitionreassignments.RequestTopic)\n\n\tfor _, assignment := range req.Assignments {\n\t\ttopic := assignment.Topic\n\t\tif topic == \"\" {\n\t\t\ttopic = req.Topic\n\t\t}\n\n\t\tapiTopic := apiTopicMap[topic]\n\t\tif apiTopic == nil {\n\t\t\tapiTopic = &alterpartitionreassignments.RequestTopic{\n\t\t\t\tName: topic,\n\t\t\t}\n\t\t\tapiTopicMap[topic] = apiTopic\n\t\t}\n\n\t\treplicas := []int32{}\n\t\tfor _, brokerID := range assignment.BrokerIDs {\n\t\t\treplicas = append(replicas, int32(brokerID))\n\t\t}\n\n\t\tapiTopic.Partitions = append(\n\t\t\tapiTopic.Partitions,\n\t\t\talterpartitionreassignments.RequestPartition{\n\t\t\t\tPartitionIndex: int32(assignment.PartitionID),\n\t\t\t\tReplicas: replicas,\n\t\t\t},\n\t\t)\n\t}\n\n\tapiReq := &alterpartitionreassignments.Request{\n\t\tTimeoutMs: int32(req.Timeout.Milliseconds()),\n\t}\n\n\tfor _, apiTopic := range apiTopicMap {\n\t\tapiReq.Topics = append(apiReq.Topics, *apiTopic)\n\t}\n\n\tprotoResp, err := c.roundTrip(\n\t\tctx,\n\t\treq.Addr,\n\t\tapiReq,\n\t)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tapiResp := protoResp.(*alterpartitionreassignments.Response)\n\n\tresp := &AlterPartitionReassignmentsResponse{\n\t\tError: makeError(apiResp.ErrorCode, apiResp.ErrorMessage),\n\t}\n\n\tfor _, topicResult := range apiResp.Results {\n\t\tfor _, partitionResult := range topicResult.Partitions {\n\t\t\tresp.PartitionResults = append(\n\t\t\t\tresp.PartitionResults,\n\t\t\t\tAlterPartitionReassignmentsResponsePartitionResult{\n\t\t\t\t\tTopic: topicResult.Name,\n\t\t\t\t\tPartitionID: int(partitionResult.PartitionIndex),\n\t\t\t\t\tError: makeError(partitionResult.ErrorCode, partitionResult.ErrorMessage),\n\t\t\t\t},\n\t\t\t)\n\t\t}\n\t}\n\n\treturn resp, nil\n}\n" }, { "path": "alterpartitionreassignments_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n\t\"time\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc TestClientAlterPartitionReassignments(t *testing.T) {\n\tif !ktesting.KafkaIsAtLeast(\"2.4.0\") {\n\t\treturn\n\t}\n\n\tctx := context.Background()\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\ttopic := makeTopic()\n\tcreateTopic(t, topic, 2)\n\tdefer deleteTopic(t, topic)\n\n\t// Local kafka only has 1 broker, so any partition reassignments are really no-ops.\n\tresp, err := client.AlterPartitionReassignments(\n\t\tctx,\n\t\t&AlterPartitionReassignmentsRequest{\n\t\t\tTopic: topic,\n\t\t\tAssignments: []AlterPartitionReassignmentsRequestAssignment{\n\t\t\t\t{\n\t\t\t\t\tPartitionID: 0,\n\t\t\t\t\tBrokerIDs: []int{1},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tPartitionID: 1,\n\t\t\t\t\tBrokerIDs: []int{1},\n\t\t\t\t},\n\t\t\t},\n\t\t\tTimeout: 5 * time.Second,\n\t\t},\n\t)\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif resp.Error != nil {\n\t\tt.Error(\n\t\t\t\"Unexpected error in response\",\n\t\t\t\"expected\", nil,\n\t\t\t\"got\", resp.Error,\n\t\t)\n\t}\n\tif len(resp.PartitionResults) != 2 {\n\t\tt.Error(\n\t\t\t\"Unexpected length of partition results\",\n\t\t\t\"expected\", 2,\n\t\t\t\"got\", len(resp.PartitionResults),\n\t\t)\n\t}\n}\n\nfunc TestClientAlterPartitionReassignmentsMultiTopics(t *testing.T) {\n\tif !ktesting.KafkaIsAtLeast(\"2.4.0\") {\n\t\treturn\n\t}\n\n\tctx := context.Background()\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\ttopic1 := makeTopic()\n\ttopic2 := makeTopic()\n\tcreateTopic(t, topic1, 2)\n\tcreateTopic(t, topic2, 2)\n\tdefer func() {\n\t\tdeleteTopic(t, topic1)\n\t\tdeleteTopic(t, topic2)\n\t}()\n\n\t// Local kafka only has 1 broker, so any partition reassignments are really no-ops.\n\tresp, err := client.AlterPartitionReassignments(\n\t\tctx,\n\t\t&AlterPartitionReassignmentsRequest{\n\t\t\tAssignments: []AlterPartitionReassignmentsRequestAssignment{\n\t\t\t\t{\n\t\t\t\t\tTopic: topic1,\n\t\t\t\t\tPartitionID: 0,\n\t\t\t\t\tBrokerIDs: []int{1},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tTopic: topic1,\n\t\t\t\t\tPartitionID: 1,\n\t\t\t\t\tBrokerIDs: []int{1},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tTopic: topic2,\n\t\t\t\t\tPartitionID: 0,\n\t\t\t\t\tBrokerIDs: []int{1},\n\t\t\t\t},\n\t\t\t},\n\t\t\tTimeout: 5 * time.Second,\n\t\t},\n\t)\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif resp.Error != nil {\n\t\tt.Error(\n\t\t\t\"Unexpected error in response\",\n\t\t\t\"expected\", nil,\n\t\t\t\"got\", resp.Error,\n\t\t)\n\t}\n\tif len(resp.PartitionResults) != 3 {\n\t\tt.Error(\n\t\t\t\"Unexpected length of partition results\",\n\t\t\t\"expected\", 3,\n\t\t\t\"got\", len(resp.PartitionResults),\n\t\t)\n\t}\n}\n" }, { "path": "alteruserscramcredentials.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/alteruserscramcredentials\"\n)\n\n// AlterUserScramCredentialsRequest represents a request sent to a kafka broker to\n// alter user scram credentials.\ntype AlterUserScramCredentialsRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// List of credentials to delete.\n\tDeletions []UserScramCredentialsDeletion\n\n\t// List of credentials to upsert.\n\tUpsertions []UserScramCredentialsUpsertion\n}\n\ntype ScramMechanism int8\n\nconst (\n\tScramMechanismUnknown ScramMechanism = iota // 0\n\tScramMechanismSha256 // 1\n\tScramMechanismSha512 // 2\n)\n\ntype UserScramCredentialsDeletion struct {\n\tName string\n\tMechanism ScramMechanism\n}\n\ntype UserScramCredentialsUpsertion struct {\n\tName string\n\tMechanism ScramMechanism\n\tIterations int\n\tSalt []byte\n\tSaltedPassword []byte\n}\n\n// AlterUserScramCredentialsResponse represents a response from a kafka broker to an alter user\n// credentials request.\ntype AlterUserScramCredentialsResponse struct {\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// List of altered user scram credentials.\n\tResults []AlterUserScramCredentialsResponseUser\n}\n\ntype AlterUserScramCredentialsResponseUser struct {\n\tUser string\n\tError error\n}\n\n// AlterUserScramCredentials sends user scram credentials alteration request to a kafka broker and returns\n// the response.\nfunc (c *Client) AlterUserScramCredentials(ctx context.Context, req *AlterUserScramCredentialsRequest) (*AlterUserScramCredentialsResponse, error) {\n\tdeletions := make([]alteruserscramcredentials.RequestUserScramCredentialsDeletion, len(req.Deletions))\n\tupsertions := make([]alteruserscramcredentials.RequestUserScramCredentialsUpsertion, len(req.Upsertions))\n\n\tfor deletionIdx, deletion := range req.Deletions {\n\t\tdeletions[deletionIdx] = alteruserscramcredentials.RequestUserScramCredentialsDeletion{\n\t\t\tName: deletion.Name,\n\t\t\tMechanism: int8(deletion.Mechanism),\n\t\t}\n\t}\n\n\tfor upsertionIdx, upsertion := range req.Upsertions {\n\t\tupsertions[upsertionIdx] = alteruserscramcredentials.RequestUserScramCredentialsUpsertion{\n\t\t\tName: upsertion.Name,\n\t\t\tMechanism: int8(upsertion.Mechanism),\n\t\t\tIterations: int32(upsertion.Iterations),\n\t\t\tSalt: upsertion.Salt,\n\t\t\tSaltedPassword: upsertion.SaltedPassword,\n\t\t}\n\t}\n\n\tm, err := c.roundTrip(ctx, req.Addr, &alteruserscramcredentials.Request{\n\t\tDeletions: deletions,\n\t\tUpsertions: upsertions,\n\t})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).AlterUserScramCredentials: %w\", err)\n\t}\n\n\tres := m.(*alteruserscramcredentials.Response)\n\tresponseEntries := make([]AlterUserScramCredentialsResponseUser, len(res.Results))\n\n\tfor responseIdx, responseResult := range res.Results {\n\t\tresponseEntries[responseIdx] = AlterUserScramCredentialsResponseUser{\n\t\t\tUser: responseResult.User,\n\t\t\tError: makeError(responseResult.ErrorCode, responseResult.ErrorMessage),\n\t\t}\n\t}\n\tret := &AlterUserScramCredentialsResponse{\n\t\tThrottle: makeDuration(res.ThrottleTimeMs),\n\t\tResults: responseEntries,\n\t}\n\n\treturn ret, nil\n}\n" }, { "path": "alteruserscramcredentials_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc TestAlterUserScramCredentials(t *testing.T) {\n\t// https://issues.apache.org/jira/browse/KAFKA-10259\n\tif !ktesting.KafkaIsAtLeast(\"2.7.0\") {\n\t\treturn\n\t}\n\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\tname := makeTopic()\n\n\tcreateRes, err := client.AlterUserScramCredentials(context.Background(), &AlterUserScramCredentialsRequest{\n\t\tUpsertions: []UserScramCredentialsUpsertion{\n\t\t\t{\n\t\t\t\tName: name,\n\t\t\t\tMechanism: ScramMechanismSha512,\n\t\t\t\tIterations: 15000,\n\t\t\t\tSalt: []byte(\"my-salt\"),\n\t\t\t\tSaltedPassword: []byte(\"my-salted-password\"),\n\t\t\t},\n\t\t},\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(createRes.Results) != 1 {\n\t\tt.Fatalf(\"expected 1 createResult; got %d\", len(createRes.Results))\n\t}\n\n\tif createRes.Results[0].User != name {\n\t\tt.Fatalf(\"expected createResult with user: %s, got %s\", name, createRes.Results[0].User)\n\t}\n\n\tif createRes.Results[0].Error != nil {\n\t\tt.Fatalf(\"didn't expect an error in createResult, got %v\", createRes.Results[0].Error)\n\t}\n\n\tdeleteRes, err := client.AlterUserScramCredentials(context.Background(), &AlterUserScramCredentialsRequest{\n\t\tDeletions: []UserScramCredentialsDeletion{\n\t\t\t{\n\t\t\t\tName: name,\n\t\t\t\tMechanism: ScramMechanismSha512,\n\t\t\t},\n\t\t},\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(deleteRes.Results) != 1 {\n\t\tt.Fatalf(\"expected 1 deleteResult; got %d\", len(deleteRes.Results))\n\t}\n\n\tif deleteRes.Results[0].User != name {\n\t\tt.Fatalf(\"expected deleteResult with user: %s, got %s\", name, deleteRes.Results[0].User)\n\t}\n\n\tif deleteRes.Results[0].Error != nil {\n\t\tt.Fatalf(\"didn't expect an error in deleteResult, got %v\", deleteRes.Results[0].Error)\n\t}\n}\n" }, { "path": "apiversions.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"net\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n\t\"github.com/segmentio/kafka-go/protocol/apiversions\"\n)\n\n// ApiVersionsRequest is a request to the ApiVersions API.\ntype ApiVersionsRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n}\n\n// ApiVersionsResponse is a response from the ApiVersions API.\ntype ApiVersionsResponse struct {\n\t// Error is set to a non-nil value if an error was encountered.\n\tError error\n\n\t// ApiKeys contains the specific details of each supported API.\n\tApiKeys []ApiVersionsResponseApiKey\n}\n\n// ApiVersionsResponseApiKey includes the details of which versions are supported for a single API.\ntype ApiVersionsResponseApiKey struct {\n\t// ApiKey is the ID of the API.\n\tApiKey int\n\n\t// ApiName is a human-friendly description of the API.\n\tApiName string\n\n\t// MinVersion is the minimum API version supported by the broker.\n\tMinVersion int\n\n\t// MaxVersion is the maximum API version supported by the broker.\n\tMaxVersion int\n}\n\nfunc (c *Client) ApiVersions(\n\tctx context.Context,\n\treq *ApiVersionsRequest,\n) (*ApiVersionsResponse, error) {\n\tapiReq := &apiversions.Request{}\n\tprotoResp, err := c.roundTrip(\n\t\tctx,\n\t\treq.Addr,\n\t\tapiReq,\n\t)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tapiResp := protoResp.(*apiversions.Response)\n\n\tresp := &ApiVersionsResponse{\n\t\tError: makeError(apiResp.ErrorCode, \"\"),\n\t}\n\tfor _, apiKey := range apiResp.ApiKeys {\n\t\tresp.ApiKeys = append(\n\t\t\tresp.ApiKeys,\n\t\t\tApiVersionsResponseApiKey{\n\t\t\t\tApiKey: int(apiKey.ApiKey),\n\t\t\t\tApiName: protocol.ApiKey(apiKey.ApiKey).String(),\n\t\t\t\tMinVersion: int(apiKey.MinVersion),\n\t\t\t\tMaxVersion: int(apiKey.MaxVersion),\n\t\t\t},\n\t\t)\n\t}\n\n\treturn resp, err\n}\n" }, { "path": "apiversions_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n)\n\nfunc TestClientApiVersions(t *testing.T) {\n\tctx := context.Background()\n\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\tresp, err := client.ApiVersions(ctx, &ApiVersionsRequest{})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif resp.Error != nil {\n\t\tt.Error(\n\t\t\t\"Unexpected error in response\",\n\t\t\t\"expected\", nil,\n\t\t\t\"got\", resp.Error,\n\t\t)\n\t}\n\n\tif len(resp.ApiKeys) == 0 {\n\t\tt.Error(\n\t\t\t\"Unexpected apiKeys length\",\n\t\t\t\"expected greater than\", 0,\n\t\t\t\"got\", 0,\n\t\t)\n\t}\n}\n" }, { "path": "balancer.go", "content": "package kafka\n\nimport (\n\t\"hash\"\n\t\"hash/crc32\"\n\t\"hash/fnv\"\n\t\"math/rand\"\n\t\"sort\"\n\t\"sync\"\n)\n\n// The Balancer interface provides an abstraction of the message distribution\n// logic used by Writer instances to route messages to the partitions available\n// on a kafka cluster.\n//\n// Balancers must be safe to use concurrently from multiple goroutines.\ntype Balancer interface {\n\t// Balance receives a message and a set of available partitions and\n\t// returns the partition number that the message should be routed to.\n\t//\n\t// An application should refrain from using a balancer to manage multiple\n\t// sets of partitions (from different topics for examples), use one balancer\n\t// instance for each partition set, so the balancer can detect when the\n\t// partitions change and assume that the kafka topic has been rebalanced.\n\tBalance(msg Message, partitions ...int) (partition int)\n}\n\n// BalancerFunc is an implementation of the Balancer interface that makes it\n// possible to use regular functions to distribute messages across partitions.\ntype BalancerFunc func(Message, ...int) int\n\n// Balance calls f, satisfies the Balancer interface.\nfunc (f BalancerFunc) Balance(msg Message, partitions ...int) int {\n\treturn f(msg, partitions...)\n}\n\n// RoundRobin is an Balancer implementation that equally distributes messages\n// across all available partitions. It can take an optional chunk size to send\n// ChunkSize messages to the same partition before moving to the next partition.\n// This can be used to improve batch sizes.\ntype RoundRobin struct {\n\tChunkSize int\n\t// Use a 32 bits integer so RoundRobin values don't need to be aligned to\n\t// apply increments.\n\tcounter uint32\n\n\tmutex sync.Mutex\n}\n\n// Balance satisfies the Balancer interface.\nfunc (rr *RoundRobin) Balance(msg Message, partitions ...int) int {\n\treturn rr.balance(partitions)\n}\n\nfunc (rr *RoundRobin) balance(partitions []int) int {\n\trr.mutex.Lock()\n\tdefer rr.mutex.Unlock()\n\n\tif rr.ChunkSize < 1 {\n\t\trr.ChunkSize = 1\n\t}\n\n\tlength := len(partitions)\n\tcounterNow := rr.counter\n\toffset := int(counterNow / uint32(rr.ChunkSize))\n\trr.counter++\n\treturn partitions[offset%length]\n}\n\n// LeastBytes is a Balancer implementation that routes messages to the partition\n// that has received the least amount of data.\n//\n// Note that no coordination is done between multiple producers, having good\n// balancing relies on the fact that each producer using a LeastBytes balancer\n// should produce well balanced messages.\ntype LeastBytes struct {\n\tmutex sync.Mutex\n\tcounters []leastBytesCounter\n}\n\ntype leastBytesCounter struct {\n\tpartition int\n\tbytes uint64\n}\n\n// Balance satisfies the Balancer interface.\nfunc (lb *LeastBytes) Balance(msg Message, partitions ...int) int {\n\tlb.mutex.Lock()\n\tdefer lb.mutex.Unlock()\n\n\t// partitions change\n\tif len(partitions) != len(lb.counters) {\n\t\tlb.counters = lb.makeCounters(partitions...)\n\t}\n\n\tminBytes := lb.counters[0].bytes\n\tminIndex := 0\n\n\tfor i, c := range lb.counters[1:] {\n\t\tif c.bytes < minBytes {\n\t\t\tminIndex = i + 1\n\t\t\tminBytes = c.bytes\n\t\t}\n\t}\n\n\tc := &lb.counters[minIndex]\n\tc.bytes += uint64(len(msg.Key)) + uint64(len(msg.Value))\n\treturn c.partition\n}\n\nfunc (lb *LeastBytes) makeCounters(partitions ...int) (counters []leastBytesCounter) {\n\tcounters = make([]leastBytesCounter, len(partitions))\n\n\tfor i, p := range partitions {\n\t\tcounters[i].partition = p\n\t}\n\n\tsort.Slice(counters, func(i int, j int) bool {\n\t\treturn counters[i].partition < counters[j].partition\n\t})\n\treturn\n}\n\nvar (\n\tfnv1aPool = &sync.Pool{\n\t\tNew: func() interface{} {\n\t\t\treturn fnv.New32a()\n\t\t},\n\t}\n)\n\n// Hash is a Balancer that uses the provided hash function to determine which\n// partition to route messages to. This ensures that messages with the same key\n// are routed to the same partition.\n//\n// The logic to calculate the partition is:\n//\n//\thasher.Sum32() % len(partitions) => partition\n//\n// By default, Hash uses the FNV-1a algorithm. This is the same algorithm used\n// by the Sarama Producer and ensures that messages produced by kafka-go will\n// be delivered to the same topics that the Sarama producer would be delivered to.\ntype Hash struct {\n\trr RoundRobin\n\tHasher hash.Hash32\n\n\t// lock protects Hasher while calculating the hash code. It is assumed that\n\t// the Hasher field is read-only once the Balancer is created, so as a\n\t// performance optimization, reads of the field are not protected.\n\tlock sync.Mutex\n}\n\nfunc (h *Hash) Balance(msg Message, partitions ...int) int {\n\tif msg.Key == nil {\n\t\treturn h.rr.Balance(msg, partitions...)\n\t}\n\n\thasher := h.Hasher\n\tif hasher != nil {\n\t\th.lock.Lock()\n\t\tdefer h.lock.Unlock()\n\t} else {\n\t\thasher = fnv1aPool.Get().(hash.Hash32)\n\t\tdefer fnv1aPool.Put(hasher)\n\t}\n\n\thasher.Reset()\n\tif _, err := hasher.Write(msg.Key); err != nil {\n\t\tpanic(err)\n\t}\n\n\t// uses same algorithm that Sarama's hashPartitioner uses\n\t// note the type conversions here. if the uint32 hash code is not cast to\n\t// an int32, we do not get the same result as sarama.\n\tpartition := int32(hasher.Sum32()) % int32(len(partitions))\n\tif partition < 0 {\n\t\tpartition = -partition\n\t}\n\n\treturn int(partition)\n}\n\n// ReferenceHash is a Balancer that uses the provided hash function to determine which\n// partition to route messages to. This ensures that messages with the same key\n// are routed to the same partition.\n//\n// The logic to calculate the partition is:\n//\n//\t(int32(hasher.Sum32()) & 0x7fffffff) % len(partitions) => partition\n//\n// By default, ReferenceHash uses the FNV-1a algorithm. This is the same algorithm as\n// the Sarama NewReferenceHashPartitioner and ensures that messages produced by kafka-go will\n// be delivered to the same topics that the Sarama producer would be delivered to.\ntype ReferenceHash struct {\n\trr randomBalancer\n\tHasher hash.Hash32\n\n\t// lock protects Hasher while calculating the hash code. It is assumed that\n\t// the Hasher field is read-only once the Balancer is created, so as a\n\t// performance optimization, reads of the field are not protected.\n\tlock sync.Mutex\n}\n\nfunc (h *ReferenceHash) Balance(msg Message, partitions ...int) int {\n\tif msg.Key == nil {\n\t\treturn h.rr.Balance(msg, partitions...)\n\t}\n\n\thasher := h.Hasher\n\tif hasher != nil {\n\t\th.lock.Lock()\n\t\tdefer h.lock.Unlock()\n\t} else {\n\t\thasher = fnv1aPool.Get().(hash.Hash32)\n\t\tdefer fnv1aPool.Put(hasher)\n\t}\n\n\thasher.Reset()\n\tif _, err := hasher.Write(msg.Key); err != nil {\n\t\tpanic(err)\n\t}\n\n\t// uses the same algorithm as the Sarama's referenceHashPartitioner.\n\t// note the type conversions here. if the uint32 hash code is not cast to\n\t// an int32, we do not get the same result as sarama.\n\tpartition := (int32(hasher.Sum32()) & 0x7fffffff) % int32(len(partitions))\n\treturn int(partition)\n}\n\ntype randomBalancer struct {\n\tmock int // mocked return value, used for testing\n}\n\nfunc (b randomBalancer) Balance(msg Message, partitions ...int) (partition int) {\n\tif b.mock != 0 {\n\t\treturn b.mock\n\t}\n\treturn partitions[rand.Int()%len(partitions)]\n}\n\n// CRC32Balancer is a Balancer that uses the CRC32 hash function to determine\n// which partition to route messages to. This ensures that messages with the\n// same key are routed to the same partition. This balancer is compatible with\n// the built-in hash partitioners in librdkafka and the language bindings that\n// are built on top of it, including the\n// github.com/confluentinc/confluent-kafka-go Go package.\n//\n// With the Consistent field false (default), this partitioner is equivalent to\n// the \"consistent_random\" setting in librdkafka. When Consistent is true, this\n// partitioner is equivalent to the \"consistent\" setting. The latter will hash\n// empty or nil keys into the same partition.\n//\n// Unless you are absolutely certain that all your messages will have keys, it's\n// best to leave the Consistent flag off. Otherwise, you run the risk of\n// creating a very hot partition.\ntype CRC32Balancer struct {\n\tConsistent bool\n\trandom randomBalancer\n}\n\nfunc (b CRC32Balancer) Balance(msg Message, partitions ...int) (partition int) {\n\t// NOTE: the crc32 balancers in librdkafka don't differentiate between nil\n\t// and empty keys. both cases are treated as unset.\n\tif len(msg.Key) == 0 && !b.Consistent {\n\t\treturn b.random.Balance(msg, partitions...)\n\t}\n\n\tidx := crc32.ChecksumIEEE(msg.Key) % uint32(len(partitions))\n\treturn partitions[idx]\n}\n\n// Murmur2Balancer is a Balancer that uses the Murmur2 hash function to\n// determine which partition to route messages to. This ensures that messages\n// with the same key are routed to the same partition. This balancer is\n// compatible with the partitioner used by the Java library and by librdkafka's\n// \"murmur2\" and \"murmur2_random\" partitioners.\n//\n// With the Consistent field false (default), this partitioner is equivalent to\n// the \"murmur2_random\" setting in librdkafka. When Consistent is true, this\n// partitioner is equivalent to the \"murmur2\" setting. The latter will hash\n// nil keys into the same partition. Empty, non-nil keys are always hashed to\n// the same partition regardless of configuration.\n//\n// Unless you are absolutely certain that all your messages will have keys, it's\n// best to leave the Consistent flag off. Otherwise, you run the risk of\n// creating a very hot partition.\n//\n// Note that the librdkafka documentation states that the \"murmur2_random\" is\n// functionally equivalent to the default Java partitioner. That's because the\n// Java partitioner will use a round robin balancer instead of random on nil\n// keys. We choose librdkafka's implementation because it arguably has a larger\n// install base.\ntype Murmur2Balancer struct {\n\tConsistent bool\n\trandom randomBalancer\n}\n\nfunc (b Murmur2Balancer) Balance(msg Message, partitions ...int) (partition int) {\n\t// NOTE: the murmur2 balancers in java and librdkafka treat a nil key as\n\t// non-existent while treating an empty slice as a defined value.\n\tif msg.Key == nil && !b.Consistent {\n\t\treturn b.random.Balance(msg, partitions...)\n\t}\n\n\tidx := (murmur2(msg.Key) & 0x7fffffff) % uint32(len(partitions))\n\treturn partitions[idx]\n}\n\n// Go port of the Java library's murmur2 function.\n// https://github.com/apache/kafka/blob/1.0/clients/src/main/java/org/apache/kafka/common/utils/Utils.java#L353\nfunc murmur2(data []byte) uint32 {\n\tlength := len(data)\n\tconst (\n\t\tseed uint32 = 0x9747b28c\n\t\t// 'm' and 'r' are mixing constants generated offline.\n\t\t// They're not really 'magic', they just happen to work well.\n\t\tm = 0x5bd1e995\n\t\tr = 24\n\t)\n\n\t// Initialize the hash to a random value\n\th := seed ^ uint32(length)\n\tlength4 := length / 4\n\n\tfor i := 0; i < length4; i++ {\n\t\ti4 := i * 4\n\t\tk := (uint32(data[i4+0]) & 0xff) + ((uint32(data[i4+1]) & 0xff) << 8) + ((uint32(data[i4+2]) & 0xff) << 16) + ((uint32(data[i4+3]) & 0xff) << 24)\n\t\tk *= m\n\t\tk ^= k >> r\n\t\tk *= m\n\t\th *= m\n\t\th ^= k\n\t}\n\n\t// Handle the last few bytes of the input array\n\textra := length % 4\n\tif extra >= 3 {\n\t\th ^= (uint32(data[(length & ^3)+2]) & 0xff) << 16\n\t}\n\tif extra >= 2 {\n\t\th ^= (uint32(data[(length & ^3)+1]) & 0xff) << 8\n\t}\n\tif extra >= 1 {\n\t\th ^= uint32(data[length & ^3]) & 0xff\n\t\th *= m\n\t}\n\n\th ^= h >> 13\n\th *= m\n\th ^= h >> 15\n\n\treturn h\n}\n" }, { "path": "balancer_test.go", "content": "package kafka\n\nimport (\n\t\"fmt\"\n\t\"hash\"\n\t\"hash/crc32\"\n\t\"testing\"\n)\n\nfunc TestHashBalancer(t *testing.T) {\n\ttestCases := map[string]struct {\n\t\tKey []byte\n\t\tHasher hash.Hash32\n\t\tPartitions []int\n\t\tPartition int\n\t}{\n\t\t\"nil\": {\n\t\t\tKey: nil,\n\t\t\tPartitions: []int{0, 1, 2},\n\t\t\tPartition: 0,\n\t\t},\n\t\t\"partition-0\": {\n\t\t\tKey: []byte(\"blah\"),\n\t\t\tPartitions: []int{0, 1},\n\t\t\tPartition: 0,\n\t\t},\n\t\t\"partition-1\": {\n\t\t\tKey: []byte(\"blah\"),\n\t\t\tPartitions: []int{0, 1, 2},\n\t\t\tPartition: 1,\n\t\t},\n\t\t\"partition-2\": {\n\t\t\tKey: []byte(\"boop\"),\n\t\t\tPartitions: []int{0, 1, 2},\n\t\t\tPartition: 2,\n\t\t},\n\t\t\"custom hash\": {\n\t\t\tKey: []byte(\"boop\"),\n\t\t\tHasher: crc32.NewIEEE(),\n\t\t\tPartitions: []int{0, 1, 2},\n\t\t\tPartition: 1,\n\t\t},\n\t\t// in a previous version, this test would select a different partition\n\t\t// than sarama's hash partitioner.\n\t\t\"hash code with MSB set\": {\n\t\t\tKey: []byte(\"20\"),\n\t\t\tPartitions: []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},\n\t\t\tPartition: 1,\n\t\t},\n\t}\n\n\tfor label, test := range testCases {\n\t\tt.Run(label, func(t *testing.T) {\n\t\t\tmsg := Message{Key: test.Key}\n\t\t\th := Hash{\n\t\t\t\tHasher: test.Hasher,\n\t\t\t}\n\t\t\tpartition := h.Balance(msg, test.Partitions...)\n\t\t\tif partition != test.Partition {\n\t\t\t\tt.Errorf(\"expected %v; got %v\", test.Partition, partition)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestReferenceHashBalancer(t *testing.T) {\n\ttestCases := map[string]struct {\n\t\tKey []byte\n\t\tHasher hash.Hash32\n\t\tPartitions []int\n\t\tPartition int\n\t\tRndBalancerResult int\n\t}{\n\t\t\"nil\": {\n\t\t\tKey: nil, // nil key means random partition\n\t\t\tPartitions: []int{0, 1, 2},\n\t\t\tPartition: 123,\n\t\t\tRndBalancerResult: 123,\n\t\t},\n\t\t\"partition-0\": {\n\t\t\tKey: []byte(\"blah\"),\n\t\t\tPartitions: []int{0, 1},\n\t\t\tPartition: 0,\n\t\t},\n\t\t\"partition-1\": {\n\t\t\tKey: []byte(\"blah\"),\n\t\t\tPartitions: []int{0, 1, 2},\n\t\t\tPartition: 1,\n\t\t},\n\t\t\"partition-2\": {\n\t\t\tKey: []byte(\"castle\"),\n\t\t\tPartitions: []int{0, 1, 2},\n\t\t\tPartition: 2,\n\t\t},\n\t\t\"custom hash\": {\n\t\t\tKey: []byte(\"boop\"),\n\t\t\tHasher: crc32.NewIEEE(),\n\t\t\tPartitions: []int{0, 1, 2},\n\t\t\tPartition: 1,\n\t\t},\n\t\t\"hash code with MSB set\": {\n\t\t\tKey: []byte(\"20\"),\n\t\t\tPartitions: []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},\n\t\t\tPartition: 15,\n\t\t},\n\t}\n\n\tfor label, test := range testCases {\n\t\tt.Run(label, func(t *testing.T) {\n\t\t\tvar rr randomBalancer\n\t\t\tif test.Key == nil {\n\t\t\t\trr.mock = test.RndBalancerResult\n\t\t\t}\n\n\t\t\tmsg := Message{Key: test.Key}\n\t\t\th := ReferenceHash{Hasher: test.Hasher, rr: rr}\n\t\t\tpartition := h.Balance(msg, test.Partitions...)\n\t\t\tif partition != test.Partition {\n\t\t\t\tt.Errorf(\"expected %v; got %v\", test.Partition, partition)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestCRC32Balancer(t *testing.T) {\n\t// These tests are taken from the default \"consistent_random\" partitioner from\n\t// https://github.com/edenhill/librdkafka/blob/master/tests/0048-partitioner.c\n\tpartitionCount := 17\n\tvar partitions []int\n\tfor i := 0; i < partitionCount; i++ {\n\t\tpartitions = append(partitions, i*i)\n\t}\n\n\ttestCases := map[string]struct {\n\t\tKey []byte\n\t\tPartitions []int\n\t\tPartition int\n\t}{\n\t\t\"nil\": {\n\t\t\tKey: nil,\n\t\t\tPartitions: partitions,\n\t\t\tPartition: -1,\n\t\t},\n\t\t\"empty\": {\n\t\t\tKey: []byte{},\n\t\t\tPartitions: partitions,\n\t\t\tPartition: -1,\n\t\t},\n\t\t\"unaligned\": {\n\t\t\tKey: []byte(\"23456\"),\n\t\t\tPartitions: partitions,\n\t\t\tPartition: partitions[0xb1b451d7%partitionCount],\n\t\t},\n\t\t\"long key\": {\n\t\t\tKey: []byte(\"this is another string with more length to it perhaps\"),\n\t\t\tPartitions: partitions,\n\t\t\tPartition: partitions[0xb0150df7%partitionCount],\n\t\t},\n\t\t\"short key\": {\n\t\t\tKey: []byte(\"hejsan\"),\n\t\t\tPartitions: partitions,\n\t\t\tPartition: partitions[0xd077037e%partitionCount],\n\t\t},\n\t}\n\n\tt.Run(\"default\", func(t *testing.T) {\n\t\tfor label, test := range testCases {\n\t\t\tt.Run(label, func(t *testing.T) {\n\t\t\t\tb := CRC32Balancer{}\n\t\t\t\tb.random.mock = -1\n\n\t\t\t\tmsg := Message{Key: test.Key}\n\t\t\t\tpartition := b.Balance(msg, test.Partitions...)\n\t\t\t\tif partition != test.Partition {\n\t\t\t\t\tt.Errorf(\"expected %v; got %v\", test.Partition, partition)\n\t\t\t\t}\n\t\t\t})\n\t\t}\n\t})\n\n\tt.Run(\"consistent\", func(t *testing.T) {\n\t\tb := CRC32Balancer{Consistent: true}\n\t\tb.random.mock = -1\n\n\t\tp := b.Balance(Message{}, partitions...)\n\t\tif p < 0 {\n\t\t\tt.Fatal(\"should not have gotten a random partition\")\n\t\t}\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tif p != b.Balance(Message{}, partitions...) {\n\t\t\t\tt.Fatal(\"nil key should always hash consistently\")\n\t\t\t}\n\t\t\tif p != b.Balance(Message{Key: []byte{}}, partitions...) {\n\t\t\t\tt.Fatal(\"empty key should always hash consistently and have same result as nil key\")\n\t\t\t}\n\t\t}\n\t})\n}\n\nfunc TestMurmur2(t *testing.T) {\n\t// These tests are taken from the \"murmur2\" implementation from\n\t// https://github.com/edenhill/librdkafka/blob/master/src/rdmurmur2.c\n\ttestCases := []struct {\n\t\tKey []byte\n\t\tJavaMurmur2Result uint32\n\t}{\n\t\t{Key: []byte(\"kafka\"), JavaMurmur2Result: 0xd067cf64},\n\t\t{Key: []byte(\"giberish123456789\"), JavaMurmur2Result: 0x8f552b0c},\n\t\t{Key: []byte(\"1234\"), JavaMurmur2Result: 0x9fc97b14},\n\t\t{Key: []byte(\"234\"), JavaMurmur2Result: 0xe7c009ca},\n\t\t{Key: []byte(\"34\"), JavaMurmur2Result: 0x873930da},\n\t\t{Key: []byte(\"4\"), JavaMurmur2Result: 0x5a4b5ca1},\n\t\t{Key: []byte(\"PreAmbleWillBeRemoved,ThePrePartThatIs\"), JavaMurmur2Result: 0x78424f1c},\n\t\t{Key: []byte(\"reAmbleWillBeRemoved,ThePrePartThatIs\"), JavaMurmur2Result: 0x4a62b377},\n\t\t{Key: []byte(\"eAmbleWillBeRemoved,ThePrePartThatIs\"), JavaMurmur2Result: 0xe0e4e09e},\n\t\t{Key: []byte(\"AmbleWillBeRemoved,ThePrePartThatIs\"), JavaMurmur2Result: 0x62b8b43f},\n\t\t{Key: []byte(\"\"), JavaMurmur2Result: 0x106e08d9},\n\t\t{Key: nil, JavaMurmur2Result: 0x106e08d9},\n\t}\n\n\tfor _, test := range testCases {\n\t\tt.Run(fmt.Sprintf(\"key:%s\", test.Key), func(t *testing.T) {\n\t\t\tgot := murmur2(test.Key)\n\t\t\tif got != test.JavaMurmur2Result {\n\t\t\t\tt.Errorf(\"expected %v; got %v\", test.JavaMurmur2Result, got)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestMurmur2Balancer(t *testing.T) {\n\t// These tests are taken from the \"murmur2_random\" partitioner from\n\t// https://github.com/edenhill/librdkafka/blob/master/tests/0048-partitioner.c\n\tpartitionCount := 17\n\tlibrdkafkaPartitions := make([]int, partitionCount)\n\tfor i := 0; i < partitionCount; i++ {\n\t\tlibrdkafkaPartitions[i] = i * i\n\t}\n\n\t// These tests are taken from the Murmur2Partitioner Python class from\n\t// https://github.com/dpkp/kafka-python/blob/master/test/test_partitioner.py\n\tpythonPartitions := make([]int, 1000)\n\tfor i := 0; i < 1000; i++ {\n\t\tpythonPartitions[i] = i\n\t}\n\n\ttestCases := map[string]struct {\n\t\tKey []byte\n\t\tPartitions []int\n\t\tPartition int\n\t}{\n\t\t\"librdkafka-nil\": {\n\t\t\tKey: nil,\n\t\t\tPartitions: librdkafkaPartitions,\n\t\t\tPartition: 123,\n\t\t},\n\t\t\"librdkafka-empty\": {\n\t\t\tKey: []byte{},\n\t\t\tPartitions: librdkafkaPartitions,\n\t\t\tPartition: librdkafkaPartitions[0x106e08d9%partitionCount],\n\t\t},\n\t\t\"librdkafka-unaligned\": {\n\t\t\tKey: []byte(\"23456\"),\n\t\t\tPartitions: librdkafkaPartitions,\n\t\t\tPartition: librdkafkaPartitions[0x058d780f%partitionCount],\n\t\t},\n\t\t\"librdkafka-long key\": {\n\t\t\tKey: []byte(\"this is another string with more length to it perhaps\"),\n\t\t\tPartitions: librdkafkaPartitions,\n\t\t\tPartition: librdkafkaPartitions[0x4f7703da%partitionCount],\n\t\t},\n\t\t\"librdkafka-short key\": {\n\t\t\tKey: []byte(\"hejsan\"),\n\t\t\tPartitions: librdkafkaPartitions,\n\t\t\tPartition: librdkafkaPartitions[0x5ec19395%partitionCount],\n\t\t},\n\t\t\"python-empty\": {\n\t\t\tKey: []byte(\"\"),\n\t\t\tPartitions: pythonPartitions,\n\t\t\tPartition: 681,\n\t\t},\n\t\t\"python-a\": {\n\t\t\tKey: []byte(\"a\"),\n\t\t\tPartitions: pythonPartitions,\n\t\t\tPartition: 524,\n\t\t},\n\t\t\"python-ab\": {\n\t\t\tKey: []byte(\"ab\"),\n\t\t\tPartitions: pythonPartitions,\n\t\t\tPartition: 434,\n\t\t},\n\t\t\"python-abc\": {\n\t\t\tKey: []byte(\"abc\"),\n\t\t\tPartitions: pythonPartitions,\n\t\t\tPartition: 107,\n\t\t},\n\t\t\"python-123456789\": {\n\t\t\tKey: []byte(\"123456789\"),\n\t\t\tPartitions: pythonPartitions,\n\t\t\tPartition: 566,\n\t\t},\n\t\t\"python-\\x00 \": {\n\t\t\tKey: []byte{0, 32},\n\t\t\tPartitions: pythonPartitions,\n\t\t\tPartition: 742,\n\t\t},\n\t}\n\n\tt.Run(\"default\", func(t *testing.T) {\n\t\tfor label, test := range testCases {\n\t\t\tt.Run(label, func(t *testing.T) {\n\t\t\t\tb := Murmur2Balancer{}\n\t\t\t\tb.random.mock = 123\n\n\t\t\t\tmsg := Message{Key: test.Key}\n\t\t\t\tpartition := b.Balance(msg, test.Partitions...)\n\t\t\t\tif partition != test.Partition {\n\t\t\t\t\tt.Errorf(\"expected %v; got %v\", test.Partition, partition)\n\t\t\t\t}\n\t\t\t})\n\t\t}\n\t})\n\n\tt.Run(\"consistent\", func(t *testing.T) {\n\t\tb := Murmur2Balancer{Consistent: true}\n\t\tb.random.mock = -1\n\n\t\tp := b.Balance(Message{}, librdkafkaPartitions...)\n\t\tif p < 0 {\n\t\t\tt.Fatal(\"should not have gotten a random partition\")\n\t\t}\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tif p != b.Balance(Message{}, librdkafkaPartitions...) {\n\t\t\t\tt.Fatal(\"nil key should always hash consistently\")\n\t\t\t}\n\t\t}\n\t})\n}\n\nfunc TestLeastBytes(t *testing.T) {\n\ttestCases := map[string]struct {\n\t\tKeys [][]byte\n\t\tPartitions [][]int\n\t\tPartition int\n\t}{\n\t\t\"single message\": {\n\t\t\tKeys: [][]byte{\n\t\t\t\t[]byte(\"key\"),\n\t\t\t},\n\t\t\tPartitions: [][]int{\n\t\t\t\t{0, 1, 2},\n\t\t\t},\n\t\t\tPartition: 0,\n\t\t},\n\t\t\"multiple messages, no partition change\": {\n\t\t\tKeys: [][]byte{\n\t\t\t\t[]byte(\"a\"),\n\t\t\t\t[]byte(\"ab\"),\n\t\t\t\t[]byte(\"abc\"),\n\t\t\t\t[]byte(\"abcd\"),\n\t\t\t},\n\t\t\tPartitions: [][]int{\n\t\t\t\t{0, 1, 2},\n\t\t\t\t{0, 1, 2},\n\t\t\t\t{0, 1, 2},\n\t\t\t\t{0, 1, 2},\n\t\t\t},\n\t\t\tPartition: 0,\n\t\t},\n\t\t\"partition gained\": {\n\t\t\tKeys: [][]byte{\n\t\t\t\t[]byte(\"hello world 1\"),\n\t\t\t\t[]byte(\"hello world 2\"),\n\t\t\t\t[]byte(\"hello world 3\"),\n\t\t\t},\n\t\t\tPartitions: [][]int{\n\t\t\t\t{0, 1},\n\t\t\t\t{0, 1},\n\t\t\t\t{0, 1, 2},\n\t\t\t},\n\t\t\tPartition: 0,\n\t\t},\n\t\t\"partition lost\": {\n\t\t\tKeys: [][]byte{\n\t\t\t\t[]byte(\"hello world 1\"),\n\t\t\t\t[]byte(\"hello world 2\"),\n\t\t\t\t[]byte(\"hello world 3\"),\n\t\t\t},\n\t\t\tPartitions: [][]int{\n\t\t\t\t{0, 1, 2},\n\t\t\t\t{0, 1, 2},\n\t\t\t\t{0, 1},\n\t\t\t},\n\t\t\tPartition: 0,\n\t\t},\n\t}\n\n\tfor label, test := range testCases {\n\t\tt.Run(label, func(t *testing.T) {\n\t\t\tlb := &LeastBytes{}\n\n\t\t\tvar partition int\n\t\t\tfor i, key := range test.Keys {\n\t\t\t\tmsg := Message{Key: key}\n\t\t\t\tpartition = lb.Balance(msg, test.Partitions[i]...)\n\t\t\t}\n\n\t\t\tif partition != test.Partition {\n\t\t\t\tt.Errorf(\"expected %v; got %v\", test.Partition, partition)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestRoundRobin(t *testing.T) {\n\ttestCases := map[string]struct {\n\t\tPartitions []int\n\t\tChunkSize int\n\t}{\n\t\t\"default - odd partition count\": {\n\t\t\tPartitions: []int{0, 1, 2, 3, 4, 5, 6},\n\t\t},\n\t\t\"negative chunk size - odd partition count\": {\n\t\t\tPartitions: []int{0, 1, 2, 3, 4, 5, 6},\n\t\t\tChunkSize: -1,\n\t\t},\n\t\t\"0 chunk size - odd partition count\": {\n\t\t\tPartitions: []int{0, 1, 2, 3, 4, 5, 6},\n\t\t\tChunkSize: 0,\n\t\t},\n\t\t\"5 chunk size - odd partition count\": {\n\t\t\tPartitions: []int{0, 1, 2, 3, 4, 5, 6},\n\t\t\tChunkSize: 5,\n\t\t},\n\t\t\"12 chunk size - odd partition count\": {\n\t\t\tPartitions: []int{0, 1, 2, 3, 4, 5, 6},\n\t\t\tChunkSize: 12,\n\t\t},\n\t\t\"default - even partition count\": {\n\t\t\tPartitions: []int{0, 1, 2, 3, 4, 5, 6, 7},\n\t\t},\n\t\t\"negative chunk size - even partition count\": {\n\t\t\tPartitions: []int{0, 1, 2, 3, 4, 5, 6, 7},\n\t\t\tChunkSize: -1,\n\t\t},\n\t\t\"0 chunk size - even partition count\": {\n\t\t\tPartitions: []int{0, 1, 2, 3, 4, 5, 6, 7},\n\t\t\tChunkSize: 0,\n\t\t},\n\t\t\"5 chunk size - even partition count\": {\n\t\t\tPartitions: []int{0, 1, 2, 3, 4, 5, 6, 7},\n\t\t\tChunkSize: 5,\n\t\t},\n\t\t\"12 chunk size - even partition count\": {\n\t\t\tPartitions: []int{0, 1, 2, 3, 4, 5, 6, 7},\n\t\t\tChunkSize: 12,\n\t\t},\n\t}\n\tfor label, test := range testCases {\n\t\tt.Run(label, func(t *testing.T) {\n\t\t\tlb := &RoundRobin{ChunkSize: test.ChunkSize}\n\t\t\tmsg := Message{}\n\t\t\tvar partition int\n\t\t\tvar i int\n\t\t\texpectedChunkSize := test.ChunkSize\n\t\t\tif expectedChunkSize < 1 {\n\t\t\t\texpectedChunkSize = 1\n\t\t\t}\n\t\t\tpartitions := test.Partitions\n\t\t\tfor i = 0; i < 50; i++ {\n\t\t\t\tpartition = lb.Balance(msg, partitions...)\n\t\t\t\tif partition != i/expectedChunkSize%len(partitions) {\n\t\t\t\t\tt.Error(\"Returned partition\", partition, \"expecting\", i/expectedChunkSize%len(partitions))\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "batch.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"errors\"\n\t\"io\"\n\t\"sync\"\n\t\"time\"\n)\n\n// A Batch is an iterator over a sequence of messages fetched from a kafka\n// server.\n//\n// Batches are created by calling (*Conn).ReadBatch. They hold a internal lock\n// on the connection, which is released when the batch is closed. Failing to\n// call a batch's Close method will likely result in a dead-lock when trying to\n// use the connection.\n//\n// Batches are safe to use concurrently from multiple goroutines.\ntype Batch struct {\n\tmutex sync.Mutex\n\tconn *Conn\n\tlock *sync.Mutex\n\tmsgs *messageSetReader\n\tdeadline time.Time\n\tthrottle time.Duration\n\ttopic string\n\tpartition int\n\toffset int64\n\thighWaterMark int64\n\terr error\n\t// The last offset in the batch.\n\t//\n\t// We use lastOffset to skip offsets that have been compacted away.\n\t//\n\t// We store lastOffset because we get lastOffset when we read a new message\n\t// but only try to handle compaction when we receive an EOF. However, when\n\t// we get an EOF we do not get the lastOffset. So there is a mismatch\n\t// between when we receive it and need to use it.\n\tlastOffset int64\n}\n\n// Throttle gives the throttling duration applied by the kafka server on the\n// connection.\nfunc (batch *Batch) Throttle() time.Duration {\n\treturn batch.throttle\n}\n\n// HighWaterMark returns the current highest watermark in a partition.\nfunc (batch *Batch) HighWaterMark() int64 {\n\treturn batch.highWaterMark\n}\n\n// Partition returns the batch partition.\nfunc (batch *Batch) Partition() int {\n\treturn batch.partition\n}\n\n// Offset returns the offset of the next message in the batch.\nfunc (batch *Batch) Offset() int64 {\n\tbatch.mutex.Lock()\n\toffset := batch.offset\n\tbatch.mutex.Unlock()\n\treturn offset\n}\n\n// Close closes the batch, releasing the connection lock and returning an error\n// if reading the batch failed for any reason.\nfunc (batch *Batch) Close() error {\n\tbatch.mutex.Lock()\n\terr := batch.close()\n\tbatch.mutex.Unlock()\n\treturn err\n}\n\nfunc (batch *Batch) close() (err error) {\n\tconn := batch.conn\n\tlock := batch.lock\n\n\tbatch.conn = nil\n\tbatch.lock = nil\n\n\tif batch.msgs != nil {\n\t\tbatch.msgs.discard()\n\t}\n\n\tif batch.msgs != nil && batch.msgs.decompressed != nil {\n\t\treleaseBuffer(batch.msgs.decompressed)\n\t\tbatch.msgs.decompressed = nil\n\t}\n\n\tif err = batch.err; errors.Is(batch.err, io.EOF) {\n\t\terr = nil\n\t}\n\n\tif conn != nil {\n\t\tconn.rdeadline.unsetConnReadDeadline()\n\t\tconn.mutex.Lock()\n\t\tconn.offset = batch.offset\n\t\tconn.mutex.Unlock()\n\n\t\tif err != nil {\n\t\t\tvar kafkaError Error\n\t\t\tif !errors.As(err, &kafkaError) && !errors.Is(err, io.ErrShortBuffer) {\n\t\t\t\tconn.Close()\n\t\t\t}\n\t\t}\n\t}\n\n\tif lock != nil {\n\t\tlock.Unlock()\n\t}\n\n\treturn\n}\n\n// Err returns a non-nil error if the batch is broken. This is the same error\n// that would be returned by Read, ReadMessage or Close (except in the case of\n// io.EOF which is never returned by Close).\n//\n// This method is useful when building retry mechanisms for (*Conn).ReadBatch,\n// the program can check whether the batch carried a error before attempting to\n// read the first message.\n//\n// Note that checking errors on a batch is optional, calling Read or ReadMessage\n// is always valid and can be used to either read a message or an error in cases\n// where that's convenient.\nfunc (batch *Batch) Err() error { return batch.err }\n\n// Read reads the value of the next message from the batch into b, returning the\n// number of bytes read, or an error if the next message couldn't be read.\n//\n// If an error is returned the batch cannot be used anymore and calling Read\n// again will keep returning that error. All errors except io.EOF (indicating\n// that the program consumed all messages from the batch) are also returned by\n// Close.\n//\n// The method fails with io.ErrShortBuffer if the buffer passed as argument is\n// too small to hold the message value.\nfunc (batch *Batch) Read(b []byte) (int, error) {\n\tn := 0\n\n\tbatch.mutex.Lock()\n\toffset := batch.offset\n\n\t_, _, _, err := batch.readMessage(\n\t\tfunc(r *bufio.Reader, size int, nbytes int) (int, error) {\n\t\t\tif nbytes < 0 {\n\t\t\t\treturn size, nil\n\t\t\t}\n\t\t\treturn discardN(r, size, nbytes)\n\t\t},\n\t\tfunc(r *bufio.Reader, size int, nbytes int) (int, error) {\n\t\t\tif nbytes < 0 {\n\t\t\t\treturn size, nil\n\t\t\t}\n\t\t\t// make sure there are enough bytes for the message value. return\n\t\t\t// errShortRead if the message is truncated.\n\t\t\tif nbytes > size {\n\t\t\t\treturn size, errShortRead\n\t\t\t}\n\t\t\tn = nbytes // return value\n\t\t\tif nbytes > cap(b) {\n\t\t\t\tnbytes = cap(b)\n\t\t\t}\n\t\t\tif nbytes > len(b) {\n\t\t\t\tb = b[:nbytes]\n\t\t\t}\n\t\t\tnbytes, err := io.ReadFull(r, b[:nbytes])\n\t\t\tif err != nil {\n\t\t\t\treturn size - nbytes, err\n\t\t\t}\n\t\t\treturn discardN(r, size-nbytes, n-nbytes)\n\t\t},\n\t)\n\n\tif err == nil && n > len(b) {\n\t\tn, err = len(b), io.ErrShortBuffer\n\t\tbatch.err = io.ErrShortBuffer\n\t\tbatch.offset = offset // rollback\n\t}\n\n\tbatch.mutex.Unlock()\n\treturn n, err\n}\n\n// ReadMessage reads and return the next message from the batch.\n//\n// Because this method allocate memory buffers for the message key and value\n// it is less memory-efficient than Read, but has the advantage of never\n// failing with io.ErrShortBuffer.\nfunc (batch *Batch) ReadMessage() (Message, error) {\n\tmsg := Message{}\n\tbatch.mutex.Lock()\n\n\tvar offset, timestamp int64\n\tvar headers []Header\n\tvar err error\n\n\toffset, timestamp, headers, err = batch.readMessage(\n\t\tfunc(r *bufio.Reader, size int, nbytes int) (remain int, err error) {\n\t\t\tmsg.Key, remain, err = readNewBytes(r, size, nbytes)\n\t\t\treturn\n\t\t},\n\t\tfunc(r *bufio.Reader, size int, nbytes int) (remain int, err error) {\n\t\t\tmsg.Value, remain, err = readNewBytes(r, size, nbytes)\n\t\t\treturn\n\t\t},\n\t)\n\t// A batch may start before the requested offset so skip messages\n\t// until the requested offset is reached.\n\tfor batch.conn != nil && offset < batch.conn.offset {\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\t\toffset, timestamp, headers, err = batch.readMessage(\n\t\t\tfunc(r *bufio.Reader, size int, nbytes int) (remain int, err error) {\n\t\t\t\tmsg.Key, remain, err = readNewBytes(r, size, nbytes)\n\t\t\t\treturn\n\t\t\t},\n\t\t\tfunc(r *bufio.Reader, size int, nbytes int) (remain int, err error) {\n\t\t\t\tmsg.Value, remain, err = readNewBytes(r, size, nbytes)\n\t\t\t\treturn\n\t\t\t},\n\t\t)\n\t}\n\n\tbatch.mutex.Unlock()\n\tmsg.Topic = batch.topic\n\tmsg.Partition = batch.partition\n\tmsg.Offset = offset\n\tmsg.HighWaterMark = batch.highWaterMark\n\tmsg.Time = makeTime(timestamp)\n\tmsg.Headers = headers\n\n\treturn msg, err\n}\n\nfunc (batch *Batch) readMessage(\n\tkey func(*bufio.Reader, int, int) (int, error),\n\tval func(*bufio.Reader, int, int) (int, error),\n) (offset int64, timestamp int64, headers []Header, err error) {\n\tif err = batch.err; err != nil {\n\t\treturn\n\t}\n\n\tvar lastOffset int64\n\toffset, lastOffset, timestamp, headers, err = batch.msgs.readMessage(batch.offset, key, val)\n\tswitch {\n\tcase err == nil:\n\t\tbatch.offset = offset + 1\n\t\tbatch.lastOffset = lastOffset\n\tcase errors.Is(err, errShortRead):\n\t\t// As an \"optimization\" kafka truncates the returned response after\n\t\t// producing MaxBytes, which could then cause the code to return\n\t\t// errShortRead.\n\t\terr = batch.msgs.discard()\n\t\tswitch {\n\t\tcase err != nil:\n\t\t\t// Since io.EOF is used by the batch to indicate that there is are\n\t\t\t// no more messages to consume, it is crucial that any io.EOF errors\n\t\t\t// on the underlying connection are repackaged. Otherwise, the\n\t\t\t// caller can't tell the difference between a batch that was fully\n\t\t\t// consumed or a batch whose connection is in an error state.\n\t\t\tbatch.err = dontExpectEOF(err)\n\t\tcase batch.msgs.remaining() == 0:\n\t\t\t// Because we use the adjusted deadline we could end up returning\n\t\t\t// before the actual deadline occurred. This is necessary otherwise\n\t\t\t// timing out the connection for real could end up leaving it in an\n\t\t\t// unpredictable state, which would require closing it.\n\t\t\t// This design decision was made to maximize the chances of keeping\n\t\t\t// the connection open, the trade off being to lose precision on the\n\t\t\t// read deadline management.\n\t\t\terr = checkTimeoutErr(batch.deadline)\n\t\t\tbatch.err = err\n\n\t\t\t// Checks the following:\n\t\t\t// - `batch.err` for a \"success\" from the previous timeout check\n\t\t\t// - `batch.msgs.lengthRemain` to ensure that this EOF is not due\n\t\t\t// to MaxBytes truncation\n\t\t\t// - `batch.lastOffset` to ensure that the message format contains\n\t\t\t// `lastOffset`\n\t\t\tif errors.Is(batch.err, io.EOF) && batch.msgs.lengthRemain == 0 && batch.lastOffset != -1 {\n\t\t\t\t// Log compaction can create batches that end with compacted\n\t\t\t\t// records so the normal strategy that increments the \"next\"\n\t\t\t\t// offset as records are read doesn't work as the compacted\n\t\t\t\t// records are \"missing\" and never get \"read\".\n\t\t\t\t//\n\t\t\t\t// In order to reliably reach the next non-compacted offset we\n\t\t\t\t// jump past the saved lastOffset.\n\t\t\t\tbatch.offset = batch.lastOffset + 1\n\t\t\t}\n\t\t}\n\tdefault:\n\t\t// Since io.EOF is used by the batch to indicate that there is are\n\t\t// no more messages to consume, it is crucial that any io.EOF errors\n\t\t// on the underlying connection are repackaged. Otherwise, the\n\t\t// caller can't tell the difference between a batch that was fully\n\t\t// consumed or a batch whose connection is in an error state.\n\t\tbatch.err = dontExpectEOF(err)\n\t}\n\n\treturn\n}\n\nfunc checkTimeoutErr(deadline time.Time) (err error) {\n\tif !deadline.IsZero() && time.Now().After(deadline) {\n\t\terr = RequestTimedOut\n\t} else {\n\t\terr = io.EOF\n\t}\n\treturn\n}\n" }, { "path": "batch_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"io\"\n\t\"net\"\n\t\"strconv\"\n\t\"testing\"\n)\n\nfunc TestBatchDontExpectEOF(t *testing.T) {\n\ttopic := makeTopic()\n\n\tbroker, err := (&Dialer{\n\t\tResolver: &net.Resolver{},\n\t}).LookupLeader(context.Background(), \"tcp\", \"localhost:9092\", topic, 0)\n\tif err != nil {\n\t\tt.Fatal(\"failed to open a new kafka connection:\", err)\n\t}\n\n\tnc, err := net.Dial(\"tcp\", net.JoinHostPort(broker.Host, strconv.Itoa(broker.Port)))\n\tif err != nil {\n\t\tt.Fatalf(\"cannot connect to partition leader at %s:%d: %s\", broker.Host, broker.Port, err)\n\t}\n\n\tconn := NewConn(nc, topic, 0)\n\tdefer conn.Close()\n\n\tnc.(*net.TCPConn).CloseRead()\n\n\tbatch := conn.ReadBatch(1024, 8192)\n\n\tif _, err := batch.ReadMessage(); !errors.Is(err, io.ErrUnexpectedEOF) {\n\t\tt.Error(\"bad error when reading message:\", err)\n\t}\n\n\tif err := batch.Close(); !errors.Is(err, io.ErrUnexpectedEOF) {\n\t\tt.Error(\"bad error when closing the batch:\", err)\n\t}\n}\n" }, { "path": "buffer.go", "content": "package kafka\n\nimport (\n\t\"bytes\"\n\t\"sync\"\n)\n\nvar bufferPool = sync.Pool{\n\tNew: func() interface{} { return newBuffer() },\n}\n\nfunc newBuffer() *bytes.Buffer {\n\tb := new(bytes.Buffer)\n\tb.Grow(65536)\n\treturn b\n}\n\nfunc acquireBuffer() *bytes.Buffer {\n\treturn bufferPool.Get().(*bytes.Buffer)\n}\n\nfunc releaseBuffer(b *bytes.Buffer) {\n\tif b != nil {\n\t\tb.Reset()\n\t\tbufferPool.Put(b)\n\t}\n}\n" }, { "path": "builder_test.go", "content": "package kafka\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"io\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/compress\"\n)\n\n// This file defines builders to assist in creating kafka payloads for unit testing.\n\n// fetchResponseBuilder builds v10 fetch responses. The version of the v10 fetch\n// responses are not as important as the message sets contained within, as this\n// type is ultimately used to unit test the message set reader that consumes the\n// rest of the response once the header has been parsed.\ntype fetchResponseBuilder struct {\n\theader fetchResponseHeader\n\tmsgSets []messageSetBuilder\n\trendered []byte\n}\n\ntype fetchResponseHeader struct {\n\tthrottle int32\n\terrorCode int16\n\tsessionID int32\n\ttopic string\n\tpartition int32\n\tpartitionErrorCode int16\n\thighWatermarkOffset int64\n\tlastStableOffset int64\n\tlogStartOffset int64\n}\n\nfunc (b *fetchResponseBuilder) messages() (res []Message) {\n\tfor _, set := range b.msgSets {\n\t\tres = append(res, set.messages()...)\n\t}\n\treturn\n}\n\nfunc (b *fetchResponseBuilder) bytes() []byte {\n\tif b.rendered == nil {\n\t\tb.rendered = newWB().call(func(wb *kafkaWriteBuffer) {\n\t\t\twb.writeInt32(b.header.throttle)\n\t\t\twb.writeInt16(b.header.errorCode)\n\t\t\twb.writeInt32(b.header.sessionID)\n\t\t\twb.writeInt32(1) // num topics\n\t\t\twb.writeString(b.header.topic)\n\t\t\twb.writeInt32(1) // how many partitions\n\t\t\twb.writeInt32(b.header.partition)\n\t\t\twb.writeInt16(b.header.partitionErrorCode)\n\t\t\twb.writeInt64(b.header.highWatermarkOffset)\n\t\t\twb.writeInt64(b.header.lastStableOffset)\n\t\t\twb.writeInt64(b.header.logStartOffset)\n\t\t\twb.writeInt32(-1) // num aborted tx\n\t\t\twb.writeBytes(newWB().call(func(wb *kafkaWriteBuffer) {\n\t\t\t\tfor _, msgSet := range b.msgSets {\n\t\t\t\t\twb.Write(msgSet.bytes())\n\t\t\t\t}\n\t\t\t}))\n\t\t})\n\t}\n\treturn b.rendered\n}\n\nfunc (b *fetchResponseBuilder) Len() int {\n\treturn len(b.bytes())\n}\n\ntype messageSetBuilder interface {\n\tbytes() []byte\n\tmessages() []Message\n}\n\ntype v0MessageSetBuilder struct {\n\tmsgs []Message\n\tcodec CompressionCodec\n}\n\nfunc (f v0MessageSetBuilder) messages() []Message {\n\treturn f.msgs\n}\n\nfunc (f v0MessageSetBuilder) bytes() []byte {\n\tbs := newWB().call(func(wb *kafkaWriteBuffer) {\n\t\tfor _, msg := range f.msgs {\n\t\t\tbs := newWB().call(func(wb *kafkaWriteBuffer) {\n\t\t\t\twb.writeInt64(msg.Offset) // offset\n\t\t\t\twb.writeBytes(newWB().call(func(wb *kafkaWriteBuffer) {\n\t\t\t\t\twb.writeInt32(-1) // crc, unused\n\t\t\t\t\twb.writeInt8(0) // magic\n\t\t\t\t\twb.writeInt8(0) // attributes -- zero, no compression for the inner message\n\t\t\t\t\twb.writeBytes(msg.Key)\n\t\t\t\t\twb.writeBytes(msg.Value)\n\t\t\t\t}))\n\t\t\t})\n\t\t\twb.Write(bs)\n\t\t}\n\t})\n\tif f.codec != nil {\n\t\tbs = newWB().call(func(wb *kafkaWriteBuffer) {\n\t\t\twb.writeInt64(f.msgs[0].Offset) // offset\n\t\t\twb.writeBytes(newWB().call(func(wb *kafkaWriteBuffer) {\n\t\t\t\tcompressed := mustCompress(bs, f.codec)\n\t\t\t\twb.writeInt32(-1) // crc, unused\n\t\t\t\twb.writeInt8(0) // magic\n\t\t\t\twb.writeInt8(f.codec.Code()) // attributes\n\t\t\t\twb.writeBytes(nil) // key is always nil for compressed\n\t\t\t\twb.writeBytes(compressed) // the value is the compressed message\n\t\t\t}))\n\t\t})\n\t}\n\treturn bs\n}\n\ntype v1MessageSetBuilder struct {\n\tmsgs []Message\n\tcodec CompressionCodec\n}\n\nfunc (f v1MessageSetBuilder) messages() []Message {\n\treturn f.msgs\n}\n\nfunc (f v1MessageSetBuilder) bytes() []byte {\n\tbs := newWB().call(func(wb *kafkaWriteBuffer) {\n\t\tfor i, msg := range f.msgs {\n\t\t\tbs := newWB().call(func(wb *kafkaWriteBuffer) {\n\t\t\t\tif f.codec != nil {\n\t\t\t\t\twb.writeInt64(int64(i)) // compressed inner message offsets are relative\n\t\t\t\t} else {\n\t\t\t\t\twb.writeInt64(msg.Offset) // offset\n\t\t\t\t}\n\t\t\t\twb.writeBytes(newWB().call(func(wb *kafkaWriteBuffer) {\n\t\t\t\t\twb.writeInt32(-1) // crc, unused\n\t\t\t\t\twb.writeInt8(1) // magic\n\t\t\t\t\twb.writeInt8(0) // attributes -- zero, no compression for the inner message\n\t\t\t\t\twb.writeInt64(1000 * msg.Time.Unix()) // timestamp\n\t\t\t\t\twb.writeBytes(msg.Key)\n\t\t\t\t\twb.writeBytes(msg.Value)\n\t\t\t\t}))\n\t\t\t})\n\t\t\twb.Write(bs)\n\t\t}\n\t})\n\tif f.codec != nil {\n\t\tbs = newWB().call(func(wb *kafkaWriteBuffer) {\n\t\t\twb.writeInt64(f.msgs[len(f.msgs)-1].Offset) // offset of the wrapper message is the last offset of the inner messages\n\t\t\twb.writeBytes(newWB().call(func(wb *kafkaWriteBuffer) {\n\t\t\t\tbs := mustCompress(bs, f.codec)\n\t\t\t\twb.writeInt32(-1) // crc, unused\n\t\t\t\twb.writeInt8(1) // magic\n\t\t\t\twb.writeInt8(f.codec.Code()) // attributes\n\t\t\t\twb.writeInt64(1000 * f.msgs[0].Time.Unix()) // timestamp\n\t\t\t\twb.writeBytes(nil) // key is always nil for compressed\n\t\t\t\twb.writeBytes(bs) // the value is the compressed message\n\t\t\t}))\n\t\t})\n\t}\n\treturn bs\n}\n\ntype v2MessageSetBuilder struct {\n\tmsgs []Message\n\tcodec CompressionCodec\n}\n\nfunc (f v2MessageSetBuilder) messages() []Message {\n\treturn f.msgs\n}\n\nfunc (f v2MessageSetBuilder) bytes() []byte {\n\tattributes := int16(0)\n\tif f.codec != nil {\n\t\tattributes = int16(f.codec.Code()) // set codec code on attributes\n\t}\n\treturn newWB().call(func(wb *kafkaWriteBuffer) {\n\t\twb.writeInt64(f.msgs[0].Offset)\n\t\twb.writeBytes(newWB().call(func(wb *kafkaWriteBuffer) {\n\t\t\twb.writeInt32(0) // leader epoch\n\t\t\twb.writeInt8(2) // magic = 2\n\t\t\twb.writeInt32(0) // crc, unused\n\t\t\twb.writeInt16(attributes) // record set attributes\n\t\t\twb.writeInt32(0) // record set last offset delta\n\t\t\twb.writeInt64(1000 * f.msgs[0].Time.Unix()) // record set first timestamp\n\t\t\twb.writeInt64(1000 * f.msgs[0].Time.Unix()) // record set last timestamp\n\t\t\twb.writeInt64(0) // record set producer id\n\t\t\twb.writeInt16(0) // record set producer epoch\n\t\t\twb.writeInt32(0) // record set base sequence\n\t\t\twb.writeInt32(int32(len(f.msgs))) // record set count\n\t\t\tbs := newWB().call(func(wb *kafkaWriteBuffer) {\n\t\t\t\tfor i, msg := range f.msgs {\n\t\t\t\t\twb.Write(newWB().call(func(wb *kafkaWriteBuffer) {\n\t\t\t\t\t\tbs := newWB().call(func(wb *kafkaWriteBuffer) {\n\t\t\t\t\t\t\twb.writeInt8(0) // record attributes, not used here\n\t\t\t\t\t\t\twb.writeVarInt(1000 * (time.Now().Unix() - msg.Time.Unix())) // timestamp\n\t\t\t\t\t\t\twb.writeVarInt(int64(i)) // offset delta\n\t\t\t\t\t\t\twb.writeVarInt(int64(len(msg.Key))) // key len\n\t\t\t\t\t\t\twb.Write(msg.Key) // key bytes\n\t\t\t\t\t\t\twb.writeVarInt(int64(len(msg.Value))) // value len\n\t\t\t\t\t\t\twb.Write(msg.Value) // value bytes\n\t\t\t\t\t\t\twb.writeVarInt(int64(len(msg.Headers))) // number of headers\n\t\t\t\t\t\t\tfor _, header := range msg.Headers {\n\t\t\t\t\t\t\t\twb.writeVarInt(int64(len(header.Key)))\n\t\t\t\t\t\t\t\twb.Write([]byte(header.Key))\n\t\t\t\t\t\t\t\twb.writeVarInt(int64(len(header.Value)))\n\t\t\t\t\t\t\t\twb.Write(header.Value)\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t})\n\t\t\t\t\t\twb.writeVarInt(int64(len(bs)))\n\t\t\t\t\t\twb.Write(bs)\n\t\t\t\t\t}))\n\t\t\t\t}\n\t\t\t})\n\t\t\tif f.codec != nil {\n\t\t\t\tbs = mustCompress(bs, f.codec)\n\t\t\t}\n\t\t\twb.Write(bs)\n\t\t}))\n\t})\n}\n\n// kafkaWriteBuffer is a write buffer that helps writing fetch responses.\ntype kafkaWriteBuffer struct {\n\twriteBuffer\n\tbuf bytes.Buffer\n}\n\nfunc newWB() *kafkaWriteBuffer {\n\tres := kafkaWriteBuffer{}\n\tres.writeBuffer.w = &res.buf\n\treturn &res\n}\n\nfunc (f *kafkaWriteBuffer) Bytes() []byte {\n\treturn f.buf.Bytes()\n}\n\n// call is a convenience method that allows the kafkaWriteBuffer to be used\n// in a functional manner. This is helpful when building\n// nested structures, as the return value can be fed into\n// other fwWB APIs.\nfunc (f *kafkaWriteBuffer) call(cb func(wb *kafkaWriteBuffer)) []byte {\n\tcb(f)\n\tbs := f.Bytes()\n\tif bs == nil {\n\t\tbs = []byte{}\n\t}\n\treturn bs\n}\n\nfunc mustCompress(bs []byte, codec compress.Codec) (res []byte) {\n\tbuf := bytes.Buffer{}\n\tcodecWriter := codec.NewWriter(&buf)\n\t_, err := io.Copy(codecWriter, bytes.NewReader(bs))\n\tif err != nil {\n\t\tpanic(fmt.Errorf(\"compress: %w\", err))\n\t}\n\terr = codecWriter.Close()\n\tif err != nil {\n\t\tpanic(fmt.Errorf(\"close codec writer: %w\", err))\n\t}\n\tres = buf.Bytes()\n\treturn\n}\n" }, { "path": "client.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n)\n\nconst (\n\tdefaultCreateTopicsTimeout = 2 * time.Second\n\tdefaultDeleteTopicsTimeout = 2 * time.Second\n\tdefaultCreatePartitionsTimeout = 2 * time.Second\n\tdefaultProduceTimeout = 500 * time.Millisecond\n\tdefaultMaxWait = 500 * time.Millisecond\n)\n\n// Client is a high-level API to interract with kafka brokers.\n//\n// All methods of the Client type accept a context as first argument, which may\n// be used to asynchronously cancel the requests.\n//\n// Clients are safe to use concurrently from multiple goroutines, as long as\n// their configuration is not changed after first use.\ntype Client struct {\n\t// Address of the kafka cluster (or specific broker) that the client will be\n\t// sending requests to.\n\t//\n\t// This field is optional, the address may be provided in each request\n\t// instead. The request address takes precedence if both were specified.\n\tAddr net.Addr\n\n\t// Time limit for requests sent by this client.\n\t//\n\t// If zero, no timeout is applied.\n\tTimeout time.Duration\n\n\t// A transport used to communicate with the kafka brokers.\n\t//\n\t// If nil, DefaultTransport is used.\n\tTransport RoundTripper\n}\n\n// A ConsumerGroup and Topic as these are both strings we define a type for\n// clarity when passing to the Client as a function argument\n//\n// N.B TopicAndGroup is currently experimental! Therefore, it is subject to\n// change, including breaking changes between MINOR and PATCH releases.\n//\n// DEPRECATED: this type will be removed in version 1.0, programs should\n// migrate to use kafka.(*Client).OffsetFetch instead.\ntype TopicAndGroup struct {\n\tTopic string\n\tGroupId string\n}\n\n// ConsumerOffsets returns a map[int]int64 of partition to committed offset for\n// a consumer group id and topic.\n//\n// DEPRECATED: this method will be removed in version 1.0, programs should\n// migrate to use kafka.(*Client).OffsetFetch instead.\nfunc (c *Client) ConsumerOffsets(ctx context.Context, tg TopicAndGroup) (map[int]int64, error) {\n\tmetadata, err := c.Metadata(ctx, &MetadataRequest{\n\t\tTopics: []string{tg.Topic},\n\t})\n\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to get topic metadata :%w\", err)\n\t}\n\n\ttopic := metadata.Topics[0]\n\tpartitions := make([]int, len(topic.Partitions))\n\n\tfor i := range topic.Partitions {\n\t\tpartitions[i] = topic.Partitions[i].ID\n\t}\n\n\toffsets, err := c.OffsetFetch(ctx, &OffsetFetchRequest{\n\t\tGroupID: tg.GroupId,\n\t\tTopics: map[string][]int{\n\t\t\ttg.Topic: partitions,\n\t\t},\n\t})\n\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to get offsets: %w\", err)\n\t}\n\n\ttopicOffsets := offsets.Topics[topic.Name]\n\tpartitionOffsets := make(map[int]int64, len(topicOffsets))\n\n\tfor _, off := range topicOffsets {\n\t\tpartitionOffsets[off.Partition] = off.CommittedOffset\n\t}\n\n\treturn partitionOffsets, nil\n}\n\nfunc (c *Client) roundTrip(ctx context.Context, addr net.Addr, msg protocol.Message) (protocol.Message, error) {\n\tif c.Timeout > 0 {\n\t\tvar cancel context.CancelFunc\n\t\tctx, cancel = context.WithTimeout(ctx, c.Timeout)\n\t\tdefer cancel()\n\t}\n\n\tif addr == nil {\n\t\tif addr = c.Addr; addr == nil {\n\t\t\treturn nil, errors.New(\"no address was given for the kafka cluster in the request or on the client\")\n\t\t}\n\t}\n\n\treturn c.transport().RoundTrip(ctx, addr, msg)\n}\n\nfunc (c *Client) transport() RoundTripper {\n\tif c.Transport != nil {\n\t\treturn c.Transport\n\t}\n\treturn DefaultTransport\n}\n\nfunc (c *Client) timeout(ctx context.Context, defaultTimeout time.Duration) time.Duration {\n\ttimeout := c.Timeout\n\n\tif deadline, ok := ctx.Deadline(); ok {\n\t\tif remain := time.Until(deadline); remain < timeout {\n\t\t\ttimeout = remain\n\t\t}\n\t}\n\n\tif timeout > 0 {\n\t\t// Half the timeout because it is communicated to kafka in multiple\n\t\t// requests (e.g. Fetch, Produce, etc...), this adds buffer to account\n\t\t// for network latency when waiting for the response from kafka.\n\t\treturn timeout / 2\n\t}\n\n\treturn defaultTimeout\n}\n\nfunc (c *Client) timeoutMs(ctx context.Context, defaultTimeout time.Duration) int32 {\n\treturn milliseconds(c.timeout(ctx, defaultTimeout))\n}\n" }, { "path": "client_test.go", "content": "package kafka\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"io\"\n\t\"math/rand\"\n\t\"net\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/compress\"\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc newLocalClientAndTopic() (*Client, string, func()) {\n\ttopic := makeTopic()\n\tclient, shutdown := newLocalClientWithTopic(topic, 1)\n\treturn client, topic, shutdown\n}\n\nfunc newLocalClientWithTopic(topic string, partitions int) (*Client, func()) {\n\tclient, shutdown := newLocalClient()\n\tif err := clientCreateTopic(client, topic, partitions); err != nil {\n\t\tshutdown()\n\t\tpanic(err)\n\t}\n\treturn client, func() {\n\t\tclient.DeleteTopics(context.Background(), &DeleteTopicsRequest{\n\t\t\tTopics: []string{topic},\n\t\t})\n\t\tshutdown()\n\t}\n}\n\nfunc clientCreateTopic(client *Client, topic string, partitions int) error {\n\t_, err := client.CreateTopics(context.Background(), &CreateTopicsRequest{\n\t\tTopics: []TopicConfig{{\n\t\t\tTopic: topic,\n\t\t\tNumPartitions: partitions,\n\t\t\tReplicationFactor: 1,\n\t\t}},\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Topic creation seems to be asynchronous. Metadata for the topic partition\n\t// layout in the cluster is available in the controller before being synced\n\t// with the other brokers, which causes \"Error:[3] Unknown Topic Or Partition\"\n\t// when sending requests to the partition leaders.\n\t//\n\t// This loop will wait up to 2 seconds polling the cluster until no errors\n\t// are returned.\n\tfor i := 0; i < 20; i++ {\n\t\tr, err := client.Fetch(context.Background(), &FetchRequest{\n\t\t\tTopic: topic,\n\t\t\tPartition: 0,\n\t\t\tOffset: 0,\n\t\t})\n\t\tif err == nil && r.Error == nil {\n\t\t\tbreak\n\t\t}\n\t\ttime.Sleep(100 * time.Millisecond)\n\t}\n\n\treturn nil\n}\n\nfunc clientEndTxn(client *Client, req *EndTxnRequest) error {\n\tctx, cancel := context.WithTimeout(context.Background(), time.Second*30)\n\tdefer cancel()\n\tresp, err := client.EndTxn(ctx, req)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\treturn resp.Error\n}\n\nfunc newLocalClient() (*Client, func()) {\n\treturn newClient(TCP(\"localhost\"))\n}\n\nfunc newClient(addr net.Addr) (*Client, func()) {\n\tconns := &ktesting.ConnWaitGroup{\n\t\tDialFunc: (&net.Dialer{}).DialContext,\n\t}\n\n\ttransport := &Transport{\n\t\tDial: conns.Dial,\n\t\tResolver: NewBrokerResolver(nil),\n\t}\n\n\tclient := &Client{\n\t\tAddr: addr,\n\t\tTimeout: 5 * time.Second,\n\t\tTransport: transport,\n\t}\n\n\treturn client, func() { transport.CloseIdleConnections(); conns.Wait() }\n}\n\nfunc TestClient(t *testing.T) {\n\ttests := []struct {\n\t\tscenario string\n\t\tfunction func(*testing.T, context.Context, *Client)\n\t}{\n\t\t{\n\t\t\tscenario: \"retrieve committed offsets for a consumer group and topic\",\n\t\t\tfunction: testConsumerGroupFetchOffsets,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\ttestFunc := test.function\n\t\tt.Run(test.scenario, func(t *testing.T) {\n\t\t\tctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)\n\t\t\tdefer cancel()\n\n\t\t\tclient, shutdown := newLocalClient()\n\t\t\tdefer shutdown()\n\n\t\t\ttestFunc(t, ctx, client)\n\t\t})\n\t}\n}\n\nfunc testConsumerGroupFetchOffsets(t *testing.T, ctx context.Context, client *Client) {\n\tconst totalMessages = 144\n\tconst partitions = 12\n\tconst msgPerPartition = totalMessages / partitions\n\n\ttopic := makeTopic()\n\tif err := clientCreateTopic(client, topic, partitions); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tgroupId := makeGroupID()\n\tbrokers := []string{\"localhost:9092\"}\n\n\twriter := &Writer{\n\t\tAddr: TCP(brokers...),\n\t\tTopic: topic,\n\t\tBalancer: &RoundRobin{},\n\t\tBatchSize: 1,\n\t\tTransport: client.Transport,\n\t}\n\tif err := writer.WriteMessages(ctx, makeTestSequence(totalMessages)...); err != nil {\n\t\tt.Fatalf(\"bad write messages: %v\", err)\n\t}\n\tif err := writer.Close(); err != nil {\n\t\tt.Fatalf(\"bad write err: %v\", err)\n\t}\n\n\tr := NewReader(ReaderConfig{\n\t\tBrokers: brokers,\n\t\tTopic: topic,\n\t\tGroupID: groupId,\n\t\tMinBytes: 1,\n\t\tMaxBytes: 10e6,\n\t\tMaxWait: 100 * time.Millisecond,\n\t})\n\tdefer r.Close()\n\n\tfor i := 0; i < totalMessages; i++ {\n\t\tm, err := r.FetchMessage(ctx)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"error fetching message: %s\", err)\n\t\t}\n\t\tif err := r.CommitMessages(context.Background(), m); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\toffsets, err := client.ConsumerOffsets(ctx, TopicAndGroup{GroupId: groupId, Topic: topic})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(offsets) != partitions {\n\t\tt.Fatalf(\"expected %d partitions but only received offsets for %d\", partitions, len(offsets))\n\t}\n\n\tfor i := 0; i < partitions; i++ {\n\t\tcommittedOffset := offsets[i]\n\t\tif committedOffset != msgPerPartition {\n\t\t\tt.Errorf(\"expected partition %d with committed offset of %d but received %d\", i, msgPerPartition, committedOffset)\n\t\t}\n\t}\n}\n\nfunc TestClientProduceAndConsume(t *testing.T) {\n\tctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)\n\tdefer cancel()\n\t// Tests a typical kafka use case, data is produced to a partition,\n\t// then consumed back sequentially. We use snappy compression because\n\t// kafka stream are often compressed, and verify that each record\n\t// produced is exposed to the consumer, and order is preserved.\n\tclient, topic, shutdown := newLocalClientAndTopic()\n\tdefer shutdown()\n\n\tepoch := time.Now()\n\tseed := int64(0) // deterministic\n\tprng := rand.New(rand.NewSource(seed))\n\toffset := int64(0)\n\n\tconst numBatches = 100\n\tconst recordsPerBatch = 320\n\tt.Logf(\"producing %d batches of %d records...\", numBatches, recordsPerBatch)\n\n\tfor i := 0; i < numBatches; i++ { // produce 100 batches\n\t\trecords := make([]Record, recordsPerBatch)\n\n\t\tfor i := range records {\n\t\t\tv := make([]byte, prng.Intn(999)+1)\n\t\t\tio.ReadFull(prng, v)\n\t\t\trecords[i].Time = epoch\n\t\t\trecords[i].Value = NewBytes(v)\n\t\t}\n\n\t\tres, err := client.Produce(ctx, &ProduceRequest{\n\t\t\tTopic: topic,\n\t\t\tPartition: 0,\n\t\t\tRequiredAcks: -1,\n\t\t\tRecords: NewRecordReader(records...),\n\t\t\tCompression: compress.Snappy,\n\t\t})\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif res.Error != nil {\n\t\t\tt.Fatal(res.Error)\n\t\t}\n\t\tif res.BaseOffset != offset {\n\t\t\tt.Fatalf(\"records were produced at an unexpected offset, want %d but got %d\", offset, res.BaseOffset)\n\t\t}\n\t\toffset += int64(len(records))\n\t}\n\n\tprng.Seed(seed)\n\toffset = 0 // reset\n\tnumFetches := 0\n\tnumRecords := 0\n\n\tfor numRecords < (numBatches * recordsPerBatch) {\n\t\tres, err := client.Fetch(ctx, &FetchRequest{\n\t\t\tTopic: topic,\n\t\t\tPartition: 0,\n\t\t\tOffset: offset,\n\t\t\tMinBytes: 1,\n\t\t\tMaxBytes: 256 * 1024,\n\t\t\tMaxWait: 100 * time.Millisecond, // should only hit on the last fetch\n\t\t})\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif res.Error != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tfor {\n\t\t\tr, err := res.Records.ReadRecord()\n\t\t\tif err != nil {\n\t\t\t\tif !errors.Is(err, io.EOF) {\n\t\t\t\t\tt.Fatal(err)\n\t\t\t\t}\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t\tif r.Key != nil {\n\t\t\t\tr.Key.Close()\n\t\t\t\tt.Error(\"unexpected non-null key on record at offset\", r.Offset)\n\t\t\t}\n\n\t\t\tn := prng.Intn(999) + 1\n\t\t\ta := make([]byte, n)\n\t\t\tb := make([]byte, n)\n\t\t\tio.ReadFull(prng, a)\n\n\t\t\t_, err = io.ReadFull(r.Value, b)\n\t\t\tr.Value.Close()\n\t\t\tif err != nil {\n\t\t\t\tt.Fatalf(\"reading record at offset %d: %v\", r.Offset, err)\n\t\t\t}\n\n\t\t\tif !bytes.Equal(a, b) {\n\t\t\t\tt.Fatalf(\"value of record at offset %d mismatches\", r.Offset)\n\t\t\t}\n\n\t\t\tif r.Offset != offset {\n\t\t\t\tt.Fatalf(\"record at offset %d was expected to have offset %d\", r.Offset, offset)\n\t\t\t}\n\n\t\t\toffset = r.Offset + 1\n\t\t\tnumRecords++\n\t\t}\n\n\t\tnumFetches++\n\t}\n\n\tt.Logf(\"%d records were read in %d fetches\", numRecords, numFetches)\n}\n" }, { "path": "commit.go", "content": "package kafka\n\n// A commit represents the instruction of publishing an update of the last\n// offset read by a program for a topic and partition.\ntype commit struct {\n\ttopic string\n\tpartition int\n\toffset int64\n}\n\n// makeCommit builds a commit value from a message, the resulting commit takes\n// its topic, partition, and offset from the message.\nfunc makeCommit(msg Message) commit {\n\treturn commit{\n\t\ttopic: msg.Topic,\n\t\tpartition: msg.Partition,\n\t\toffset: msg.Offset + 1,\n\t}\n}\n\n// makeCommits generates a slice of commits from a list of messages, it extracts\n// the topic, partition, and offset of each message and builds the corresponding\n// commit slice.\nfunc makeCommits(msgs ...Message) []commit {\n\tcommits := make([]commit, len(msgs))\n\n\tfor i, m := range msgs {\n\t\tcommits[i] = makeCommit(m)\n\t}\n\n\treturn commits\n}\n\n// commitRequest is the data type exchanged between the CommitMessages method\n// and internals of the reader's implementation.\ntype commitRequest struct {\n\tcommits []commit\n\terrch chan<- error\n}\n" }, { "path": "commit_test.go", "content": "package kafka\n\nimport \"testing\"\n\nfunc TestMakeCommit(t *testing.T) {\n\tmsg := Message{\n\t\tTopic: \"blah\",\n\t\tPartition: 1,\n\t\tOffset: 2,\n\t}\n\n\tcommit := makeCommit(msg)\n\tif commit.topic != msg.Topic {\n\t\tt.Errorf(\"bad topic: expected %v; got %v\", msg.Topic, commit.topic)\n\t}\n\tif commit.partition != msg.Partition {\n\t\tt.Errorf(\"bad partition: expected %v; got %v\", msg.Partition, commit.partition)\n\t}\n\tif commit.offset != msg.Offset+1 {\n\t\tt.Errorf(\"expected committed offset to be 1 greater than msg offset\")\n\t}\n}\n" }, { "path": "compress/compress.go", "content": "package compress\n\nimport (\n\t\"encoding\"\n\t\"fmt\"\n\t\"io\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/segmentio/kafka-go/compress/gzip\"\n\t\"github.com/segmentio/kafka-go/compress/lz4\"\n\t\"github.com/segmentio/kafka-go/compress/snappy\"\n\t\"github.com/segmentio/kafka-go/compress/zstd\"\n)\n\n// Compression represents the compression applied to a record set.\ntype Compression int8\n\nconst (\n\tNone Compression = 0\n\tGzip Compression = 1\n\tSnappy Compression = 2\n\tLz4 Compression = 3\n\tZstd Compression = 4\n)\n\nfunc (c Compression) Codec() Codec {\n\tif i := int(c); i >= 0 && i < len(Codecs) {\n\t\treturn Codecs[i]\n\t}\n\treturn nil\n}\n\nfunc (c Compression) String() string {\n\tif codec := c.Codec(); codec != nil {\n\t\treturn codec.Name()\n\t}\n\treturn \"uncompressed\"\n}\n\nfunc (c Compression) MarshalText() ([]byte, error) {\n\treturn []byte(c.String()), nil\n}\n\nfunc (c *Compression) UnmarshalText(b []byte) error {\n\tswitch string(b) {\n\tcase \"none\", \"uncompressed\":\n\t\t*c = None\n\t\treturn nil\n\t}\n\n\tfor _, codec := range Codecs[None+1:] {\n\t\tif codec.Name() == string(b) {\n\t\t\t*c = Compression(codec.Code())\n\t\t\treturn nil\n\t\t}\n\t}\n\n\ti, err := strconv.ParseInt(string(b), 10, 64)\n\tif err == nil && i >= 0 && i < int64(len(Codecs)) {\n\t\t*c = Compression(i)\n\t\treturn nil\n\t}\n\n\ts := &strings.Builder{}\n\ts.WriteString(\"none, uncompressed\")\n\n\tfor i, codec := range Codecs[None+1:] {\n\t\tif i < (len(Codecs) - 1) {\n\t\t\ts.WriteString(\", \")\n\t\t} else {\n\t\t\ts.WriteString(\", or \")\n\t\t}\n\t\ts.WriteString(codec.Name())\n\t}\n\n\treturn fmt.Errorf(\"compression format must be one of %s, not %q\", s, b)\n}\n\nvar (\n\t_ encoding.TextMarshaler = Compression(0)\n\t_ encoding.TextUnmarshaler = (*Compression)(nil)\n)\n\n// Codec represents a compression codec to encode and decode the messages.\n// See : https://cwiki.apache.org/confluence/display/KAFKA/Compression\n//\n// A Codec must be safe for concurrent access by multiple go routines.\ntype Codec interface {\n\t// Code returns the compression codec code\n\tCode() int8\n\n\t// Human-readable name for the codec.\n\tName() string\n\n\t// Constructs a new reader which decompresses data from r.\n\tNewReader(r io.Reader) io.ReadCloser\n\n\t// Constructs a new writer which writes compressed data to w.\n\tNewWriter(w io.Writer) io.WriteCloser\n}\n\nvar (\n\t// The global gzip codec installed on the Codecs table.\n\tGzipCodec gzip.Codec\n\n\t// The global snappy codec installed on the Codecs table.\n\tSnappyCodec snappy.Codec\n\n\t// The global lz4 codec installed on the Codecs table.\n\tLz4Codec lz4.Codec\n\n\t// The global zstd codec installed on the Codecs table.\n\tZstdCodec zstd.Codec\n\n\t// The global table of compression codecs supported by the kafka protocol.\n\tCodecs = [...]Codec{\n\t\tNone: nil,\n\t\tGzip: &GzipCodec,\n\t\tSnappy: &SnappyCodec,\n\t\tLz4: &Lz4Codec,\n\t\tZstd: &ZstdCodec,\n\t}\n)\n" }, { "path": "compress/compress_test.go", "content": "package compress_test\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"io/ioutil\"\n\t\"math/rand\"\n\t\"net\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strconv\"\n\t\"testing\"\n\t\"text/tabwriter\"\n\t\"time\"\n\n\tgz \"github.com/klauspost/compress/gzip\"\n\t\"github.com/segmentio/kafka-go\"\n\tpkg \"github.com/segmentio/kafka-go/compress\"\n\t\"github.com/segmentio/kafka-go/compress/gzip\"\n\t\"github.com/segmentio/kafka-go/compress/lz4\"\n\t\"github.com/segmentio/kafka-go/compress/snappy\"\n\t\"github.com/segmentio/kafka-go/compress/zstd\"\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc init() {\n\t// Seeding the random source is important to prevent multiple test runs from\n\t// reusing the same topic names.\n\trand.Seed(time.Now().UnixNano())\n}\n\nfunc TestCodecs(t *testing.T) {\n\tfor i, c := range pkg.Codecs {\n\t\tif c != nil {\n\t\t\tif code := c.Code(); int8(code) != int8(i) {\n\t\t\t\tt.Fatal(\"default compression codec table is misconfigured for\", c.Name())\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestCompression(t *testing.T) {\n\tmsg := kafka.Message{\n\t\tValue: []byte(\"message\"),\n\t}\n\n\ttestEncodeDecode(t, msg, new(gzip.Codec))\n\ttestEncodeDecode(t, msg, new(snappy.Codec))\n\ttestEncodeDecode(t, msg, new(lz4.Codec))\n\tif ktesting.KafkaIsAtLeast(\"2.1.0\") {\n\t\ttestEncodeDecode(t, msg, new(zstd.Codec))\n\t}\n}\n\nfunc compress(codec pkg.Codec, src []byte) ([]byte, error) {\n\tb := new(bytes.Buffer)\n\tr := bytes.NewReader(src)\n\tw := codec.NewWriter(b)\n\tif _, err := io.Copy(w, r); err != nil {\n\t\tw.Close()\n\t\treturn nil, err\n\t}\n\tif err := w.Close(); err != nil {\n\t\treturn nil, err\n\t}\n\treturn b.Bytes(), nil\n}\n\nfunc decompress(codec pkg.Codec, src []byte) ([]byte, error) {\n\tb := new(bytes.Buffer)\n\tr := codec.NewReader(bytes.NewReader(src))\n\tif _, err := io.Copy(b, r); err != nil {\n\t\tr.Close()\n\t\treturn nil, err\n\t}\n\tif err := r.Close(); err != nil {\n\t\treturn nil, err\n\t}\n\treturn b.Bytes(), nil\n}\n\nfunc testEncodeDecode(t *testing.T, m kafka.Message, codec pkg.Codec) {\n\tvar r1, r2 []byte\n\tvar err error\n\n\tt.Run(\"text format of \"+codec.Name(), func(t *testing.T) {\n\t\tc := pkg.Compression(codec.Code())\n\t\ta := strconv.Itoa(int(c))\n\t\tx := pkg.Compression(-1)\n\t\ty := pkg.Compression(-1)\n\t\tb, err := c.MarshalText()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tif err := x.UnmarshalText([]byte(a)); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif err := y.UnmarshalText(b); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tif x != c {\n\t\t\tt.Errorf(\"compression mismatch after marshal/unmarshal: want=%s got=%s\", c, x)\n\t\t}\n\t\tif y != c {\n\t\t\tt.Errorf(\"compression mismatch after marshal/unmarshal: want=%s got=%s\", c, y)\n\t\t}\n\t})\n\n\tt.Run(\"encode with \"+codec.Name(), func(t *testing.T) {\n\t\tr1, err = compress(codec, m.Value)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t})\n\n\tt.Run(\"decode with \"+codec.Name(), func(t *testing.T) {\n\t\tif r1 == nil {\n\t\t\tif r1, err = compress(codec, m.Value); err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t}\n\t\tr2, err = decompress(codec, r1)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif string(r2) != \"message\" {\n\t\t\tt.Error(\"bad message\")\n\t\t\tt.Logf(\"expected: %q\", string(m.Value))\n\t\t\tt.Logf(\"got: %q\", string(r2))\n\t\t}\n\t})\n}\n\nfunc TestCompressedMessages(t *testing.T) {\n\ttestCompressedMessages(t, new(gzip.Codec))\n\ttestCompressedMessages(t, new(snappy.Codec))\n\ttestCompressedMessages(t, new(lz4.Codec))\n\n\tif ktesting.KafkaIsAtLeast(\"2.1.0\") {\n\t\ttestCompressedMessages(t, new(zstd.Codec))\n\t}\n}\n\nfunc testCompressedMessages(t *testing.T, codec pkg.Codec) {\n\tt.Run(codec.Name(), func(t *testing.T) {\n\t\tclient, topic, shutdown := newLocalClientAndTopic()\n\t\tdefer shutdown()\n\n\t\tw := &kafka.Writer{\n\t\t\tAddr: kafka.TCP(\"127.0.0.1:9092\"),\n\t\t\tTopic: topic,\n\t\t\tCompression: kafka.Compression(codec.Code()),\n\t\t\tBatchTimeout: 10 * time.Millisecond,\n\t\t\tTransport: client.Transport,\n\t\t}\n\t\tdefer w.Close()\n\n\t\toffset := 0\n\t\tvar values []string\n\t\tfor i := 0; i < 10; i++ {\n\t\t\tbatch := make([]kafka.Message, i+1)\n\t\t\tfor j := range batch {\n\t\t\t\tvalue := fmt.Sprintf(\"Hello World %d!\", offset)\n\t\t\t\tvalues = append(values, value)\n\t\t\t\tbatch[j] = kafka.Message{\n\t\t\t\t\tKey: []byte(strconv.Itoa(offset)),\n\t\t\t\t\tValue: []byte(value),\n\t\t\t\t}\n\t\t\t\toffset++\n\t\t\t}\n\t\t\tctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)\n\t\t\tif err := w.WriteMessages(ctx, batch...); err != nil {\n\t\t\t\tt.Errorf(\"error sending batch %d, reason: %+v\", i+1, err)\n\t\t\t}\n\t\t\tcancel()\n\t\t}\n\n\t\tr := kafka.NewReader(kafka.ReaderConfig{\n\t\t\tBrokers: []string{\"127.0.0.1:9092\"},\n\t\t\tTopic: topic,\n\t\t\tPartition: 0,\n\t\t\tMaxWait: 10 * time.Millisecond,\n\t\t\tMinBytes: 1,\n\t\t\tMaxBytes: 1024,\n\t\t})\n\t\tdefer r.Close()\n\n\t\tctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)\n\t\tdefer cancel()\n\n\t\t// in order to ensure proper handling of decompressing message, read at\n\t\t// offsets that we know to be in the middle of compressed message sets.\n\t\tfor base := range values {\n\t\t\tr.SetOffset(int64(base))\n\t\t\tfor i := base; i < len(values); i++ {\n\t\t\t\tmsg, err := r.ReadMessage(ctx)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"error receiving message at loop %d, offset %d, reason: %+v\", base, i, err)\n\t\t\t\t}\n\t\t\t\tif msg.Offset != int64(i) {\n\t\t\t\t\tt.Fatalf(\"wrong offset at loop %d...expected %d but got %d\", base, i, msg.Offset)\n\t\t\t\t}\n\t\t\t\tif strconv.Itoa(i) != string(msg.Key) {\n\t\t\t\t\tt.Fatalf(\"wrong message key at loop %d...expected %d but got %s\", base, i, string(msg.Key))\n\t\t\t\t}\n\t\t\t\tif values[i] != string(msg.Value) {\n\t\t\t\t\tt.Fatalf(\"wrong message value at loop %d...expected %s but got %s\", base, values[i], string(msg.Value))\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t})\n}\n\nfunc TestMixedCompressedMessages(t *testing.T) {\n\tclient, topic, shutdown := newLocalClientAndTopic()\n\tdefer shutdown()\n\n\toffset := 0\n\tvar values []string\n\tproduce := func(n int, codec pkg.Codec) {\n\t\tw := &kafka.Writer{\n\t\t\tAddr: kafka.TCP(\"127.0.0.1:9092\"),\n\t\t\tTopic: topic,\n\t\t\tTransport: client.Transport,\n\t\t}\n\t\tdefer w.Close()\n\n\t\tif codec != nil {\n\t\t\tw.Compression = kafka.Compression(codec.Code())\n\t\t}\n\n\t\tmsgs := make([]kafka.Message, n)\n\t\tfor i := range msgs {\n\t\t\tvalue := fmt.Sprintf(\"Hello World %d!\", offset)\n\t\t\tvalues = append(values, value)\n\t\t\toffset++\n\t\t\tmsgs[i] = kafka.Message{Value: []byte(value)}\n\t\t}\n\n\t\tctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)\n\t\tdefer cancel()\n\t\tif err := w.WriteMessages(ctx, msgs...); err != nil {\n\t\t\tt.Errorf(\"failed to produce messages: %+v\", err)\n\t\t}\n\t}\n\n\t// produce messages that interleave uncompressed messages and messages with\n\t// different compression codecs. reader should be able to properly handle\n\t// all of them.\n\tproduce(10, nil)\n\tproduce(20, new(gzip.Codec))\n\tproduce(5, nil)\n\tproduce(10, new(snappy.Codec))\n\tproduce(10, new(lz4.Codec))\n\tproduce(5, nil)\n\n\tr := kafka.NewReader(kafka.ReaderConfig{\n\t\tBrokers: []string{\"127.0.0.1:9092\"},\n\t\tTopic: topic,\n\t\tPartition: 0,\n\t\tMaxWait: 10 * time.Millisecond,\n\t\tMinBytes: 1,\n\t\tMaxBytes: 1024,\n\t})\n\tdefer r.Close()\n\n\tctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)\n\tdefer cancel()\n\n\t// in order to ensure proper handling of decompressing message, read at\n\t// offsets that we know to be in the middle of compressed message sets.\n\tfor base := range values {\n\t\tr.SetOffset(int64(base))\n\t\tfor i := base; i < len(values); i++ {\n\t\t\tmsg, err := r.ReadMessage(ctx)\n\t\t\tif err != nil {\n\t\t\t\tt.Errorf(\"error receiving message at loop %d, offset %d, reason: %+v\", base, i, err)\n\t\t\t}\n\t\t\tif msg.Offset != int64(i) {\n\t\t\t\tt.Errorf(\"wrong offset at loop %d...expected %d but got %d\", base, i, msg.Offset)\n\t\t\t}\n\t\t\tif values[i] != string(msg.Value) {\n\t\t\t\tt.Errorf(\"wrong message value at loop %d...expected %s but got %s\", base, values[i], string(msg.Value))\n\t\t\t}\n\t\t}\n\t}\n}\n\ntype noopCodec struct{}\n\nfunc (noopCodec) Code() int8 {\n\treturn 0\n}\n\nfunc (noopCodec) Name() string {\n\treturn \"none\"\n}\n\nfunc (noopCodec) NewReader(r io.Reader) io.ReadCloser {\n\treturn ioutil.NopCloser(r)\n}\n\nfunc (noopCodec) NewWriter(w io.Writer) io.WriteCloser {\n\treturn nopWriteCloser{w}\n}\n\ntype nopWriteCloser struct{ io.Writer }\n\nfunc (nopWriteCloser) Close() error { return nil }\n\nfunc BenchmarkCompression(b *testing.B) {\n\tbenchmarks := []struct {\n\t\tcodec pkg.Codec\n\t\tfunction func(*testing.B, pkg.Codec, *bytes.Buffer, []byte) float64\n\t}{\n\t\t{\n\t\t\tcodec: &noopCodec{},\n\t\t\tfunction: benchmarkCompression,\n\t\t},\n\t\t{\n\t\t\tcodec: new(gzip.Codec),\n\t\t\tfunction: benchmarkCompression,\n\t\t},\n\t\t{\n\t\t\tcodec: new(snappy.Codec),\n\t\t\tfunction: benchmarkCompression,\n\t\t},\n\t\t{\n\t\t\tcodec: new(lz4.Codec),\n\t\t\tfunction: benchmarkCompression,\n\t\t},\n\t\t{\n\t\t\tcodec: new(zstd.Codec),\n\t\t\tfunction: benchmarkCompression,\n\t\t},\n\t}\n\n\tf, err := os.Open(filepath.Join(os.Getenv(\"GOROOT\"), \"src/encoding/json/testdata/code.json.gz\"))\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\tdefer f.Close()\n\n\tz, err := gz.NewReader(f)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\n\tpayload, err := ioutil.ReadAll(z)\n\tif err != nil {\n\t\tb.Fatal(err)\n\t}\n\n\tbuffer := bytes.Buffer{}\n\tbuffer.Grow(len(payload))\n\n\tts := &bytes.Buffer{}\n\ttw := tabwriter.NewWriter(ts, 0, 8, 0, '\\t', 0)\n\tdefer func() {\n\t\ttw.Flush()\n\t\tfmt.Printf(\"input => %.2f MB\\n\", float64(len(payload))/(1024*1024))\n\t\tfmt.Println(ts)\n\t}()\n\n\tfor i := range benchmarks {\n\t\tbenchmark := &benchmarks[i]\n\t\tratio := 0.0\n\n\t\tb.Run(benchmark.codec.Name(), func(b *testing.B) {\n\t\t\tratio = benchmark.function(b, benchmark.codec, &buffer, payload)\n\t\t})\n\n\t\tfmt.Fprintf(tw, \" %s:\\t%.2f%%\\n\", benchmark.codec.Name(), 100*ratio)\n\t}\n}\n\nfunc benchmarkCompression(b *testing.B, codec pkg.Codec, buf *bytes.Buffer, payload []byte) float64 {\n\t// In case only the decompression benchmark are run, we use this flags to\n\t// detect whether we have to compress the payload before the decompression\n\t// benchmarks.\n\tcompressed := false\n\n\tb.Run(\"compress\", func(b *testing.B) {\n\t\tcompressed = true\n\t\tr := bytes.NewReader(payload)\n\t\tb.ReportAllocs()\n\n\t\tfor i := 0; i < b.N; i++ {\n\t\t\tbuf.Reset()\n\t\t\tr.Reset(payload)\n\t\t\tw := codec.NewWriter(buf)\n\n\t\t\t_, err := io.Copy(w, r)\n\t\t\tif err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\t\t\tif err := w.Close(); err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\t\t}\n\n\t\tb.SetBytes(int64(buf.Len()))\n\t})\n\n\tif !compressed {\n\t\tr := bytes.NewReader(payload)\n\t\tw := codec.NewWriter(buf)\n\n\t\t_, err := io.Copy(w, r)\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t\tif err := w.Close(); err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n\n\tb.Run(\"decompress\", func(b *testing.B) {\n\t\tc := bytes.NewReader(buf.Bytes())\n\t\tb.ReportAllocs()\n\t\tfor i := 0; i < b.N; i++ {\n\t\t\tc.Reset(buf.Bytes())\n\t\t\tr := codec.NewReader(c)\n\n\t\t\tn, err := io.Copy(ioutil.Discard, r)\n\t\t\tif err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\t\t\tif err := r.Close(); err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\n\t\t\tb.SetBytes(n)\n\t\t}\n\t})\n\n\treturn 1 - (float64(buf.Len()) / float64(len(payload)))\n}\n\nfunc init() {\n\trand.Seed(time.Now().UnixNano())\n}\n\nfunc makeTopic() string {\n\treturn fmt.Sprintf(\"kafka-go-%016x\", rand.Int63())\n}\n\nfunc newLocalClientAndTopic() (*kafka.Client, string, func()) {\n\ttopic := makeTopic()\n\tclient, shutdown := newLocalClient()\n\n\t_, err := client.CreateTopics(context.Background(), &kafka.CreateTopicsRequest{\n\t\tTopics: []kafka.TopicConfig{{\n\t\t\tTopic: topic,\n\t\t\tNumPartitions: 1,\n\t\t\tReplicationFactor: 1,\n\t\t}},\n\t})\n\tif err != nil {\n\t\tshutdown()\n\t\tpanic(err)\n\t}\n\n\t// Topic creation seems to be asynchronous. Metadata for the topic partition\n\t// layout in the cluster is available in the controller before being synced\n\t// with the other brokers, which causes \"Error:[3] Unknown Topic Or Partition\"\n\t// when sending requests to the partition leaders.\n\tfor i := 0; i < 20; i++ {\n\t\tr, err := client.Fetch(context.Background(), &kafka.FetchRequest{\n\t\t\tTopic: topic,\n\t\t\tPartition: 0,\n\t\t\tOffset: 0,\n\t\t})\n\t\tif err == nil && r.Error == nil {\n\t\t\tbreak\n\t\t}\n\t\ttime.Sleep(100 * time.Millisecond)\n\t}\n\n\treturn client, topic, func() {\n\t\tclient.DeleteTopics(context.Background(), &kafka.DeleteTopicsRequest{\n\t\t\tTopics: []string{topic},\n\t\t})\n\t\tshutdown()\n\t}\n}\n\nfunc newLocalClient() (*kafka.Client, func()) {\n\treturn newClient(kafka.TCP(\"127.0.0.1:9092\"))\n}\n\nfunc newClient(addr net.Addr) (*kafka.Client, func()) {\n\tconns := &ktesting.ConnWaitGroup{\n\t\tDialFunc: (&net.Dialer{}).DialContext,\n\t}\n\n\ttransport := &kafka.Transport{\n\t\tDial: conns.Dial,\n\t}\n\n\tclient := &kafka.Client{\n\t\tAddr: addr,\n\t\tTimeout: 5 * time.Second,\n\t\tTransport: transport,\n\t}\n\n\treturn client, func() { transport.CloseIdleConnections(); conns.Wait() }\n}\n" }, { "path": "compress/gzip/gzip.go", "content": "package gzip\n\nimport (\n\t\"io\"\n\t\"sync\"\n\n\t\"github.com/klauspost/compress/gzip\"\n)\n\nvar (\n\treaderPool sync.Pool\n)\n\n// Codec is the implementation of a compress.Codec which supports creating\n// readers and writers for kafka messages compressed with gzip.\ntype Codec struct {\n\t// The compression level to configure on writers created by this codec.\n\t// Acceptable values are defined in the standard gzip package.\n\t//\n\t// Default to gzip.DefaultCompressionLevel.\n\tLevel int\n\n\twriterPool sync.Pool\n}\n\n// Code implements the compress.Codec interface.\nfunc (c *Codec) Code() int8 { return 1 }\n\n// Name implements the compress.Codec interface.\nfunc (c *Codec) Name() string { return \"gzip\" }\n\n// NewReader implements the compress.Codec interface.\nfunc (c *Codec) NewReader(r io.Reader) io.ReadCloser {\n\tvar err error\n\tz, _ := readerPool.Get().(*gzip.Reader)\n\tif z != nil {\n\t\terr = z.Reset(r)\n\t} else {\n\t\tz, err = gzip.NewReader(r)\n\t}\n\tif err != nil {\n\t\tif z != nil {\n\t\t\treaderPool.Put(z)\n\t\t}\n\t\treturn &errorReader{err: err}\n\t}\n\treturn &reader{Reader: z}\n}\n\n// NewWriter implements the compress.Codec interface.\nfunc (c *Codec) NewWriter(w io.Writer) io.WriteCloser {\n\tx := c.writerPool.Get()\n\tz, _ := x.(*gzip.Writer)\n\tif z == nil {\n\t\tx, err := gzip.NewWriterLevel(w, c.level())\n\t\tif err != nil {\n\t\t\treturn &errorWriter{err: err}\n\t\t}\n\t\tz = x\n\t} else {\n\t\tz.Reset(w)\n\t}\n\treturn &writer{codec: c, Writer: z}\n}\n\nfunc (c *Codec) level() int {\n\tif c.Level != 0 {\n\t\treturn c.Level\n\t}\n\treturn gzip.DefaultCompression\n}\n\ntype reader struct{ *gzip.Reader }\n\nfunc (r *reader) Close() (err error) {\n\tif z := r.Reader; z != nil {\n\t\tr.Reader = nil\n\t\terr = z.Close()\n\t\t// Pass it an empty reader, which is a zero-size value implementing the\n\t\t// flate.Reader interface to avoid the construction of a bufio.Reader in\n\t\t// the call to Reset.\n\t\t//\n\t\t// Note: we could also not reset the reader at all, but that would cause\n\t\t// the underlying reader to be retained until the gzip.Reader is freed,\n\t\t// which may not be desirable.\n\t\tz.Reset(emptyReader{})\n\t\treaderPool.Put(z)\n\t}\n\treturn\n}\n\ntype writer struct {\n\tcodec *Codec\n\t*gzip.Writer\n}\n\nfunc (w *writer) Close() (err error) {\n\tif z := w.Writer; z != nil {\n\t\tw.Writer = nil\n\t\terr = z.Close()\n\t\tz.Reset(nil)\n\t\tw.codec.writerPool.Put(z)\n\t}\n\treturn\n}\n\ntype emptyReader struct{}\n\nfunc (emptyReader) ReadByte() (byte, error) { return 0, io.EOF }\n\nfunc (emptyReader) Read([]byte) (int, error) { return 0, io.EOF }\n\ntype errorReader struct{ err error }\n\nfunc (r *errorReader) Close() error { return r.err }\n\nfunc (r *errorReader) Read([]byte) (int, error) { return 0, r.err }\n\ntype errorWriter struct{ err error }\n\nfunc (w *errorWriter) Close() error { return w.err }\n\nfunc (w *errorWriter) Write([]byte) (int, error) { return 0, w.err }\n" }, { "path": "compress/lz4/lz4.go", "content": "package lz4\n\nimport (\n\t\"io\"\n\t\"sync\"\n\n\t\"github.com/pierrec/lz4/v4\"\n)\n\nvar (\n\treaderPool sync.Pool\n\twriterPool sync.Pool\n)\n\n// Codec is the implementation of a compress.Codec which supports creating\n// readers and writers for kafka messages compressed with lz4.\ntype Codec struct{}\n\n// Code implements the compress.Codec interface.\nfunc (c *Codec) Code() int8 { return 3 }\n\n// Name implements the compress.Codec interface.\nfunc (c *Codec) Name() string { return \"lz4\" }\n\n// NewReader implements the compress.Codec interface.\nfunc (c *Codec) NewReader(r io.Reader) io.ReadCloser {\n\tz, _ := readerPool.Get().(*lz4.Reader)\n\tif z != nil {\n\t\tz.Reset(r)\n\t} else {\n\t\tz = lz4.NewReader(r)\n\t}\n\treturn &reader{Reader: z}\n}\n\n// NewWriter implements the compress.Codec interface.\nfunc (c *Codec) NewWriter(w io.Writer) io.WriteCloser {\n\tz, _ := writerPool.Get().(*lz4.Writer)\n\tif z != nil {\n\t\tz.Reset(w)\n\t} else {\n\t\tz = lz4.NewWriter(w)\n\t}\n\treturn &writer{Writer: z}\n}\n\ntype reader struct{ *lz4.Reader }\n\nfunc (r *reader) Close() (err error) {\n\tif z := r.Reader; z != nil {\n\t\tr.Reader = nil\n\t\tz.Reset(nil)\n\t\treaderPool.Put(z)\n\t}\n\treturn\n}\n\ntype writer struct{ *lz4.Writer }\n\nfunc (w *writer) Close() (err error) {\n\tif z := w.Writer; z != nil {\n\t\tw.Writer = nil\n\t\terr = z.Close()\n\t\tz.Reset(nil)\n\t\twriterPool.Put(z)\n\t}\n\treturn\n}\n" }, { "path": "compress/snappy/go-xerial-snappy/LICENSE", "content": "The MIT License (MIT)\n\nCopyright (c) 2016 Evan Huus\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" }, { "path": "compress/snappy/go-xerial-snappy/README.md", "content": "# go-xerial-snappy\n\n[![Build Status](https://travis-ci.org/eapache/go-xerial-snappy.svg?branch=master)](https://travis-ci.org/eapache/go-xerial-snappy)\n\nXerial-compatible Snappy framing support for golang.\n\nPackages using Xerial for snappy encoding use a framing format incompatible with\nbasically everything else in existence. This package wraps Go's built-in snappy\npackage to support it.\n\nApps that use this format include Apache Kafka (see\nhttps://github.com/dpkp/kafka-python/issues/126#issuecomment-35478921 for\ndetails).\n" }, { "path": "compress/snappy/go-xerial-snappy/corpus/05979b224be0294bf350310d4ba5257c9bb815db-3", "content": "���Y" }, { "path": "compress/snappy/go-xerial-snappy/corpus/521e7e67b6063a75e0eeb24b0d1dd20731d34ad8-4", "content": "����Y" }, { "path": "compress/snappy/go-xerial-snappy/corpus/581b8fe7088f921567811fdf30e1f527c9f48e5e", "content": "package " }, { "path": "compress/snappy/go-xerial-snappy/corpus/80881d1b911b95e0203b3b0e7dc6360c35f7620f-7", "content": "墳￩��￩￩���꿽" }, { "path": "compress/snappy/go-xerial-snappy/corpus/b2419fcb7a9aef359de67cb6bd2b8a8c1f5c100f-4", "content": "��Y" }, { "path": "compress/snappy/go-xerial-snappy/corpus/cb806bc4f67316af02d6ae677332a3b6005a18da-5", "content": "墳�濽" }, { "path": "compress/snappy/go-xerial-snappy/corpus/ce3c6f4c31f74d72fbf74c17d14a8d29aa62059e-6", "content": "墳￩���꿽" }, { "path": "compress/snappy/go-xerial-snappy/corpus/da39a3ee5e6b4b0d3255bfef95601890afd80709-1", "content": "" }, { "path": "compress/snappy/go-xerial-snappy/fuzz.go", "content": "// +build gofuzz\n\npackage snappy\n\nfunc Fuzz(data []byte) int {\n\tdecode, err := Decode(data)\n\tif decode == nil && err == nil {\n\t\tpanic(\"nil error with nil result\")\n\t}\n\n\tif err != nil {\n\t\treturn 0\n\t}\n\n\treturn 1\n}\n" }, { "path": "compress/snappy/go-xerial-snappy/snappy.go", "content": "package snappy\n\nimport (\n\t\"bytes\"\n\t\"encoding/binary\"\n\t\"errors\"\n\n\tmaster \"github.com/klauspost/compress/snappy\"\n)\n\nconst (\n\tsizeOffset = 16\n\tsizeBytes = 4\n)\n\nvar (\n\txerialHeader = []byte{130, 83, 78, 65, 80, 80, 89, 0}\n\n\t// This is xerial version 1 and minimally compatible with version 1.\n\txerialVersionInfo = []byte{0, 0, 0, 1, 0, 0, 0, 1}\n\n\t// ErrMalformed is returned by the decoder when the xerial framing\n\t// is malformed.\n\tErrMalformed = errors.New(\"malformed xerial framing\")\n)\n\nfunc min(x, y int) int {\n\tif x < y {\n\t\treturn x\n\t}\n\treturn y\n}\n\n// Encode encodes data as snappy with no framing header.\nfunc Encode(src []byte) []byte {\n\treturn master.Encode(nil, src)\n}\n\n// EncodeStream *appends* to the specified 'dst' the compressed\n// 'src' in xerial framing format. If 'dst' does not have enough\n// capacity, then a new slice will be allocated. If 'dst' has\n// non-zero length, then if *must* have been built using this function.\nfunc EncodeStream(dst, src []byte) []byte {\n\tif len(dst) == 0 {\n\t\tdst = append(dst, xerialHeader...)\n\t\tdst = append(dst, xerialVersionInfo...)\n\t}\n\n\t// Snappy encode in blocks of maximum 32KB\n\tvar (\n\t\tmax = len(src)\n\t\tblockSize = 32 * 1024\n\t\tpos = 0\n\t\tchunk []byte\n\t)\n\n\tfor pos < max {\n\t\tnewPos := min(pos+blockSize, max)\n\t\tchunk = master.Encode(chunk[:cap(chunk)], src[pos:newPos])\n\n\t\t// First encode the compressed size (big-endian)\n\t\t// Put* panics if the buffer is too small, so pad 4 bytes first\n\t\torigLen := len(dst)\n\t\tdst = append(dst, dst[0:4]...)\n\t\tbinary.BigEndian.PutUint32(dst[origLen:], uint32(len(chunk)))\n\n\t\t// And now the compressed data\n\t\tdst = append(dst, chunk...)\n\t\tpos = newPos\n\t}\n\treturn dst\n}\n\n// Decode decodes snappy data whether it is traditional unframed\n// or includes the xerial framing format.\nfunc Decode(src []byte) ([]byte, error) {\n\treturn DecodeInto(nil, src)\n}\n\n// DecodeInto decodes snappy data whether it is traditional unframed\n// or includes the xerial framing format into the specified `dst`.\n// It is assumed that the entirety of `dst` including all capacity is available\n// for use by this function. If `dst` is nil *or* insufficiently large to hold\n// the decoded `src`, new space will be allocated.\nfunc DecodeInto(dst, src []byte) ([]byte, error) {\n\tvar max = len(src)\n\tif max < len(xerialHeader) {\n\t\treturn nil, ErrMalformed\n\t}\n\n\tif !bytes.Equal(src[:8], xerialHeader) {\n\t\treturn master.Decode(dst[:cap(dst)], src)\n\t}\n\n\tif max < sizeOffset+sizeBytes {\n\t\treturn nil, ErrMalformed\n\t}\n\n\tif dst == nil {\n\t\tdst = make([]byte, 0, len(src))\n\t}\n\n\tdst = dst[:0]\n\tvar (\n\t\tpos = sizeOffset\n\t\tchunk []byte\n\t\terr error\n\t)\n\n\tfor pos+sizeBytes <= max {\n\t\tsize := int(binary.BigEndian.Uint32(src[pos : pos+sizeBytes]))\n\t\tpos += sizeBytes\n\n\t\tnextPos := pos + size\n\t\t// On architectures where int is 32-bytes wide size + pos could\n\t\t// overflow so we need to check the low bound as well as the\n\t\t// high\n\t\tif nextPos < pos || nextPos > max {\n\t\t\treturn nil, ErrMalformed\n\t\t}\n\n\t\tchunk, err = master.Decode(chunk[:cap(chunk)], src[pos:nextPos])\n\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tpos = nextPos\n\t\tdst = append(dst, chunk...)\n\t}\n\treturn dst, nil\n}\n" }, { "path": "compress/snappy/go-xerial-snappy/snappy_test.go", "content": "package snappy\n\nimport (\n\t\"bytes\"\n\t\"errors\"\n\t\"testing\"\n)\n\nconst largeString = `Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur? At vero eos et accusamus et iusto odio dignissimos ducimus qui blanditiis praesentium voluptatum deleniti atque corrupti quos dolores et quas molestias except`\n\nvar snappyTestCases = map[string][]byte{\n\t\"REPEATREPEATREPEATREPEATREPEATREPEAT\": {36, 20, 82, 69, 80, 69, 65, 84, 118, 6, 0},\n\t\"REALLY SHORT\": {12, 44, 82, 69, 65, 76, 76, 89, 32, 83, 72, 79, 82, 84},\n\t\"AXBXCXDXEXFX\": {12, 44, 65, 88, 66, 88, 67, 88, 68, 88, 69, 88, 70, 88},\n}\n\nvar snappyStreamTestCases = map[string][]byte{\n\t\"PLAINDATA\": {130, 83, 78, 65, 80, 80, 89, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 11, 9, 32, 80, 76, 65, 73, 78, 68, 65, 84, 65},\n\t`{\"a\":\"UtaitILHMDAAAAfU\",\"b\":\"日本\"}`: {130, 83, 78, 65, 80, 80, 89, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 39, 37, 144, 123, 34, 97, 34, 58, 34, 85, 116, 97, 105, 116, 73, 76, 72, 77, 68, 65, 65, 65, 65, 102, 85, 34, 44, 34, 98, 34, 58, 34, 230, 151, 165, 230, 156, 172, 34, 125},\n\tlargeString: {130, 83, 78, 65, 80, 80, 89, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 3, 89, 128, 8, 240, 90, 83, 101, 100, 32, 117, 116, 32, 112, 101, 114, 115, 112, 105, 99, 105, 97, 116, 105, 115, 32, 117, 110, 100, 101, 32, 111, 109, 110, 105, 115, 32, 105, 115, 116, 101, 32, 110, 97, 116, 117, 115, 32, 101, 114, 114, 111, 114, 32, 115, 105, 116, 32, 118, 111, 108, 117, 112, 116, 97, 116, 101, 109, 32, 97, 99, 99, 117, 115, 97, 110, 116, 105, 117, 109, 32, 100, 111, 108, 111, 114, 101, 109, 113, 117, 101, 32, 108, 97, 117, 100, 97, 5, 22, 240, 60, 44, 32, 116, 111, 116, 97, 109, 32, 114, 101, 109, 32, 97, 112, 101, 114, 105, 97, 109, 44, 32, 101, 97, 113, 117, 101, 32, 105, 112, 115, 97, 32, 113, 117, 97, 101, 32, 97, 98, 32, 105, 108, 108, 111, 32, 105, 110, 118, 101, 110, 116, 111, 114, 101, 32, 118, 101, 114, 105, 116, 97, 1, 141, 4, 101, 116, 1, 36, 88, 115, 105, 32, 97, 114, 99, 104, 105, 116, 101, 99, 116, 111, 32, 98, 101, 97, 116, 97, 101, 32, 118, 105, 1, 6, 120, 100, 105, 99, 116, 97, 32, 115, 117, 110, 116, 32, 101, 120, 112, 108, 105, 99, 97, 98, 111, 46, 32, 78, 101, 109, 111, 32, 101, 110, 105, 109, 5, 103, 0, 109, 46, 180, 0, 12, 113, 117, 105, 97, 17, 16, 0, 115, 5, 209, 72, 97, 115, 112, 101, 114, 110, 97, 116, 117, 114, 32, 97, 117, 116, 32, 111, 100, 105, 116, 5, 9, 36, 102, 117, 103, 105, 116, 44, 32, 115, 101, 100, 9, 53, 32, 99, 111, 110, 115, 101, 113, 117, 117, 110, 1, 42, 20, 109, 97, 103, 110, 105, 32, 9, 245, 16, 115, 32, 101, 111, 115, 1, 36, 28, 32, 114, 97, 116, 105, 111, 110, 101, 17, 96, 33, 36, 1, 51, 36, 105, 32, 110, 101, 115, 99, 105, 117, 110, 116, 1, 155, 1, 254, 16, 112, 111, 114, 114, 111, 1, 51, 36, 115, 113, 117, 97, 109, 32, 101, 115, 116, 44, 1, 14, 13, 81, 5, 183, 4, 117, 109, 1, 18, 0, 97, 9, 19, 4, 32, 115, 1, 149, 12, 109, 101, 116, 44, 9, 135, 76, 99, 116, 101, 116, 117, 114, 44, 32, 97, 100, 105, 112, 105, 115, 99, 105, 32, 118, 101, 108, 50, 173, 0, 24, 110, 111, 110, 32, 110, 117, 109, 9, 94, 84, 105, 117, 115, 32, 109, 111, 100, 105, 32, 116, 101, 109, 112, 111, 114, 97, 32, 105, 110, 99, 105, 100, 33, 52, 20, 117, 116, 32, 108, 97, 98, 33, 116, 4, 101, 116, 9, 106, 0, 101, 5, 219, 20, 97, 109, 32, 97, 108, 105, 5, 62, 33, 164, 8, 114, 97, 116, 29, 212, 12, 46, 32, 85, 116, 41, 94, 52, 97, 100, 32, 109, 105, 110, 105, 109, 97, 32, 118, 101, 110, 105, 33, 221, 72, 113, 117, 105, 115, 32, 110, 111, 115, 116, 114, 117, 109, 32, 101, 120, 101, 114, 99, 105, 33, 202, 104, 111, 110, 101, 109, 32, 117, 108, 108, 97, 109, 32, 99, 111, 114, 112, 111, 114, 105, 115, 32, 115, 117, 115, 99, 105, 112, 105, 13, 130, 8, 105, 111, 115, 1, 64, 12, 110, 105, 115, 105, 1, 150, 5, 126, 44, 105, 100, 32, 101, 120, 32, 101, 97, 32, 99, 111, 109, 5, 192, 0, 99, 41, 131, 33, 172, 8, 63, 32, 81, 1, 107, 4, 97, 117, 33, 101, 96, 118, 101, 108, 32, 101, 117, 109, 32, 105, 117, 114, 101, 32, 114, 101, 112, 114, 101, 104, 101, 110, 100, 101, 114, 105, 65, 63, 12, 105, 32, 105, 110, 1, 69, 16, 118, 111, 108, 117, 112, 65, 185, 1, 47, 24, 105, 116, 32, 101, 115, 115, 101, 1, 222, 64, 109, 32, 110, 105, 104, 105, 108, 32, 109, 111, 108, 101, 115, 116, 105, 97, 101, 46, 103, 0, 0, 44, 1, 45, 16, 32, 105, 108, 108, 117, 37, 143, 45, 36, 0, 109, 5, 110, 65, 33, 20, 97, 116, 32, 113, 117, 111, 17, 92, 44, 115, 32, 110, 117, 108, 108, 97, 32, 112, 97, 114, 105, 9, 165, 24, 65, 116, 32, 118, 101, 114, 111, 69, 34, 44, 101, 116, 32, 97, 99, 99, 117, 115, 97, 109, 117, 115, 1, 13, 104, 105, 117, 115, 116, 111, 32, 111, 100, 105, 111, 32, 100, 105, 103, 110, 105, 115, 115, 105, 109, 111, 115, 32, 100, 117, 99, 105, 1, 34, 80, 113, 117, 105, 32, 98, 108, 97, 110, 100, 105, 116, 105, 105, 115, 32, 112, 114, 97, 101, 115, 101, 101, 87, 17, 111, 56, 116, 117, 109, 32, 100, 101, 108, 101, 110, 105, 116, 105, 32, 97, 116, 65, 89, 28, 99, 111, 114, 114, 117, 112, 116, 105, 1, 150, 0, 115, 13, 174, 5, 109, 8, 113, 117, 97, 65, 5, 52, 108, 101, 115, 116, 105, 97, 115, 32, 101, 120, 99, 101, 112, 116, 0, 0, 0, 1, 0},\n}\n\nfunc makeMassive(input string, numCopies int) string {\n\toutBuff := make([]byte, len(input)*numCopies)\n\n\tfor i := 0; i < numCopies; i++ {\n\t\tcopy(outBuff[len(outBuff):], input)\n\t}\n\n\treturn string(outBuff)\n}\n\nfunc TestSnappyEncode(t *testing.T) {\n\tfor src, exp := range snappyTestCases {\n\t\tdst := Encode([]byte(src))\n\t\tif !bytes.Equal(dst, exp) {\n\t\t\tt.Errorf(\"Expected %s to generate %v, but was %v\", src, exp, dst)\n\t\t}\n\t}\n}\n\nfunc TestSnappyEncodeStream(t *testing.T) {\n\tfor src := range snappyStreamTestCases {\n\t\tdst := EncodeStream(nil, []byte(src))\n\n\t\t// Block size can change the bytes generated, so let's just decode and make sure in matches out\n\t\tdec, err := Decode(dst)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t\tif src != string(dec) {\n\t\t\tt.Errorf(\"Expected decode to match encode orig = %s, decoded = %s\", src, string(dec))\n\t\t}\n\t}\n}\n\nfunc TestSnappyLargeStringEncodeStream(t *testing.T) {\n\tmassiveString := makeMassive(largeString, 10000)\n\tdst := EncodeStream(nil, []byte(massiveString))\n\tdec, err := Decode(dst)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\tif massiveString != string(dec) {\n\t\tt.Errorf(\"Decoded string didn't match original input (not printing due to size)\")\n\t}\n}\n\nfunc TestSnappyDecode(t *testing.T) {\n\tfor exp, src := range snappyTestCases {\n\t\tdst, err := Decode(src)\n\t\tif err != nil {\n\t\t\tt.Error(\"Encoding error: \", err)\n\t\t} else if !bytes.Equal(dst, []byte(exp)) {\n\t\t\tt.Errorf(\"Expected %s to be generated from %v, but was %s\", exp, src, string(dst))\n\t\t}\n\t}\n}\n\nfunc TestSnappyDecodeStreams(t *testing.T) {\n\tfor exp, src := range snappyStreamTestCases {\n\t\tdst, err := Decode(src)\n\t\tif err != nil {\n\t\t\tt.Error(\"Encoding error: \", err)\n\t\t} else if !bytes.Equal(dst, []byte(exp)) {\n\t\t\tt.Errorf(\"Expected %s to be generated from [%d]byte, but was %s\", exp, len(src), string(dst))\n\t\t}\n\t}\n}\n\nfunc TestSnappyDecodeMalformedTruncatedHeader(t *testing.T) {\n\t// Truncated headers should not cause a panic.\n\tfor i := 0; i < len(xerialHeader); i++ {\n\t\tbuf := make([]byte, i)\n\t\tcopy(buf, xerialHeader[:i])\n\t\tif _, err := Decode(buf); !errors.Is(err, ErrMalformed) {\n\t\t\tt.Errorf(\"expected ErrMalformed got %v\", err)\n\t\t}\n\t}\n}\n\nfunc TestSnappyDecodeMalformedTruncatedSize(t *testing.T) {\n\t// Inputs with valid Xerial header but truncated \"size\" field\n\tsizes := []int{sizeOffset + 1, sizeOffset + 2, sizeOffset + 3}\n\tfor _, size := range sizes {\n\t\tbuf := make([]byte, size)\n\t\tcopy(buf, xerialHeader)\n\t\tif _, err := Decode(buf); !errors.Is(err, ErrMalformed) {\n\t\t\tt.Errorf(\"expected ErrMalformed got %v\", err)\n\t\t}\n\t}\n}\n\nfunc TestSnappyDecodeMalformedBNoData(t *testing.T) {\n\t// No data after the size field\n\tbuf := make([]byte, 20)\n\tcopy(buf, xerialHeader)\n\t// indicate that there's one byte of data to be read\n\tbuf[len(buf)-1] = 1\n\tif _, err := Decode(buf); !errors.Is(err, ErrMalformed) {\n\t\tt.Errorf(\"expected ErrMalformed got %v\", err)\n\t}\n}\n\nfunc TestSnappyMasterDecodeFailed(t *testing.T) {\n\tbuf := make([]byte, 21)\n\tcopy(buf, xerialHeader)\n\t// indicate that there's one byte of data to be read\n\tbuf[len(buf)-2] = 1\n\t// A payload which will not decode\n\tbuf[len(buf)-1] = 1\n\tif _, err := Decode(buf); errors.Is(err, ErrMalformed) || err == nil {\n\t\tt.Errorf(\"unexpected err: %v\", err)\n\t}\n}\n\nfunc BenchmarkSnappyDecode(b *testing.B) {\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\n\tfor n := 0; n < b.N; n++ {\n\t\tbytes := 0\n\t\tfor _, test := range snappyTestCases {\n\t\t\tdst, err := Decode(test)\n\t\t\tif err != nil {\n\t\t\t\tb.Error(\"Decoding error: \", err)\n\t\t\t}\n\t\t\tbytes += len(dst)\n\t\t}\n\t\tb.SetBytes(int64(bytes))\n\t}\n}\n\nfunc BenchmarkSnappyDecodeInto(b *testing.B) {\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\n\tvar (\n\t\tdst []byte\n\t\terr error\n\t)\n\n\tfor n := 0; n < b.N; n++ {\n\t\tbytes := 0\n\t\tfor _, test := range snappyTestCases {\n\n\t\t\tdst, err = DecodeInto(dst, test)\n\t\t\tif err != nil {\n\t\t\t\tb.Error(\"Decoding error: \", err)\n\t\t\t}\n\t\t\tbytes += len(dst)\n\t\t}\n\t\tb.SetBytes(int64(bytes))\n\t}\n}\n\nfunc BenchmarkSnappyStreamDecode(b *testing.B) {\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\n\tfor n := 0; n < b.N; n++ {\n\t\tbytes := 0\n\t\tfor _, test := range snappyStreamTestCases {\n\t\t\tdst, err := Decode(test)\n\t\t\tif err != nil {\n\t\t\t\tb.Error(\"Decoding error: \", err)\n\t\t\t}\n\t\t\tbytes += len(dst)\n\t\t}\n\t\tb.SetBytes(int64(bytes))\n\t}\n}\n\nfunc BenchmarkSnappyStreamDecodeInto(b *testing.B) {\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\n\tvar (\n\t\tdst = make([]byte, 1024)\n\t\terr error\n\t)\n\n\tfor n := 0; n < b.N; n++ {\n\t\tbytes := 0\n\t\tfor _, test := range snappyStreamTestCases {\n\t\t\tdst, err = DecodeInto(dst, test)\n\t\t\tif err != nil {\n\t\t\t\tb.Error(\"Decoding error: \", err)\n\t\t\t}\n\t\t\tbytes += len(dst)\n\t\t}\n\t\tb.SetBytes(int64(bytes))\n\t}\n}\nfunc BenchmarkSnappyStreamDecodeMassive(b *testing.B) {\n\tmassiveString := makeMassive(largeString, 10000)\n\tenc := EncodeStream(nil, []byte(massiveString))\n\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\tb.SetBytes(int64(len(massiveString)))\n\n\tfor n := 0; n < b.N; n++ {\n\t\t_, err := Decode(enc)\n\t\tif err != nil {\n\t\t\tb.Error(\"Decoding error: \", err)\n\t\t}\n\t}\n}\n\nfunc BenchmarkSnappyStreamDecodeIntoMassive(b *testing.B) {\n\tmassiveString := makeMassive(largeString, 10000)\n\tenc := EncodeStream(nil, []byte(massiveString))\n\n\tb.ReportAllocs()\n\tb.ResetTimer()\n\tb.SetBytes(int64(len(massiveString)))\n\n\tvar (\n\t\tdst = make([]byte, 1024)\n\t\terr error\n\t)\n\n\tfor n := 0; n < b.N; n++ {\n\t\tdst, err = DecodeInto(dst, enc)\n\t\tif err != nil {\n\t\t\tb.Error(\"Decoding error: \", err)\n\t\t}\n\t}\n}\n" }, { "path": "compress/snappy/snappy.go", "content": "package snappy\n\nimport (\n\t\"io\"\n\t\"sync\"\n\n\t\"github.com/klauspost/compress/s2\"\n\t\"github.com/klauspost/compress/snappy\"\n)\n\n// Framing is an enumeration type used to enable or disable xerial framing of\n// snappy messages.\ntype Framing int\n\nconst (\n\tFramed Framing = iota\n\tUnframed\n)\n\n// Compression level.\ntype Compression int\n\nconst (\n\tDefaultCompression Compression = iota\n\tFasterCompression\n\tBetterCompression\n\tBestCompression\n)\n\nvar (\n\treaderPool sync.Pool\n\twriterPool sync.Pool\n)\n\n// Codec is the implementation of a compress.Codec which supports creating\n// readers and writers for kafka messages compressed with snappy.\ntype Codec struct {\n\t// An optional framing to apply to snappy compression.\n\t//\n\t// Default to Framed.\n\tFraming Framing\n\n\t// Compression level.\n\tCompression Compression\n}\n\n// Code implements the compress.Codec interface.\nfunc (c *Codec) Code() int8 { return 2 }\n\n// Name implements the compress.Codec interface.\nfunc (c *Codec) Name() string { return \"snappy\" }\n\n// NewReader implements the compress.Codec interface.\nfunc (c *Codec) NewReader(r io.Reader) io.ReadCloser {\n\tx, _ := readerPool.Get().(*xerialReader)\n\tif x != nil {\n\t\tx.Reset(r)\n\t} else {\n\t\tx = &xerialReader{\n\t\t\treader: r,\n\t\t\tdecode: snappy.Decode,\n\t\t}\n\t}\n\treturn &reader{xerialReader: x}\n}\n\n// NewWriter implements the compress.Codec interface.\nfunc (c *Codec) NewWriter(w io.Writer) io.WriteCloser {\n\tx, _ := writerPool.Get().(*xerialWriter)\n\tif x != nil {\n\t\tx.Reset(w)\n\t} else {\n\t\tx = &xerialWriter{writer: w}\n\t}\n\tx.framed = c.Framing == Framed\n\tswitch c.Compression {\n\tcase FasterCompression:\n\t\tx.encode = s2.EncodeSnappy\n\tcase BetterCompression:\n\t\tx.encode = s2.EncodeSnappyBetter\n\tcase BestCompression:\n\t\tx.encode = s2.EncodeSnappyBest\n\tdefault:\n\t\tx.encode = snappy.Encode // aka. s2.EncodeSnappyBetter\n\t}\n\treturn &writer{xerialWriter: x}\n}\n\ntype reader struct{ *xerialReader }\n\nfunc (r *reader) Close() (err error) {\n\tif x := r.xerialReader; x != nil {\n\t\tr.xerialReader = nil\n\t\tx.Reset(nil)\n\t\treaderPool.Put(x)\n\t}\n\treturn\n}\n\ntype writer struct{ *xerialWriter }\n\nfunc (w *writer) Close() (err error) {\n\tif x := w.xerialWriter; x != nil {\n\t\tw.xerialWriter = nil\n\t\terr = x.Flush()\n\t\tx.Reset(nil)\n\t\twriterPool.Put(x)\n\t}\n\treturn\n}\n" }, { "path": "compress/snappy/xerial.go", "content": "package snappy\n\nimport (\n\t\"bytes\"\n\t\"encoding/binary\"\n\t\"errors\"\n\t\"io\"\n\n\t\"github.com/klauspost/compress/snappy\"\n)\n\nconst defaultBufferSize = 32 * 1024\n\n// An implementation of io.Reader which consumes a stream of xerial-framed\n// snappy-encoeded data. The framing is optional, if no framing is detected\n// the reader will simply forward the bytes from its underlying stream.\ntype xerialReader struct {\n\treader io.Reader\n\theader [16]byte\n\tinput []byte\n\toutput []byte\n\toffset int64\n\tnbytes int64\n\tdecode func([]byte, []byte) ([]byte, error)\n}\n\nfunc (x *xerialReader) Reset(r io.Reader) {\n\tx.reader = r\n\tx.input = x.input[:0]\n\tx.output = x.output[:0]\n\tx.header = [16]byte{}\n\tx.offset = 0\n\tx.nbytes = 0\n}\n\nfunc (x *xerialReader) Read(b []byte) (int, error) {\n\tfor {\n\t\tif x.offset < int64(len(x.output)) {\n\t\t\tn := copy(b, x.output[x.offset:])\n\t\t\tx.offset += int64(n)\n\t\t\treturn n, nil\n\t\t}\n\n\t\tn, err := x.readChunk(b)\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\t\tif n > 0 {\n\t\t\treturn n, nil\n\t\t}\n\t}\n}\n\nfunc (x *xerialReader) WriteTo(w io.Writer) (int64, error) {\n\twn := int64(0)\n\n\tfor {\n\t\tfor x.offset < int64(len(x.output)) {\n\t\t\tn, err := w.Write(x.output[x.offset:])\n\t\t\twn += int64(n)\n\t\t\tx.offset += int64(n)\n\t\t\tif err != nil {\n\t\t\t\treturn wn, err\n\t\t\t}\n\t\t}\n\n\t\tif _, err := x.readChunk(nil); err != nil {\n\t\t\tif errors.Is(err, io.EOF) {\n\t\t\t\terr = nil\n\t\t\t}\n\t\t\treturn wn, err\n\t\t}\n\t}\n}\n\nfunc (x *xerialReader) readChunk(dst []byte) (int, error) {\n\tx.output = x.output[:0]\n\tx.offset = 0\n\tprefix := 0\n\n\tif x.nbytes == 0 {\n\t\tn, err := x.readFull(x.header[:])\n\t\tif err != nil && n == 0 {\n\t\t\treturn 0, err\n\t\t}\n\t\tprefix = n\n\t}\n\n\tif isXerialHeader(x.header[:]) {\n\t\tif cap(x.input) < 4 {\n\t\t\tx.input = make([]byte, 4, defaultBufferSize)\n\t\t} else {\n\t\t\tx.input = x.input[:4]\n\t\t}\n\n\t\t_, err := x.readFull(x.input)\n\t\tif err != nil {\n\t\t\treturn 0, err\n\t\t}\n\n\t\tframe := int(binary.BigEndian.Uint32(x.input))\n\t\tif cap(x.input) < frame {\n\t\t\tx.input = make([]byte, frame, align(frame, defaultBufferSize))\n\t\t} else {\n\t\t\tx.input = x.input[:frame]\n\t\t}\n\n\t\tif _, err := x.readFull(x.input); err != nil {\n\t\t\treturn 0, err\n\t\t}\n\t} else {\n\t\tif cap(x.input) == 0 {\n\t\t\tx.input = make([]byte, 0, defaultBufferSize)\n\t\t} else {\n\t\t\tx.input = x.input[:0]\n\t\t}\n\n\t\tif prefix > 0 {\n\t\t\tx.input = append(x.input, x.header[:prefix]...)\n\t\t}\n\n\t\tfor {\n\t\t\tif len(x.input) == cap(x.input) {\n\t\t\t\tb := make([]byte, len(x.input), 2*cap(x.input))\n\t\t\t\tcopy(b, x.input)\n\t\t\t\tx.input = b\n\t\t\t}\n\n\t\t\tn, err := x.read(x.input[len(x.input):cap(x.input)])\n\t\t\tx.input = x.input[:len(x.input)+n]\n\t\t\tif err != nil {\n\t\t\t\tif errors.Is(err, io.EOF) && len(x.input) > 0 {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\treturn 0, err\n\t\t\t}\n\t\t}\n\t}\n\n\tvar n int\n\tvar err error\n\n\tif x.decode == nil {\n\t\tx.output, x.input, err = x.input, x.output, nil\n\t} else if n, err = snappy.DecodedLen(x.input); n <= len(dst) && err == nil {\n\t\t// If the output buffer is large enough to hold the decode value,\n\t\t// write it there directly instead of using the intermediary output\n\t\t// buffer.\n\t\t_, err = x.decode(dst, x.input)\n\t} else {\n\t\tvar b []byte\n\t\tn = 0\n\t\tb, err = x.decode(x.output[:cap(x.output)], x.input)\n\t\tif err == nil {\n\t\t\tx.output = b\n\t\t}\n\t}\n\n\treturn n, err\n}\n\nfunc (x *xerialReader) read(b []byte) (int, error) {\n\tn, err := x.reader.Read(b)\n\tx.nbytes += int64(n)\n\treturn n, err\n}\n\nfunc (x *xerialReader) readFull(b []byte) (int, error) {\n\tn, err := io.ReadFull(x.reader, b)\n\tx.nbytes += int64(n)\n\treturn n, err\n}\n\n// An implementation of a xerial-framed snappy-encoded output stream.\n// Each Write made to the writer is framed with a xerial header.\ntype xerialWriter struct {\n\twriter io.Writer\n\theader [16]byte\n\tinput []byte\n\toutput []byte\n\tnbytes int64\n\tframed bool\n\tencode func([]byte, []byte) []byte\n}\n\nfunc (x *xerialWriter) Reset(w io.Writer) {\n\tx.writer = w\n\tx.input = x.input[:0]\n\tx.output = x.output[:0]\n\tx.nbytes = 0\n}\n\nfunc (x *xerialWriter) ReadFrom(r io.Reader) (int64, error) {\n\twn := int64(0)\n\n\tif cap(x.input) == 0 {\n\t\tx.input = make([]byte, 0, defaultBufferSize)\n\t}\n\n\tfor {\n\t\tif x.full() {\n\t\t\tx.grow()\n\t\t}\n\n\t\tn, err := r.Read(x.input[len(x.input):cap(x.input)])\n\t\twn += int64(n)\n\t\tx.input = x.input[:len(x.input)+n]\n\n\t\tif x.fullEnough() {\n\t\t\tif err := x.Flush(); err != nil {\n\t\t\t\treturn wn, err\n\t\t\t}\n\t\t}\n\n\t\tif err != nil {\n\t\t\tif errors.Is(err, io.EOF) {\n\t\t\t\terr = nil\n\t\t\t}\n\t\t\treturn wn, err\n\t\t}\n\t}\n}\n\nfunc (x *xerialWriter) Write(b []byte) (int, error) {\n\twn := 0\n\n\tif cap(x.input) == 0 {\n\t\tx.input = make([]byte, 0, defaultBufferSize)\n\t}\n\n\tfor len(b) > 0 {\n\t\tif x.full() {\n\t\t\tx.grow()\n\t\t}\n\n\t\tn := copy(x.input[len(x.input):cap(x.input)], b)\n\t\tb = b[n:]\n\t\twn += n\n\t\tx.input = x.input[:len(x.input)+n]\n\n\t\tif x.fullEnough() {\n\t\t\tif err := x.Flush(); err != nil {\n\t\t\t\treturn wn, err\n\t\t\t}\n\t\t}\n\t}\n\n\treturn wn, nil\n}\n\nfunc (x *xerialWriter) Flush() error {\n\tif len(x.input) == 0 {\n\t\treturn nil\n\t}\n\n\tvar b []byte\n\tif x.encode == nil {\n\t\tb = x.input\n\t} else {\n\t\tx.output = x.encode(x.output[:cap(x.output)], x.input)\n\t\tb = x.output\n\t}\n\n\tx.input = x.input[:0]\n\tx.output = x.output[:0]\n\n\tif x.framed && x.nbytes == 0 {\n\t\twriteXerialHeader(x.header[:])\n\t\t_, err := x.write(x.header[:])\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif x.framed {\n\t\twriteXerialFrame(x.header[:4], len(b))\n\t\t_, err := x.write(x.header[:4])\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\t_, err := x.write(b)\n\treturn err\n}\n\nfunc (x *xerialWriter) write(b []byte) (int, error) {\n\tn, err := x.writer.Write(b)\n\tx.nbytes += int64(n)\n\treturn n, err\n}\n\nfunc (x *xerialWriter) full() bool {\n\treturn len(x.input) == cap(x.input)\n}\n\nfunc (x *xerialWriter) fullEnough() bool {\n\treturn x.framed && (cap(x.input)-len(x.input)) < 1024\n}\n\nfunc (x *xerialWriter) grow() {\n\ttmp := make([]byte, len(x.input), 2*cap(x.input))\n\tcopy(tmp, x.input)\n\tx.input = tmp\n}\n\nfunc align(n, a int) int {\n\tif (n % a) == 0 {\n\t\treturn n\n\t}\n\treturn ((n / a) + 1) * a\n}\n\nvar (\n\txerialHeader = [...]byte{130, 83, 78, 65, 80, 80, 89, 0}\n\txerialVersionInfo = [...]byte{0, 0, 0, 1, 0, 0, 0, 1}\n)\n\nfunc isXerialHeader(src []byte) bool {\n\treturn len(src) >= 16 && bytes.Equal(src[:8], xerialHeader[:])\n}\n\nfunc writeXerialHeader(b []byte) {\n\tcopy(b[:8], xerialHeader[:])\n\tcopy(b[8:], xerialVersionInfo[:])\n}\n\nfunc writeXerialFrame(b []byte, n int) {\n\tbinary.BigEndian.PutUint32(b, uint32(n))\n}\n" }, { "path": "compress/snappy/xerial_test.go", "content": "package snappy\n\nimport (\n\t\"bytes\"\n\t\"crypto/rand\"\n\t\"io\"\n\t\"testing\"\n\n\t\"github.com/klauspost/compress/snappy\"\n\tgoxerialsnappy \"github.com/segmentio/kafka-go/compress/snappy/go-xerial-snappy\"\n)\n\n// Wrap an io.Reader or io.Writer to disable all copy optimizations like\n// io.WriterTo or io.ReaderFrom.\n// We use this to ensure writes are chunked by io.Copy's internal buffer\n// in the tests.\ntype simpleReader struct{ io.Reader }\ntype simpleWriter struct{ io.Writer }\n\nfunc TestXerialReaderSnappy(t *testing.T) {\n\trawData := new(bytes.Buffer)\n\trawData.Grow(1024 * 1024)\n\tio.CopyN(rawData, rand.Reader, 1024*1024)\n\n\tcompressedRawData := bytes.NewReader(snappy.Encode(nil, rawData.Bytes()))\n\n\tdecompressedData := new(bytes.Buffer)\n\tio.Copy(decompressedData,\n\t\t&xerialReader{reader: compressedRawData, decode: snappy.Decode})\n\n\tb0 := rawData.Bytes()\n\tb1 := decompressedData.Bytes()\n\n\tif !bytes.Equal(b0, b1) {\n\t\tt.Error(\"data mismatch\")\n\t}\n}\n\nfunc TestXerialReaderWriter(t *testing.T) {\n\trawData := new(bytes.Buffer)\n\trawData.Grow(1024 * 1024)\n\tio.CopyN(rawData, rand.Reader, 1024*1024)\n\n\tframedData := new(bytes.Buffer)\n\tframedData.Grow(rawData.Len() + 1024)\n\tw := simpleWriter{&xerialWriter{writer: framedData}}\n\tr := simpleReader{bytes.NewReader(rawData.Bytes())}\n\tio.Copy(w, r)\n\tw.Writer.(*xerialWriter).Flush()\n\n\tunframedData := new(bytes.Buffer)\n\tunframedData.Grow(rawData.Len())\n\tio.Copy(unframedData, &xerialReader{reader: framedData})\n\n\tb0 := rawData.Bytes()\n\tb1 := unframedData.Bytes()\n\n\tif !bytes.Equal(b0, b1) {\n\t\tt.Error(\"data mismatch\")\n\t}\n}\n\nfunc TestXerialFramedCompression(t *testing.T) {\n\trawData := new(bytes.Buffer)\n\trawData.Grow(1024 * 1024)\n\tio.CopyN(rawData, rand.Reader, 1024*1024)\n\n\tframedAndCompressedData := new(bytes.Buffer)\n\tframedAndCompressedData.Grow(rawData.Len())\n\tw := simpleWriter{&xerialWriter{writer: framedAndCompressedData, framed: true, encode: snappy.Encode}}\n\tr := simpleReader{bytes.NewReader(rawData.Bytes())}\n\tio.Copy(w, r)\n\tw.Writer.(*xerialWriter).Flush()\n\n\tunframedAndDecompressedData := new(bytes.Buffer)\n\tunframedAndDecompressedData.Grow(rawData.Len())\n\tio.Copy(unframedAndDecompressedData,\n\t\tsimpleReader{&xerialReader{reader: framedAndCompressedData, decode: snappy.Decode}})\n\n\tb0 := rawData.Bytes()\n\tb1 := unframedAndDecompressedData.Bytes()\n\n\tif !bytes.Equal(b0, b1) {\n\t\tt.Error(\"data mismatch\")\n\t}\n}\n\nfunc TestXerialFramedCompressionOptimized(t *testing.T) {\n\trawData := new(bytes.Buffer)\n\trawData.Grow(1024 * 1024)\n\tio.CopyN(rawData, rand.Reader, 1024*1024)\n\n\tframedAndCompressedData := new(bytes.Buffer)\n\tframedAndCompressedData.Grow(rawData.Len())\n\tw := &xerialWriter{writer: framedAndCompressedData, framed: true, encode: snappy.Encode}\n\tr := simpleReader{bytes.NewReader(rawData.Bytes())}\n\tio.Copy(w, r)\n\tw.Flush()\n\n\tunframedAndDecompressedData := new(bytes.Buffer)\n\tunframedAndDecompressedData.Grow(rawData.Len())\n\tio.Copy(unframedAndDecompressedData,\n\t\t&xerialReader{reader: framedAndCompressedData, decode: snappy.Decode})\n\n\tb0 := rawData.Bytes()\n\tb1 := unframedAndDecompressedData.Bytes()\n\n\tif !bytes.Equal(b0, b1) {\n\t\tt.Error(\"data mismatch\")\n\t}\n}\n\nfunc TestXerialReaderAgainstGoXerialSnappy(t *testing.T) {\n\trawData := new(bytes.Buffer)\n\trawData.Grow(1024 * 1024)\n\tio.CopyN(rawData, rand.Reader, 1024*1024)\n\trawBytes := rawData.Bytes()\n\n\tframedAndCompressedData := []byte{}\n\tconst chunkSize = 999\n\tfor i := 0; i < len(rawBytes); i += chunkSize {\n\t\tj := i + chunkSize\n\t\tif j > len(rawBytes) {\n\t\t\tj = len(rawBytes)\n\t\t}\n\t\tframedAndCompressedData = goxerialsnappy.EncodeStream(framedAndCompressedData, rawBytes[i:j])\n\t}\n\n\tunframedAndDecompressedData := new(bytes.Buffer)\n\tunframedAndDecompressedData.Grow(rawData.Len())\n\tio.Copy(unframedAndDecompressedData,\n\t\t&xerialReader{reader: bytes.NewReader(framedAndCompressedData), decode: snappy.Decode})\n\n\tb0 := rawBytes\n\tb1 := unframedAndDecompressedData.Bytes()\n\n\tif !bytes.Equal(b0, b1) {\n\t\tt.Error(\"data mismatch\")\n\t}\n}\n\nfunc TestXerialWriterAgainstGoXerialSnappy(t *testing.T) {\n\trawData := new(bytes.Buffer)\n\trawData.Grow(1024 * 1024)\n\tio.CopyN(rawData, rand.Reader, 1024*1024)\n\n\tframedAndCompressedData := new(bytes.Buffer)\n\tframedAndCompressedData.Grow(rawData.Len())\n\tw := &xerialWriter{writer: framedAndCompressedData, framed: true, encode: snappy.Encode}\n\tr := simpleReader{bytes.NewReader(rawData.Bytes())}\n\tio.Copy(w, r)\n\tw.Flush()\n\n\tunframedAndDecompressedData, err := goxerialsnappy.Decode(framedAndCompressedData.Bytes())\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tb0 := rawData.Bytes()\n\tb1 := unframedAndDecompressedData\n\n\tif !bytes.Equal(b0, b1) {\n\t\tt.Error(\"data mismatch\")\n\t}\n}\n" }, { "path": "compress/zstd/zstd.go", "content": "// Package zstd implements Zstandard compression.\npackage zstd\n\nimport (\n\t\"io\"\n\t\"sync\"\n\n\t\"github.com/klauspost/compress/zstd\"\n)\n\n// Codec is the implementation of a compress.Codec which supports creating\n// readers and writers for kafka messages compressed with zstd.\ntype Codec struct {\n\t// The compression level configured on writers created by the codec.\n\t//\n\t// Default to 3.\n\tLevel int\n\n\tencoderPool sync.Pool // *encoder\n}\n\n// Code implements the compress.Codec interface.\nfunc (c *Codec) Code() int8 { return 4 }\n\n// Name implements the compress.Codec interface.\nfunc (c *Codec) Name() string { return \"zstd\" }\n\n// NewReader implements the compress.Codec interface.\nfunc (c *Codec) NewReader(r io.Reader) io.ReadCloser {\n\tp := new(reader)\n\tif p.dec, _ = decoderPool.Get().(*zstd.Decoder); p.dec != nil {\n\t\tp.dec.Reset(r)\n\t} else {\n\t\tz, err := zstd.NewReader(r,\n\t\t\tzstd.WithDecoderConcurrency(1),\n\t\t)\n\t\tif err != nil {\n\t\t\tp.err = err\n\t\t} else {\n\t\t\tp.dec = z\n\t\t}\n\t}\n\treturn p\n}\n\nfunc (c *Codec) level() int {\n\tif c.Level != 0 {\n\t\treturn c.Level\n\t}\n\treturn 3\n}\n\nfunc (c *Codec) zstdLevel() zstd.EncoderLevel {\n\treturn zstd.EncoderLevelFromZstd(c.level())\n}\n\nvar decoderPool sync.Pool // *zstd.Decoder\n\ntype reader struct {\n\tdec *zstd.Decoder\n\terr error\n}\n\n// Close implements the io.Closer interface.\nfunc (r *reader) Close() error {\n\tif r.dec != nil {\n\t\tr.dec.Reset(devNull{}) // don't retain the underlying reader\n\t\tdecoderPool.Put(r.dec)\n\t\tr.dec = nil\n\t\tr.err = io.ErrClosedPipe\n\t}\n\treturn nil\n}\n\n// Read implements the io.Reader interface.\nfunc (r *reader) Read(p []byte) (int, error) {\n\tif r.err != nil {\n\t\treturn 0, r.err\n\t}\n\tif r.dec == nil {\n\t\treturn 0, io.EOF\n\t}\n\treturn r.dec.Read(p)\n}\n\n// WriteTo implements the io.WriterTo interface.\nfunc (r *reader) WriteTo(w io.Writer) (int64, error) {\n\tif r.err != nil {\n\t\treturn 0, r.err\n\t}\n\tif r.dec == nil {\n\t\treturn 0, io.ErrClosedPipe\n\t}\n\treturn r.dec.WriteTo(w)\n}\n\n// NewWriter implements the compress.Codec interface.\nfunc (c *Codec) NewWriter(w io.Writer) io.WriteCloser {\n\tp := new(writer)\n\tif enc, _ := c.encoderPool.Get().(*zstd.Encoder); enc == nil {\n\t\tz, err := zstd.NewWriter(w,\n\t\t\tzstd.WithEncoderLevel(c.zstdLevel()),\n\t\t\tzstd.WithEncoderConcurrency(1),\n\t\t\tzstd.WithZeroFrames(true),\n\t\t)\n\t\tif err != nil {\n\t\t\tp.err = err\n\t\t} else {\n\t\t\tp.enc = z\n\t\t}\n\t} else {\n\t\tp.enc = enc\n\t\tp.enc.Reset(w)\n\t}\n\tp.c = c\n\treturn p\n}\n\ntype writer struct {\n\tc *Codec\n\tenc *zstd.Encoder\n\terr error\n}\n\n// Close implements the io.Closer interface.\nfunc (w *writer) Close() error {\n\tif w.enc != nil {\n\t\t// Close needs to be called to write the end of stream marker and flush\n\t\t// the buffers. The zstd package documents that the encoder is re-usable\n\t\t// after being closed.\n\t\terr := w.enc.Close()\n\t\tif err != nil {\n\t\t\tw.err = err\n\t\t}\n\t\tw.enc.Reset(devNull{}) // don't retain the underlying writer\n\t\tw.c.encoderPool.Put(w.enc)\n\t\tw.enc = nil\n\t\treturn err\n\t}\n\treturn w.err\n}\n\n// WriteTo implements the io.WriterTo interface.\nfunc (w *writer) Write(p []byte) (int, error) {\n\tif w.err != nil {\n\t\treturn 0, w.err\n\t}\n\tif w.enc == nil {\n\t\treturn 0, io.ErrClosedPipe\n\t}\n\treturn w.enc.Write(p)\n}\n\n// ReadFrom implements the io.ReaderFrom interface.\nfunc (w *writer) ReadFrom(r io.Reader) (int64, error) {\n\tif w.err != nil {\n\t\treturn 0, w.err\n\t}\n\tif w.enc == nil {\n\t\treturn 0, io.ErrClosedPipe\n\t}\n\treturn w.enc.ReadFrom(r)\n}\n\ntype devNull struct{}\n\nfunc (devNull) Read([]byte) (int, error) { return 0, io.EOF }\nfunc (devNull) Write([]byte) (int, error) { return 0, nil }\n" }, { "path": "compression.go", "content": "package kafka\n\nimport (\n\t\"errors\"\n\n\t\"github.com/segmentio/kafka-go/compress\"\n)\n\ntype Compression = compress.Compression\n\nconst (\n\tGzip Compression = compress.Gzip\n\tSnappy Compression = compress.Snappy\n\tLz4 Compression = compress.Lz4\n\tZstd Compression = compress.Zstd\n)\n\ntype CompressionCodec = compress.Codec\n\nvar (\n\terrUnknownCodec = errors.New(\"the compression code is invalid or its codec has not been imported\")\n)\n\n// resolveCodec looks up a codec by Code().\nfunc resolveCodec(code int8) (CompressionCodec, error) {\n\tcodec := compress.Compression(code).Codec()\n\tif codec == nil {\n\t\treturn nil, errUnknownCodec\n\t}\n\treturn codec, nil\n}\n" }, { "path": "conn.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"net\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n)\n\nvar (\n\terrInvalidWriteTopic = errors.New(\"writes must NOT set Topic on kafka.Message\")\n\terrInvalidWritePartition = errors.New(\"writes must NOT set Partition on kafka.Message\")\n)\n\n// Conn represents a connection to a kafka broker.\n//\n// Instances of Conn are safe to use concurrently from multiple goroutines.\ntype Conn struct {\n\t// base network connection\n\tconn net.Conn\n\n\t// number of inflight requests on the connection.\n\tinflight int32\n\n\t// offset management (synchronized on the mutex field)\n\tmutex sync.Mutex\n\toffset int64\n\n\t// read buffer (synchronized on rlock)\n\trlock sync.Mutex\n\trbuf bufio.Reader\n\n\t// write buffer (synchronized on wlock)\n\twlock sync.Mutex\n\twbuf bufio.Writer\n\twb writeBuffer\n\n\t// deadline management\n\twdeadline connDeadline\n\trdeadline connDeadline\n\n\t// immutable values of the connection object\n\tclientID string\n\ttopic string\n\tpartition int32\n\tfetchMaxBytes int32\n\tfetchMinSize int32\n\tbroker int32\n\track string\n\n\t// correlation ID generator (synchronized on wlock)\n\tcorrelationID int32\n\n\t// number of replica acks required when publishing to a partition\n\trequiredAcks int32\n\n\t// lazily loaded API versions used by this connection\n\tapiVersions atomic.Value // apiVersionMap\n\n\ttransactionalID *string\n}\n\ntype apiVersionMap map[apiKey]ApiVersion\n\nfunc (v apiVersionMap) negotiate(key apiKey, sortedSupportedVersions ...apiVersion) apiVersion {\n\tx := v[key]\n\n\tfor i := len(sortedSupportedVersions) - 1; i >= 0; i-- {\n\t\ts := sortedSupportedVersions[i]\n\n\t\tif apiVersion(x.MaxVersion) >= s {\n\t\t\treturn s\n\t\t}\n\t}\n\n\treturn -1\n}\n\n// ConnConfig is a configuration object used to create new instances of Conn.\ntype ConnConfig struct {\n\tClientID string\n\tTopic string\n\tPartition int\n\tBroker int\n\tRack string\n\n\t// The transactional id to use for transactional delivery. Idempotent\n\t// deliver should be enabled if transactional id is configured.\n\t// For more details look at transactional.id description here: http://kafka.apache.org/documentation.html#producerconfigs\n\t// Empty string means that this connection can't be transactional.\n\tTransactionalID string\n}\n\n// ReadBatchConfig is a configuration object used for reading batches of messages.\ntype ReadBatchConfig struct {\n\t// MinBytes indicates to the broker the minimum batch size that the consumer\n\t// will accept. Setting a high minimum when consuming from a low-volume topic\n\t// may result in delayed delivery when the broker does not have enough data to\n\t// satisfy the defined minimum.\n\tMinBytes int\n\n\t// MaxBytes indicates to the broker the maximum batch size that the consumer\n\t// will accept. The broker will truncate a message to satisfy this maximum, so\n\t// choose a value that is high enough for your largest message size.\n\tMaxBytes int\n\n\t// IsolationLevel controls the visibility of transactional records.\n\t// ReadUncommitted makes all records visible. With ReadCommitted only\n\t// non-transactional and committed records are visible.\n\tIsolationLevel IsolationLevel\n\n\t// MaxWait is the amount of time for the broker while waiting to hit the\n\t// min/max byte targets. This setting is independent of any network-level\n\t// timeouts or deadlines.\n\t//\n\t// For backward compatibility, when this field is left zero, kafka-go will\n\t// infer the max wait from the connection's read deadline.\n\tMaxWait time.Duration\n}\n\ntype IsolationLevel int8\n\nconst (\n\tReadUncommitted IsolationLevel = 0\n\tReadCommitted IsolationLevel = 1\n)\n\nvar (\n\t// DefaultClientID is the default value used as ClientID of kafka\n\t// connections.\n\tDefaultClientID string\n)\n\nfunc init() {\n\tprogname := filepath.Base(os.Args[0])\n\thostname, _ := os.Hostname()\n\tDefaultClientID = fmt.Sprintf(\"%s@%s (github.com/segmentio/kafka-go)\", progname, hostname)\n}\n\n// NewConn returns a new kafka connection for the given topic and partition.\nfunc NewConn(conn net.Conn, topic string, partition int) *Conn {\n\treturn NewConnWith(conn, ConnConfig{\n\t\tTopic: topic,\n\t\tPartition: partition,\n\t})\n}\n\nfunc emptyToNullable(transactionalID string) (result *string) {\n\tif transactionalID != \"\" {\n\t\tresult = &transactionalID\n\t}\n\treturn result\n}\n\n// NewConnWith returns a new kafka connection configured with config.\n// The offset is initialized to FirstOffset.\nfunc NewConnWith(conn net.Conn, config ConnConfig) *Conn {\n\tif len(config.ClientID) == 0 {\n\t\tconfig.ClientID = DefaultClientID\n\t}\n\n\tif config.Partition < 0 || config.Partition > math.MaxInt32 {\n\t\tpanic(fmt.Sprintf(\"invalid partition number: %d\", config.Partition))\n\t}\n\n\tc := &Conn{\n\t\tconn: conn,\n\t\trbuf: *bufio.NewReader(conn),\n\t\twbuf: *bufio.NewWriter(conn),\n\t\tclientID: config.ClientID,\n\t\ttopic: config.Topic,\n\t\tpartition: int32(config.Partition),\n\t\tbroker: int32(config.Broker),\n\t\track: config.Rack,\n\t\toffset: FirstOffset,\n\t\trequiredAcks: -1,\n\t\ttransactionalID: emptyToNullable(config.TransactionalID),\n\t}\n\n\tc.wb.w = &c.wbuf\n\n\t// The fetch request needs to ask for a MaxBytes value that is at least\n\t// enough to load the control data of the response. To avoid having to\n\t// recompute it on every read, it is cached here in the Conn value.\n\tc.fetchMinSize = (fetchResponseV2{\n\t\tTopics: []fetchResponseTopicV2{{\n\t\t\tTopicName: config.Topic,\n\t\t\tPartitions: []fetchResponsePartitionV2{{\n\t\t\t\tPartition: int32(config.Partition),\n\t\t\t\tMessageSet: messageSet{{}},\n\t\t\t}},\n\t\t}},\n\t}).size()\n\tc.fetchMaxBytes = math.MaxInt32 - c.fetchMinSize\n\treturn c\n}\n\nfunc (c *Conn) negotiateVersion(key apiKey, sortedSupportedVersions ...apiVersion) (apiVersion, error) {\n\tv, err := c.loadVersions()\n\tif err != nil {\n\t\treturn -1, err\n\t}\n\ta := v.negotiate(key, sortedSupportedVersions...)\n\tif a < 0 {\n\t\treturn -1, fmt.Errorf(\"no matching versions were found between the client and the broker for API key %d\", key)\n\t}\n\treturn a, nil\n}\n\nfunc (c *Conn) loadVersions() (apiVersionMap, error) {\n\tv, _ := c.apiVersions.Load().(apiVersionMap)\n\tif v != nil {\n\t\treturn v, nil\n\t}\n\n\tbrokerVersions, err := c.ApiVersions()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tv = make(apiVersionMap, len(brokerVersions))\n\n\tfor _, a := range brokerVersions {\n\t\tv[apiKey(a.ApiKey)] = a\n\t}\n\n\tc.apiVersions.Store(v)\n\treturn v, nil\n}\n\n// Broker returns a Broker value representing the kafka broker that this\n// connection was established to.\nfunc (c *Conn) Broker() Broker {\n\taddr := c.conn.RemoteAddr()\n\thost, port, _ := splitHostPortNumber(addr.String())\n\treturn Broker{\n\t\tHost: host,\n\t\tPort: port,\n\t\tID: int(c.broker),\n\t\tRack: c.rack,\n\t}\n}\n\n// Controller requests kafka for the current controller and returns its URL.\nfunc (c *Conn) Controller() (broker Broker, err error) {\n\terr = c.readOperation(\n\t\tfunc(deadline time.Time, id int32) error {\n\t\t\treturn c.writeRequest(metadata, v1, id, topicMetadataRequestV1([]string{}))\n\t\t},\n\t\tfunc(deadline time.Time, size int) error {\n\t\t\tvar res metadataResponseV1\n\n\t\t\tif err := c.readResponse(size, &res); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tfor _, brokerMeta := range res.Brokers {\n\t\t\t\tif brokerMeta.NodeID == res.ControllerID {\n\t\t\t\t\tbroker = Broker{ID: int(brokerMeta.NodeID),\n\t\t\t\t\t\tPort: int(brokerMeta.Port),\n\t\t\t\t\t\tHost: brokerMeta.Host,\n\t\t\t\t\t\tRack: brokerMeta.Rack}\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t},\n\t)\n\treturn broker, err\n}\n\n// Brokers retrieve the broker list from the Kafka metadata.\nfunc (c *Conn) Brokers() ([]Broker, error) {\n\tvar brokers []Broker\n\terr := c.readOperation(\n\t\tfunc(deadline time.Time, id int32) error {\n\t\t\treturn c.writeRequest(metadata, v1, id, topicMetadataRequestV1([]string{}))\n\t\t},\n\t\tfunc(deadline time.Time, size int) error {\n\t\t\tvar res metadataResponseV1\n\n\t\t\tif err := c.readResponse(size, &res); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\tbrokers = make([]Broker, len(res.Brokers))\n\t\t\tfor i, brokerMeta := range res.Brokers {\n\t\t\t\tbrokers[i] = Broker{\n\t\t\t\t\tID: int(brokerMeta.NodeID),\n\t\t\t\t\tPort: int(brokerMeta.Port),\n\t\t\t\t\tHost: brokerMeta.Host,\n\t\t\t\t\tRack: brokerMeta.Rack,\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn nil\n\t\t},\n\t)\n\treturn brokers, err\n}\n\n// DeleteTopics deletes the specified topics.\nfunc (c *Conn) DeleteTopics(topics ...string) error {\n\t_, err := c.deleteTopics(deleteTopicsRequest{\n\t\tTopics: topics,\n\t})\n\treturn err\n}\n\n// findCoordinator finds the coordinator for the specified group or transaction\n//\n// See http://kafka.apache.org/protocol.html#The_Messages_FindCoordinator\nfunc (c *Conn) findCoordinator(request findCoordinatorRequestV0) (findCoordinatorResponseV0, error) {\n\tvar response findCoordinatorResponseV0\n\n\terr := c.readOperation(\n\t\tfunc(deadline time.Time, id int32) error {\n\t\t\treturn c.writeRequest(findCoordinator, v0, id, request)\n\n\t\t},\n\t\tfunc(deadline time.Time, size int) error {\n\t\t\treturn expectZeroSize(func() (remain int, err error) {\n\t\t\t\treturn (&response).readFrom(&c.rbuf, size)\n\t\t\t}())\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn findCoordinatorResponseV0{}, err\n\t}\n\tif response.ErrorCode != 0 {\n\t\treturn findCoordinatorResponseV0{}, Error(response.ErrorCode)\n\t}\n\n\treturn response, nil\n}\n\n// heartbeat sends a heartbeat message required by consumer groups\n//\n// See http://kafka.apache.org/protocol.html#The_Messages_Heartbeat\nfunc (c *Conn) heartbeat(request heartbeatRequestV0) (heartbeatResponseV0, error) {\n\tvar response heartbeatResponseV0\n\n\terr := c.writeOperation(\n\t\tfunc(deadline time.Time, id int32) error {\n\t\t\treturn c.writeRequest(heartbeat, v0, id, request)\n\t\t},\n\t\tfunc(deadline time.Time, size int) error {\n\t\t\treturn expectZeroSize(func() (remain int, err error) {\n\t\t\t\treturn (&response).readFrom(&c.rbuf, size)\n\t\t\t}())\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn heartbeatResponseV0{}, err\n\t}\n\tif response.ErrorCode != 0 {\n\t\treturn heartbeatResponseV0{}, Error(response.ErrorCode)\n\t}\n\n\treturn response, nil\n}\n\n// joinGroup attempts to join a consumer group\n//\n// See http://kafka.apache.org/protocol.html#The_Messages_JoinGroup\nfunc (c *Conn) joinGroup(request joinGroupRequest) (joinGroupResponse, error) {\n\tversion, err := c.negotiateVersion(joinGroup, v1, v2)\n\tif err != nil {\n\t\treturn joinGroupResponse{}, err\n\t}\n\n\tresponse := joinGroupResponse{v: version}\n\n\terr = c.writeOperation(\n\t\tfunc(deadline time.Time, id int32) error {\n\t\t\treturn c.writeRequest(joinGroup, version, id, request)\n\t\t},\n\t\tfunc(deadline time.Time, size int) error {\n\t\t\treturn expectZeroSize(func() (remain int, err error) {\n\t\t\t\treturn (&response).readFrom(&c.rbuf, size)\n\t\t\t}())\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn joinGroupResponse{}, err\n\t}\n\tif response.ErrorCode != 0 {\n\t\treturn joinGroupResponse{}, Error(response.ErrorCode)\n\t}\n\n\treturn response, nil\n}\n\n// leaveGroup leaves the consumer from the consumer group\n//\n// See http://kafka.apache.org/protocol.html#The_Messages_LeaveGroup\nfunc (c *Conn) leaveGroup(request leaveGroupRequestV0) (leaveGroupResponseV0, error) {\n\tvar response leaveGroupResponseV0\n\n\terr := c.writeOperation(\n\t\tfunc(deadline time.Time, id int32) error {\n\t\t\treturn c.writeRequest(leaveGroup, v0, id, request)\n\t\t},\n\t\tfunc(deadline time.Time, size int) error {\n\t\t\treturn expectZeroSize(func() (remain int, err error) {\n\t\t\t\treturn (&response).readFrom(&c.rbuf, size)\n\t\t\t}())\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn leaveGroupResponseV0{}, err\n\t}\n\tif response.ErrorCode != 0 {\n\t\treturn leaveGroupResponseV0{}, Error(response.ErrorCode)\n\t}\n\n\treturn response, nil\n}\n\n// listGroups lists all the consumer groups\n//\n// See http://kafka.apache.org/protocol.html#The_Messages_ListGroups\nfunc (c *Conn) listGroups(request listGroupsRequestV1) (listGroupsResponseV1, error) {\n\tvar response listGroupsResponseV1\n\n\terr := c.readOperation(\n\t\tfunc(deadline time.Time, id int32) error {\n\t\t\treturn c.writeRequest(listGroups, v1, id, request)\n\t\t},\n\t\tfunc(deadline time.Time, size int) error {\n\t\t\treturn expectZeroSize(func() (remain int, err error) {\n\t\t\t\treturn (&response).readFrom(&c.rbuf, size)\n\t\t\t}())\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn listGroupsResponseV1{}, err\n\t}\n\tif response.ErrorCode != 0 {\n\t\treturn listGroupsResponseV1{}, Error(response.ErrorCode)\n\t}\n\n\treturn response, nil\n}\n\n// offsetCommit commits the specified topic partition offsets\n//\n// See http://kafka.apache.org/protocol.html#The_Messages_OffsetCommit\nfunc (c *Conn) offsetCommit(request offsetCommitRequestV2) (offsetCommitResponseV2, error) {\n\tvar response offsetCommitResponseV2\n\n\terr := c.writeOperation(\n\t\tfunc(deadline time.Time, id int32) error {\n\t\t\treturn c.writeRequest(offsetCommit, v2, id, request)\n\t\t},\n\t\tfunc(deadline time.Time, size int) error {\n\t\t\treturn expectZeroSize(func() (remain int, err error) {\n\t\t\t\treturn (&response).readFrom(&c.rbuf, size)\n\t\t\t}())\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn offsetCommitResponseV2{}, err\n\t}\n\tfor _, r := range response.Responses {\n\t\tfor _, pr := range r.PartitionResponses {\n\t\t\tif pr.ErrorCode != 0 {\n\t\t\t\treturn offsetCommitResponseV2{}, Error(pr.ErrorCode)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn response, nil\n}\n\n// offsetFetch fetches the offsets for the specified topic partitions.\n// -1 indicates that there is no offset saved for the partition.\n//\n// See http://kafka.apache.org/protocol.html#The_Messages_OffsetFetch\nfunc (c *Conn) offsetFetch(request offsetFetchRequestV1) (offsetFetchResponseV1, error) {\n\tvar response offsetFetchResponseV1\n\n\terr := c.readOperation(\n\t\tfunc(deadline time.Time, id int32) error {\n\t\t\treturn c.writeRequest(offsetFetch, v1, id, request)\n\t\t},\n\t\tfunc(deadline time.Time, size int) error {\n\t\t\treturn expectZeroSize(func() (remain int, err error) {\n\t\t\t\treturn (&response).readFrom(&c.rbuf, size)\n\t\t\t}())\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn offsetFetchResponseV1{}, err\n\t}\n\tfor _, r := range response.Responses {\n\t\tfor _, pr := range r.PartitionResponses {\n\t\t\tif pr.ErrorCode != 0 {\n\t\t\t\treturn offsetFetchResponseV1{}, Error(pr.ErrorCode)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn response, nil\n}\n\n// syncGroup completes the handshake to join a consumer group\n//\n// See http://kafka.apache.org/protocol.html#The_Messages_SyncGroup\nfunc (c *Conn) syncGroup(request syncGroupRequestV0) (syncGroupResponseV0, error) {\n\tvar response syncGroupResponseV0\n\n\terr := c.readOperation(\n\t\tfunc(deadline time.Time, id int32) error {\n\t\t\treturn c.writeRequest(syncGroup, v0, id, request)\n\t\t},\n\t\tfunc(deadline time.Time, size int) error {\n\t\t\treturn expectZeroSize(func() (remain int, err error) {\n\t\t\t\treturn (&response).readFrom(&c.rbuf, size)\n\t\t\t}())\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn syncGroupResponseV0{}, err\n\t}\n\tif response.ErrorCode != 0 {\n\t\treturn syncGroupResponseV0{}, Error(response.ErrorCode)\n\t}\n\n\treturn response, nil\n}\n\n// Close closes the kafka connection.\nfunc (c *Conn) Close() error {\n\treturn c.conn.Close()\n}\n\n// LocalAddr returns the local network address.\nfunc (c *Conn) LocalAddr() net.Addr {\n\treturn c.conn.LocalAddr()\n}\n\n// RemoteAddr returns the remote network address.\nfunc (c *Conn) RemoteAddr() net.Addr {\n\treturn c.conn.RemoteAddr()\n}\n\n// SetDeadline sets the read and write deadlines associated with the connection.\n// It is equivalent to calling both SetReadDeadline and SetWriteDeadline.\n//\n// A deadline is an absolute time after which I/O operations fail with a timeout\n// (see type Error) instead of blocking. The deadline applies to all future and\n// pending I/O, not just the immediately following call to Read or Write. After\n// a deadline has been exceeded, the connection may be closed if it was found to\n// be in an unrecoverable state.\n//\n// A zero value for t means I/O operations will not time out.\nfunc (c *Conn) SetDeadline(t time.Time) error {\n\tc.rdeadline.setDeadline(t)\n\tc.wdeadline.setDeadline(t)\n\treturn nil\n}\n\n// SetReadDeadline sets the deadline for future Read calls and any\n// currently-blocked Read call.\n// A zero value for t means Read will not time out.\nfunc (c *Conn) SetReadDeadline(t time.Time) error {\n\tc.rdeadline.setDeadline(t)\n\treturn nil\n}\n\n// SetWriteDeadline sets the deadline for future Write calls and any\n// currently-blocked Write call.\n// Even if write times out, it may return n > 0, indicating that some of the\n// data was successfully written.\n// A zero value for t means Write will not time out.\nfunc (c *Conn) SetWriteDeadline(t time.Time) error {\n\tc.wdeadline.setDeadline(t)\n\treturn nil\n}\n\n// Offset returns the current offset of the connection as pair of integers,\n// where the first one is an offset value and the second one indicates how\n// to interpret it.\n//\n// See Seek for more details about the offset and whence values.\nfunc (c *Conn) Offset() (offset int64, whence int) {\n\tc.mutex.Lock()\n\toffset = c.offset\n\tc.mutex.Unlock()\n\n\tswitch offset {\n\tcase FirstOffset:\n\t\toffset = 0\n\t\twhence = SeekStart\n\tcase LastOffset:\n\t\toffset = 0\n\t\twhence = SeekEnd\n\tdefault:\n\t\twhence = SeekAbsolute\n\t}\n\treturn\n}\n\nconst (\n\tSeekStart = 0 // Seek relative to the first offset available in the partition.\n\tSeekAbsolute = 1 // Seek to an absolute offset.\n\tSeekEnd = 2 // Seek relative to the last offset available in the partition.\n\tSeekCurrent = 3 // Seek relative to the current offset.\n\n\t// This flag may be combined to any of the SeekAbsolute and SeekCurrent\n\t// constants to skip the bound check that the connection would do otherwise.\n\t// Programs can use this flag to avoid making a metadata request to the kafka\n\t// broker to read the current first and last offsets of the partition.\n\tSeekDontCheck = 1 << 30\n)\n\n// Seek sets the offset for the next read or write operation according to whence, which\n// should be one of SeekStart, SeekAbsolute, SeekEnd, or SeekCurrent.\n// When seeking relative to the end, the offset is subtracted from the current offset.\n// Note that for historical reasons, these do not align with the usual whence constants\n// as in lseek(2) or os.Seek.\n// The method returns the new absolute offset of the connection.\nfunc (c *Conn) Seek(offset int64, whence int) (int64, error) {\n\tseekDontCheck := (whence & SeekDontCheck) != 0\n\twhence &= ^SeekDontCheck\n\n\tswitch whence {\n\tcase SeekStart, SeekAbsolute, SeekEnd, SeekCurrent:\n\tdefault:\n\t\treturn 0, fmt.Errorf(\"whence must be one of 0, 1, 2, or 3. (whence = %d)\", whence)\n\t}\n\n\tif seekDontCheck {\n\t\tif whence == SeekAbsolute {\n\t\t\tc.mutex.Lock()\n\t\t\tc.offset = offset\n\t\t\tc.mutex.Unlock()\n\t\t\treturn offset, nil\n\t\t}\n\n\t\tif whence == SeekCurrent {\n\t\t\tc.mutex.Lock()\n\t\t\tc.offset += offset\n\t\t\toffset = c.offset\n\t\t\tc.mutex.Unlock()\n\t\t\treturn offset, nil\n\t\t}\n\t}\n\n\tif whence == SeekAbsolute {\n\t\tc.mutex.Lock()\n\t\tunchanged := offset == c.offset\n\t\tc.mutex.Unlock()\n\t\tif unchanged {\n\t\t\treturn offset, nil\n\t\t}\n\t}\n\n\tif whence == SeekCurrent {\n\t\tc.mutex.Lock()\n\t\toffset = c.offset + offset\n\t\tc.mutex.Unlock()\n\t}\n\n\tfirst, last, err := c.ReadOffsets()\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tswitch whence {\n\tcase SeekStart:\n\t\toffset = first + offset\n\tcase SeekEnd:\n\t\toffset = last - offset\n\t}\n\n\tif offset < first || offset > last {\n\t\treturn 0, OffsetOutOfRange\n\t}\n\n\tc.mutex.Lock()\n\tc.offset = offset\n\tc.mutex.Unlock()\n\treturn offset, nil\n}\n\n// Read reads the message at the current offset from the connection, advancing\n// the offset on success so the next call to a read method will produce the next\n// message.\n// The method returns the number of bytes read, or an error if something went\n// wrong.\n//\n// While it is safe to call Read concurrently from multiple goroutines it may\n// be hard for the program to predict the results as the connection offset will\n// be read and written by multiple goroutines, they could read duplicates, or\n// messages may be seen by only some of the goroutines.\n//\n// The method fails with io.ErrShortBuffer if the buffer passed as argument is\n// too small to hold the message value.\n//\n// This method is provided to satisfy the net.Conn interface but is much less\n// efficient than using the more general purpose ReadBatch method.\nfunc (c *Conn) Read(b []byte) (int, error) {\n\tbatch := c.ReadBatch(1, len(b))\n\tn, err := batch.Read(b)\n\treturn n, coalesceErrors(silentEOF(err), batch.Close())\n}\n\n// ReadMessage reads the message at the current offset from the connection,\n// advancing the offset on success so the next call to a read method will\n// produce the next message.\n//\n// Because this method allocate memory buffers for the message key and value\n// it is less memory-efficient than Read, but has the advantage of never\n// failing with io.ErrShortBuffer.\n//\n// While it is safe to call Read concurrently from multiple goroutines it may\n// be hard for the program to predict the results as the connection offset will\n// be read and written by multiple goroutines, they could read duplicates, or\n// messages may be seen by only some of the goroutines.\n//\n// This method is provided for convenience purposes but is much less efficient\n// than using the more general purpose ReadBatch method.\nfunc (c *Conn) ReadMessage(maxBytes int) (Message, error) {\n\tbatch := c.ReadBatch(1, maxBytes)\n\tmsg, err := batch.ReadMessage()\n\treturn msg, coalesceErrors(silentEOF(err), batch.Close())\n}\n\n// ReadBatch reads a batch of messages from the kafka server. The method always\n// returns a non-nil Batch value. If an error occurred, either sending the fetch\n// request or reading the response, the error will be made available by the\n// returned value of the batch's Close method.\n//\n// While it is safe to call ReadBatch concurrently from multiple goroutines it\n// may be hard for the program to predict the results as the connection offset\n// will be read and written by multiple goroutines, they could read duplicates,\n// or messages may be seen by only some of the goroutines.\n//\n// A program doesn't specify the number of messages in wants from a batch, but\n// gives the minimum and maximum number of bytes that it wants to receive from\n// the kafka server.\nfunc (c *Conn) ReadBatch(minBytes, maxBytes int) *Batch {\n\treturn c.ReadBatchWith(ReadBatchConfig{\n\t\tMinBytes: minBytes,\n\t\tMaxBytes: maxBytes,\n\t})\n}\n\n// ReadBatchWith in every way is similar to ReadBatch. ReadBatch is configured\n// with the default values in ReadBatchConfig except for minBytes and maxBytes.\nfunc (c *Conn) ReadBatchWith(cfg ReadBatchConfig) *Batch {\n\n\tvar adjustedDeadline time.Time\n\tvar maxFetch = int(c.fetchMaxBytes)\n\n\tif cfg.MinBytes < 0 || cfg.MinBytes > maxFetch {\n\t\treturn &Batch{err: fmt.Errorf(\"kafka.(*Conn).ReadBatch: minBytes of %d out of [1,%d] bounds\", cfg.MinBytes, maxFetch)}\n\t}\n\tif cfg.MaxBytes < 0 || cfg.MaxBytes > maxFetch {\n\t\treturn &Batch{err: fmt.Errorf(\"kafka.(*Conn).ReadBatch: maxBytes of %d out of [1,%d] bounds\", cfg.MaxBytes, maxFetch)}\n\t}\n\tif cfg.MinBytes > cfg.MaxBytes {\n\t\treturn &Batch{err: fmt.Errorf(\"kafka.(*Conn).ReadBatch: minBytes (%d) > maxBytes (%d)\", cfg.MinBytes, cfg.MaxBytes)}\n\t}\n\n\toffset, whence := c.Offset()\n\n\toffset, err := c.Seek(offset, whence|SeekDontCheck)\n\tif err != nil {\n\t\treturn &Batch{err: dontExpectEOF(err)}\n\t}\n\n\tfetchVersion, err := c.negotiateVersion(fetch, v2, v5, v10)\n\tif err != nil {\n\t\treturn &Batch{err: dontExpectEOF(err)}\n\t}\n\n\tid, err := c.doRequest(&c.rdeadline, func(deadline time.Time, id int32) error {\n\t\tnow := time.Now()\n\t\tvar timeout time.Duration\n\t\tif cfg.MaxWait > 0 {\n\t\t\t// explicitly-configured case: no changes are made to the deadline,\n\t\t\t// and the timeout is sent exactly as specified.\n\t\t\ttimeout = cfg.MaxWait\n\t\t} else {\n\t\t\t// default case: use the original logic to adjust the conn's\n\t\t\t// deadline.T\n\t\t\tdeadline = adjustDeadlineForRTT(deadline, now, defaultRTT)\n\t\t\ttimeout = deadlineToTimeout(deadline, now)\n\t\t}\n\t\t// save this variable outside of the closure for later use in detecting\n\t\t// truncated messages.\n\t\tadjustedDeadline = deadline\n\t\tswitch fetchVersion {\n\t\tcase v10:\n\t\t\treturn c.wb.writeFetchRequestV10(\n\t\t\t\tid,\n\t\t\t\tc.clientID,\n\t\t\t\tc.topic,\n\t\t\t\tc.partition,\n\t\t\t\toffset,\n\t\t\t\tcfg.MinBytes,\n\t\t\t\tcfg.MaxBytes+int(c.fetchMinSize),\n\t\t\t\ttimeout,\n\t\t\t\tint8(cfg.IsolationLevel),\n\t\t\t)\n\t\tcase v5:\n\t\t\treturn c.wb.writeFetchRequestV5(\n\t\t\t\tid,\n\t\t\t\tc.clientID,\n\t\t\t\tc.topic,\n\t\t\t\tc.partition,\n\t\t\t\toffset,\n\t\t\t\tcfg.MinBytes,\n\t\t\t\tcfg.MaxBytes+int(c.fetchMinSize),\n\t\t\t\ttimeout,\n\t\t\t\tint8(cfg.IsolationLevel),\n\t\t\t)\n\t\tdefault:\n\t\t\treturn c.wb.writeFetchRequestV2(\n\t\t\t\tid,\n\t\t\t\tc.clientID,\n\t\t\t\tc.topic,\n\t\t\t\tc.partition,\n\t\t\t\toffset,\n\t\t\t\tcfg.MinBytes,\n\t\t\t\tcfg.MaxBytes+int(c.fetchMinSize),\n\t\t\t\ttimeout,\n\t\t\t)\n\t\t}\n\t})\n\tif err != nil {\n\t\treturn &Batch{err: dontExpectEOF(err)}\n\t}\n\n\t_, size, lock, err := c.waitResponse(&c.rdeadline, id)\n\tif err != nil {\n\t\treturn &Batch{err: dontExpectEOF(err)}\n\t}\n\n\tvar throttle int32\n\tvar highWaterMark int64\n\tvar remain int\n\n\tswitch fetchVersion {\n\tcase v10:\n\t\tthrottle, highWaterMark, remain, err = readFetchResponseHeaderV10(&c.rbuf, size)\n\tcase v5:\n\t\tthrottle, highWaterMark, remain, err = readFetchResponseHeaderV5(&c.rbuf, size)\n\tdefault:\n\t\tthrottle, highWaterMark, remain, err = readFetchResponseHeaderV2(&c.rbuf, size)\n\t}\n\tif errors.Is(err, errShortRead) {\n\t\terr = checkTimeoutErr(adjustedDeadline)\n\t}\n\n\tvar msgs *messageSetReader\n\tif err == nil {\n\t\tif highWaterMark == offset {\n\t\t\tmsgs = &messageSetReader{empty: true}\n\t\t} else {\n\t\t\tmsgs, err = newMessageSetReader(&c.rbuf, remain)\n\t\t}\n\t}\n\tif errors.Is(err, errShortRead) {\n\t\terr = checkTimeoutErr(adjustedDeadline)\n\t}\n\n\treturn &Batch{\n\t\tconn: c,\n\t\tmsgs: msgs,\n\t\tdeadline: adjustedDeadline,\n\t\tthrottle: makeDuration(throttle),\n\t\tlock: lock,\n\t\ttopic: c.topic, // topic is copied to Batch to prevent race with Batch.close\n\t\tpartition: int(c.partition), // partition is copied to Batch to prevent race with Batch.close\n\t\toffset: offset,\n\t\thighWaterMark: highWaterMark,\n\t\t// there shouldn't be a short read on initially setting up the batch.\n\t\t// as such, any io.EOF is re-mapped to an io.ErrUnexpectedEOF so that we\n\t\t// don't accidentally signal that we successfully reached the end of the\n\t\t// batch.\n\t\terr: dontExpectEOF(err),\n\t}\n}\n\n// ReadOffset returns the offset of the first message with a timestamp equal or\n// greater to t.\nfunc (c *Conn) ReadOffset(t time.Time) (int64, error) {\n\treturn c.readOffset(timestamp(t))\n}\n\n// ReadFirstOffset returns the first offset available on the connection.\nfunc (c *Conn) ReadFirstOffset() (int64, error) {\n\treturn c.readOffset(FirstOffset)\n}\n\n// ReadLastOffset returns the last offset available on the connection.\nfunc (c *Conn) ReadLastOffset() (int64, error) {\n\treturn c.readOffset(LastOffset)\n}\n\n// ReadOffsets returns the absolute first and last offsets of the topic used by\n// the connection.\nfunc (c *Conn) ReadOffsets() (first, last int64, err error) {\n\t// We have to submit two different requests to fetch the first and last\n\t// offsets because kafka refuses requests that ask for multiple offsets\n\t// on the same topic and partition.\n\tif first, err = c.ReadFirstOffset(); err != nil {\n\t\treturn\n\t}\n\tif last, err = c.ReadLastOffset(); err != nil {\n\t\tfirst = 0 // don't leak the value on error\n\t\treturn\n\t}\n\treturn\n}\n\nfunc (c *Conn) readOffset(t int64) (offset int64, err error) {\n\terr = c.readOperation(\n\t\tfunc(deadline time.Time, id int32) error {\n\t\t\treturn c.wb.writeListOffsetRequestV1(id, c.clientID, c.topic, c.partition, t)\n\t\t},\n\t\tfunc(deadline time.Time, size int) error {\n\t\t\treturn expectZeroSize(readArrayWith(&c.rbuf, size, func(r *bufio.Reader, size int) (int, error) {\n\t\t\t\t// We skip the topic name because we've made a request for\n\t\t\t\t// a single topic.\n\t\t\t\tsize, err := discardString(r, size)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn size, err\n\t\t\t\t}\n\n\t\t\t\t// Reading the array of partitions, there will be only one\n\t\t\t\t// partition which gives the offset we're looking for.\n\t\t\t\treturn readArrayWith(r, size, func(r *bufio.Reader, size int) (int, error) {\n\t\t\t\t\tvar p partitionOffsetV1\n\t\t\t\t\tsize, err := p.readFrom(r, size)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\treturn size, err\n\t\t\t\t\t}\n\t\t\t\t\tif p.ErrorCode != 0 {\n\t\t\t\t\t\treturn size, Error(p.ErrorCode)\n\t\t\t\t\t}\n\t\t\t\t\toffset = p.Offset\n\t\t\t\t\treturn size, nil\n\t\t\t\t})\n\t\t\t}))\n\t\t},\n\t)\n\treturn\n}\n\n// ReadPartitions returns the list of available partitions for the given list of\n// topics.\n//\n// If the method is called with no topic, it uses the topic configured on the\n// connection. If there are none, the method fetches all partitions of the kafka\n// cluster.\nfunc (c *Conn) ReadPartitions(topics ...string) (partitions []Partition, err error) {\n\n\tif len(topics) == 0 {\n\t\tif len(c.topic) != 0 {\n\t\t\tdefaultTopics := [...]string{c.topic}\n\t\t\ttopics = defaultTopics[:]\n\t\t} else {\n\t\t\t// topics needs to be explicitly nil-ed out or the broker will\n\t\t\t// interpret it as a request for 0 partitions instead of all.\n\t\t\ttopics = nil\n\t\t}\n\t}\n\tmetadataVersion, err := c.negotiateVersion(metadata, v1, v6)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\terr = c.readOperation(\n\t\tfunc(deadline time.Time, id int32) error {\n\t\t\tswitch metadataVersion {\n\t\t\tcase v6:\n\t\t\t\treturn c.writeRequest(metadata, v6, id, topicMetadataRequestV6{Topics: topics, AllowAutoTopicCreation: true})\n\t\t\tdefault:\n\t\t\t\treturn c.writeRequest(metadata, v1, id, topicMetadataRequestV1(topics))\n\t\t\t}\n\t\t},\n\t\tfunc(deadline time.Time, size int) error {\n\t\t\tpartitions, err = c.readPartitionsResponse(metadataVersion, size)\n\t\t\treturn err\n\t\t},\n\t)\n\treturn\n}\n\nfunc (c *Conn) readPartitionsResponse(metadataVersion apiVersion, size int) ([]Partition, error) {\n\tswitch metadataVersion {\n\tcase v6:\n\t\tvar res metadataResponseV6\n\t\tif err := c.readResponse(size, &res); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tbrokers := readBrokerMetadata(res.Brokers)\n\t\treturn c.readTopicMetadatav6(brokers, res.Topics)\n\tdefault:\n\t\tvar res metadataResponseV1\n\t\tif err := c.readResponse(size, &res); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tbrokers := readBrokerMetadata(res.Brokers)\n\t\treturn c.readTopicMetadatav1(brokers, res.Topics)\n\t}\n}\n\nfunc readBrokerMetadata(brokerMetadata []brokerMetadataV1) map[int32]Broker {\n\tbrokers := make(map[int32]Broker, len(brokerMetadata))\n\tfor _, b := range brokerMetadata {\n\t\tbrokers[b.NodeID] = Broker{\n\t\t\tHost: b.Host,\n\t\t\tPort: int(b.Port),\n\t\t\tID: int(b.NodeID),\n\t\t\tRack: b.Rack,\n\t\t}\n\t}\n\treturn brokers\n}\n\nfunc (c *Conn) readTopicMetadatav1(brokers map[int32]Broker, topicMetadata []topicMetadataV1) (partitions []Partition, err error) {\n\tfor _, t := range topicMetadata {\n\t\tif t.TopicErrorCode != 0 && (c.topic == \"\" || t.TopicName == c.topic) {\n\t\t\t// We only report errors if they happened for the topic of\n\t\t\t// the connection, otherwise the topic will simply have no\n\t\t\t// partitions in the result set.\n\t\t\treturn nil, Error(t.TopicErrorCode)\n\t\t}\n\t\tfor _, p := range t.Partitions {\n\t\t\tpartitions = append(partitions, Partition{\n\t\t\t\tTopic: t.TopicName,\n\t\t\t\tLeader: brokers[p.Leader],\n\t\t\t\tReplicas: makeBrokers(brokers, p.Replicas...),\n\t\t\t\tIsr: makeBrokers(brokers, p.Isr...),\n\t\t\t\tID: int(p.PartitionID),\n\t\t\t\tOfflineReplicas: []Broker{},\n\t\t\t})\n\t\t}\n\t}\n\treturn\n}\n\nfunc (c *Conn) readTopicMetadatav6(brokers map[int32]Broker, topicMetadata []topicMetadataV6) (partitions []Partition, err error) {\n\tfor _, t := range topicMetadata {\n\t\tif t.TopicErrorCode != 0 && (c.topic == \"\" || t.TopicName == c.topic) {\n\t\t\t// We only report errors if they happened for the topic of\n\t\t\t// the connection, otherwise the topic will simply have no\n\t\t\t// partitions in the result set.\n\t\t\treturn nil, Error(t.TopicErrorCode)\n\t\t}\n\t\tfor _, p := range t.Partitions {\n\t\t\tpartitions = append(partitions, Partition{\n\t\t\t\tTopic: t.TopicName,\n\t\t\t\tLeader: brokers[p.Leader],\n\t\t\t\tReplicas: makeBrokers(brokers, p.Replicas...),\n\t\t\t\tIsr: makeBrokers(brokers, p.Isr...),\n\t\t\t\tID: int(p.PartitionID),\n\t\t\t\tOfflineReplicas: makeBrokers(brokers, p.OfflineReplicas...),\n\t\t\t})\n\t\t}\n\t}\n\treturn\n}\n\nfunc makeBrokers(brokers map[int32]Broker, ids ...int32) []Broker {\n\tb := make([]Broker, len(ids))\n\tfor i, id := range ids {\n\t\tbr, ok := brokers[id]\n\t\tif !ok {\n\t\t\t// When the broker id isn't found in the current list of known\n\t\t\t// brokers, use a placeholder to report that the cluster has\n\t\t\t// logical knowledge of the broker but no information about the\n\t\t\t// physical host where it is running.\n\t\t\tbr.ID = int(id)\n\t\t}\n\t\tb[i] = br\n\t}\n\treturn b\n}\n\n// Write writes a message to the kafka broker that this connection was\n// established to. The method returns the number of bytes written, or an error\n// if something went wrong.\n//\n// The operation either succeeds or fail, it never partially writes the message.\n//\n// This method is exposed to satisfy the net.Conn interface but is less efficient\n// than the more general purpose WriteMessages method.\nfunc (c *Conn) Write(b []byte) (int, error) {\n\treturn c.WriteCompressedMessages(nil, Message{Value: b})\n}\n\n// WriteMessages writes a batch of messages to the connection's topic and\n// partition, returning the number of bytes written. The write is an atomic\n// operation, it either fully succeeds or fails.\nfunc (c *Conn) WriteMessages(msgs ...Message) (int, error) {\n\treturn c.WriteCompressedMessages(nil, msgs...)\n}\n\n// WriteCompressedMessages writes a batch of messages to the connection's topic\n// and partition, returning the number of bytes written. The write is an atomic\n// operation, it either fully succeeds or fails.\n//\n// If the compression codec is not nil, the messages will be compressed.\nfunc (c *Conn) WriteCompressedMessages(codec CompressionCodec, msgs ...Message) (nbytes int, err error) {\n\tnbytes, _, _, _, err = c.writeCompressedMessages(codec, msgs...)\n\treturn\n}\n\n// WriteCompressedMessagesAt writes a batch of messages to the connection's topic\n// and partition, returning the number of bytes written, partition and offset numbers\n// and timestamp assigned by the kafka broker to the message set. The write is an atomic\n// operation, it either fully succeeds or fails.\n//\n// If the compression codec is not nil, the messages will be compressed.\nfunc (c *Conn) WriteCompressedMessagesAt(codec CompressionCodec, msgs ...Message) (nbytes int, partition int32, offset int64, appendTime time.Time, err error) {\n\treturn c.writeCompressedMessages(codec, msgs...)\n}\n\nfunc (c *Conn) writeCompressedMessages(codec CompressionCodec, msgs ...Message) (nbytes int, partition int32, offset int64, appendTime time.Time, err error) {\n\tif len(msgs) == 0 {\n\t\treturn\n\t}\n\n\twriteTime := time.Now()\n\tfor i, msg := range msgs {\n\t\t// users may believe they can set the Topic and/or Partition\n\t\t// on the kafka message.\n\t\tif msg.Topic != \"\" && msg.Topic != c.topic {\n\t\t\terr = errInvalidWriteTopic\n\t\t\treturn\n\t\t}\n\t\tif msg.Partition != 0 {\n\t\t\terr = errInvalidWritePartition\n\t\t\treturn\n\t\t}\n\n\t\tif msg.Time.IsZero() {\n\t\t\tmsgs[i].Time = writeTime\n\t\t}\n\n\t\tnbytes += len(msg.Key) + len(msg.Value)\n\t}\n\n\tvar produceVersion apiVersion\n\tif produceVersion, err = c.negotiateVersion(produce, v2, v3, v7); err != nil {\n\t\treturn\n\t}\n\n\terr = c.writeOperation(\n\t\tfunc(deadline time.Time, id int32) error {\n\t\t\tnow := time.Now()\n\t\t\tdeadline = adjustDeadlineForRTT(deadline, now, defaultRTT)\n\t\t\tswitch produceVersion {\n\t\t\tcase v7:\n\t\t\t\trecordBatch, err :=\n\t\t\t\t\tnewRecordBatch(\n\t\t\t\t\t\tcodec,\n\t\t\t\t\t\tmsgs...,\n\t\t\t\t\t)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\treturn c.wb.writeProduceRequestV7(\n\t\t\t\t\tid,\n\t\t\t\t\tc.clientID,\n\t\t\t\t\tc.topic,\n\t\t\t\t\tc.partition,\n\t\t\t\t\tdeadlineToTimeout(deadline, now),\n\t\t\t\t\tint16(atomic.LoadInt32(&c.requiredAcks)),\n\t\t\t\t\tc.transactionalID,\n\t\t\t\t\trecordBatch,\n\t\t\t\t)\n\t\t\tcase v3:\n\t\t\t\trecordBatch, err :=\n\t\t\t\t\tnewRecordBatch(\n\t\t\t\t\t\tcodec,\n\t\t\t\t\t\tmsgs...,\n\t\t\t\t\t)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\treturn c.wb.writeProduceRequestV3(\n\t\t\t\t\tid,\n\t\t\t\t\tc.clientID,\n\t\t\t\t\tc.topic,\n\t\t\t\t\tc.partition,\n\t\t\t\t\tdeadlineToTimeout(deadline, now),\n\t\t\t\t\tint16(atomic.LoadInt32(&c.requiredAcks)),\n\t\t\t\t\tc.transactionalID,\n\t\t\t\t\trecordBatch,\n\t\t\t\t)\n\t\t\tdefault:\n\t\t\t\treturn c.wb.writeProduceRequestV2(\n\t\t\t\t\tcodec,\n\t\t\t\t\tid,\n\t\t\t\t\tc.clientID,\n\t\t\t\t\tc.topic,\n\t\t\t\t\tc.partition,\n\t\t\t\t\tdeadlineToTimeout(deadline, now),\n\t\t\t\t\tint16(atomic.LoadInt32(&c.requiredAcks)),\n\t\t\t\t\tmsgs...,\n\t\t\t\t)\n\t\t\t}\n\t\t},\n\t\tfunc(deadline time.Time, size int) error {\n\t\t\treturn expectZeroSize(readArrayWith(&c.rbuf, size, func(r *bufio.Reader, size int) (int, error) {\n\t\t\t\t// Skip the topic, we've produced the message to only one topic,\n\t\t\t\t// no need to waste resources loading it in memory.\n\t\t\t\tsize, err := discardString(r, size)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn size, err\n\t\t\t\t}\n\n\t\t\t\t// Read the list of partitions, there should be only one since\n\t\t\t\t// we've produced a message to a single partition.\n\t\t\t\tsize, err = readArrayWith(r, size, func(r *bufio.Reader, size int) (int, error) {\n\t\t\t\t\tswitch produceVersion {\n\t\t\t\t\tcase v7:\n\t\t\t\t\t\tvar p produceResponsePartitionV7\n\t\t\t\t\t\tsize, err := p.readFrom(r, size)\n\t\t\t\t\t\tif err == nil && p.ErrorCode != 0 {\n\t\t\t\t\t\t\terr = Error(p.ErrorCode)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif err == nil {\n\t\t\t\t\t\t\tpartition = p.Partition\n\t\t\t\t\t\t\toffset = p.Offset\n\t\t\t\t\t\t\tappendTime = time.Unix(0, p.Timestamp*int64(time.Millisecond))\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn size, err\n\t\t\t\t\tdefault:\n\t\t\t\t\t\tvar p produceResponsePartitionV2\n\t\t\t\t\t\tsize, err := p.readFrom(r, size)\n\t\t\t\t\t\tif err == nil && p.ErrorCode != 0 {\n\t\t\t\t\t\t\terr = Error(p.ErrorCode)\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif err == nil {\n\t\t\t\t\t\t\tpartition = p.Partition\n\t\t\t\t\t\t\toffset = p.Offset\n\t\t\t\t\t\t\tappendTime = time.Unix(0, p.Timestamp*int64(time.Millisecond))\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn size, err\n\t\t\t\t\t}\n\n\t\t\t\t})\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn size, err\n\t\t\t\t}\n\n\t\t\t\t// The response is trailed by the throttle time, also skipping\n\t\t\t\t// since it's not interesting here.\n\t\t\t\treturn discardInt32(r, size)\n\t\t\t}))\n\t\t},\n\t)\n\n\tif err != nil {\n\t\tnbytes = 0\n\t}\n\n\treturn\n}\n\n// SetRequiredAcks sets the number of acknowledges from replicas that the\n// connection requests when producing messages.\nfunc (c *Conn) SetRequiredAcks(n int) error {\n\tswitch n {\n\tcase -1, 1:\n\t\tatomic.StoreInt32(&c.requiredAcks, int32(n))\n\t\treturn nil\n\tdefault:\n\t\treturn InvalidRequiredAcks\n\t}\n}\n\nfunc (c *Conn) writeRequest(apiKey apiKey, apiVersion apiVersion, correlationID int32, req request) error {\n\thdr := c.requestHeader(apiKey, apiVersion, correlationID)\n\thdr.Size = (hdr.size() + req.size()) - 4\n\thdr.writeTo(&c.wb)\n\treq.writeTo(&c.wb)\n\treturn c.wbuf.Flush()\n}\n\nfunc (c *Conn) readResponse(size int, res interface{}) error {\n\tsize, err := read(&c.rbuf, size, res)\n\tif err != nil {\n\t\tvar kafkaError Error\n\t\tif errors.As(err, &kafkaError) {\n\t\t\tsize, err = discardN(&c.rbuf, size, size)\n\t\t}\n\t}\n\treturn expectZeroSize(size, err)\n}\n\nfunc (c *Conn) peekResponseSizeAndID() (int32, int32, error) {\n\tb, err := c.rbuf.Peek(8)\n\tif err != nil {\n\t\treturn 0, 0, err\n\t}\n\tsize, id := makeInt32(b[:4]), makeInt32(b[4:])\n\treturn size, id, nil\n}\n\nfunc (c *Conn) skipResponseSizeAndID() {\n\tc.rbuf.Discard(8)\n}\n\nfunc (c *Conn) readDeadline() time.Time {\n\treturn c.rdeadline.deadline()\n}\n\nfunc (c *Conn) writeDeadline() time.Time {\n\treturn c.wdeadline.deadline()\n}\n\nfunc (c *Conn) readOperation(write func(time.Time, int32) error, read func(time.Time, int) error) error {\n\treturn c.do(&c.rdeadline, write, read)\n}\n\nfunc (c *Conn) writeOperation(write func(time.Time, int32) error, read func(time.Time, int) error) error {\n\treturn c.do(&c.wdeadline, write, read)\n}\n\nfunc (c *Conn) enter() {\n\tatomic.AddInt32(&c.inflight, +1)\n}\n\nfunc (c *Conn) leave() {\n\tatomic.AddInt32(&c.inflight, -1)\n}\n\nfunc (c *Conn) concurrency() int {\n\treturn int(atomic.LoadInt32(&c.inflight))\n}\n\nfunc (c *Conn) do(d *connDeadline, write func(time.Time, int32) error, read func(time.Time, int) error) error {\n\tid, err := c.doRequest(d, write)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tdeadline, size, lock, err := c.waitResponse(d, id)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif err = read(deadline, size); err != nil {\n\t\tvar kafkaError Error\n\t\tif !errors.As(err, &kafkaError) {\n\t\t\tc.conn.Close()\n\t\t}\n\t}\n\n\td.unsetConnReadDeadline()\n\tlock.Unlock()\n\treturn err\n}\n\nfunc (c *Conn) doRequest(d *connDeadline, write func(time.Time, int32) error) (id int32, err error) {\n\tc.enter()\n\tc.wlock.Lock()\n\tc.correlationID++\n\tid = c.correlationID\n\terr = write(d.setConnWriteDeadline(c.conn), id)\n\td.unsetConnWriteDeadline()\n\n\tif err != nil {\n\t\t// When an error occurs there's no way to know if the connection is in a\n\t\t// recoverable state so we're better off just giving up at this point to\n\t\t// avoid any risk of corrupting the following operations.\n\t\tc.conn.Close()\n\t\tc.leave()\n\t}\n\n\tc.wlock.Unlock()\n\treturn\n}\n\nfunc (c *Conn) waitResponse(d *connDeadline, id int32) (deadline time.Time, size int, lock *sync.Mutex, err error) {\n\tfor {\n\t\tvar rsz int32\n\t\tvar rid int32\n\n\t\tc.rlock.Lock()\n\t\tdeadline = d.setConnReadDeadline(c.conn)\n\t\trsz, rid, err = c.peekResponseSizeAndID()\n\n\t\tif err != nil {\n\t\t\td.unsetConnReadDeadline()\n\t\t\tc.conn.Close()\n\t\t\tc.rlock.Unlock()\n\t\t\tbreak\n\t\t}\n\n\t\tif id == rid {\n\t\t\tc.skipResponseSizeAndID()\n\t\t\tsize, lock = int(rsz-4), &c.rlock\n\t\t\t// Don't unlock the read mutex to yield ownership to the caller.\n\t\t\tbreak\n\t\t}\n\n\t\tif c.concurrency() == 1 {\n\t\t\t// If the goroutine is the only one waiting on this connection it\n\t\t\t// should be impossible to read a correlation id different from the\n\t\t\t// one it expects. This is a sign that the data we are reading on\n\t\t\t// the wire is corrupted and the connection needs to be closed.\n\t\t\terr = io.ErrNoProgress\n\t\t\tc.rlock.Unlock()\n\t\t\tbreak\n\t\t}\n\n\t\t// Optimistically release the read lock if a response has already\n\t\t// been received but the current operation is not the target for it.\n\t\tc.rlock.Unlock()\n\t}\n\n\tc.leave()\n\treturn\n}\n\nfunc (c *Conn) requestHeader(apiKey apiKey, apiVersion apiVersion, correlationID int32) requestHeader {\n\treturn requestHeader{\n\t\tApiKey: int16(apiKey),\n\t\tApiVersion: int16(apiVersion),\n\t\tCorrelationID: correlationID,\n\t\tClientID: c.clientID,\n\t}\n}\n\nfunc (c *Conn) ApiVersions() ([]ApiVersion, error) {\n\tdeadline := &c.rdeadline\n\n\tif deadline.deadline().IsZero() {\n\t\t// ApiVersions is called automatically when API version negotiation\n\t\t// needs to happen, so we are not guaranteed that a read deadline has\n\t\t// been set yet. Fallback to use the write deadline in case it was\n\t\t// set, for example when version negotiation is initiated during a\n\t\t// produce request.\n\t\tdeadline = &c.wdeadline\n\t}\n\n\tid, err := c.doRequest(deadline, func(_ time.Time, id int32) error {\n\t\th := requestHeader{\n\t\t\tApiKey: int16(apiVersions),\n\t\t\tApiVersion: int16(v0),\n\t\t\tCorrelationID: id,\n\t\t\tClientID: c.clientID,\n\t\t}\n\t\th.Size = (h.size() - 4)\n\t\th.writeTo(&c.wb)\n\t\treturn c.wbuf.Flush()\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\t_, size, lock, err := c.waitResponse(deadline, id)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer lock.Unlock()\n\n\tvar errorCode int16\n\tif size, err = readInt16(&c.rbuf, size, &errorCode); err != nil {\n\t\treturn nil, err\n\t}\n\tvar arrSize int32\n\tif size, err = readInt32(&c.rbuf, size, &arrSize); err != nil {\n\t\treturn nil, err\n\t}\n\tr := make([]ApiVersion, arrSize)\n\tfor i := 0; i < int(arrSize); i++ {\n\t\tif size, err = readInt16(&c.rbuf, size, &r[i].ApiKey); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif size, err = readInt16(&c.rbuf, size, &r[i].MinVersion); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tif size, err = readInt16(&c.rbuf, size, &r[i].MaxVersion); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tif errorCode != 0 {\n\t\treturn r, Error(errorCode)\n\t}\n\n\treturn r, nil\n}\n\n// connDeadline is a helper type to implement read/write deadline management on\n// the kafka connection.\ntype connDeadline struct {\n\tmutex sync.Mutex\n\tvalue time.Time\n\trconn net.Conn\n\twconn net.Conn\n}\n\nfunc (d *connDeadline) deadline() time.Time {\n\td.mutex.Lock()\n\tt := d.value\n\td.mutex.Unlock()\n\treturn t\n}\n\nfunc (d *connDeadline) setDeadline(t time.Time) {\n\td.mutex.Lock()\n\td.value = t\n\n\tif d.rconn != nil {\n\t\td.rconn.SetReadDeadline(t)\n\t}\n\n\tif d.wconn != nil {\n\t\td.wconn.SetWriteDeadline(t)\n\t}\n\n\td.mutex.Unlock()\n}\n\nfunc (d *connDeadline) setConnReadDeadline(conn net.Conn) time.Time {\n\td.mutex.Lock()\n\tdeadline := d.value\n\td.rconn = conn\n\td.rconn.SetReadDeadline(deadline)\n\td.mutex.Unlock()\n\treturn deadline\n}\n\nfunc (d *connDeadline) setConnWriteDeadline(conn net.Conn) time.Time {\n\td.mutex.Lock()\n\tdeadline := d.value\n\td.wconn = conn\n\td.wconn.SetWriteDeadline(deadline)\n\td.mutex.Unlock()\n\treturn deadline\n}\n\nfunc (d *connDeadline) unsetConnReadDeadline() {\n\td.mutex.Lock()\n\td.rconn = nil\n\td.mutex.Unlock()\n}\n\nfunc (d *connDeadline) unsetConnWriteDeadline() {\n\td.mutex.Lock()\n\td.wconn = nil\n\td.mutex.Unlock()\n}\n\n// saslHandshake sends the SASL handshake message. This will determine whether\n// the Mechanism is supported by the cluster. If it's not, this function will\n// error out with UnsupportedSASLMechanism.\n//\n// If the mechanism is unsupported, the handshake request will reply with the\n// list of the cluster's configured mechanisms, which could potentially be used\n// to facilitate negotiation. At the moment, we are not negotiating the\n// mechanism as we believe that brokers are usually known to the client, and\n// therefore the client should already know which mechanisms are supported.\n//\n// See http://kafka.apache.org/protocol.html#The_Messages_SaslHandshake\nfunc (c *Conn) saslHandshake(mechanism string) error {\n\t// The wire format for V0 and V1 is identical, but the version\n\t// number will affect how the SASL authentication\n\t// challenge/responses are sent\n\tvar resp saslHandshakeResponseV0\n\n\tversion, err := c.negotiateVersion(saslHandshake, v0, v1)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\terr = c.writeOperation(\n\t\tfunc(deadline time.Time, id int32) error {\n\t\t\treturn c.writeRequest(saslHandshake, version, id, &saslHandshakeRequestV0{Mechanism: mechanism})\n\t\t},\n\t\tfunc(deadline time.Time, size int) error {\n\t\t\treturn expectZeroSize(func() (int, error) {\n\t\t\t\treturn (&resp).readFrom(&c.rbuf, size)\n\t\t\t}())\n\t\t},\n\t)\n\tif err == nil && resp.ErrorCode != 0 {\n\t\terr = Error(resp.ErrorCode)\n\t}\n\treturn err\n}\n\n// saslAuthenticate sends the SASL authenticate message. This function must\n// be immediately preceded by a successful saslHandshake.\n//\n// See http://kafka.apache.org/protocol.html#The_Messages_SaslAuthenticate\nfunc (c *Conn) saslAuthenticate(data []byte) ([]byte, error) {\n\t// if we sent a v1 handshake, then we must encapsulate the authentication\n\t// request in a saslAuthenticateRequest. otherwise, we read and write raw\n\t// bytes.\n\tversion, err := c.negotiateVersion(saslHandshake, v0, v1)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif version == v1 {\n\t\tvar request = saslAuthenticateRequestV0{Data: data}\n\t\tvar response saslAuthenticateResponseV0\n\n\t\terr := c.writeOperation(\n\t\t\tfunc(deadline time.Time, id int32) error {\n\t\t\t\treturn c.writeRequest(saslAuthenticate, v0, id, request)\n\t\t\t},\n\t\t\tfunc(deadline time.Time, size int) error {\n\t\t\t\treturn expectZeroSize(func() (remain int, err error) {\n\t\t\t\t\treturn (&response).readFrom(&c.rbuf, size)\n\t\t\t\t}())\n\t\t\t},\n\t\t)\n\t\tif err == nil && response.ErrorCode != 0 {\n\t\t\terr = Error(response.ErrorCode)\n\t\t}\n\t\treturn response.Data, err\n\t}\n\n\t// fall back to opaque bytes on the wire. the broker is expecting these if\n\t// it just processed a v0 sasl handshake.\n\tc.wb.writeInt32(int32(len(data)))\n\tif _, err := c.wb.Write(data); err != nil {\n\t\treturn nil, err\n\t}\n\tif err := c.wb.Flush(); err != nil {\n\t\treturn nil, err\n\t}\n\n\tvar respLen int32\n\tif _, err := readInt32(&c.rbuf, 4, &respLen); err != nil {\n\t\treturn nil, err\n\t}\n\n\tresp, _, err := readNewBytes(&c.rbuf, int(respLen), int(respLen))\n\treturn resp, err\n}\n" }, { "path": "conn_test.go", "content": "package kafka\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math/rand\"\n\t\"net\"\n\t\"os\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\t\"golang.org/x/net/nettest\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\ntype timeout struct{}\n\nfunc (*timeout) Error() string { return \"timeout\" }\nfunc (*timeout) Temporary() bool { return true }\nfunc (*timeout) Timeout() bool { return true }\n\n// connPipe is an adapter that implements the net.Conn interface on top of\n// two client kafka connections to pass the nettest.TestConn test suite.\ntype connPipe struct {\n\trconn *Conn\n\twconn *Conn\n}\n\nfunc (c *connPipe) Close() error {\n\tb := [1]byte{} // marker that the connection has been closed\n\tc.wconn.SetWriteDeadline(time.Time{})\n\tc.wconn.Write(b[:])\n\tc.wconn.Close()\n\tc.rconn.Close()\n\treturn nil\n}\n\nfunc (c *connPipe) Read(b []byte) (int, error) {\n\t// See comments in Write.\n\ttime.Sleep(time.Millisecond)\n\tif t := c.rconn.readDeadline(); !t.IsZero() {\n\t\treturn 0, &timeout{}\n\t}\n\tn, err := c.rconn.Read(b)\n\tif n == 1 && b[0] == 0 {\n\t\tc.rconn.Close()\n\t\tn, err = 0, io.EOF\n\t}\n\treturn n, err\n}\n\nfunc (c *connPipe) Write(b []byte) (int, error) {\n\t// The nettest/ConcurrentMethods test spawns a bunch of goroutines that do\n\t// random stuff on the connection, if a Read or Write was issued before a\n\t// deadline was set then it could cancel an inflight request to kafka,\n\t// resulting in the connection being closed.\n\t// To prevent this from happening we wait a little while to give the other\n\t// goroutines a chance to start and set the deadline.\n\ttime.Sleep(time.Millisecond)\n\n\t// The nettest code only sets deadlines when it expects the write to time\n\t// out. The broker connection is alive and able to accept data, so we need\n\t// to simulate the timeout in order to get the tests to pass.\n\tif t := c.wconn.writeDeadline(); !t.IsZero() {\n\t\treturn 0, &timeout{}\n\t}\n\n\treturn c.wconn.Write(b)\n}\n\nfunc (c *connPipe) LocalAddr() net.Addr {\n\treturn c.rconn.LocalAddr()\n}\n\nfunc (c *connPipe) RemoteAddr() net.Addr {\n\treturn c.wconn.LocalAddr()\n}\n\nfunc (c *connPipe) SetDeadline(t time.Time) error {\n\tc.rconn.SetDeadline(t)\n\tc.wconn.SetDeadline(t)\n\treturn nil\n}\n\nfunc (c *connPipe) SetReadDeadline(t time.Time) error {\n\treturn c.rconn.SetReadDeadline(t)\n}\n\nfunc (c *connPipe) SetWriteDeadline(t time.Time) error {\n\treturn c.wconn.SetWriteDeadline(t)\n}\n\nfunc init() {\n\trand.Seed(time.Now().UnixNano())\n}\n\nfunc makeTopic() string {\n\treturn fmt.Sprintf(\"kafka-go-%016x\", rand.Int63())\n}\n\nfunc makeGroupID() string {\n\treturn fmt.Sprintf(\"kafka-go-group-%016x\", rand.Int63())\n}\n\nfunc makeTransactionalID() string {\n\treturn fmt.Sprintf(\"kafka-go-transactional-id-%016x\", rand.Int63())\n}\n\nfunc TestConn(t *testing.T) {\n\ttests := []struct {\n\t\tscenario string\n\t\tfunction func(*testing.T, *Conn)\n\t\tminVersion string\n\t}{\n\t\t{\n\t\t\tscenario: \"close right away\",\n\t\t\tfunction: testConnClose,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"ensure the initial offset of a connection is the first offset\",\n\t\t\tfunction: testConnFirstOffset,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"write a single message to kafka should succeed\",\n\t\t\tfunction: testConnWrite,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"writing a message to a closed kafka connection should fail\",\n\t\t\tfunction: testConnCloseAndWrite,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"ensure the connection can seek to the first offset\",\n\t\t\tfunction: testConnSeekFirstOffset,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"ensure the connection can seek to the last offset\",\n\t\t\tfunction: testConnSeekLastOffset,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"ensure the connection can seek relative to the current offset\",\n\t\t\tfunction: testConnSeekCurrentOffset,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"ensure the connection can seek to a random offset\",\n\t\t\tfunction: testConnSeekRandomOffset,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"unchecked seeks allow the connection to be positioned outside the boundaries of the partition\",\n\t\t\tfunction: testConnSeekDontCheck,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"writing and reading messages sequentially should preserve the order\",\n\t\t\tfunction: testConnWriteReadSequentially,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"writing a batch of messages and reading it sequentially should preserve the order\",\n\t\t\tfunction: testConnWriteBatchReadSequentially,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"writing and reading messages concurrently should preserve the order\",\n\t\t\tfunction: testConnWriteReadConcurrently,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"reading messages with a buffer that is too short should return io.ErrShortBuffer and maintain the connection open\",\n\t\t\tfunction: testConnReadShortBuffer,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"reading messages from an empty partition should timeout after reaching the deadline\",\n\t\t\tfunction: testConnReadEmptyWithDeadline,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"write batch of messages and read the highest offset (watermark)\",\n\t\t\tfunction: testConnReadWatermarkFromBatch,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"read a batch with no explicit min or max bytes\",\n\t\t\tfunction: testConnReadBatchWithNoMinMaxBytes,\n\t\t\tminVersion: \"0.11.0\",\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"read a batch using explicit max wait time\",\n\t\t\tfunction: testConnReadBatchWithMaxWait,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"find the group coordinator\",\n\t\t\tfunction: testConnFindCoordinator,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"test join group with an invalid groupID\",\n\t\t\tfunction: testConnJoinGroupInvalidGroupID,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"test join group with an invalid sessionTimeout\",\n\t\t\tfunction: testConnJoinGroupInvalidSessionTimeout,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"test join group with an invalid refreshTimeout\",\n\t\t\tfunction: testConnJoinGroupInvalidRefreshTimeout,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"test heartbeat once group has been created\",\n\t\t\tfunction: testConnHeartbeatErr,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"test leave group returns error when called outside group\",\n\t\t\tfunction: testConnLeaveGroupErr,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"test sync group with bad memberID\",\n\t\t\tfunction: testConnSyncGroupErr,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"test list groups\",\n\t\t\tfunction: testConnListGroupsReturnsGroups,\n\t\t\tminVersion: \"0.11.0\",\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"test fetch and commit offset\",\n\t\t\tfunction: testConnFetchAndCommitOffsets,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"test delete topics\",\n\t\t\tfunction: testDeleteTopics,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"test delete topics with an invalid topic\",\n\t\t\tfunction: testDeleteTopicsInvalidTopic,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"test retrieve controller\",\n\t\t\tfunction: testController,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"test list brokers\",\n\t\t\tfunction: testBrokers,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"the connection advertises the broker that it is connected to\",\n\t\t\tfunction: testConnBroker,\n\t\t},\n\t}\n\n\tconst (\n\t\ttcp = \"tcp\"\n\t\tkafka = \"localhost:9092\"\n\t)\n\n\tfor _, test := range tests {\n\t\tif !ktesting.KafkaIsAtLeast(test.minVersion) {\n\t\t\tt.Log(\"skipping \" + test.scenario + \" because broker is not at least version \" + test.minVersion)\n\t\t\tcontinue\n\t\t}\n\n\t\ttestFunc := test.function\n\t\tt.Run(test.scenario, func(t *testing.T) {\n\t\t\tt.Parallel()\n\n\t\t\tctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)\n\t\t\tdefer cancel()\n\n\t\t\ttopic := makeTopic()\n\n\t\t\tconn, err := (&Dialer{\n\t\t\t\tResolver: &net.Resolver{},\n\t\t\t}).DialLeader(ctx, tcp, kafka, topic, 0)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(\"failed to open a new kafka connection:\", err)\n\t\t\t}\n\t\t\tdefer conn.Close()\n\t\t\ttestFunc(t, conn)\n\t\t})\n\t}\n\n\tt.Run(\"nettest\", func(t *testing.T) {\n\t\t// Need ability to skip nettest on newer Kafka versions to avoid these kinds of errors:\n\t\t// --- FAIL: TestConn/nettest (17.56s)\n\t\t// --- FAIL: TestConn/nettest/PingPong (7.40s)\n\t\t// conntest.go:112: unexpected Read error: [7] Request Timed Out: the request exceeded the user-specified time limit in the request\n\t\t// conntest.go:118: mismatching value: got 77, want 78\n\t\t// conntest.go:118: mismatching value: got 78, want 79\n\t\t// ...\n\t\t//\n\t\t// TODO: Figure out why these are happening and fix them (they don't appear to be new).\n\t\tif _, ok := os.LookupEnv(\"KAFKA_SKIP_NETTEST\"); ok {\n\t\t\tt.Log(\"skipping nettest because KAFKA_SKIP_NETTEST is set\")\n\t\t\tt.Skip()\n\t\t}\n\n\t\tt.Parallel()\n\n\t\tnettest.TestConn(t, func() (c1 net.Conn, c2 net.Conn, stop func(), err error) {\n\t\t\ttopic1 := makeTopic()\n\t\t\ttopic2 := makeTopic()\n\t\t\tvar t1Reader *Conn\n\t\t\tvar t2Reader *Conn\n\t\t\tvar t1Writer *Conn\n\t\t\tvar t2Writer *Conn\n\t\t\tdialer := &Dialer{}\n\n\t\t\tctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)\n\t\t\tdefer cancel()\n\n\t\t\tif t1Reader, err = dialer.DialLeader(ctx, tcp, kafka, topic1, 0); err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif t2Reader, err = dialer.DialLeader(ctx, tcp, kafka, topic2, 0); err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif t1Writer, err = dialer.DialLeader(ctx, tcp, kafka, topic1, 0); err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif t2Writer, err = dialer.DialLeader(ctx, tcp, kafka, topic2, 0); err != nil {\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tstop = func() {\n\t\t\t\tt1Reader.Close()\n\t\t\t\tt1Writer.Close()\n\t\t\t\tt2Reader.Close()\n\t\t\t\tt2Writer.Close()\n\t\t\t}\n\t\t\tc1 = &connPipe{rconn: t1Reader, wconn: t2Writer}\n\t\t\tc2 = &connPipe{rconn: t2Reader, wconn: t1Writer}\n\t\t\treturn\n\t\t})\n\t})\n}\n\nfunc testConnClose(t *testing.T, conn *Conn) {\n\tif err := conn.Close(); err != nil {\n\t\tt.Error(err)\n\t}\n}\n\nfunc testConnFirstOffset(t *testing.T, conn *Conn) {\n\toffset, whence := conn.Offset()\n\n\tif offset != 0 && whence != 0 {\n\t\tt.Error(\"bad first offset:\", offset, whence)\n\t}\n}\n\nfunc testConnWrite(t *testing.T, conn *Conn) {\n\tb := []byte(\"Hello World!\")\n\tn, err := conn.Write(b)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tif n != len(b) {\n\t\tt.Error(\"bad length returned by (*Conn).Write:\", n)\n\t}\n}\n\nfunc testConnCloseAndWrite(t *testing.T, conn *Conn) {\n\tconn.Close()\n\n\t_, err := conn.Write([]byte(\"Hello World!\"))\n\n\t// expect a network error\n\tvar netOpError *net.OpError\n\tif !errors.As(err, &netOpError) {\n\t\tt.Error(err)\n\t}\n}\n\nfunc testConnSeekFirstOffset(t *testing.T, conn *Conn) {\n\tfor i := 0; i != 10; i++ {\n\t\tif _, err := conn.Write([]byte(strconv.Itoa(i))); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\toffset, err := conn.Seek(0, SeekStart)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tif offset != 0 {\n\t\tt.Error(\"bad offset:\", offset)\n\t}\n}\n\nfunc testConnSeekLastOffset(t *testing.T, conn *Conn) {\n\tfor i := 0; i != 10; i++ {\n\t\tif _, err := conn.Write([]byte(strconv.Itoa(i))); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\toffset, err := conn.Seek(0, SeekEnd)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tif offset != 10 {\n\t\tt.Error(\"bad offset:\", offset)\n\t}\n}\n\nfunc testConnSeekCurrentOffset(t *testing.T, conn *Conn) {\n\tfor i := 0; i != 10; i++ {\n\t\tif _, err := conn.Write([]byte(strconv.Itoa(i))); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\toffset, err := conn.Seek(5, SeekStart)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tif offset != 5 {\n\t\tt.Error(\"bad offset:\", offset)\n\t}\n\n\toffset, err = conn.Seek(-2, SeekCurrent)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tif offset != 3 {\n\t\tt.Error(\"bad offset:\", offset)\n\t}\n}\n\nfunc testConnSeekRandomOffset(t *testing.T, conn *Conn) {\n\tfor i := 0; i != 10; i++ {\n\t\tif _, err := conn.Write([]byte(strconv.Itoa(i))); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\toffset, err := conn.Seek(3, SeekAbsolute)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tif offset != 3 {\n\t\tt.Error(\"bad offset:\", offset)\n\t}\n}\n\nfunc testConnSeekDontCheck(t *testing.T, conn *Conn) {\n\tfor i := 0; i != 10; i++ {\n\t\tif _, err := conn.Write([]byte(strconv.Itoa(i))); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\toffset, err := conn.Seek(42, SeekAbsolute|SeekDontCheck)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tif offset != 42 {\n\t\tt.Error(\"bad offset:\", offset)\n\t}\n\n\tif _, err := conn.ReadMessage(1024); !errors.Is(err, OffsetOutOfRange) {\n\t\tt.Error(\"unexpected error:\", err)\n\t}\n}\n\nfunc testConnWriteReadSequentially(t *testing.T, conn *Conn) {\n\tfor i := 0; i != 10; i++ {\n\t\tif _, err := conn.Write([]byte(strconv.Itoa(i))); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\tb := make([]byte, 128)\n\n\tfor i := 0; i != 10; i++ {\n\t\tn, err := conn.Read(b)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t\tcontinue\n\t\t}\n\t\ts := string(b[:n])\n\t\tif v, err := strconv.Atoi(s); err != nil {\n\t\t\tt.Error(err)\n\t\t} else if v != i {\n\t\t\tt.Errorf(\"bad message read at offset %d: %s\", i, s)\n\t\t}\n\t}\n}\n\nfunc testConnWriteBatchReadSequentially(t *testing.T, conn *Conn) {\n\tmsgs := makeTestSequence(10)\n\n\tif _, err := conn.WriteMessages(msgs...); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfor i := 0; i != 10; i++ {\n\t\tmsg, err := conn.ReadMessage(128)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t\tcontinue\n\t\t}\n\t\tif !bytes.Equal(msg.Key, msgs[i].Key) {\n\t\t\tt.Errorf(\"bad message key at offset %d: %q != %q\", i, msg.Key, msgs[i].Key)\n\t\t}\n\t\tif !bytes.Equal(msg.Value, msgs[i].Value) {\n\t\t\tt.Errorf(\"bad message value at offset %d: %q != %q\", i, msg.Value, msgs[i].Value)\n\t\t}\n\t\tif !msg.Time.Equal(msgs[i].Time) {\n\t\t\tt.Errorf(\"bad message time at offset %d: %s != %s\", i, msg.Time, msgs[i].Time)\n\t\t}\n\t}\n}\n\nfunc testConnReadWatermarkFromBatch(t *testing.T, conn *Conn) {\n\tif _, err := conn.WriteMessages(makeTestSequence(10)...); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tconst minBytes = 1\n\tconst maxBytes = 10e6 // 10 MB\n\n\tvalue := make([]byte, 10e3) // 10 KB\n\n\tbatch := conn.ReadBatch(minBytes, maxBytes)\n\n\tfor i := 0; i < 10; i++ {\n\t\t_, err := batch.Read(value)\n\t\tif err != nil {\n\t\t\tif err = batch.Close(); err != nil {\n\t\t\t\tt.Fatalf(\"error trying to read batch message: %s\", err)\n\t\t\t}\n\t\t}\n\n\t\tif batch.HighWaterMark() != 10 {\n\t\t\tt.Fatal(\"expected highest offset (watermark) to be 10\")\n\t\t}\n\t}\n\n\tbatch.Close()\n}\n\nfunc testConnReadBatchWithNoMinMaxBytes(t *testing.T, conn *Conn) {\n\tif _, err := conn.WriteMessages(makeTestSequence(10)...); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tvalue := make([]byte, 10e3) // 10 KB\n\n\tbatch := conn.ReadBatchWith(ReadBatchConfig{})\n\n\tfor i := 0; i < 10; i++ {\n\t\t_, err := batch.Read(value)\n\t\tif err != nil {\n\t\t\tif err = batch.Close(); err != nil {\n\t\t\t\tt.Fatalf(\"error trying to read batch message: %s\", err)\n\t\t\t}\n\t\t}\n\n\t\tif batch.HighWaterMark() != 10 {\n\t\t\tt.Fatal(\"expected highest offset (watermark) to be 10\")\n\t\t}\n\t}\n\n\tif err := batch.Close(); err != nil {\n\t\tt.Fatalf(\"error trying to close batch: %s\", err)\n\t}\n\n\tif err := batch.Err(); err != nil {\n\t\tt.Fatalf(\"broken batch: %s\", err)\n\t}\n}\n\nfunc testConnReadBatchWithMaxWait(t *testing.T, conn *Conn) {\n\tif _, err := conn.WriteMessages(makeTestSequence(10)...); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tconst maxBytes = 10e6 // 10 MB\n\n\tvalue := make([]byte, 10e3) // 10 KB\n\n\tcfg := ReadBatchConfig{\n\t\tMinBytes: maxBytes, // use max for both so that we hit max wait time\n\t\tMaxBytes: maxBytes,\n\t\tMaxWait: 500 * time.Millisecond,\n\t}\n\n\t// set aa read deadline so the batch will succeed.\n\tconn.SetDeadline(time.Now().Add(time.Second))\n\tbatch := conn.ReadBatchWith(cfg)\n\n\tfor i := 0; i < 10; i++ {\n\t\t_, err := batch.Read(value)\n\t\tif err != nil {\n\t\t\tif err = batch.Close(); err != nil {\n\t\t\t\tt.Fatalf(\"error trying to read batch message: %s\", err)\n\t\t\t}\n\t\t}\n\n\t\tif batch.HighWaterMark() != 10 {\n\t\t\tt.Fatal(\"expected highest offset (watermark) to be 10\")\n\t\t}\n\t}\n\n\tbatch.Close()\n\n\t// reset the offset and ensure that the conn deadline takes precedence over\n\t// the max wait\n\tconn.Seek(0, SeekAbsolute)\n\tconn.SetDeadline(time.Now().Add(50 * time.Millisecond))\n\tbatch = conn.ReadBatchWith(cfg)\n\tvar netErr net.Error\n\tif err := batch.Err(); err == nil {\n\t\tt.Fatal(\"should have timed out, but got no error\")\n\t} else if errors.As(err, &netErr) {\n\t\tif !netErr.Timeout() {\n\t\t\tt.Fatalf(\"should have timed out, but got: %v\", err)\n\t\t}\n\t}\n}\n\nfunc waitForCoordinator(t *testing.T, conn *Conn, groupID string) {\n\t// ensure that kafka has allocated a group coordinator. oddly, issue doesn't\n\t// appear to happen if the kafka been running for a while.\n\tconst maxAttempts = 20\n\tfor attempt := 1; attempt <= maxAttempts; attempt++ {\n\t\t_, err := conn.findCoordinator(findCoordinatorRequestV0{\n\t\t\tCoordinatorKey: groupID,\n\t\t})\n\t\tif err != nil {\n\t\t\tif errors.Is(err, GroupCoordinatorNotAvailable) {\n\t\t\t\ttime.Sleep(250 * time.Millisecond)\n\t\t\t\tcontinue\n\t\t\t} else {\n\t\t\t\tt.Fatalf(\"unable to find coordinator for group: %v\", err)\n\t\t\t}\n\t\t} else {\n\t\t\treturn\n\t\t}\n\t}\n\n\tt.Fatalf(\"unable to connect to coordinator after %v attempts\", maxAttempts)\n}\n\nfunc createGroup(t *testing.T, conn *Conn, groupID string) (generationID int32, memberID string, stop func()) {\n\twaitForCoordinator(t, conn, groupID)\n\n\tjoin := func() (joinGroup joinGroupResponse) {\n\t\tvar err error\n\t\tfor attempt := 0; attempt < 10; attempt++ {\n\t\t\tjoinGroup, err = conn.joinGroup(joinGroupRequest{\n\t\t\t\tGroupID: groupID,\n\t\t\t\tSessionTimeout: int32(time.Minute / time.Millisecond),\n\t\t\t\tRebalanceTimeout: int32(time.Second / time.Millisecond),\n\t\t\t\tProtocolType: \"roundrobin\",\n\t\t\t\tGroupProtocols: []joinGroupRequestGroupProtocolV1{\n\t\t\t\t\t{\n\t\t\t\t\t\tProtocolName: \"roundrobin\",\n\t\t\t\t\t\tProtocolMetadata: []byte(\"blah\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tif errors.Is(err, NotCoordinatorForGroup) {\n\t\t\t\t\ttime.Sleep(250 * time.Millisecond)\n\t\t\t\t\tcontinue\n\t\t\t\t} else {\n\t\t\t\t\tt.Fatalf(\"bad joinGroup: %s\", err)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\n\t\treturn\n\t}\n\n\t// join the group\n\tjoinGroup := join()\n\n\t// sync the group\n\t_, err := conn.syncGroup(syncGroupRequestV0{\n\t\tGroupID: groupID,\n\t\tGenerationID: joinGroup.GenerationID,\n\t\tMemberID: joinGroup.MemberID,\n\t\tGroupAssignments: []syncGroupRequestGroupAssignmentV0{\n\t\t\t{\n\t\t\t\tMemberID: joinGroup.MemberID,\n\t\t\t\tMemberAssignments: []byte(\"blah\"),\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"bad syncGroup: %s\", err)\n\t}\n\n\tgenerationID = joinGroup.GenerationID\n\tmemberID = joinGroup.MemberID\n\tstop = func() {\n\t\tconn.leaveGroup(leaveGroupRequestV0{\n\t\t\tGroupID: groupID,\n\t\t\tMemberID: joinGroup.MemberID,\n\t\t})\n\t}\n\n\treturn\n}\n\nfunc testConnFindCoordinator(t *testing.T, conn *Conn) {\n\tgroupID := makeGroupID()\n\n\tfor attempt := 0; attempt < 10; attempt++ {\n\t\tif attempt != 0 {\n\t\t\ttime.Sleep(time.Millisecond * 50)\n\t\t}\n\t\tresponse, err := conn.findCoordinator(findCoordinatorRequestV0{CoordinatorKey: groupID})\n\t\tif err != nil {\n\t\t\tif errors.Is(err, GroupCoordinatorNotAvailable) {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tt.Fatalf(\"bad findCoordinator: %s\", err)\n\t\t}\n\n\t\tif response.Coordinator.NodeID == 0 {\n\t\t\tt.Errorf(\"bad NodeID\")\n\t\t}\n\t\tif response.Coordinator.Host == \"\" {\n\t\t\tt.Errorf(\"bad Host\")\n\t\t}\n\t\tif response.Coordinator.Port == 0 {\n\t\t\tt.Errorf(\"bad Port\")\n\t\t}\n\t\treturn\n\t}\n}\n\nfunc testConnJoinGroupInvalidGroupID(t *testing.T, conn *Conn) {\n\t_, err := conn.joinGroup(joinGroupRequest{})\n\tif !errors.Is(err, InvalidGroupId) && !errors.Is(err, NotCoordinatorForGroup) {\n\t\tt.Fatalf(\"expected %v or %v; got %v\", InvalidGroupId, NotCoordinatorForGroup, err)\n\t}\n}\n\nfunc testConnJoinGroupInvalidSessionTimeout(t *testing.T, conn *Conn) {\n\tgroupID := makeGroupID()\n\twaitForCoordinator(t, conn, groupID)\n\n\t_, err := conn.joinGroup(joinGroupRequest{\n\t\tGroupID: groupID,\n\t})\n\tif !errors.Is(err, InvalidSessionTimeout) && !errors.Is(err, NotCoordinatorForGroup) {\n\t\tt.Fatalf(\"expected %v or %v; got %v\", InvalidSessionTimeout, NotCoordinatorForGroup, err)\n\t}\n}\n\nfunc testConnJoinGroupInvalidRefreshTimeout(t *testing.T, conn *Conn) {\n\tgroupID := makeGroupID()\n\twaitForCoordinator(t, conn, groupID)\n\n\t_, err := conn.joinGroup(joinGroupRequest{\n\t\tGroupID: groupID,\n\t\tSessionTimeout: int32(3 * time.Second / time.Millisecond),\n\t})\n\tif !errors.Is(err, InvalidSessionTimeout) && !errors.Is(err, NotCoordinatorForGroup) {\n\t\tt.Fatalf(\"expected %v or %v; got %v\", InvalidSessionTimeout, NotCoordinatorForGroup, err)\n\t}\n}\n\nfunc testConnHeartbeatErr(t *testing.T, conn *Conn) {\n\tgroupID := makeGroupID()\n\tcreateGroup(t, conn, groupID)\n\n\t_, err := conn.syncGroup(syncGroupRequestV0{\n\t\tGroupID: groupID,\n\t})\n\tif !errors.Is(err, UnknownMemberId) && !errors.Is(err, NotCoordinatorForGroup) {\n\t\tt.Fatalf(\"expected %v or %v; got %v\", UnknownMemberId, NotCoordinatorForGroup, err)\n\t}\n}\n\nfunc testConnLeaveGroupErr(t *testing.T, conn *Conn) {\n\tgroupID := makeGroupID()\n\twaitForCoordinator(t, conn, groupID)\n\n\t_, err := conn.leaveGroup(leaveGroupRequestV0{\n\t\tGroupID: groupID,\n\t})\n\tif !errors.Is(err, UnknownMemberId) && !errors.Is(err, NotCoordinatorForGroup) {\n\t\tt.Fatalf(\"expected %v or %v; got %v\", UnknownMemberId, NotCoordinatorForGroup, err)\n\t}\n}\n\nfunc testConnSyncGroupErr(t *testing.T, conn *Conn) {\n\tgroupID := makeGroupID()\n\twaitForCoordinator(t, conn, groupID)\n\n\t_, err := conn.syncGroup(syncGroupRequestV0{\n\t\tGroupID: groupID,\n\t})\n\tif !errors.Is(err, UnknownMemberId) && !errors.Is(err, NotCoordinatorForGroup) {\n\t\tt.Fatalf(\"expected %v or %v; got %v\", UnknownMemberId, NotCoordinatorForGroup, err)\n\t}\n}\n\nfunc testConnListGroupsReturnsGroups(t *testing.T, conn *Conn) {\n\tgroup1 := makeGroupID()\n\t_, _, stop1 := createGroup(t, conn, group1)\n\tdefer stop1()\n\n\tgroup2 := makeGroupID()\n\t_, _, stop2 := createGroup(t, conn, group2)\n\tdefer stop2()\n\n\tout, err := conn.listGroups(listGroupsRequestV1{})\n\tif err != nil {\n\t\tt.Fatalf(\"bad err: %v\", err)\n\t}\n\n\tcontainsGroup := func(groupID string) bool {\n\t\tfor _, group := range out.Groups {\n\t\t\tif group.GroupID == groupID {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t\treturn false\n\t}\n\n\tif !containsGroup(group1) {\n\t\tt.Errorf(\"expected groups to contain group1\")\n\t}\n\n\tif !containsGroup(group2) {\n\t\tt.Errorf(\"expected groups to contain group2\")\n\t}\n}\n\nfunc testConnFetchAndCommitOffsets(t *testing.T, conn *Conn) {\n\tconst N = 10\n\tif _, err := conn.WriteMessages(makeTestSequence(N)...); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tgroupID := makeGroupID()\n\tgenerationID, memberID, stop := createGroup(t, conn, groupID)\n\tdefer stop()\n\n\trequest := offsetFetchRequestV1{\n\t\tGroupID: groupID,\n\t\tTopics: []offsetFetchRequestV1Topic{\n\t\t\t{\n\t\t\t\tTopic: conn.topic,\n\t\t\t\tPartitions: []int32{0},\n\t\t\t},\n\t\t},\n\t}\n\tfetch, err := conn.offsetFetch(request)\n\tif err != nil {\n\t\tt.Fatalf(\"bad err: %v\", err)\n\t}\n\n\tif v := len(fetch.Responses); v != 1 {\n\t\tt.Fatalf(\"expected 1 Response; got %v\", v)\n\t}\n\n\tif v := len(fetch.Responses[0].PartitionResponses); v != 1 {\n\t\tt.Fatalf(\"expected 1 PartitionResponses; got %v\", v)\n\t}\n\n\tif offset := fetch.Responses[0].PartitionResponses[0].Offset; offset != -1 {\n\t\tt.Fatalf(\"expected initial offset of -1; got %v\", offset)\n\t}\n\n\tcommittedOffset := int64(N - 1)\n\t_, err = conn.offsetCommit(offsetCommitRequestV2{\n\t\tGroupID: groupID,\n\t\tGenerationID: generationID,\n\t\tMemberID: memberID,\n\t\tRetentionTime: int64(time.Hour / time.Millisecond),\n\t\tTopics: []offsetCommitRequestV2Topic{\n\t\t\t{\n\t\t\t\tTopic: conn.topic,\n\t\t\t\tPartitions: []offsetCommitRequestV2Partition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\tOffset: committedOffset,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"bad error: %v\", err)\n\t}\n\n\tfetch, err = conn.offsetFetch(request)\n\tif err != nil {\n\t\tt.Fatalf(\"bad error: %v\", err)\n\t}\n\n\tfetchedOffset := fetch.Responses[0].PartitionResponses[0].Offset\n\tif committedOffset != fetchedOffset {\n\t\tt.Fatalf(\"bad offset. expected %v; got %v\", committedOffset, fetchedOffset)\n\t}\n}\n\nfunc testConnWriteReadConcurrently(t *testing.T, conn *Conn) {\n\tconst N = 1000\n\tmsgs := make([]string, N)\n\tdone := make(chan struct{})\n\twritten := make(chan struct{}, N/10)\n\n\tfor i := 0; i != N; i++ {\n\t\tmsgs[i] = strconv.Itoa(i)\n\t}\n\n\tgo func() {\n\t\tdefer close(done)\n\t\tfor _, msg := range msgs {\n\t\t\tif _, err := conn.Write([]byte(msg)); err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t}\n\t\t\twritten <- struct{}{}\n\t\t}\n\t}()\n\n\tb := make([]byte, 128)\n\n\tfor i := 0; i != N; i++ {\n\t\t// wait until at least one message has been written. the reason for\n\t\t// this synchronization is that we aren't using deadlines. as such, if\n\t\t// the read happens before a message is available, it will cause a\n\t\t// deadlock because the read request will never hit the one byte minimum\n\t\t// in order to return and release the lock on the conn. by ensuring\n\t\t// that there's at least one message produced, we don't hit that\n\t\t// condition.\n\t\t<-written\n\t\tn, err := conn.Read(b)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t\tif s := string(b[:n]); s != strconv.Itoa(i) {\n\t\t\tt.Errorf(\"bad message read at offset %d: %s\", i, s)\n\t\t}\n\t}\n\n\t<-done\n}\n\nfunc testConnReadShortBuffer(t *testing.T, conn *Conn) {\n\tif _, err := conn.Write([]byte(\"Hello World!\")); err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tb := make([]byte, 4)\n\n\tfor i := 0; i != 10; i++ {\n\t\tb[0] = 0\n\t\tb[1] = 0\n\t\tb[2] = 0\n\t\tb[3] = 0\n\n\t\tn, err := conn.Read(b)\n\t\tif !errors.Is(err, io.ErrShortBuffer) {\n\t\t\tt.Error(\"bad error:\", i, err)\n\t\t}\n\t\tif n != 4 {\n\t\t\tt.Error(\"bad byte count:\", i, n)\n\t\t}\n\t\tif s := string(b); s != \"Hell\" {\n\t\t\tt.Error(\"bad content:\", i, s)\n\t\t}\n\t}\n}\n\nfunc testConnReadEmptyWithDeadline(t *testing.T, conn *Conn) {\n\tb := make([]byte, 100)\n\n\tstart := time.Now()\n\tdeadline := start.Add(time.Second)\n\n\tconn.SetReadDeadline(deadline)\n\tn, err := conn.Read(b)\n\n\tif n != 0 {\n\t\tt.Error(\"bad byte count:\", n)\n\t}\n\n\tif !isTimeout(err) {\n\t\tt.Error(\"expected timeout error but got\", err)\n\t}\n}\n\nfunc testDeleteTopics(t *testing.T, conn *Conn) {\n\ttopic1 := makeTopic()\n\ttopic2 := makeTopic()\n\terr := conn.CreateTopics(\n\t\tTopicConfig{\n\t\t\tTopic: topic1,\n\t\t\tNumPartitions: 1,\n\t\t\tReplicationFactor: 1,\n\t\t},\n\t\tTopicConfig{\n\t\t\tTopic: topic2,\n\t\t\tNumPartitions: 1,\n\t\t\tReplicationFactor: 1,\n\t\t},\n\t)\n\tif err != nil {\n\t\tt.Fatalf(\"bad CreateTopics: %v\", err)\n\t}\n\tconn.SetDeadline(time.Now().Add(time.Second))\n\terr = conn.DeleteTopics(topic1, topic2)\n\tif err != nil {\n\t\tt.Fatalf(\"bad DeleteTopics: %v\", err)\n\t}\n\tpartitions, err := conn.ReadPartitions(topic1, topic2)\n\tif err != nil {\n\t\tt.Fatalf(\"bad ReadPartitions: %v\", err)\n\t}\n\tif len(partitions) != 0 {\n\t\tt.Fatal(\"exepected partitions to be empty \")\n\t}\n}\n\nfunc testDeleteTopicsInvalidTopic(t *testing.T, conn *Conn) {\n\ttopic := makeTopic()\n\terr := conn.CreateTopics(\n\t\tTopicConfig{\n\t\t\tTopic: topic,\n\t\t\tNumPartitions: 1,\n\t\t\tReplicationFactor: 1,\n\t\t},\n\t)\n\tif err != nil {\n\t\tt.Fatalf(\"bad CreateTopics: %v\", err)\n\t}\n\tconn.SetDeadline(time.Now().Add(5 * time.Second))\n\terr = conn.DeleteTopics(\"invalid-topic\", topic)\n\tif !errors.Is(err, UnknownTopicOrPartition) {\n\t\tt.Fatalf(\"expected UnknownTopicOrPartition error, but got %v\", err)\n\t}\n\tpartitions, err := conn.ReadPartitions(topic)\n\tif err != nil {\n\t\tt.Fatalf(\"bad ReadPartitions: %v\", err)\n\t}\n\tif len(partitions) != 0 {\n\t\tt.Fatal(\"expected partitions to be empty\")\n\t}\n}\n\nfunc testController(t *testing.T, conn *Conn) {\n\tb, err := conn.Controller()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tif b.Host != \"localhost\" {\n\t\tt.Errorf(\"expected localhost received %s\", b.Host)\n\t}\n\tif b.Port != 9092 {\n\t\tt.Errorf(\"expected 9092 received %d\", b.Port)\n\t}\n\tif b.ID != 1 {\n\t\tt.Errorf(\"expected 1 received %d\", b.ID)\n\t}\n\tif b.Rack != \"\" {\n\t\tt.Errorf(\"expected empty string for rack received %s\", b.Rack)\n\t}\n}\n\nfunc testBrokers(t *testing.T, conn *Conn) {\n\tbrokers, err := conn.Brokers()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tif len(brokers) != 1 {\n\t\tt.Errorf(\"expected 1 broker in %+v\", brokers)\n\t}\n\n\tif brokers[0].ID != 1 {\n\t\tt.Errorf(\"expected ID 1 received %d\", brokers[0].ID)\n\t}\n}\n\nfunc testConnBroker(t *testing.T, conn *Conn) {\n\tbroker := conn.Broker()\n\t// Depending on the environment the test is being run, IPv4 or IPv6 may be used.\n\tif broker.Host != \"::1\" && broker.Host != \"127.0.0.1\" {\n\t\tt.Errorf(\"invalid broker address: %q\", broker.Host)\n\t}\n\tif broker.Port != 9092 {\n\t\tt.Errorf(\"invalid broker port: %d\", broker.Port)\n\t}\n\tif broker.ID != 1 {\n\t\tt.Errorf(\"invalid broker id: %d\", broker.ID)\n\t}\n\tif broker.Rack != \"\" {\n\t\tt.Errorf(\"invalid broker rack: %q\", broker.Rack)\n\t}\n}\n\nfunc TestReadPartitionsNoTopic(t *testing.T) {\n\tconn, err := Dial(\"tcp\", \"127.0.0.1:9092\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer conn.Close()\n\n\tparts, err := conn.ReadPartitions()\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tif len(parts) == 0 {\n\t\tt.Errorf(\"no partitions were returned\")\n\t}\n}\n\nfunc TestUnsupportedSASLMechanism(t *testing.T) {\n\tctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)\n\tdefer cancel()\n\n\tconn, err := (&Dialer{\n\t\tResolver: &net.Resolver{},\n\t}).DialContext(ctx, \"tcp\", \"127.0.0.1:9093\")\n\tif err != nil {\n\t\tt.Fatal(\"failed to open a new kafka connection:\", err)\n\t}\n\tdefer conn.Close()\n\n\tif err := conn.saslHandshake(\"FOO\"); !errors.Is(err, UnsupportedSASLMechanism) {\n\t\tt.Errorf(\"Expected UnsupportedSASLMechanism but got %v\", err)\n\t}\n}\n\nconst benchmarkMessageCount = 100\n\nfunc BenchmarkConn(b *testing.B) {\n\tbenchmarks := []struct {\n\t\tscenario string\n\t\tfunction func(*testing.B, *Conn, []byte)\n\t}{\n\t\t{\n\t\t\tscenario: \"Seek\",\n\t\t\tfunction: benchmarkConnSeek,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"Read\",\n\t\t\tfunction: benchmarkConnRead,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"ReadBatch\",\n\t\t\tfunction: benchmarkConnReadBatch,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"ReadOffsets\",\n\t\t\tfunction: benchmarkConnReadOffsets,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"Write\",\n\t\t\tfunction: benchmarkConnWrite,\n\t\t},\n\t}\n\n\ttopic := makeTopic()\n\tvalue := make([]byte, 10e3) // 10 KB\n\tmsgs := make([]Message, benchmarkMessageCount)\n\n\tfor i := range msgs {\n\t\tmsgs[i].Value = value\n\t}\n\n\tconn, _ := DialLeader(context.Background(), \"tcp\", \"localhost:9092\", topic, 0)\n\tdefer conn.Close()\n\n\tif _, err := conn.WriteMessages(msgs...); err != nil {\n\t\tb.Fatal(err)\n\t}\n\n\tfor _, benchmark := range benchmarks {\n\t\tb.Run(benchmark.scenario, func(b *testing.B) {\n\t\t\tif _, err := conn.Seek(0, SeekStart); err != nil {\n\t\t\t\tb.Error(err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tbenchmark.function(b, conn, value)\n\t\t})\n\t}\n}\n\nfunc benchmarkConnSeek(b *testing.B, conn *Conn, _ []byte) {\n\tfor i := 0; i != b.N; i++ {\n\t\tif _, err := conn.Seek(int64(i%benchmarkMessageCount), SeekAbsolute); err != nil {\n\t\t\tb.Error(err)\n\t\t\treturn\n\t\t}\n\t}\n}\n\nfunc benchmarkConnRead(b *testing.B, conn *Conn, a []byte) {\n\tn := 0\n\ti := 0\n\n\tfor i != b.N {\n\t\tif (i % benchmarkMessageCount) == 0 {\n\t\t\tif _, err := conn.Seek(0, SeekStart); err != nil {\n\t\t\t\tb.Error(err)\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\n\t\tc, err := conn.Read(a)\n\t\tif err != nil {\n\t\t\tb.Error(err)\n\t\t\treturn\n\t\t}\n\n\t\tn += c\n\t\ti++\n\t}\n\n\tb.SetBytes(int64(n / i))\n}\n\nfunc benchmarkConnReadBatch(b *testing.B, conn *Conn, a []byte) {\n\tconst minBytes = 1\n\tconst maxBytes = 10e6 // 10 MB\n\n\tbatch := conn.ReadBatch(minBytes, maxBytes)\n\ti := 0\n\tn := 0\n\n\tfor i != b.N {\n\t\tc, err := batch.Read(a)\n\t\tif err != nil {\n\t\t\tif err = batch.Close(); err != nil {\n\t\t\t\tb.Error(err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif _, err = conn.Seek(0, SeekStart); err != nil {\n\t\t\t\tb.Error(err)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tbatch = conn.ReadBatch(minBytes, maxBytes)\n\t\t}\n\t\tn += c\n\t\ti++\n\t}\n\n\tbatch.Close()\n\tb.SetBytes(int64(n / i))\n}\n\nfunc benchmarkConnReadOffsets(b *testing.B, conn *Conn, _ []byte) {\n\tfor i := 0; i != b.N; i++ {\n\t\t_, _, err := conn.ReadOffsets()\n\t\tif err != nil {\n\t\t\tb.Error(err)\n\t\t\treturn\n\t\t}\n\t}\n}\n\nfunc benchmarkConnWrite(b *testing.B, conn *Conn, _ []byte) {\n\ta := make([]byte, 10e3) // 10 KB\n\tn := 0\n\ti := 0\n\n\tfor i != b.N {\n\t\tc, err := conn.Write(a)\n\t\tif err != nil {\n\t\t\tb.Error(err)\n\t\t\treturn\n\t\t}\n\t\tn += c\n\t\ti++\n\t}\n\n\tb.SetBytes(int64(n / i))\n}\n\nfunc TestEmptyToNullableReturnsNil(t *testing.T) {\n\tif emptyToNullable(\"\") != nil {\n\t\tt.Error(\"Empty string is not converted to nil\")\n\t}\n}\n\nfunc TestEmptyToNullableLeavesStringsIntact(t *testing.T) {\n\tconst s = \"abc\"\n\tr := emptyToNullable(s)\n\tif *r != s {\n\t\tt.Error(\"Non empty string is not equal to the original string\")\n\t}\n}\n\nfunc TestMakeBrokersAllPresent(t *testing.T) {\n\tbrokers := make(map[int32]Broker)\n\tbrokers[1] = Broker{ID: 1, Host: \"203.0.113.101\", Port: 9092}\n\tbrokers[2] = Broker{ID: 1, Host: \"203.0.113.102\", Port: 9092}\n\tbrokers[3] = Broker{ID: 1, Host: \"203.0.113.103\", Port: 9092}\n\n\tb := makeBrokers(brokers, 1, 2, 3)\n\tif len(b) != 3 {\n\t\tt.Errorf(\"Expected 3 brokers, got %d\", len(b))\n\t}\n\tfor _, i := range []int32{1, 2, 3} {\n\t\tif b[i-1] != brokers[i] {\n\t\t\tt.Errorf(\"Expected broker %d at index %d, got %d\", i, i-1, b[i].ID)\n\t\t}\n\t}\n}\n\nfunc TestMakeBrokersOneMissing(t *testing.T) {\n\tbrokers := make(map[int32]Broker)\n\tbrokers[1] = Broker{ID: 1, Host: \"203.0.113.101\", Port: 9092}\n\tbrokers[3] = Broker{ID: 3, Host: \"203.0.113.103\", Port: 9092}\n\n\tb := makeBrokers(brokers, 1, 2, 3)\n\tif len(b) != 3 {\n\t\tt.Errorf(\"Expected 3 brokers, got %d\", len(b))\n\t}\n\tif b[0] != brokers[1] {\n\t\tt.Errorf(\"Expected broker 1 at index 0, got %d\", b[0].ID)\n\t}\n\tif b[1] != (Broker{ID: 2}) {\n\t\tt.Errorf(\"Expected broker 2 at index 1, got %d\", b[1].ID)\n\t}\n\tif b[2] != brokers[3] {\n\t\tt.Errorf(\"Expected broker 3 at index 1, got %d\", b[2].ID)\n\t}\n}\n" }, { "path": "consumergroup.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"net\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n)\n\n// ErrGroupClosed is returned by ConsumerGroup.Next when the group has already\n// been closed.\nvar ErrGroupClosed = errors.New(\"consumer group is closed\")\n\n// ErrGenerationEnded is returned by the context.Context issued by the\n// Generation's Start function when the context has been closed.\nvar ErrGenerationEnded = errors.New(\"consumer group generation has ended\")\n\nconst (\n\t// defaultProtocolType holds the default protocol type documented in the\n\t// kafka protocol\n\t//\n\t// See https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-GroupMembershipAPI\n\tdefaultProtocolType = \"consumer\"\n\n\t// defaultHeartbeatInterval contains the default time between heartbeats. If\n\t// the coordinator does not receive a heartbeat within the session timeout interval,\n\t// the consumer will be considered dead and the coordinator will rebalance the\n\t// group.\n\t//\n\t// As a rule, the heartbeat interval should be no greater than 1/3 the session timeout.\n\tdefaultHeartbeatInterval = 3 * time.Second\n\n\t// defaultSessionTimeout contains the default interval the coordinator will wait\n\t// for a heartbeat before marking a consumer as dead.\n\tdefaultSessionTimeout = 30 * time.Second\n\n\t// defaultRebalanceTimeout contains the amount of time the coordinator will wait\n\t// for consumers to issue a join group once a rebalance has been requested.\n\tdefaultRebalanceTimeout = 30 * time.Second\n\n\t// defaultJoinGroupBackoff is the amount of time to wait after a failed\n\t// consumer group generation before attempting to re-join.\n\tdefaultJoinGroupBackoff = 5 * time.Second\n\n\t// defaultRetentionTime holds the length of time a the consumer group will be\n\t// saved by kafka. This value tells the broker to use its configured value.\n\tdefaultRetentionTime = -1 * time.Millisecond\n\n\t// defaultPartitionWatchTime contains the amount of time the kafka-go will wait to\n\t// query the brokers looking for partition changes.\n\tdefaultPartitionWatchTime = 5 * time.Second\n\n\t// defaultTimeout is the deadline to set when interacting with the\n\t// consumer group coordinator.\n\tdefaultTimeout = 5 * time.Second\n)\n\n// ConsumerGroupConfig is a configuration object used to create new instances of\n// ConsumerGroup.\ntype ConsumerGroupConfig struct {\n\t// ID is the consumer group ID. It must not be empty.\n\tID string\n\n\t// The list of broker addresses used to connect to the kafka cluster. It\n\t// must not be empty.\n\tBrokers []string\n\n\t// An dialer used to open connections to the kafka server. This field is\n\t// optional, if nil, the default dialer is used instead.\n\tDialer *Dialer\n\n\t// Topics is the list of topics that will be consumed by this group. It\n\t// will usually have a single value, but it is permitted to have multiple\n\t// for more complex use cases.\n\tTopics []string\n\n\t// GroupBalancers is the priority-ordered list of client-side consumer group\n\t// balancing strategies that will be offered to the coordinator. The first\n\t// strategy that all group members support will be chosen by the leader.\n\t//\n\t// Default: [Range, RoundRobin]\n\tGroupBalancers []GroupBalancer\n\n\t// HeartbeatInterval sets the optional frequency at which the reader sends the consumer\n\t// group heartbeat update.\n\t//\n\t// Default: 3s\n\tHeartbeatInterval time.Duration\n\n\t// PartitionWatchInterval indicates how often a reader checks for partition changes.\n\t// If a reader sees a partition change (such as a partition add) it will rebalance the group\n\t// picking up new partitions.\n\t//\n\t// Default: 5s\n\tPartitionWatchInterval time.Duration\n\n\t// WatchForPartitionChanges is used to inform kafka-go that a consumer group should be\n\t// polling the brokers and rebalancing if any partition changes happen to the topic.\n\tWatchPartitionChanges bool\n\n\t// SessionTimeout optionally sets the length of time that may pass without a heartbeat\n\t// before the coordinator considers the consumer dead and initiates a rebalance.\n\t//\n\t// Default: 30s\n\tSessionTimeout time.Duration\n\n\t// RebalanceTimeout optionally sets the length of time the coordinator will wait\n\t// for members to join as part of a rebalance. For kafka servers under higher\n\t// load, it may be useful to set this value higher.\n\t//\n\t// Default: 30s\n\tRebalanceTimeout time.Duration\n\n\t// JoinGroupBackoff optionally sets the length of time to wait before re-joining\n\t// the consumer group after an error.\n\t//\n\t// Default: 5s\n\tJoinGroupBackoff time.Duration\n\n\t// RetentionTime optionally sets the length of time the consumer group will\n\t// be saved by the broker. -1 will disable the setting and leave the\n\t// retention up to the broker's offsets.retention.minutes property. By\n\t// default, that setting is 1 day for kafka < 2.0 and 7 days for kafka >=\n\t// 2.0.\n\t//\n\t// Default: -1\n\tRetentionTime time.Duration\n\n\t// StartOffset determines from whence the consumer group should begin\n\t// consuming when it finds a partition without a committed offset. If\n\t// non-zero, it must be set to one of FirstOffset or LastOffset.\n\t//\n\t// Default: FirstOffset\n\tStartOffset int64\n\n\t// If not nil, specifies a logger used to report internal changes within the\n\t// reader.\n\tLogger Logger\n\n\t// ErrorLogger is the logger used to report errors. If nil, the reader falls\n\t// back to using Logger instead.\n\tErrorLogger Logger\n\n\t// Timeout is the network timeout used when communicating with the consumer\n\t// group coordinator. This value should not be too small since errors\n\t// communicating with the broker will generally cause a consumer group\n\t// rebalance, and it's undesirable that a transient network error intoduce\n\t// that overhead. Similarly, it should not be too large or the consumer\n\t// group may be slow to respond to the coordinator failing over to another\n\t// broker.\n\t//\n\t// Default: 5s\n\tTimeout time.Duration\n\n\t// connect is a function for dialing the coordinator. This is provided for\n\t// unit testing to mock broker connections.\n\tconnect func(dialer *Dialer, brokers ...string) (coordinator, error)\n}\n\n// Validate method validates ConsumerGroupConfig properties and sets relevant\n// defaults.\nfunc (config *ConsumerGroupConfig) Validate() error {\n\n\tif len(config.Brokers) == 0 {\n\t\treturn errors.New(\"cannot create a consumer group with an empty list of broker addresses\")\n\t}\n\n\tif len(config.Topics) == 0 {\n\t\treturn errors.New(\"cannot create a consumer group without a topic\")\n\t}\n\n\tif config.ID == \"\" {\n\t\treturn errors.New(\"cannot create a consumer group without an ID\")\n\t}\n\n\tif config.Dialer == nil {\n\t\tconfig.Dialer = DefaultDialer\n\t}\n\n\tif len(config.GroupBalancers) == 0 {\n\t\tconfig.GroupBalancers = []GroupBalancer{\n\t\t\tRangeGroupBalancer{},\n\t\t\tRoundRobinGroupBalancer{},\n\t\t}\n\t}\n\n\tif config.HeartbeatInterval == 0 {\n\t\tconfig.HeartbeatInterval = defaultHeartbeatInterval\n\t}\n\n\tif config.SessionTimeout == 0 {\n\t\tconfig.SessionTimeout = defaultSessionTimeout\n\t}\n\n\tif config.PartitionWatchInterval == 0 {\n\t\tconfig.PartitionWatchInterval = defaultPartitionWatchTime\n\t}\n\n\tif config.RebalanceTimeout == 0 {\n\t\tconfig.RebalanceTimeout = defaultRebalanceTimeout\n\t}\n\n\tif config.JoinGroupBackoff == 0 {\n\t\tconfig.JoinGroupBackoff = defaultJoinGroupBackoff\n\t}\n\n\tif config.RetentionTime == 0 {\n\t\tconfig.RetentionTime = defaultRetentionTime\n\t}\n\n\tif config.HeartbeatInterval < 0 || (config.HeartbeatInterval/time.Millisecond) >= math.MaxInt32 {\n\t\treturn fmt.Errorf(\"HeartbeatInterval out of bounds: %d\", config.HeartbeatInterval)\n\t}\n\n\tif config.SessionTimeout < 0 || (config.SessionTimeout/time.Millisecond) >= math.MaxInt32 {\n\t\treturn fmt.Errorf(\"SessionTimeout out of bounds: %d\", config.SessionTimeout)\n\t}\n\n\tif config.RebalanceTimeout < 0 || (config.RebalanceTimeout/time.Millisecond) >= math.MaxInt32 {\n\t\treturn fmt.Errorf(\"RebalanceTimeout out of bounds: %d\", config.RebalanceTimeout)\n\t}\n\n\tif config.JoinGroupBackoff < 0 || (config.JoinGroupBackoff/time.Millisecond) >= math.MaxInt32 {\n\t\treturn fmt.Errorf(\"JoinGroupBackoff out of bounds: %d\", config.JoinGroupBackoff)\n\t}\n\n\tif config.RetentionTime < 0 && config.RetentionTime != defaultRetentionTime {\n\t\treturn fmt.Errorf(\"RetentionTime out of bounds: %d\", config.RetentionTime)\n\t}\n\n\tif config.PartitionWatchInterval < 0 || (config.PartitionWatchInterval/time.Millisecond) >= math.MaxInt32 {\n\t\treturn fmt.Errorf(\"PartitionWachInterval out of bounds %d\", config.PartitionWatchInterval)\n\t}\n\n\tif config.StartOffset == 0 {\n\t\tconfig.StartOffset = FirstOffset\n\t}\n\n\tif config.StartOffset != FirstOffset && config.StartOffset != LastOffset {\n\t\treturn fmt.Errorf(\"StartOffset is not valid %d\", config.StartOffset)\n\t}\n\n\tif config.Timeout == 0 {\n\t\tconfig.Timeout = defaultTimeout\n\t}\n\n\tif config.connect == nil {\n\t\tconfig.connect = makeConnect(*config)\n\t}\n\n\treturn nil\n}\n\n// PartitionAssignment represents the starting state of a partition that has\n// been assigned to a consumer.\ntype PartitionAssignment struct {\n\t// ID is the partition ID.\n\tID int\n\n\t// Offset is the initial offset at which this assignment begins. It will\n\t// either be an absolute offset if one has previously been committed for\n\t// the consumer group or a relative offset such as FirstOffset when this\n\t// is the first time the partition have been assigned to a member of the\n\t// group.\n\tOffset int64\n}\n\n// genCtx adapts the done channel of the generation to a context.Context. This\n// is used by Generation.Start so that we can pass a context to go routines\n// instead of passing around channels.\ntype genCtx struct {\n\tgen *Generation\n}\n\nfunc (c genCtx) Done() <-chan struct{} {\n\treturn c.gen.done\n}\n\nfunc (c genCtx) Err() error {\n\tselect {\n\tcase <-c.gen.done:\n\t\treturn ErrGenerationEnded\n\tdefault:\n\t\treturn nil\n\t}\n}\n\nfunc (c genCtx) Deadline() (time.Time, bool) {\n\treturn time.Time{}, false\n}\n\nfunc (c genCtx) Value(interface{}) interface{} {\n\treturn nil\n}\n\n// Generation represents a single consumer group generation. The generation\n// carries the topic+partition assignments for the given. It also provides\n// facilities for committing offsets and for running functions whose lifecycles\n// are bound to the generation.\ntype Generation struct {\n\t// ID is the generation ID as assigned by the consumer group coordinator.\n\tID int32\n\n\t// GroupID is the name of the consumer group.\n\tGroupID string\n\n\t// MemberID is the ID assigned to this consumer by the consumer group\n\t// coordinator.\n\tMemberID string\n\n\t// Assignments is the initial state of this Generation. The partition\n\t// assignments are grouped by topic.\n\tAssignments map[string][]PartitionAssignment\n\n\tconn coordinator\n\n\t// the following fields are used for process accounting to synchronize\n\t// between Start and close. lock protects all of them. done is closed\n\t// when the generation is ending in order to signal that the generation\n\t// should start self-desructing. closed protects against double-closing\n\t// the done chan. routines is a count of running go routines that have been\n\t// launched by Start. joined will be closed by the last go routine to exit.\n\tlock sync.Mutex\n\tdone chan struct{}\n\tclosed bool\n\troutines int\n\tjoined chan struct{}\n\n\tretentionMillis int64\n\tlog func(func(Logger))\n\tlogError func(func(Logger))\n}\n\n// close stops the generation and waits for all functions launched via Start to\n// terminate.\nfunc (g *Generation) close() {\n\tg.lock.Lock()\n\tif !g.closed {\n\t\tclose(g.done)\n\t\tg.closed = true\n\t}\n\t// determine whether any go routines are running that we need to wait for.\n\t// waiting needs to happen outside of the critical section.\n\tr := g.routines\n\tg.lock.Unlock()\n\n\t// NOTE: r will be zero if no go routines were ever launched. no need to\n\t// wait in that case.\n\tif r > 0 {\n\t\t<-g.joined\n\t}\n}\n\n// Start launches the provided function in a go routine and adds accounting such\n// that when the function exits, it stops the current generation (if not\n// already in the process of doing so).\n//\n// The provided function MUST support cancellation via the ctx argument and exit\n// in a timely manner once the ctx is complete. When the context is closed, the\n// context's Error() function will return ErrGenerationEnded.\n//\n// When closing out a generation, the consumer group will wait for all functions\n// launched by Start to exit before the group can move on and join the next\n// generation. If the function does not exit promptly, it will stop forward\n// progress for this consumer and potentially cause consumer group membership\n// churn.\nfunc (g *Generation) Start(fn func(ctx context.Context)) {\n\tg.lock.Lock()\n\tdefer g.lock.Unlock()\n\n\t// this is an edge case: if the generation has already closed, then it's\n\t// possible that the close func has already waited on outstanding go\n\t// routines and exited.\n\t//\n\t// nonetheless, it's important to honor that the fn is invoked in case the\n\t// calling function is waiting e.g. on a channel send or a WaitGroup. in\n\t// such a case, fn should immediately exit because ctx.Err() will return\n\t// ErrGenerationEnded.\n\tif g.closed {\n\t\tgo fn(genCtx{g})\n\t\treturn\n\t}\n\n\t// register that there is one more go routine that's part of this gen.\n\tg.routines++\n\n\tgo func() {\n\t\tfn(genCtx{g})\n\t\tg.lock.Lock()\n\t\t// shut down the generation as soon as one function exits. this is\n\t\t// different from close() in that it doesn't wait for all go routines in\n\t\t// the generation to exit.\n\t\tif !g.closed {\n\t\t\tclose(g.done)\n\t\t\tg.closed = true\n\t\t}\n\t\tg.routines--\n\t\t// if this was the last go routine in the generation, close the joined\n\t\t// chan so that close() can exit if it's waiting.\n\t\tif g.routines == 0 {\n\t\t\tclose(g.joined)\n\t\t}\n\t\tg.lock.Unlock()\n\t}()\n}\n\n// CommitOffsets commits the provided topic+partition+offset combos to the\n// consumer group coordinator. This can be used to reset the consumer to\n// explicit offsets.\nfunc (g *Generation) CommitOffsets(offsets map[string]map[int]int64) error {\n\tif len(offsets) == 0 {\n\t\treturn nil\n\t}\n\n\ttopics := make([]offsetCommitRequestV2Topic, 0, len(offsets))\n\tfor topic, partitions := range offsets {\n\t\tt := offsetCommitRequestV2Topic{Topic: topic}\n\t\tfor partition, offset := range partitions {\n\t\t\tt.Partitions = append(t.Partitions, offsetCommitRequestV2Partition{\n\t\t\t\tPartition: int32(partition),\n\t\t\t\tOffset: offset,\n\t\t\t})\n\t\t}\n\t\ttopics = append(topics, t)\n\t}\n\n\trequest := offsetCommitRequestV2{\n\t\tGroupID: g.GroupID,\n\t\tGenerationID: g.ID,\n\t\tMemberID: g.MemberID,\n\t\tRetentionTime: g.retentionMillis,\n\t\tTopics: topics,\n\t}\n\n\t_, err := g.conn.offsetCommit(request)\n\tif err == nil {\n\t\t// if logging is enabled, print out the partitions that were committed.\n\t\tg.log(func(l Logger) {\n\t\t\tvar report []string\n\t\t\tfor _, t := range request.Topics {\n\t\t\t\treport = append(report, fmt.Sprintf(\"\\ttopic: %s\", t.Topic))\n\t\t\t\tfor _, p := range t.Partitions {\n\t\t\t\t\treport = append(report, fmt.Sprintf(\"\\t\\tpartition %d: %d\", p.Partition, p.Offset))\n\t\t\t\t}\n\t\t\t}\n\t\t\tl.Printf(\"committed offsets for group %s: \\n%s\", g.GroupID, strings.Join(report, \"\\n\"))\n\t\t})\n\t}\n\n\treturn err\n}\n\n// heartbeatLoop checks in with the consumer group coordinator at the provided\n// interval. It exits if it ever encounters an error, which would signal the\n// end of the generation.\nfunc (g *Generation) heartbeatLoop(interval time.Duration) {\n\tg.Start(func(ctx context.Context) {\n\t\tg.log(func(l Logger) {\n\t\t\tl.Printf(\"started heartbeat for group, %v [%v]\", g.GroupID, interval)\n\t\t})\n\t\tdefer g.log(func(l Logger) {\n\t\t\tl.Printf(\"stopped heartbeat for group %s\\n\", g.GroupID)\n\t\t})\n\n\t\tticker := time.NewTicker(interval)\n\t\tdefer ticker.Stop()\n\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn\n\t\t\tcase <-ticker.C:\n\t\t\t\t_, err := g.conn.heartbeat(heartbeatRequestV0{\n\t\t\t\t\tGroupID: g.GroupID,\n\t\t\t\t\tGenerationID: g.ID,\n\t\t\t\t\tMemberID: g.MemberID,\n\t\t\t\t})\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t})\n}\n\n// partitionWatcher queries kafka and watches for partition changes, triggering\n// a rebalance if changes are found. Similar to heartbeat it's okay to return on\n// error here as if you are unable to ask a broker for basic metadata you're in\n// a bad spot and should rebalance. Commonly you will see an error here if there\n// is a problem with the connection to the coordinator and a rebalance will\n// establish a new connection to the coordinator.\nfunc (g *Generation) partitionWatcher(interval time.Duration, topic string) {\n\tg.Start(func(ctx context.Context) {\n\t\tg.log(func(l Logger) {\n\t\t\tl.Printf(\"started partition watcher for group, %v, topic %v [%v]\", g.GroupID, topic, interval)\n\t\t})\n\t\tdefer g.log(func(l Logger) {\n\t\t\tl.Printf(\"stopped partition watcher for group, %v, topic %v\", g.GroupID, topic)\n\t\t})\n\n\t\tticker := time.NewTicker(interval)\n\t\tdefer ticker.Stop()\n\n\t\tops, err := g.conn.readPartitions(topic)\n\t\tif err != nil {\n\t\t\tg.logError(func(l Logger) {\n\t\t\t\tl.Printf(\"Problem getting partitions during startup, %v\\n, Returning and setting up nextGeneration\", err)\n\t\t\t})\n\t\t\treturn\n\t\t}\n\t\toParts := len(ops)\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn\n\t\t\tcase <-ticker.C:\n\t\t\t\tops, err := g.conn.readPartitions(topic)\n\t\t\t\tswitch {\n\t\t\t\tcase err == nil, errors.Is(err, UnknownTopicOrPartition):\n\t\t\t\t\tif len(ops) != oParts {\n\t\t\t\t\t\tg.log(func(l Logger) {\n\t\t\t\t\t\t\tl.Printf(\"Partition changes found, rebalancing group: %v.\", g.GroupID)\n\t\t\t\t\t\t})\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\n\t\t\t\tdefault:\n\t\t\t\t\tg.logError(func(l Logger) {\n\t\t\t\t\t\tl.Printf(\"Problem getting partitions while checking for changes, %v\", err)\n\t\t\t\t\t})\n\t\t\t\t\tvar kafkaError Error\n\t\t\t\t\tif errors.As(err, &kafkaError) {\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\t// other errors imply that we lost the connection to the coordinator, so we\n\t\t\t\t\t// should abort and reconnect.\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t})\n}\n\n// coordinator is a subset of the functionality in Conn in order to facilitate\n// testing the consumer group...especially for error conditions that are\n// difficult to instigate with a live broker running in docker.\ntype coordinator interface {\n\tio.Closer\n\tfindCoordinator(findCoordinatorRequestV0) (findCoordinatorResponseV0, error)\n\tjoinGroup(joinGroupRequest) (joinGroupResponse, error)\n\tsyncGroup(syncGroupRequestV0) (syncGroupResponseV0, error)\n\tleaveGroup(leaveGroupRequestV0) (leaveGroupResponseV0, error)\n\theartbeat(heartbeatRequestV0) (heartbeatResponseV0, error)\n\toffsetFetch(offsetFetchRequestV1) (offsetFetchResponseV1, error)\n\toffsetCommit(offsetCommitRequestV2) (offsetCommitResponseV2, error)\n\treadPartitions(...string) ([]Partition, error)\n}\n\n// timeoutCoordinator wraps the Conn to ensure that every operation has a\n// deadline. Otherwise, it would be possible for requests to block indefinitely\n// if the remote server never responds. There are many spots where the consumer\n// group needs to interact with the broker, so it feels less error prone to\n// factor all of the deadline management into this shared location as opposed to\n// peppering it all through where the code actually interacts with the broker.\ntype timeoutCoordinator struct {\n\ttimeout time.Duration\n\tsessionTimeout time.Duration\n\trebalanceTimeout time.Duration\n\tconn *Conn\n}\n\nfunc (t *timeoutCoordinator) Close() error {\n\treturn t.conn.Close()\n}\n\nfunc (t *timeoutCoordinator) findCoordinator(req findCoordinatorRequestV0) (findCoordinatorResponseV0, error) {\n\tif err := t.conn.SetDeadline(time.Now().Add(t.timeout)); err != nil {\n\t\treturn findCoordinatorResponseV0{}, err\n\t}\n\treturn t.conn.findCoordinator(req)\n}\n\nfunc (t *timeoutCoordinator) joinGroup(req joinGroupRequest) (joinGroupResponse, error) {\n\t// in the case of join group, the consumer group coordinator may wait up\n\t// to rebalance timeout in order to wait for all members to join.\n\tif err := t.conn.SetDeadline(time.Now().Add(t.timeout + t.rebalanceTimeout)); err != nil {\n\t\treturn joinGroupResponse{}, err\n\t}\n\treturn t.conn.joinGroup(req)\n}\n\nfunc (t *timeoutCoordinator) syncGroup(req syncGroupRequestV0) (syncGroupResponseV0, error) {\n\t// in the case of sync group, the consumer group leader is given up to\n\t// the session timeout to respond before the coordinator will give up.\n\tif err := t.conn.SetDeadline(time.Now().Add(t.timeout + t.sessionTimeout)); err != nil {\n\t\treturn syncGroupResponseV0{}, err\n\t}\n\treturn t.conn.syncGroup(req)\n}\n\nfunc (t *timeoutCoordinator) leaveGroup(req leaveGroupRequestV0) (leaveGroupResponseV0, error) {\n\tif err := t.conn.SetDeadline(time.Now().Add(t.timeout)); err != nil {\n\t\treturn leaveGroupResponseV0{}, err\n\t}\n\treturn t.conn.leaveGroup(req)\n}\n\nfunc (t *timeoutCoordinator) heartbeat(req heartbeatRequestV0) (heartbeatResponseV0, error) {\n\tif err := t.conn.SetDeadline(time.Now().Add(t.timeout)); err != nil {\n\t\treturn heartbeatResponseV0{}, err\n\t}\n\treturn t.conn.heartbeat(req)\n}\n\nfunc (t *timeoutCoordinator) offsetFetch(req offsetFetchRequestV1) (offsetFetchResponseV1, error) {\n\tif err := t.conn.SetDeadline(time.Now().Add(t.timeout)); err != nil {\n\t\treturn offsetFetchResponseV1{}, err\n\t}\n\treturn t.conn.offsetFetch(req)\n}\n\nfunc (t *timeoutCoordinator) offsetCommit(req offsetCommitRequestV2) (offsetCommitResponseV2, error) {\n\tif err := t.conn.SetDeadline(time.Now().Add(t.timeout)); err != nil {\n\t\treturn offsetCommitResponseV2{}, err\n\t}\n\treturn t.conn.offsetCommit(req)\n}\n\nfunc (t *timeoutCoordinator) readPartitions(topics ...string) ([]Partition, error) {\n\tif err := t.conn.SetDeadline(time.Now().Add(t.timeout)); err != nil {\n\t\treturn nil, err\n\t}\n\treturn t.conn.ReadPartitions(topics...)\n}\n\n// NewConsumerGroup creates a new ConsumerGroup. It returns an error if the\n// provided configuration is invalid. It does not attempt to connect to the\n// Kafka cluster. That happens asynchronously, and any errors will be reported\n// by Next.\nfunc NewConsumerGroup(config ConsumerGroupConfig) (*ConsumerGroup, error) {\n\tif err := config.Validate(); err != nil {\n\t\treturn nil, err\n\t}\n\n\tcg := &ConsumerGroup{\n\t\tconfig: config,\n\t\tnext: make(chan *Generation),\n\t\terrs: make(chan error),\n\t\tdone: make(chan struct{}),\n\t}\n\tcg.wg.Add(1)\n\tgo func() {\n\t\tcg.run()\n\t\tcg.wg.Done()\n\t}()\n\treturn cg, nil\n}\n\n// ConsumerGroup models a Kafka consumer group. A caller doesn't interact with\n// the group directly. Rather, they interact with a Generation. Every time a\n// member enters or exits the group, it results in a new Generation. The\n// Generation is where partition assignments and offset management occur.\n// Callers will use Next to get a handle to the Generation.\ntype ConsumerGroup struct {\n\tconfig ConsumerGroupConfig\n\tnext chan *Generation\n\terrs chan error\n\n\tcloseOnce sync.Once\n\twg sync.WaitGroup\n\tdone chan struct{}\n}\n\n// Close terminates the current generation by causing this member to leave and\n// releases all local resources used to participate in the consumer group.\n// Close will also end the current generation if it is still active.\nfunc (cg *ConsumerGroup) Close() error {\n\tcg.closeOnce.Do(func() {\n\t\tclose(cg.done)\n\t})\n\tcg.wg.Wait()\n\treturn nil\n}\n\n// Next waits for the next consumer group generation. There will never be two\n// active generations. Next will never return a new generation until the\n// previous one has completed.\n//\n// If there are errors setting up the next generation, they will be surfaced\n// here.\n//\n// If the ConsumerGroup has been closed, then Next will return ErrGroupClosed.\nfunc (cg *ConsumerGroup) Next(ctx context.Context) (*Generation, error) {\n\tselect {\n\tcase <-ctx.Done():\n\t\treturn nil, ctx.Err()\n\tcase <-cg.done:\n\t\treturn nil, ErrGroupClosed\n\tcase err := <-cg.errs:\n\t\treturn nil, err\n\tcase next := <-cg.next:\n\t\treturn next, nil\n\t}\n}\n\nfunc (cg *ConsumerGroup) run() {\n\t// the memberID is the only piece of information that is maintained across\n\t// generations. it starts empty and will be assigned on the first nextGeneration\n\t// when the joinGroup request is processed. it may change again later if\n\t// the CG coordinator fails over or if the member is evicted. otherwise, it\n\t// will be constant for the lifetime of this group.\n\tvar memberID string\n\tvar err error\n\tfor {\n\t\tmemberID, err = cg.nextGeneration(memberID)\n\n\t\t// backoff will be set if this go routine should sleep before continuing\n\t\t// to the next generation. it will be non-nil in the case of an error\n\t\t// joining or syncing the group.\n\t\tvar backoff <-chan time.Time\n\n\t\tswitch {\n\t\tcase err == nil:\n\t\t\t// no error...the previous generation finished normally.\n\t\t\tcontinue\n\n\t\tcase errors.Is(err, ErrGroupClosed):\n\t\t\t// the CG has been closed...leave the group and exit loop.\n\t\t\t_ = cg.leaveGroup(memberID)\n\t\t\treturn\n\n\t\tcase errors.Is(err, RebalanceInProgress):\n\t\t\t// in case of a RebalanceInProgress, don't leave the group or\n\t\t\t// change the member ID, but report the error. the next attempt\n\t\t\t// to join the group will then be subject to the rebalance\n\t\t\t// timeout, so the broker will be responsible for throttling\n\t\t\t// this loop.\n\n\t\tdefault:\n\t\t\t// leave the group and report the error if we had gotten far\n\t\t\t// enough so as to have a member ID. also clear the member id\n\t\t\t// so we don't attempt to use it again. in order to avoid\n\t\t\t// a tight error loop, backoff before the next attempt to join\n\t\t\t// the group.\n\t\t\t_ = cg.leaveGroup(memberID)\n\t\t\tmemberID = \"\"\n\t\t\tbackoff = time.After(cg.config.JoinGroupBackoff)\n\t\t}\n\t\t// ensure that we exit cleanly in case the CG is done and no one is\n\t\t// waiting to receive on the unbuffered error channel.\n\t\tselect {\n\t\tcase <-cg.done:\n\t\t\treturn\n\t\tcase cg.errs <- err:\n\t\t}\n\t\t// backoff if needed, being sure to exit cleanly if the CG is done.\n\t\tif backoff != nil {\n\t\t\tselect {\n\t\t\tcase <-cg.done:\n\t\t\t\t// exit cleanly if the group is closed.\n\t\t\t\treturn\n\t\t\tcase <-backoff:\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (cg *ConsumerGroup) nextGeneration(memberID string) (string, error) {\n\t// get a new connection to the coordinator on each loop. the previous\n\t// generation could have exited due to losing the connection, so this\n\t// ensures that we always have a clean starting point. it means we will\n\t// re-connect in certain cases, but that shouldn't be an issue given that\n\t// rebalances are relatively infrequent under normal operating\n\t// conditions.\n\tconn, err := cg.coordinator()\n\tif err != nil {\n\t\tcg.withErrorLogger(func(log Logger) {\n\t\t\tlog.Printf(\"Unable to establish connection to consumer group coordinator for group %s: %v\", cg.config.ID, err)\n\t\t})\n\t\treturn memberID, err // a prior memberID may still be valid, so don't return \"\"\n\t}\n\tdefer conn.Close()\n\n\tvar generationID int32\n\tvar groupAssignments GroupMemberAssignments\n\tvar assignments map[string][]int32\n\n\t// join group. this will join the group and prepare assignments if our\n\t// consumer is elected leader. it may also change or assign the member ID.\n\tmemberID, generationID, groupAssignments, err = cg.joinGroup(conn, memberID)\n\tif err != nil {\n\t\tcg.withErrorLogger(func(log Logger) {\n\t\t\tlog.Printf(\"Failed to join group %s: %v\", cg.config.ID, err)\n\t\t})\n\t\treturn memberID, err\n\t}\n\tcg.withLogger(func(log Logger) {\n\t\tlog.Printf(\"Joined group %s as member %s in generation %d\", cg.config.ID, memberID, generationID)\n\t})\n\n\t// sync group\n\tassignments, err = cg.syncGroup(conn, memberID, generationID, groupAssignments)\n\tif err != nil {\n\t\tcg.withErrorLogger(func(log Logger) {\n\t\t\tlog.Printf(\"Failed to sync group %s: %v\", cg.config.ID, err)\n\t\t})\n\t\treturn memberID, err\n\t}\n\n\t// fetch initial offsets.\n\tvar offsets map[string]map[int]int64\n\toffsets, err = cg.fetchOffsets(conn, assignments)\n\tif err != nil {\n\t\tcg.withErrorLogger(func(log Logger) {\n\t\t\tlog.Printf(\"Failed to fetch offsets for group %s: %v\", cg.config.ID, err)\n\t\t})\n\t\treturn memberID, err\n\t}\n\n\t// create the generation.\n\tgen := Generation{\n\t\tID: generationID,\n\t\tGroupID: cg.config.ID,\n\t\tMemberID: memberID,\n\t\tAssignments: cg.makeAssignments(assignments, offsets),\n\t\tconn: conn,\n\t\tdone: make(chan struct{}),\n\t\tjoined: make(chan struct{}),\n\t\tretentionMillis: int64(cg.config.RetentionTime / time.Millisecond),\n\t\tlog: cg.withLogger,\n\t\tlogError: cg.withErrorLogger,\n\t}\n\n\t// spawn all of the go routines required to facilitate this generation. if\n\t// any of these functions exit, then the generation is determined to be\n\t// complete.\n\tgen.heartbeatLoop(cg.config.HeartbeatInterval)\n\tif cg.config.WatchPartitionChanges {\n\t\tfor _, topic := range cg.config.Topics {\n\t\t\tgen.partitionWatcher(cg.config.PartitionWatchInterval, topic)\n\t\t}\n\t}\n\n\t// make this generation available for retrieval. if the CG is closed before\n\t// we can send it on the channel, exit. that case is required b/c the next\n\t// channel is unbuffered. if the caller to Next has already bailed because\n\t// it's own teardown logic has been invoked, this would deadlock otherwise.\n\tselect {\n\tcase <-cg.done:\n\t\tgen.close()\n\t\treturn memberID, ErrGroupClosed // ErrGroupClosed will trigger leave logic.\n\tcase cg.next <- &gen:\n\t}\n\n\t// wait for generation to complete. if the CG is closed before the\n\t// generation is finished, exit and leave the group.\n\tselect {\n\tcase <-cg.done:\n\t\tgen.close()\n\t\treturn memberID, ErrGroupClosed // ErrGroupClosed will trigger leave logic.\n\tcase <-gen.done:\n\t\t// time for next generation! make sure all the current go routines exit\n\t\t// before continuing onward.\n\t\tgen.close()\n\t\treturn memberID, nil\n\t}\n}\n\n// connect returns a connection to ANY broker.\nfunc makeConnect(config ConsumerGroupConfig) func(dialer *Dialer, brokers ...string) (coordinator, error) {\n\treturn func(dialer *Dialer, brokers ...string) (coordinator, error) {\n\t\tvar err error\n\t\tfor _, broker := range brokers {\n\t\t\tvar conn *Conn\n\t\t\tif conn, err = dialer.Dial(\"tcp\", broker); err == nil {\n\t\t\t\treturn &timeoutCoordinator{\n\t\t\t\t\tconn: conn,\n\t\t\t\t\ttimeout: config.Timeout,\n\t\t\t\t\tsessionTimeout: config.SessionTimeout,\n\t\t\t\t\trebalanceTimeout: config.RebalanceTimeout,\n\t\t\t\t}, nil\n\t\t\t}\n\t\t}\n\t\treturn nil, err // err will be non-nil\n\t}\n}\n\n// coordinator establishes a connection to the coordinator for this consumer\n// group.\nfunc (cg *ConsumerGroup) coordinator() (coordinator, error) {\n\t// NOTE : could try to cache the coordinator to avoid the double connect\n\t// here. since consumer group balances happen infrequently and are\n\t// an expensive operation, we're not currently optimizing that case\n\t// in order to keep the code simpler.\n\tconn, err := cg.config.connect(cg.config.Dialer, cg.config.Brokers...)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer conn.Close()\n\n\tout, err := conn.findCoordinator(findCoordinatorRequestV0{\n\t\tCoordinatorKey: cg.config.ID,\n\t})\n\tif err == nil && out.ErrorCode != 0 {\n\t\terr = Error(out.ErrorCode)\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\taddress := net.JoinHostPort(out.Coordinator.Host, strconv.Itoa(int(out.Coordinator.Port)))\n\treturn cg.config.connect(cg.config.Dialer, address)\n}\n\n// joinGroup attempts to join the reader to the consumer group.\n// Returns GroupMemberAssignments is this Reader was selected as\n// the leader. Otherwise, GroupMemberAssignments will be nil.\n//\n// Possible kafka error codes returned:\n// * GroupLoadInProgress:\n// * GroupCoordinatorNotAvailable:\n// * NotCoordinatorForGroup:\n// * InconsistentGroupProtocol:\n// * InvalidSessionTimeout:\n// * GroupAuthorizationFailed:\nfunc (cg *ConsumerGroup) joinGroup(conn coordinator, memberID string) (string, int32, GroupMemberAssignments, error) {\n\trequest, err := cg.makeJoinGroupRequest(memberID)\n\tif err != nil {\n\t\treturn \"\", 0, nil, err\n\t}\n\n\tresponse, err := conn.joinGroup(request)\n\tif err == nil && response.ErrorCode != 0 {\n\t\terr = Error(response.ErrorCode)\n\t}\n\tif err != nil {\n\t\treturn \"\", 0, nil, err\n\t}\n\n\tmemberID = response.MemberID\n\tgenerationID := response.GenerationID\n\n\tcg.withLogger(func(l Logger) {\n\t\tl.Printf(\"joined group %s as member %s in generation %d\", cg.config.ID, memberID, generationID)\n\t})\n\n\tvar assignments GroupMemberAssignments\n\tif iAmLeader := response.MemberID == response.LeaderID; iAmLeader {\n\t\tv, err := cg.assignTopicPartitions(conn, response)\n\t\tif err != nil {\n\t\t\treturn memberID, 0, nil, err\n\t\t}\n\t\tassignments = v\n\n\t\tcg.withLogger(func(l Logger) {\n\t\t\tfor memberID, assignment := range assignments {\n\t\t\t\tfor topic, partitions := range assignment {\n\t\t\t\t\tl.Printf(\"assigned member/topic/partitions %v/%v/%v\", memberID, topic, partitions)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n\n\tcg.withLogger(func(l Logger) {\n\t\tl.Printf(\"joinGroup succeeded for response, %v. generationID=%v, memberID=%v\", cg.config.ID, response.GenerationID, response.MemberID)\n\t})\n\n\treturn memberID, generationID, assignments, nil\n}\n\n// makeJoinGroupRequestV1 handles the logic of constructing a joinGroup\n// request.\nfunc (cg *ConsumerGroup) makeJoinGroupRequest(memberID string) (joinGroupRequest, error) {\n\trequest := joinGroupRequest{\n\t\tGroupID: cg.config.ID,\n\t\tMemberID: memberID,\n\t\tSessionTimeout: int32(cg.config.SessionTimeout / time.Millisecond),\n\t\tRebalanceTimeout: int32(cg.config.RebalanceTimeout / time.Millisecond),\n\t\tProtocolType: defaultProtocolType,\n\t}\n\n\tfor _, balancer := range cg.config.GroupBalancers {\n\t\tuserData, err := balancer.UserData()\n\t\tif err != nil {\n\t\t\treturn joinGroupRequest{}, fmt.Errorf(\"unable to construct protocol metadata for member, %v: %w\", balancer.ProtocolName(), err)\n\t\t}\n\t\trequest.GroupProtocols = append(request.GroupProtocols, joinGroupRequestGroupProtocolV1{\n\t\t\tProtocolName: balancer.ProtocolName(),\n\t\t\tProtocolMetadata: groupMetadata{\n\t\t\t\tVersion: 1,\n\t\t\t\tTopics: cg.config.Topics,\n\t\t\t\tUserData: userData,\n\t\t\t}.bytes(),\n\t\t})\n\t}\n\n\treturn request, nil\n}\n\n// assignTopicPartitions uses the selected GroupBalancer to assign members to\n// their various partitions.\nfunc (cg *ConsumerGroup) assignTopicPartitions(conn coordinator, group joinGroupResponse) (GroupMemberAssignments, error) {\n\tcg.withLogger(func(l Logger) {\n\t\tl.Printf(\"selected as leader for group, %s\\n\", cg.config.ID)\n\t})\n\n\tbalancer, ok := findGroupBalancer(group.GroupProtocol, cg.config.GroupBalancers)\n\tif !ok {\n\t\t// NOTE : this shouldn't happen in practice...the broker should not\n\t\t// return successfully from joinGroup unless all members support\n\t\t// at least one common protocol.\n\t\treturn nil, fmt.Errorf(\"unable to find selected balancer, %v, for group, %v\", group.GroupProtocol, cg.config.ID)\n\t}\n\n\tmembers, err := cg.makeMemberProtocolMetadata(group.Members)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\ttopics := extractTopics(members)\n\tpartitions, err := conn.readPartitions(topics...)\n\n\t// it's not a failure if the topic doesn't exist yet. it results in no\n\t// assignments for the topic. this matches the behavior of the official\n\t// clients: java, python, and librdkafka.\n\t// a topic watcher can trigger a rebalance when the topic comes into being.\n\tif err != nil && !errors.Is(err, UnknownTopicOrPartition) {\n\t\treturn nil, err\n\t}\n\n\tcg.withLogger(func(l Logger) {\n\t\tl.Printf(\"using '%v' balancer to assign group, %v\", group.GroupProtocol, cg.config.ID)\n\t\tfor _, member := range members {\n\t\t\tl.Printf(\"found member: %v/%#v\", member.ID, member.UserData)\n\t\t}\n\t\tfor _, partition := range partitions {\n\t\t\tl.Printf(\"found topic/partition: %v/%v\", partition.Topic, partition.ID)\n\t\t}\n\t})\n\n\treturn balancer.AssignGroups(members, partitions), nil\n}\n\n// makeMemberProtocolMetadata maps encoded member metadata ([]byte) into []GroupMember.\nfunc (cg *ConsumerGroup) makeMemberProtocolMetadata(in []joinGroupResponseMember) ([]GroupMember, error) {\n\tmembers := make([]GroupMember, 0, len(in))\n\tfor _, item := range in {\n\t\tmetadata := groupMetadata{}\n\t\treader := bufio.NewReader(bytes.NewReader(item.MemberMetadata))\n\t\tif remain, err := (&metadata).readFrom(reader, len(item.MemberMetadata)); err != nil || remain != 0 {\n\t\t\treturn nil, fmt.Errorf(\"unable to read metadata for member, %v: %w\", item.MemberID, err)\n\t\t}\n\n\t\tmembers = append(members, GroupMember{\n\t\t\tID: item.MemberID,\n\t\t\tTopics: metadata.Topics,\n\t\t\tUserData: metadata.UserData,\n\t\t})\n\t}\n\treturn members, nil\n}\n\n// syncGroup completes the consumer group nextGeneration by accepting the\n// memberAssignments (if this Reader is the leader) and returning this\n// Readers subscriptions topic => partitions\n//\n// Possible kafka error codes returned:\n// * GroupCoordinatorNotAvailable:\n// * NotCoordinatorForGroup:\n// * IllegalGeneration:\n// * RebalanceInProgress:\n// * GroupAuthorizationFailed:\nfunc (cg *ConsumerGroup) syncGroup(conn coordinator, memberID string, generationID int32, memberAssignments GroupMemberAssignments) (map[string][]int32, error) {\n\trequest := cg.makeSyncGroupRequestV0(memberID, generationID, memberAssignments)\n\tresponse, err := conn.syncGroup(request)\n\tif err == nil && response.ErrorCode != 0 {\n\t\terr = Error(response.ErrorCode)\n\t}\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tassignments := groupAssignment{}\n\treader := bufio.NewReader(bytes.NewReader(response.MemberAssignments))\n\tif _, err := (&assignments).readFrom(reader, len(response.MemberAssignments)); err != nil {\n\t\treturn nil, err\n\t}\n\n\tif len(assignments.Topics) == 0 {\n\t\tcg.withLogger(func(l Logger) {\n\t\t\tl.Printf(\"received empty assignments for group, %v as member %s for generation %d\", cg.config.ID, memberID, generationID)\n\t\t})\n\t}\n\n\tcg.withLogger(func(l Logger) {\n\t\tl.Printf(\"sync group finished for group, %v\", cg.config.ID)\n\t})\n\n\treturn assignments.Topics, nil\n}\n\nfunc (cg *ConsumerGroup) makeSyncGroupRequestV0(memberID string, generationID int32, memberAssignments GroupMemberAssignments) syncGroupRequestV0 {\n\trequest := syncGroupRequestV0{\n\t\tGroupID: cg.config.ID,\n\t\tGenerationID: generationID,\n\t\tMemberID: memberID,\n\t}\n\n\tif memberAssignments != nil {\n\t\trequest.GroupAssignments = make([]syncGroupRequestGroupAssignmentV0, 0, 1)\n\n\t\tfor memberID, topics := range memberAssignments {\n\t\t\ttopics32 := make(map[string][]int32)\n\t\t\tfor topic, partitions := range topics {\n\t\t\t\tpartitions32 := make([]int32, len(partitions))\n\t\t\t\tfor i := range partitions {\n\t\t\t\t\tpartitions32[i] = int32(partitions[i])\n\t\t\t\t}\n\t\t\t\ttopics32[topic] = partitions32\n\t\t\t}\n\t\t\trequest.GroupAssignments = append(request.GroupAssignments, syncGroupRequestGroupAssignmentV0{\n\t\t\t\tMemberID: memberID,\n\t\t\t\tMemberAssignments: groupAssignment{\n\t\t\t\t\tVersion: 1,\n\t\t\t\t\tTopics: topics32,\n\t\t\t\t}.bytes(),\n\t\t\t})\n\t\t}\n\n\t\tcg.withLogger(func(logger Logger) {\n\t\t\tlogger.Printf(\"Syncing %d assignments for generation %d as member %s\", len(request.GroupAssignments), generationID, memberID)\n\t\t})\n\t}\n\n\treturn request\n}\n\nfunc (cg *ConsumerGroup) fetchOffsets(conn coordinator, subs map[string][]int32) (map[string]map[int]int64, error) {\n\treq := offsetFetchRequestV1{\n\t\tGroupID: cg.config.ID,\n\t\tTopics: make([]offsetFetchRequestV1Topic, 0, len(cg.config.Topics)),\n\t}\n\tfor _, topic := range cg.config.Topics {\n\t\treq.Topics = append(req.Topics, offsetFetchRequestV1Topic{\n\t\t\tTopic: topic,\n\t\t\tPartitions: subs[topic],\n\t\t})\n\t}\n\toffsets, err := conn.offsetFetch(req)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\toffsetsByTopic := make(map[string]map[int]int64)\n\tfor _, res := range offsets.Responses {\n\t\toffsetsByPartition := map[int]int64{}\n\t\toffsetsByTopic[res.Topic] = offsetsByPartition\n\t\tfor _, pr := range res.PartitionResponses {\n\t\t\tfor _, partition := range subs[res.Topic] {\n\t\t\t\tif partition == pr.Partition {\n\t\t\t\t\toffset := pr.Offset\n\t\t\t\t\tif offset < 0 {\n\t\t\t\t\t\toffset = cg.config.StartOffset\n\t\t\t\t\t}\n\t\t\t\t\toffsetsByPartition[int(partition)] = offset\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn offsetsByTopic, nil\n}\n\nfunc (cg *ConsumerGroup) makeAssignments(assignments map[string][]int32, offsets map[string]map[int]int64) map[string][]PartitionAssignment {\n\ttopicAssignments := make(map[string][]PartitionAssignment)\n\tfor _, topic := range cg.config.Topics {\n\t\ttopicPartitions := assignments[topic]\n\t\ttopicAssignments[topic] = make([]PartitionAssignment, 0, len(topicPartitions))\n\t\tfor _, partition := range topicPartitions {\n\t\t\tvar offset int64\n\t\t\tpartitionOffsets, ok := offsets[topic]\n\t\t\tif ok {\n\t\t\t\toffset, ok = partitionOffsets[int(partition)]\n\t\t\t}\n\t\t\tif !ok {\n\t\t\t\toffset = cg.config.StartOffset\n\t\t\t}\n\t\t\ttopicAssignments[topic] = append(topicAssignments[topic], PartitionAssignment{\n\t\t\t\tID: int(partition),\n\t\t\t\tOffset: offset,\n\t\t\t})\n\t\t}\n\t}\n\treturn topicAssignments\n}\n\nfunc (cg *ConsumerGroup) leaveGroup(memberID string) error {\n\t// don't attempt to leave the group if no memberID was ever assigned.\n\tif memberID == \"\" {\n\t\treturn nil\n\t}\n\n\tcg.withLogger(func(log Logger) {\n\t\tlog.Printf(\"Leaving group %s, member %s\", cg.config.ID, memberID)\n\t})\n\n\t// IMPORTANT : leaveGroup establishes its own connection to the coordinator\n\t// because it is often called after some other operation failed.\n\t// said failure could be the result of connection-level issues,\n\t// so we want to re-establish the connection to ensure that we\n\t// are able to process the cleanup step.\n\tcoordinator, err := cg.coordinator()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t_, err = coordinator.leaveGroup(leaveGroupRequestV0{\n\t\tGroupID: cg.config.ID,\n\t\tMemberID: memberID,\n\t})\n\tif err != nil {\n\t\tcg.withErrorLogger(func(log Logger) {\n\t\t\tlog.Printf(\"leave group failed for group, %v, and member, %v: %v\", cg.config.ID, memberID, err)\n\t\t})\n\t}\n\n\t_ = coordinator.Close()\n\n\treturn err\n}\n\nfunc (cg *ConsumerGroup) withLogger(do func(Logger)) {\n\tif cg.config.Logger != nil {\n\t\tdo(cg.config.Logger)\n\t}\n}\n\nfunc (cg *ConsumerGroup) withErrorLogger(do func(Logger)) {\n\tif cg.config.ErrorLogger != nil {\n\t\tdo(cg.config.ErrorLogger)\n\t} else {\n\t\tcg.withLogger(do)\n\t}\n}\n" }, { "path": "consumergroup_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n)\n\nvar _ coordinator = mockCoordinator{}\n\ntype mockCoordinator struct {\n\tcloseFunc func() error\n\tfindCoordinatorFunc func(findCoordinatorRequestV0) (findCoordinatorResponseV0, error)\n\tjoinGroupFunc func(joinGroupRequest) (joinGroupResponse, error)\n\tsyncGroupFunc func(syncGroupRequestV0) (syncGroupResponseV0, error)\n\tleaveGroupFunc func(leaveGroupRequestV0) (leaveGroupResponseV0, error)\n\theartbeatFunc func(heartbeatRequestV0) (heartbeatResponseV0, error)\n\toffsetFetchFunc func(offsetFetchRequestV1) (offsetFetchResponseV1, error)\n\toffsetCommitFunc func(offsetCommitRequestV2) (offsetCommitResponseV2, error)\n\treadPartitionsFunc func(...string) ([]Partition, error)\n}\n\nfunc (c mockCoordinator) Close() error {\n\tif c.closeFunc != nil {\n\t\treturn c.closeFunc()\n\t}\n\treturn nil\n}\n\nfunc (c mockCoordinator) findCoordinator(req findCoordinatorRequestV0) (findCoordinatorResponseV0, error) {\n\tif c.findCoordinatorFunc == nil {\n\t\treturn findCoordinatorResponseV0{}, errors.New(\"no findCoordinator behavior specified\")\n\t}\n\treturn c.findCoordinatorFunc(req)\n}\n\nfunc (c mockCoordinator) joinGroup(req joinGroupRequest) (joinGroupResponse, error) {\n\tif c.joinGroupFunc == nil {\n\t\treturn joinGroupResponse{}, errors.New(\"no joinGroup behavior specified\")\n\t}\n\treturn c.joinGroupFunc(req)\n}\n\nfunc (c mockCoordinator) syncGroup(req syncGroupRequestV0) (syncGroupResponseV0, error) {\n\tif c.syncGroupFunc == nil {\n\t\treturn syncGroupResponseV0{}, errors.New(\"no syncGroup behavior specified\")\n\t}\n\treturn c.syncGroupFunc(req)\n}\n\nfunc (c mockCoordinator) leaveGroup(req leaveGroupRequestV0) (leaveGroupResponseV0, error) {\n\tif c.leaveGroupFunc == nil {\n\t\treturn leaveGroupResponseV0{}, errors.New(\"no leaveGroup behavior specified\")\n\t}\n\treturn c.leaveGroupFunc(req)\n}\n\nfunc (c mockCoordinator) heartbeat(req heartbeatRequestV0) (heartbeatResponseV0, error) {\n\tif c.heartbeatFunc == nil {\n\t\treturn heartbeatResponseV0{}, errors.New(\"no heartbeat behavior specified\")\n\t}\n\treturn c.heartbeatFunc(req)\n}\n\nfunc (c mockCoordinator) offsetFetch(req offsetFetchRequestV1) (offsetFetchResponseV1, error) {\n\tif c.offsetFetchFunc == nil {\n\t\treturn offsetFetchResponseV1{}, errors.New(\"no offsetFetch behavior specified\")\n\t}\n\treturn c.offsetFetchFunc(req)\n}\n\nfunc (c mockCoordinator) offsetCommit(req offsetCommitRequestV2) (offsetCommitResponseV2, error) {\n\tif c.offsetCommitFunc == nil {\n\t\treturn offsetCommitResponseV2{}, errors.New(\"no offsetCommit behavior specified\")\n\t}\n\treturn c.offsetCommitFunc(req)\n}\n\nfunc (c mockCoordinator) readPartitions(topics ...string) ([]Partition, error) {\n\tif c.readPartitionsFunc == nil {\n\t\treturn nil, errors.New(\"no Readpartitions behavior specified\")\n\t}\n\treturn c.readPartitionsFunc(topics...)\n}\n\nfunc TestValidateConsumerGroupConfig(t *testing.T) {\n\ttests := []struct {\n\t\tconfig ConsumerGroupConfig\n\t\terrorOccured bool\n\t}{\n\t\t{config: ConsumerGroupConfig{}, errorOccured: true},\n\t\t{config: ConsumerGroupConfig{Brokers: []string{\"broker1\"}, HeartbeatInterval: 2}, errorOccured: true},\n\t\t{config: ConsumerGroupConfig{Brokers: []string{\"broker1\"}, Topics: []string{\"t1\"}}, errorOccured: true},\n\t\t{config: ConsumerGroupConfig{Brokers: []string{\"broker1\"}, Topics: []string{\"t1\"}, ID: \"group1\", HeartbeatInterval: -1}, errorOccured: true},\n\t\t{config: ConsumerGroupConfig{Brokers: []string{\"broker1\"}, Topics: []string{\"t1\"}, ID: \"group1\", SessionTimeout: -1}, errorOccured: true},\n\t\t{config: ConsumerGroupConfig{Brokers: []string{\"broker1\"}, Topics: []string{\"t1\"}, ID: \"group1\", HeartbeatInterval: 2, SessionTimeout: -1}, errorOccured: true},\n\t\t{config: ConsumerGroupConfig{Brokers: []string{\"broker1\"}, Topics: []string{\"t1\"}, ID: \"group1\", HeartbeatInterval: 2, SessionTimeout: 2, RebalanceTimeout: -2}, errorOccured: true},\n\t\t{config: ConsumerGroupConfig{Brokers: []string{\"broker1\"}, Topics: []string{\"t1\"}, ID: \"group1\", HeartbeatInterval: 2, SessionTimeout: 2, RebalanceTimeout: 2, RetentionTime: -1}, errorOccured: true},\n\t\t{config: ConsumerGroupConfig{Brokers: []string{\"broker1\"}, Topics: []string{\"t1\"}, ID: \"group1\", HeartbeatInterval: 2, SessionTimeout: 2, RebalanceTimeout: 2, RetentionTime: 1, StartOffset: 123}, errorOccured: true},\n\t\t{config: ConsumerGroupConfig{Brokers: []string{\"broker1\"}, Topics: []string{\"t1\"}, ID: \"group1\", HeartbeatInterval: 2, SessionTimeout: 2, RebalanceTimeout: 2, RetentionTime: 1, PartitionWatchInterval: -1}, errorOccured: true},\n\t\t{config: ConsumerGroupConfig{Brokers: []string{\"broker1\"}, Topics: []string{\"t1\"}, ID: \"group1\", HeartbeatInterval: 2, SessionTimeout: 2, RebalanceTimeout: 2, RetentionTime: 1, PartitionWatchInterval: 1, JoinGroupBackoff: -1}, errorOccured: true},\n\t\t{config: ConsumerGroupConfig{Brokers: []string{\"broker1\"}, Topics: []string{\"t1\"}, ID: \"group1\", HeartbeatInterval: 2, SessionTimeout: 2, RebalanceTimeout: 2, RetentionTime: 1, PartitionWatchInterval: 1, JoinGroupBackoff: 1}, errorOccured: false},\n\t}\n\tfor _, test := range tests {\n\t\terr := test.config.Validate()\n\t\tif test.errorOccured && err == nil {\n\t\t\tt.Error(\"expected an error\", test.config)\n\t\t}\n\t\tif !test.errorOccured && err != nil {\n\t\t\tt.Error(\"expected no error, got\", err, test.config)\n\t\t}\n\t}\n}\n\nfunc TestReaderAssignTopicPartitions(t *testing.T) {\n\tconn := &mockCoordinator{\n\t\treadPartitionsFunc: func(...string) ([]Partition, error) {\n\t\t\treturn []Partition{\n\t\t\t\t{\n\t\t\t\t\tTopic: \"topic-1\",\n\t\t\t\t\tID: 0,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tTopic: \"topic-1\",\n\t\t\t\t\tID: 1,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tTopic: \"topic-1\",\n\t\t\t\t\tID: 2,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tTopic: \"topic-2\",\n\t\t\t\t\tID: 0,\n\t\t\t\t},\n\t\t\t}, nil\n\t\t},\n\t}\n\n\tnewJoinGroupResponse := func(topicsByMemberID map[string][]string) func(v apiVersion) joinGroupResponse {\n\t\treturn func(v apiVersion) joinGroupResponse {\n\t\t\tresp := joinGroupResponse{\n\t\t\t\tv: v,\n\t\t\t\tGroupProtocol: RoundRobinGroupBalancer{}.ProtocolName(),\n\t\t\t}\n\n\t\t\tfor memberID, topics := range topicsByMemberID {\n\t\t\t\tresp.Members = append(resp.Members, joinGroupResponseMember{\n\t\t\t\t\tMemberID: memberID,\n\t\t\t\t\tMemberMetadata: groupMetadata{\n\t\t\t\t\t\tTopics: topics,\n\t\t\t\t\t}.bytes(),\n\t\t\t\t})\n\t\t\t}\n\n\t\t\treturn resp\n\t\t}\n\t}\n\n\ttestCases := map[string]struct {\n\t\tMembersFunc func(v apiVersion) joinGroupResponse\n\t\tAssignments GroupMemberAssignments\n\t}{\n\t\t\"nil\": {\n\t\t\tMembersFunc: newJoinGroupResponse(nil),\n\t\t\tAssignments: GroupMemberAssignments{},\n\t\t},\n\t\t\"one member, one topic\": {\n\t\t\tMembersFunc: newJoinGroupResponse(map[string][]string{\n\t\t\t\t\"member-1\": {\"topic-1\"},\n\t\t\t}),\n\t\t\tAssignments: GroupMemberAssignments{\n\t\t\t\t\"member-1\": map[string][]int{\n\t\t\t\t\t\"topic-1\": {0, 1, 2},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t\"one member, two topics\": {\n\t\t\tMembersFunc: newJoinGroupResponse(map[string][]string{\n\t\t\t\t\"member-1\": {\"topic-1\", \"topic-2\"},\n\t\t\t}),\n\t\t\tAssignments: GroupMemberAssignments{\n\t\t\t\t\"member-1\": map[string][]int{\n\t\t\t\t\t\"topic-1\": {0, 1, 2},\n\t\t\t\t\t\"topic-2\": {0},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t\"two members, one topic\": {\n\t\t\tMembersFunc: newJoinGroupResponse(map[string][]string{\n\t\t\t\t\"member-1\": {\"topic-1\"},\n\t\t\t\t\"member-2\": {\"topic-1\"},\n\t\t\t}),\n\t\t\tAssignments: GroupMemberAssignments{\n\t\t\t\t\"member-1\": map[string][]int{\n\t\t\t\t\t\"topic-1\": {0, 2},\n\t\t\t\t},\n\t\t\t\t\"member-2\": map[string][]int{\n\t\t\t\t\t\"topic-1\": {1},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t\"two members, two unshared topics\": {\n\t\t\tMembersFunc: newJoinGroupResponse(map[string][]string{\n\t\t\t\t\"member-1\": {\"topic-1\"},\n\t\t\t\t\"member-2\": {\"topic-2\"},\n\t\t\t}),\n\t\t\tAssignments: GroupMemberAssignments{\n\t\t\t\t\"member-1\": map[string][]int{\n\t\t\t\t\t\"topic-1\": {0, 1, 2},\n\t\t\t\t},\n\t\t\t\t\"member-2\": map[string][]int{\n\t\t\t\t\t\"topic-2\": {0},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tsupportedVersions := []apiVersion{v1, v2} // joinGroup versions\n\tfor label, tc := range testCases {\n\t\tfor _, v := range supportedVersions {\n\t\t\tt.Run(label+\"_v\"+strconv.Itoa(int(v)), func(t *testing.T) {\n\t\t\t\tcg := ConsumerGroup{}\n\t\t\t\tcg.config.GroupBalancers = []GroupBalancer{\n\t\t\t\t\tRangeGroupBalancer{},\n\t\t\t\t\tRoundRobinGroupBalancer{},\n\t\t\t\t}\n\t\t\t\tassignments, err := cg.assignTopicPartitions(conn, tc.MembersFunc(v))\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"bad err: %v\", err)\n\t\t\t\t}\n\t\t\t\tif !reflect.DeepEqual(tc.Assignments, assignments) {\n\t\t\t\t\tt.Errorf(\"expected %v; got %v\", tc.Assignments, assignments)\n\t\t\t\t}\n\t\t\t})\n\t\t}\n\t}\n}\n\nfunc TestConsumerGroup(t *testing.T) {\n\ttests := []struct {\n\t\tscenario string\n\t\tfunction func(*testing.T, context.Context, *ConsumerGroup)\n\t}{\n\t\t{\n\t\t\tscenario: \"Next returns generations\",\n\t\t\tfunction: func(t *testing.T, ctx context.Context, cg *ConsumerGroup) {\n\t\t\t\tgen1, err := cg.Next(ctx)\n\t\t\t\tif gen1 == nil {\n\t\t\t\t\tt.Fatalf(\"expected generation 1 not to be nil\")\n\t\t\t\t}\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"expected no error, but got %+v\", err)\n\t\t\t\t}\n\t\t\t\t// returning from this function should cause the generation to\n\t\t\t\t// exit.\n\t\t\t\tgen1.Start(func(context.Context) {})\n\n\t\t\t\t// if this fails due to context timeout, it would indicate that\n\t\t\t\t// the\n\t\t\t\tgen2, err := cg.Next(ctx)\n\t\t\t\tif gen2 == nil {\n\t\t\t\t\tt.Fatalf(\"expected generation 2 not to be nil\")\n\t\t\t\t}\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Fatalf(\"expected no error, but got %+v\", err)\n\t\t\t\t}\n\n\t\t\t\tif gen1.ID == gen2.ID {\n\t\t\t\t\tt.Errorf(\"generation ID should have changed, but it stayed as %d\", gen1.ID)\n\t\t\t\t}\n\t\t\t\tif gen1.GroupID != gen2.GroupID {\n\t\t\t\t\tt.Errorf(\"mismatched group ID between generations: %s and %s\", gen1.GroupID, gen2.GroupID)\n\t\t\t\t}\n\t\t\t\tif gen1.MemberID != gen2.MemberID {\n\t\t\t\t\tt.Errorf(\"mismatched member ID between generations: %s and %s\", gen1.MemberID, gen2.MemberID)\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"Next returns ctx.Err() on canceled context\",\n\t\t\tfunction: func(t *testing.T, _ context.Context, cg *ConsumerGroup) {\n\t\t\t\tctx, cancel := context.WithCancel(context.Background())\n\t\t\t\tcancel()\n\n\t\t\t\tgen, err := cg.Next(ctx)\n\t\t\t\tif gen != nil {\n\t\t\t\t\tt.Errorf(\"expected generation to be nil\")\n\t\t\t\t}\n\t\t\t\tif !errors.Is(err, context.Canceled) {\n\t\t\t\t\tt.Errorf(\"expected context.Canceled, but got %+v\", err)\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"Next returns ErrGroupClosed on closed group\",\n\t\t\tfunction: func(t *testing.T, ctx context.Context, cg *ConsumerGroup) {\n\t\t\t\tif err := cg.Close(); err != nil {\n\t\t\t\t\tt.Fatal(err)\n\t\t\t\t}\n\t\t\t\tgen, err := cg.Next(ctx)\n\t\t\t\tif gen != nil {\n\t\t\t\t\tt.Errorf(\"expected generation to be nil\")\n\t\t\t\t}\n\t\t\t\tif !errors.Is(err, ErrGroupClosed) {\n\t\t\t\t\tt.Errorf(\"expected ErrGroupClosed, but got %+v\", err)\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t}\n\n\ttopic := makeTopic()\n\tcreateTopic(t, topic, 1)\n\tdefer deleteTopic(t, topic)\n\n\tfor _, test := range tests {\n\t\tt.Run(test.scenario, func(t *testing.T) {\n\t\t\tgroup, err := NewConsumerGroup(ConsumerGroupConfig{\n\t\t\t\tID: makeGroupID(),\n\t\t\t\tTopics: []string{topic},\n\t\t\t\tBrokers: []string{\"localhost:9092\"},\n\t\t\t\tHeartbeatInterval: 2 * time.Second,\n\t\t\t\tRebalanceTimeout: 2 * time.Second,\n\t\t\t\tRetentionTime: time.Hour,\n\t\t\t\tLogger: &testKafkaLogger{T: t},\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tdefer group.Close()\n\n\t\t\tctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)\n\t\t\tdefer cancel()\n\n\t\t\ttest.function(t, ctx, group)\n\t\t})\n\t}\n}\n\nfunc TestConsumerGroupErrors(t *testing.T) {\n\tvar left []string\n\tvar lock sync.Mutex\n\tmc := mockCoordinator{\n\t\tleaveGroupFunc: func(req leaveGroupRequestV0) (leaveGroupResponseV0, error) {\n\t\t\tlock.Lock()\n\t\t\tleft = append(left, req.MemberID)\n\t\t\tlock.Unlock()\n\t\t\treturn leaveGroupResponseV0{}, nil\n\t\t},\n\t}\n\tassertLeftGroup := func(t *testing.T, memberID string) {\n\t\tlock.Lock()\n\t\tif !reflect.DeepEqual(left, []string{memberID}) {\n\t\t\tt.Errorf(\"expected abc to have left group once, members left: %v\", left)\n\t\t}\n\t\tleft = left[0:0]\n\t\tlock.Unlock()\n\t}\n\n\t// NOTE : the mocked behavior is accumulated across the tests, so they are\n\t// \t\t NOT run in parallel. this simplifies test setup so that each test\n\t// \t \t can specify only the error behavior required and leverage setup\n\t// from previous steps.\n\ttests := []struct {\n\t\tscenario string\n\t\tprepare func(*mockCoordinator)\n\t\tfunction func(*testing.T, context.Context, *ConsumerGroup)\n\t}{\n\t\t{\n\t\t\tscenario: \"fails to find coordinator (general error)\",\n\t\t\tprepare: func(mc *mockCoordinator) {\n\t\t\t\tmc.findCoordinatorFunc = func(findCoordinatorRequestV0) (findCoordinatorResponseV0, error) {\n\t\t\t\t\treturn findCoordinatorResponseV0{}, errors.New(\"dial error\")\n\t\t\t\t}\n\t\t\t},\n\t\t\tfunction: func(t *testing.T, ctx context.Context, group *ConsumerGroup) {\n\t\t\t\tgen, err := group.Next(ctx)\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Errorf(\"expected an error\")\n\t\t\t\t} else if err.Error() != \"dial error\" {\n\t\t\t\t\tt.Errorf(\"got wrong error: %+v\", err)\n\t\t\t\t}\n\t\t\t\tif gen != nil {\n\t\t\t\t\tt.Error(\"expected a nil consumer group generation\")\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"fails to find coordinator (error code in response)\",\n\t\t\tprepare: func(mc *mockCoordinator) {\n\t\t\t\tmc.findCoordinatorFunc = func(findCoordinatorRequestV0) (findCoordinatorResponseV0, error) {\n\t\t\t\t\treturn findCoordinatorResponseV0{\n\t\t\t\t\t\tErrorCode: int16(NotCoordinatorForGroup),\n\t\t\t\t\t}, nil\n\t\t\t\t}\n\t\t\t},\n\t\t\tfunction: func(t *testing.T, ctx context.Context, group *ConsumerGroup) {\n\t\t\t\tgen, err := group.Next(ctx)\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Errorf(\"expected an error\")\n\t\t\t\t} else if !errors.Is(err, NotCoordinatorForGroup) {\n\t\t\t\t\tt.Errorf(\"got wrong error: %+v\", err)\n\t\t\t\t}\n\t\t\t\tif gen != nil {\n\t\t\t\t\tt.Error(\"expected a nil consumer group generation\")\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"fails to join group (general error)\",\n\t\t\tprepare: func(mc *mockCoordinator) {\n\t\t\t\tmc.findCoordinatorFunc = func(findCoordinatorRequestV0) (findCoordinatorResponseV0, error) {\n\t\t\t\t\treturn findCoordinatorResponseV0{\n\t\t\t\t\t\tCoordinator: findCoordinatorResponseCoordinatorV0{\n\t\t\t\t\t\t\tNodeID: 1,\n\t\t\t\t\t\t\tHost: \"foo.bar.com\",\n\t\t\t\t\t\t\tPort: 12345,\n\t\t\t\t\t\t},\n\t\t\t\t\t}, nil\n\t\t\t\t}\n\t\t\t\tmc.joinGroupFunc = func(joinGroupRequest) (joinGroupResponse, error) {\n\t\t\t\t\treturn joinGroupResponse{}, errors.New(\"join group failed\")\n\t\t\t\t}\n\t\t\t\t// NOTE : no stub for leaving the group b/c the member never joined.\n\t\t\t},\n\t\t\tfunction: func(t *testing.T, ctx context.Context, group *ConsumerGroup) {\n\t\t\t\tgen, err := group.Next(ctx)\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Errorf(\"expected an error\")\n\t\t\t\t} else if err.Error() != \"join group failed\" {\n\t\t\t\t\tt.Errorf(\"got wrong error: %+v\", err)\n\t\t\t\t}\n\t\t\t\tif gen != nil {\n\t\t\t\t\tt.Error(\"expected a nil consumer group generation\")\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"fails to join group (error code)\",\n\t\t\tprepare: func(mc *mockCoordinator) {\n\t\t\t\tmc.findCoordinatorFunc = func(findCoordinatorRequestV0) (findCoordinatorResponseV0, error) {\n\t\t\t\t\treturn findCoordinatorResponseV0{\n\t\t\t\t\t\tCoordinator: findCoordinatorResponseCoordinatorV0{\n\t\t\t\t\t\t\tNodeID: 1,\n\t\t\t\t\t\t\tHost: \"foo.bar.com\",\n\t\t\t\t\t\t\tPort: 12345,\n\t\t\t\t\t\t},\n\t\t\t\t\t}, nil\n\t\t\t\t}\n\t\t\t\tmc.joinGroupFunc = func(joinGroupRequest) (joinGroupResponse, error) {\n\t\t\t\t\treturn joinGroupResponse{\n\t\t\t\t\t\tErrorCode: int16(InvalidTopic),\n\t\t\t\t\t}, nil\n\t\t\t\t}\n\t\t\t\t// NOTE : no stub for leaving the group b/c the member never joined.\n\t\t\t},\n\t\t\tfunction: func(t *testing.T, ctx context.Context, group *ConsumerGroup) {\n\t\t\t\tgen, err := group.Next(ctx)\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Errorf(\"expected an error\")\n\t\t\t\t} else if !errors.Is(err, InvalidTopic) {\n\t\t\t\t\tt.Errorf(\"got wrong error: %+v\", err)\n\t\t\t\t}\n\t\t\t\tif gen != nil {\n\t\t\t\t\tt.Error(\"expected a nil consumer group generation\")\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"fails to join group (leader, unsupported protocol)\",\n\t\t\tprepare: func(mc *mockCoordinator) {\n\t\t\t\tmc.joinGroupFunc = func(joinGroupRequest) (joinGroupResponse, error) {\n\t\t\t\t\treturn joinGroupResponse{\n\t\t\t\t\t\tGenerationID: 12345,\n\t\t\t\t\t\tGroupProtocol: \"foo\",\n\t\t\t\t\t\tLeaderID: \"abc\",\n\t\t\t\t\t\tMemberID: \"abc\",\n\t\t\t\t\t}, nil\n\t\t\t\t}\n\t\t\t},\n\t\t\tfunction: func(t *testing.T, ctx context.Context, group *ConsumerGroup) {\n\t\t\t\tgen, err := group.Next(ctx)\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Errorf(\"expected an error\")\n\t\t\t\t} else if !strings.HasPrefix(err.Error(), \"unable to find selected balancer\") {\n\t\t\t\t\tt.Errorf(\"got wrong error: %+v\", err)\n\t\t\t\t}\n\t\t\t\tif gen != nil {\n\t\t\t\t\tt.Error(\"expected a nil consumer group generation\")\n\t\t\t\t}\n\t\t\t\tassertLeftGroup(t, \"abc\")\n\t\t\t},\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"fails to sync group (general error)\",\n\t\t\tprepare: func(mc *mockCoordinator) {\n\t\t\t\tmc.joinGroupFunc = func(joinGroupRequest) (joinGroupResponse, error) {\n\t\t\t\t\treturn joinGroupResponse{\n\t\t\t\t\t\tGenerationID: 12345,\n\t\t\t\t\t\tGroupProtocol: \"range\",\n\t\t\t\t\t\tLeaderID: \"abc\",\n\t\t\t\t\t\tMemberID: \"abc\",\n\t\t\t\t\t}, nil\n\t\t\t\t}\n\t\t\t\tmc.readPartitionsFunc = func(...string) ([]Partition, error) {\n\t\t\t\t\treturn []Partition{}, nil\n\t\t\t\t}\n\t\t\t\tmc.syncGroupFunc = func(syncGroupRequestV0) (syncGroupResponseV0, error) {\n\t\t\t\t\treturn syncGroupResponseV0{}, errors.New(\"sync group failed\")\n\t\t\t\t}\n\t\t\t},\n\t\t\tfunction: func(t *testing.T, ctx context.Context, group *ConsumerGroup) {\n\t\t\t\tgen, err := group.Next(ctx)\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Errorf(\"expected an error\")\n\t\t\t\t} else if err.Error() != \"sync group failed\" {\n\t\t\t\t\tt.Errorf(\"got wrong error: %+v\", err)\n\t\t\t\t}\n\t\t\t\tif gen != nil {\n\t\t\t\t\tt.Error(\"expected a nil consumer group generation\")\n\t\t\t\t}\n\t\t\t\tassertLeftGroup(t, \"abc\")\n\t\t\t},\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"fails to sync group (error code)\",\n\t\t\tprepare: func(mc *mockCoordinator) {\n\t\t\t\tmc.syncGroupFunc = func(syncGroupRequestV0) (syncGroupResponseV0, error) {\n\t\t\t\t\treturn syncGroupResponseV0{\n\t\t\t\t\t\tErrorCode: int16(InvalidTopic),\n\t\t\t\t\t}, nil\n\t\t\t\t}\n\t\t\t},\n\t\t\tfunction: func(t *testing.T, ctx context.Context, group *ConsumerGroup) {\n\t\t\t\tgen, err := group.Next(ctx)\n\t\t\t\tif err == nil {\n\t\t\t\t\tt.Errorf(\"expected an error\")\n\t\t\t\t} else if !errors.Is(err, InvalidTopic) {\n\t\t\t\t\tt.Errorf(\"got wrong error: %+v\", err)\n\t\t\t\t}\n\t\t\t\tif gen != nil {\n\t\t\t\t\tt.Error(\"expected a nil consumer group generation\")\n\t\t\t\t}\n\t\t\t\tassertLeftGroup(t, \"abc\")\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.scenario, func(t *testing.T) {\n\n\t\t\ttt.prepare(&mc)\n\n\t\t\tgroup, err := NewConsumerGroup(ConsumerGroupConfig{\n\t\t\t\tID: makeGroupID(),\n\t\t\t\tTopics: []string{\"test\"},\n\t\t\t\tBrokers: []string{\"no-such-broker\"}, // should not attempt to actually dial anything\n\t\t\t\tHeartbeatInterval: 2 * time.Second,\n\t\t\t\tRebalanceTimeout: time.Second,\n\t\t\t\tJoinGroupBackoff: time.Second,\n\t\t\t\tRetentionTime: time.Hour,\n\t\t\t\tconnect: func(*Dialer, ...string) (coordinator, error) {\n\t\t\t\t\treturn mc, nil\n\t\t\t\t},\n\t\t\t\tLogger: &testKafkaLogger{T: t},\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\t// these tests should all execute fairly quickly since they're\n\t\t\t// mocking the coordinator.\n\t\t\tctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)\n\t\t\tdefer cancel()\n\n\t\t\ttt.function(t, ctx, group)\n\n\t\t\tif err := group.Close(); err != nil {\n\t\t\t\tt.Errorf(\"error on close: %+v\", err)\n\t\t\t}\n\t\t})\n\t}\n}\n\n// todo : test for multi-topic?\n\nfunc TestGenerationExitsOnPartitionChange(t *testing.T) {\n\tvar count int\n\tpartitions := [][]Partition{\n\t\t{\n\t\t\tPartition{\n\t\t\t\tTopic: \"topic-1\",\n\t\t\t\tID: 0,\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tPartition{\n\t\t\t\tTopic: \"topic-1\",\n\t\t\t\tID: 0,\n\t\t\t},\n\t\t\t{\n\t\t\t\tTopic: \"topic-1\",\n\t\t\t\tID: 1,\n\t\t\t},\n\t\t},\n\t}\n\n\tconn := mockCoordinator{\n\t\treadPartitionsFunc: func(...string) ([]Partition, error) {\n\t\t\tp := partitions[count]\n\t\t\t// cap the count at len(partitions) -1 so ReadPartitions doesn't even go out of bounds\n\t\t\t// and long running tests don't fail\n\t\t\tif count < len(partitions) {\n\t\t\t\tcount++\n\t\t\t}\n\t\t\treturn p, nil\n\t\t},\n\t}\n\n\t// Sadly this test is time based, so at the end will be seeing if the runGroup run to completion within the\n\t// allotted time. The allotted time is 4x the PartitionWatchInterval.\n\tnow := time.Now()\n\twatchTime := 500 * time.Millisecond\n\n\tgen := Generation{\n\t\tconn: conn,\n\t\tdone: make(chan struct{}),\n\t\tjoined: make(chan struct{}),\n\t\tlog: func(func(Logger)) {},\n\t\tlogError: func(func(Logger)) {},\n\t}\n\n\tdone := make(chan struct{})\n\tgo func() {\n\t\tgen.partitionWatcher(watchTime, \"topic-1\")\n\t\tclose(done)\n\t}()\n\n\tselect {\n\tcase <-time.After(5 * time.Second):\n\t\tt.Fatal(\"timed out waiting for partition watcher to exit\")\n\tcase <-done:\n\t\tif time.Since(now).Seconds() > watchTime.Seconds()*4 {\n\t\t\tt.Error(\"partitionWatcher didn't see update\")\n\t\t}\n\t}\n}\n\nfunc TestGenerationStartsFunctionAfterClosed(t *testing.T) {\n\tgen := Generation{\n\t\tconn: &mockCoordinator{},\n\t\tdone: make(chan struct{}),\n\t\tjoined: make(chan struct{}),\n\t\tlog: func(func(Logger)) {},\n\t\tlogError: func(func(Logger)) {},\n\t}\n\n\tgen.close()\n\n\tch := make(chan error)\n\tgen.Start(func(ctx context.Context) {\n\t\t<-ctx.Done()\n\t\tch <- ctx.Err()\n\t})\n\n\tselect {\n\tcase <-time.After(time.Second):\n\t\tt.Fatal(\"timed out waiting for func to run\")\n\tcase err := <-ch:\n\t\tif !errors.Is(err, ErrGenerationEnded) {\n\t\t\tt.Fatalf(\"expected %v but got %v\", ErrGenerationEnded, err)\n\t\t}\n\t}\n}\n" }, { "path": "crc32.go", "content": "package kafka\n\nimport (\n\t\"encoding/binary\"\n\t\"hash/crc32\"\n)\n\ntype crc32Writer struct {\n\ttable *crc32.Table\n\tbuffer [8]byte\n\tcrc32 uint32\n}\n\nfunc (w *crc32Writer) update(b []byte) {\n\tw.crc32 = crc32.Update(w.crc32, w.table, b)\n}\n\nfunc (w *crc32Writer) writeInt8(i int8) {\n\tw.buffer[0] = byte(i)\n\tw.update(w.buffer[:1])\n}\n\nfunc (w *crc32Writer) writeInt16(i int16) {\n\tbinary.BigEndian.PutUint16(w.buffer[:2], uint16(i))\n\tw.update(w.buffer[:2])\n}\n\nfunc (w *crc32Writer) writeInt32(i int32) {\n\tbinary.BigEndian.PutUint32(w.buffer[:4], uint32(i))\n\tw.update(w.buffer[:4])\n}\n\nfunc (w *crc32Writer) writeInt64(i int64) {\n\tbinary.BigEndian.PutUint64(w.buffer[:8], uint64(i))\n\tw.update(w.buffer[:8])\n}\n\nfunc (w *crc32Writer) writeBytes(b []byte) {\n\tn := len(b)\n\tif b == nil {\n\t\tn = -1\n\t}\n\tw.writeInt32(int32(n))\n\tw.update(b)\n}\n\nfunc (w *crc32Writer) Write(b []byte) (int, error) {\n\tw.update(b)\n\treturn len(b), nil\n}\n\nfunc (w *crc32Writer) WriteString(s string) (int, error) {\n\tw.update([]byte(s))\n\treturn len(s), nil\n}\n" }, { "path": "crc32_test.go", "content": "package kafka\n\nimport (\n\t\"bytes\"\n\t\"hash/crc32\"\n\t\"testing\"\n)\n\nfunc TestMessageCRC32(t *testing.T) {\n\tm := message{\n\t\tMagicByte: 1,\n\t\tTimestamp: 42,\n\t\tKey: nil,\n\t\tValue: []byte(\"Hello World!\"),\n\t}\n\n\tb := &bytes.Buffer{}\n\tw := &writeBuffer{w: b}\n\tw.write(m)\n\n\th := crc32.New(crc32.IEEETable)\n\th.Write(b.Bytes()[4:])\n\n\tsum1 := h.Sum32()\n\tsum2 := uint32(m.crc32(&crc32Writer{table: crc32.IEEETable}))\n\n\tif sum1 != sum2 {\n\t\tt.Error(\"bad CRC32:\")\n\t\tt.Logf(\"expected: %d\", sum1)\n\t\tt.Logf(\"found: %d\", sum2)\n\t}\n}\n" }, { "path": "createacls.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/createacls\"\n)\n\n// CreateACLsRequest represents a request sent to a kafka broker to add\n// new ACLs.\ntype CreateACLsRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// List of ACL to create.\n\tACLs []ACLEntry\n}\n\n// CreateACLsResponse represents a response from a kafka broker to an ACL\n// creation request.\ntype CreateACLsResponse struct {\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// List of errors that occurred while attempting to create\n\t// the ACLs.\n\t//\n\t// The errors contain the kafka error code. Programs may use the standard\n\t// errors.Is function to test the error against kafka error codes.\n\tErrors []error\n}\n\ntype ACLPermissionType int8\n\nconst (\n\tACLPermissionTypeUnknown ACLPermissionType = 0\n\tACLPermissionTypeAny ACLPermissionType = 1\n\tACLPermissionTypeDeny ACLPermissionType = 2\n\tACLPermissionTypeAllow ACLPermissionType = 3\n)\n\nfunc (apt ACLPermissionType) String() string {\n\tmapping := map[ACLPermissionType]string{\n\t\tACLPermissionTypeUnknown: \"Unknown\",\n\t\tACLPermissionTypeAny: \"Any\",\n\t\tACLPermissionTypeDeny: \"Deny\",\n\t\tACLPermissionTypeAllow: \"Allow\",\n\t}\n\ts, ok := mapping[apt]\n\tif !ok {\n\t\ts = mapping[ACLPermissionTypeUnknown]\n\t}\n\treturn s\n}\n\n// MarshalText transforms an ACLPermissionType into its string representation.\nfunc (apt ACLPermissionType) MarshalText() ([]byte, error) {\n\treturn []byte(apt.String()), nil\n}\n\n// UnmarshalText takes a string representation of the resource type and converts it to an ACLPermissionType.\nfunc (apt *ACLPermissionType) UnmarshalText(text []byte) error {\n\tnormalized := strings.ToLower(string(text))\n\tmapping := map[string]ACLPermissionType{\n\t\t\"unknown\": ACLPermissionTypeUnknown,\n\t\t\"any\": ACLPermissionTypeAny,\n\t\t\"deny\": ACLPermissionTypeDeny,\n\t\t\"allow\": ACLPermissionTypeAllow,\n\t}\n\tparsed, ok := mapping[normalized]\n\tif !ok {\n\t\t*apt = ACLPermissionTypeUnknown\n\t\treturn fmt.Errorf(\"cannot parse %s as an ACLPermissionType\", normalized)\n\t}\n\t*apt = parsed\n\treturn nil\n}\n\ntype ACLOperationType int8\n\nconst (\n\tACLOperationTypeUnknown ACLOperationType = 0\n\tACLOperationTypeAny ACLOperationType = 1\n\tACLOperationTypeAll ACLOperationType = 2\n\tACLOperationTypeRead ACLOperationType = 3\n\tACLOperationTypeWrite ACLOperationType = 4\n\tACLOperationTypeCreate ACLOperationType = 5\n\tACLOperationTypeDelete ACLOperationType = 6\n\tACLOperationTypeAlter ACLOperationType = 7\n\tACLOperationTypeDescribe ACLOperationType = 8\n\tACLOperationTypeClusterAction ACLOperationType = 9\n\tACLOperationTypeDescribeConfigs ACLOperationType = 10\n\tACLOperationTypeAlterConfigs ACLOperationType = 11\n\tACLOperationTypeIdempotentWrite ACLOperationType = 12\n)\n\nfunc (aot ACLOperationType) String() string {\n\tmapping := map[ACLOperationType]string{\n\t\tACLOperationTypeUnknown: \"Unknown\",\n\t\tACLOperationTypeAny: \"Any\",\n\t\tACLOperationTypeAll: \"All\",\n\t\tACLOperationTypeRead: \"Read\",\n\t\tACLOperationTypeWrite: \"Write\",\n\t\tACLOperationTypeCreate: \"Create\",\n\t\tACLOperationTypeDelete: \"Delete\",\n\t\tACLOperationTypeAlter: \"Alter\",\n\t\tACLOperationTypeDescribe: \"Describe\",\n\t\tACLOperationTypeClusterAction: \"ClusterAction\",\n\t\tACLOperationTypeDescribeConfigs: \"DescribeConfigs\",\n\t\tACLOperationTypeAlterConfigs: \"AlterConfigs\",\n\t\tACLOperationTypeIdempotentWrite: \"IdempotentWrite\",\n\t}\n\ts, ok := mapping[aot]\n\tif !ok {\n\t\ts = mapping[ACLOperationTypeUnknown]\n\t}\n\treturn s\n}\n\n// MarshalText transforms an ACLOperationType into its string representation.\nfunc (aot ACLOperationType) MarshalText() ([]byte, error) {\n\treturn []byte(aot.String()), nil\n}\n\n// UnmarshalText takes a string representation of the resource type and converts it to an ACLPermissionType.\nfunc (aot *ACLOperationType) UnmarshalText(text []byte) error {\n\tnormalized := strings.ToLower(string(text))\n\tmapping := map[string]ACLOperationType{\n\t\t\"unknown\": ACLOperationTypeUnknown,\n\t\t\"any\": ACLOperationTypeAny,\n\t\t\"all\": ACLOperationTypeAll,\n\t\t\"read\": ACLOperationTypeRead,\n\t\t\"write\": ACLOperationTypeWrite,\n\t\t\"create\": ACLOperationTypeCreate,\n\t\t\"delete\": ACLOperationTypeDelete,\n\t\t\"alter\": ACLOperationTypeAlter,\n\t\t\"describe\": ACLOperationTypeDescribe,\n\t\t\"clusteraction\": ACLOperationTypeClusterAction,\n\t\t\"describeconfigs\": ACLOperationTypeDescribeConfigs,\n\t\t\"alterconfigs\": ACLOperationTypeAlterConfigs,\n\t\t\"idempotentwrite\": ACLOperationTypeIdempotentWrite,\n\t}\n\tparsed, ok := mapping[normalized]\n\tif !ok {\n\t\t*aot = ACLOperationTypeUnknown\n\t\treturn fmt.Errorf(\"cannot parse %s as an ACLOperationType\", normalized)\n\t}\n\t*aot = parsed\n\treturn nil\n\n}\n\ntype ACLEntry struct {\n\tResourceType ResourceType\n\tResourceName string\n\tResourcePatternType PatternType\n\tPrincipal string\n\tHost string\n\tOperation ACLOperationType\n\tPermissionType ACLPermissionType\n}\n\n// CreateACLs sends ACLs creation request to a kafka broker and returns the\n// response.\nfunc (c *Client) CreateACLs(ctx context.Context, req *CreateACLsRequest) (*CreateACLsResponse, error) {\n\tacls := make([]createacls.RequestACLs, 0, len(req.ACLs))\n\n\tfor _, acl := range req.ACLs {\n\t\tacls = append(acls, createacls.RequestACLs{\n\t\t\tResourceType: int8(acl.ResourceType),\n\t\t\tResourceName: acl.ResourceName,\n\t\t\tResourcePatternType: int8(acl.ResourcePatternType),\n\t\t\tPrincipal: acl.Principal,\n\t\t\tHost: acl.Host,\n\t\t\tOperation: int8(acl.Operation),\n\t\t\tPermissionType: int8(acl.PermissionType),\n\t\t})\n\t}\n\n\tm, err := c.roundTrip(ctx, req.Addr, &createacls.Request{\n\t\tCreations: acls,\n\t})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).CreateACLs: %w\", err)\n\t}\n\n\tres := m.(*createacls.Response)\n\tret := &CreateACLsResponse{\n\t\tThrottle: makeDuration(res.ThrottleTimeMs),\n\t\tErrors: make([]error, 0, len(res.Results)),\n\t}\n\n\tfor _, t := range res.Results {\n\t\tret.Errors = append(ret.Errors, makeError(t.ErrorCode, t.ErrorMessage))\n\t}\n\n\treturn ret, nil\n}\n" }, { "path": "createacls_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc TestClientCreateACLs(t *testing.T) {\n\tif !ktesting.KafkaIsAtLeast(\"2.0.1\") {\n\t\treturn\n\t}\n\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\ttopic := makeTopic()\n\tgroup := makeGroupID()\n\n\tcreateRes, err := client.CreateACLs(context.Background(), &CreateACLsRequest{\n\t\tACLs: []ACLEntry{\n\t\t\t{\n\t\t\t\tPrincipal: \"User:alice\",\n\t\t\t\tPermissionType: ACLPermissionTypeAllow,\n\t\t\t\tOperation: ACLOperationTypeRead,\n\t\t\t\tResourceType: ResourceTypeTopic,\n\t\t\t\tResourcePatternType: PatternTypeLiteral,\n\t\t\t\tResourceName: topic,\n\t\t\t\tHost: \"*\",\n\t\t\t},\n\t\t\t{\n\t\t\t\tPrincipal: \"User:bob\",\n\t\t\t\tPermissionType: ACLPermissionTypeAllow,\n\t\t\t\tOperation: ACLOperationTypeRead,\n\t\t\t\tResourceType: ResourceTypeGroup,\n\t\t\t\tResourcePatternType: PatternTypeLiteral,\n\t\t\t\tResourceName: group,\n\t\t\t\tHost: \"*\",\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfor _, err := range createRes.Errors {\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}\n}\n\nfunc TestACLPermissionTypeMarshal(t *testing.T) {\n\tfor i := ACLPermissionTypeUnknown; i <= ACLPermissionTypeAllow; i++ {\n\t\ttext, err := i.MarshalText()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"couldn't marshal %d to text: %s\", i, err)\n\t\t}\n\t\tvar got ACLPermissionType\n\t\terr = got.UnmarshalText(text)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"couldn't unmarshal %s to ACLPermissionType: %s\", text, err)\n\t\t}\n\t\tif got != i {\n\t\t\tt.Errorf(\"got %d, want %d\", got, i)\n\t\t}\n\t}\n}\n\nfunc TestACLOperationTypeMarshal(t *testing.T) {\n\tfor i := ACLOperationTypeUnknown; i <= ACLOperationTypeIdempotentWrite; i++ {\n\t\ttext, err := i.MarshalText()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"couldn't marshal %d to text: %s\", i, err)\n\t\t}\n\t\tvar got ACLOperationType\n\t\terr = got.UnmarshalText(text)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"couldn't unmarshal %s to ACLOperationType: %s\", text, err)\n\t\t}\n\t\tif got != i {\n\t\t\tt.Errorf(\"got %d, want %d\", got, i)\n\t\t}\n\t}\n}\n" }, { "path": "createpartitions.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/createpartitions\"\n)\n\n// CreatePartitionsRequest represents a request sent to a kafka broker to create\n// and update topic parititions.\ntype CreatePartitionsRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// List of topics to create and their configuration.\n\tTopics []TopicPartitionsConfig\n\n\t// When set to true, topics are not created but the configuration is\n\t// validated as if they were.\n\tValidateOnly bool\n}\n\n// CreatePartitionsResponse represents a response from a kafka broker to a partition\n// creation request.\ntype CreatePartitionsResponse struct {\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// Mapping of topic names to errors that occurred while attempting to create\n\t// the topics.\n\t//\n\t// The errors contain the kafka error code. Programs may use the standard\n\t// errors.Is function to test the error against kafka error codes.\n\tErrors map[string]error\n}\n\n// CreatePartitions sends a partitions creation request to a kafka broker and returns the\n// response.\nfunc (c *Client) CreatePartitions(ctx context.Context, req *CreatePartitionsRequest) (*CreatePartitionsResponse, error) {\n\ttopics := make([]createpartitions.RequestTopic, len(req.Topics))\n\n\tfor i, t := range req.Topics {\n\t\ttopics[i] = createpartitions.RequestTopic{\n\t\t\tName: t.Name,\n\t\t\tCount: t.Count,\n\t\t\tAssignments: t.assignments(),\n\t\t}\n\t}\n\n\tm, err := c.roundTrip(ctx, req.Addr, &createpartitions.Request{\n\t\tTopics: topics,\n\t\tTimeoutMs: c.timeoutMs(ctx, defaultCreatePartitionsTimeout),\n\t\tValidateOnly: req.ValidateOnly,\n\t})\n\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).CreatePartitions: %w\", err)\n\t}\n\n\tres := m.(*createpartitions.Response)\n\tret := &CreatePartitionsResponse{\n\t\tThrottle: makeDuration(res.ThrottleTimeMs),\n\t\tErrors: make(map[string]error, len(res.Results)),\n\t}\n\n\tfor _, t := range res.Results {\n\t\tret.Errors[t.Name] = makeError(t.ErrorCode, t.ErrorMessage)\n\t}\n\n\treturn ret, nil\n}\n\ntype TopicPartitionsConfig struct {\n\t// Topic name\n\tName string\n\n\t// Topic partition's count.\n\tCount int32\n\n\t// TopicPartitionAssignments among kafka brokers for this topic partitions.\n\tTopicPartitionAssignments []TopicPartitionAssignment\n}\n\nfunc (t *TopicPartitionsConfig) assignments() []createpartitions.RequestAssignment {\n\tif len(t.TopicPartitionAssignments) == 0 {\n\t\treturn nil\n\t}\n\tassignments := make([]createpartitions.RequestAssignment, len(t.TopicPartitionAssignments))\n\tfor i, a := range t.TopicPartitionAssignments {\n\t\tassignments[i] = createpartitions.RequestAssignment{\n\t\t\tBrokerIDs: a.BrokerIDs,\n\t\t}\n\t}\n\treturn assignments\n}\n\ntype TopicPartitionAssignment struct {\n\t// Broker IDs\n\tBrokerIDs []int32\n}\n" }, { "path": "createpartitions_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc TestClientCreatePartitions(t *testing.T) {\n\tif !ktesting.KafkaIsAtLeast(\"1.0.1\") {\n\t\treturn\n\t}\n\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\ttopic := makeTopic()\n\tcreateTopic(t, topic, 1)\n\tdefer deleteTopic(t, topic)\n\n\tres, err := client.CreatePartitions(context.Background(), &CreatePartitionsRequest{\n\t\tTopics: []TopicPartitionsConfig{\n\t\t\t{\n\t\t\t\tName: topic,\n\t\t\t\tCount: 2,\n\t\t\t\tTopicPartitionAssignments: []TopicPartitionAssignment{\n\t\t\t\t\t{\n\t\t\t\t\t\tBrokerIDs: []int32{1},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\tValidateOnly: false,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif err := res.Errors[topic]; err != nil {\n\t\tt.Error(err)\n\t}\n}\n\nfunc TestClientCreatePartitionsNoAssignments(t *testing.T) {\n\tif !ktesting.KafkaIsAtLeast(\"1.0.1\") {\n\t\treturn\n\t}\n\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\ttopic := makeTopic()\n\tcreateTopic(t, topic, 1)\n\tdefer deleteTopic(t, topic)\n\n\tres, err := client.CreatePartitions(context.Background(), &CreatePartitionsRequest{\n\t\tTopics: []TopicPartitionsConfig{\n\t\t\t{\n\t\t\t\tName: topic,\n\t\t\t\tCount: 2,\n\t\t\t},\n\t\t},\n\t\tValidateOnly: false,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif err := res.Errors[topic]; err != nil {\n\t\tt.Error(err)\n\t}\n}\n" }, { "path": "createtopics.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/createtopics\"\n)\n\n// CreateTopicsRequest represents a request sent to a kafka broker to create\n// new topics.\ntype CreateTopicsRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// List of topics to create and their configuration.\n\tTopics []TopicConfig\n\n\t// When set to true, topics are not created but the configuration is\n\t// validated as if they were.\n\t//\n\t// This field will be ignored if the kafka broker did not support the\n\t// CreateTopics API in version 1 or above.\n\tValidateOnly bool\n}\n\n// CreateTopicsResponse represents a response from a kafka broker to a topic\n// creation request.\ntype CreateTopicsResponse struct {\n\t// The amount of time that the broker throttled the request.\n\t//\n\t// This field will be zero if the kafka broker did not support the\n\t// CreateTopics API in version 2 or above.\n\tThrottle time.Duration\n\n\t// Mapping of topic names to errors that occurred while attempting to create\n\t// the topics.\n\t//\n\t// The errors contain the kafka error code. Programs may use the standard\n\t// errors.Is function to test the error against kafka error codes.\n\tErrors map[string]error\n}\n\n// CreateTopics sends a topic creation request to a kafka broker and returns the\n// response.\nfunc (c *Client) CreateTopics(ctx context.Context, req *CreateTopicsRequest) (*CreateTopicsResponse, error) {\n\ttopics := make([]createtopics.RequestTopic, len(req.Topics))\n\n\tfor i, t := range req.Topics {\n\t\ttopics[i] = createtopics.RequestTopic{\n\t\t\tName: t.Topic,\n\t\t\tNumPartitions: int32(t.NumPartitions),\n\t\t\tReplicationFactor: int16(t.ReplicationFactor),\n\t\t\tAssignments: t.assignments(),\n\t\t\tConfigs: t.configs(),\n\t\t}\n\t}\n\n\tm, err := c.roundTrip(ctx, req.Addr, &createtopics.Request{\n\t\tTopics: topics,\n\t\tTimeoutMs: c.timeoutMs(ctx, defaultCreateTopicsTimeout),\n\t\tValidateOnly: req.ValidateOnly,\n\t})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).CreateTopics: %w\", err)\n\t}\n\n\tres := m.(*createtopics.Response)\n\tret := &CreateTopicsResponse{\n\t\tThrottle: makeDuration(res.ThrottleTimeMs),\n\t\tErrors: make(map[string]error, len(res.Topics)),\n\t}\n\n\tfor _, t := range res.Topics {\n\t\tret.Errors[t.Name] = makeError(t.ErrorCode, t.ErrorMessage)\n\t}\n\n\treturn ret, nil\n}\n\ntype ConfigEntry struct {\n\tConfigName string\n\tConfigValue string\n}\n\nfunc (c ConfigEntry) toCreateTopicsRequestV0ConfigEntry() createTopicsRequestV0ConfigEntry {\n\treturn createTopicsRequestV0ConfigEntry(c)\n}\n\ntype createTopicsRequestV0ConfigEntry struct {\n\tConfigName string\n\tConfigValue string\n}\n\nfunc (t createTopicsRequestV0ConfigEntry) size() int32 {\n\treturn sizeofString(t.ConfigName) +\n\t\tsizeofString(t.ConfigValue)\n}\n\nfunc (t createTopicsRequestV0ConfigEntry) writeTo(wb *writeBuffer) {\n\twb.writeString(t.ConfigName)\n\twb.writeString(t.ConfigValue)\n}\n\ntype ReplicaAssignment struct {\n\tPartition int\n\t// The list of brokers where the partition should be allocated. There must\n\t// be as many entries in thie list as there are replicas of the partition.\n\t// The first entry represents the broker that will be the preferred leader\n\t// for the partition.\n\t//\n\t// This field changed in 0.4 from `int` to `[]int`. It was invalid to pass\n\t// a single integer as this is supposed to be a list. While this introduces\n\t// a breaking change, it probably never worked before.\n\tReplicas []int\n}\n\nfunc (a *ReplicaAssignment) partitionIndex() int32 {\n\treturn int32(a.Partition)\n}\n\nfunc (a *ReplicaAssignment) brokerIDs() []int32 {\n\tif len(a.Replicas) == 0 {\n\t\treturn nil\n\t}\n\treplicas := make([]int32, len(a.Replicas))\n\tfor i, r := range a.Replicas {\n\t\treplicas[i] = int32(r)\n\t}\n\treturn replicas\n}\n\nfunc (a ReplicaAssignment) toCreateTopicsRequestV0ReplicaAssignment() createTopicsRequestV0ReplicaAssignment {\n\treturn createTopicsRequestV0ReplicaAssignment{\n\t\tPartition: int32(a.Partition),\n\t\tReplicas: a.brokerIDs(),\n\t}\n}\n\ntype createTopicsRequestV0ReplicaAssignment struct {\n\tPartition int32\n\tReplicas []int32\n}\n\nfunc (t createTopicsRequestV0ReplicaAssignment) size() int32 {\n\treturn sizeofInt32(t.Partition) +\n\t\t(int32(len(t.Replicas)+1) * sizeofInt32(0)) // N+1 because the array length is a int32\n}\n\nfunc (t createTopicsRequestV0ReplicaAssignment) writeTo(wb *writeBuffer) {\n\twb.writeInt32(t.Partition)\n\twb.writeInt32(int32(len(t.Replicas)))\n\tfor _, r := range t.Replicas {\n\t\twb.writeInt32(int32(r))\n\t}\n}\n\ntype TopicConfig struct {\n\t// Topic name\n\tTopic string\n\n\t// NumPartitions created. -1 indicates unset.\n\tNumPartitions int\n\n\t// ReplicationFactor for the topic. -1 indicates unset.\n\tReplicationFactor int\n\n\t// ReplicaAssignments among kafka brokers for this topic partitions. If this\n\t// is set num_partitions and replication_factor must be unset.\n\tReplicaAssignments []ReplicaAssignment\n\n\t// ConfigEntries holds topic level configuration for topic to be set.\n\tConfigEntries []ConfigEntry\n}\n\nfunc (t *TopicConfig) assignments() []createtopics.RequestAssignment {\n\tif len(t.ReplicaAssignments) == 0 {\n\t\treturn nil\n\t}\n\tassignments := make([]createtopics.RequestAssignment, len(t.ReplicaAssignments))\n\tfor i, a := range t.ReplicaAssignments {\n\t\tassignments[i] = createtopics.RequestAssignment{\n\t\t\tPartitionIndex: a.partitionIndex(),\n\t\t\tBrokerIDs: a.brokerIDs(),\n\t\t}\n\t}\n\treturn assignments\n}\n\nfunc (t *TopicConfig) configs() []createtopics.RequestConfig {\n\tif len(t.ConfigEntries) == 0 {\n\t\treturn nil\n\t}\n\tconfigs := make([]createtopics.RequestConfig, len(t.ConfigEntries))\n\tfor i, c := range t.ConfigEntries {\n\t\tconfigs[i] = createtopics.RequestConfig{\n\t\t\tName: c.ConfigName,\n\t\t\tValue: c.ConfigValue,\n\t\t}\n\t}\n\treturn configs\n}\n\nfunc (t TopicConfig) toCreateTopicsRequestV0Topic() createTopicsRequestV0Topic {\n\trequestV0ReplicaAssignments := make([]createTopicsRequestV0ReplicaAssignment, 0, len(t.ReplicaAssignments))\n\tfor _, a := range t.ReplicaAssignments {\n\t\trequestV0ReplicaAssignments = append(\n\t\t\trequestV0ReplicaAssignments,\n\t\t\ta.toCreateTopicsRequestV0ReplicaAssignment())\n\t}\n\trequestV0ConfigEntries := make([]createTopicsRequestV0ConfigEntry, 0, len(t.ConfigEntries))\n\tfor _, c := range t.ConfigEntries {\n\t\trequestV0ConfigEntries = append(\n\t\t\trequestV0ConfigEntries,\n\t\t\tc.toCreateTopicsRequestV0ConfigEntry())\n\t}\n\n\treturn createTopicsRequestV0Topic{\n\t\tTopic: t.Topic,\n\t\tNumPartitions: int32(t.NumPartitions),\n\t\tReplicationFactor: int16(t.ReplicationFactor),\n\t\tReplicaAssignments: requestV0ReplicaAssignments,\n\t\tConfigEntries: requestV0ConfigEntries,\n\t}\n}\n\ntype createTopicsRequestV0Topic struct {\n\t// Topic name\n\tTopic string\n\n\t// NumPartitions created. -1 indicates unset.\n\tNumPartitions int32\n\n\t// ReplicationFactor for the topic. -1 indicates unset.\n\tReplicationFactor int16\n\n\t// ReplicaAssignments among kafka brokers for this topic partitions. If this\n\t// is set num_partitions and replication_factor must be unset.\n\tReplicaAssignments []createTopicsRequestV0ReplicaAssignment\n\n\t// ConfigEntries holds topic level configuration for topic to be set.\n\tConfigEntries []createTopicsRequestV0ConfigEntry\n}\n\nfunc (t createTopicsRequestV0Topic) size() int32 {\n\treturn sizeofString(t.Topic) +\n\t\tsizeofInt32(t.NumPartitions) +\n\t\tsizeofInt16(t.ReplicationFactor) +\n\t\tsizeofArray(len(t.ReplicaAssignments), func(i int) int32 { return t.ReplicaAssignments[i].size() }) +\n\t\tsizeofArray(len(t.ConfigEntries), func(i int) int32 { return t.ConfigEntries[i].size() })\n}\n\nfunc (t createTopicsRequestV0Topic) writeTo(wb *writeBuffer) {\n\twb.writeString(t.Topic)\n\twb.writeInt32(t.NumPartitions)\n\twb.writeInt16(t.ReplicationFactor)\n\twb.writeArray(len(t.ReplicaAssignments), func(i int) { t.ReplicaAssignments[i].writeTo(wb) })\n\twb.writeArray(len(t.ConfigEntries), func(i int) { t.ConfigEntries[i].writeTo(wb) })\n}\n\n// See http://kafka.apache.org/protocol.html#The_Messages_CreateTopics\ntype createTopicsRequest struct {\n\tv apiVersion // v0, v1, v2\n\n\t// Topics contains n array of single topic creation requests. Can not\n\t// have multiple entries for the same topic.\n\tTopics []createTopicsRequestV0Topic\n\n\t// Timeout ms to wait for a topic to be completely created on the\n\t// controller node. Values <= 0 will trigger topic creation and return immediately\n\tTimeout int32\n\n\t// If true, check that the topics can be created as specified, but don't create anything.\n\t// Internal use only for Kafka 4.0 support.\n\tValidateOnly bool\n}\n\nfunc (t createTopicsRequest) size() int32 {\n\tsz := sizeofArray(len(t.Topics), func(i int) int32 { return t.Topics[i].size() }) +\n\t\tsizeofInt32(t.Timeout)\n\tif t.v >= v1 {\n\t\tsz += 1\n\t}\n\treturn sz\n}\n\nfunc (t createTopicsRequest) writeTo(wb *writeBuffer) {\n\twb.writeArray(len(t.Topics), func(i int) { t.Topics[i].writeTo(wb) })\n\twb.writeInt32(t.Timeout)\n\tif t.v >= v1 {\n\t\twb.writeBool(t.ValidateOnly)\n\t}\n}\n\ntype createTopicsResponseTopicError struct {\n\tv apiVersion\n\n\t// Topic name\n\tTopic string\n\n\t// ErrorCode holds response error code\n\tErrorCode int16\n\n\t// ErrorMessage holds response error message string\n\tErrorMessage string\n}\n\nfunc (t createTopicsResponseTopicError) size() int32 {\n\tsz := sizeofString(t.Topic) +\n\t\tsizeofInt16(t.ErrorCode)\n\tif t.v >= v1 {\n\t\tsz += sizeofString(t.ErrorMessage)\n\t}\n\treturn sz\n}\n\nfunc (t createTopicsResponseTopicError) writeTo(wb *writeBuffer) {\n\twb.writeString(t.Topic)\n\twb.writeInt16(t.ErrorCode)\n\tif t.v >= v1 {\n\t\twb.writeString(t.ErrorMessage)\n\t}\n}\n\nfunc (t *createTopicsResponseTopicError) readFrom(r *bufio.Reader, size int) (remain int, err error) {\n\tif remain, err = readString(r, size, &t.Topic); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readInt16(r, remain, &t.ErrorCode); err != nil {\n\t\treturn\n\t}\n\tif t.v >= v1 {\n\t\tif remain, err = readString(r, remain, &t.ErrorMessage); err != nil {\n\t\t\treturn\n\t\t}\n\t}\n\treturn\n}\n\n// See http://kafka.apache.org/protocol.html#The_Messages_CreateTopics\ntype createTopicsResponse struct {\n\tv apiVersion\n\n\tThrottleTime int32 // v2+\n\tTopicErrors []createTopicsResponseTopicError\n}\n\nfunc (t createTopicsResponse) size() int32 {\n\tsz := sizeofArray(len(t.TopicErrors), func(i int) int32 { return t.TopicErrors[i].size() })\n\tif t.v >= v2 {\n\t\tsz += sizeofInt32(t.ThrottleTime)\n\t}\n\treturn sz\n}\n\nfunc (t createTopicsResponse) writeTo(wb *writeBuffer) {\n\tif t.v >= v2 {\n\t\twb.writeInt32(t.ThrottleTime)\n\t}\n\twb.writeArray(len(t.TopicErrors), func(i int) { t.TopicErrors[i].writeTo(wb) })\n}\n\nfunc (t *createTopicsResponse) readFrom(r *bufio.Reader, size int) (remain int, err error) {\n\tfn := func(r *bufio.Reader, size int) (fnRemain int, fnErr error) {\n\t\ttopic := createTopicsResponseTopicError{v: t.v}\n\t\tif fnRemain, fnErr = (&topic).readFrom(r, size); fnErr != nil {\n\t\t\treturn\n\t\t}\n\t\tt.TopicErrors = append(t.TopicErrors, topic)\n\t\treturn\n\t}\n\tremain = size\n\tif t.v >= v2 {\n\t\tif remain, err = readInt32(r, size, &t.ThrottleTime); err != nil {\n\t\t\treturn\n\t\t}\n\t}\n\tif remain, err = readArrayWith(r, remain, fn); err != nil {\n\t\treturn\n\t}\n\n\treturn\n}\n\nfunc (c *Conn) createTopics(request createTopicsRequest) (createTopicsResponse, error) {\n\tversion, err := c.negotiateVersion(createTopics, v0, v1, v2)\n\tif err != nil {\n\t\treturn createTopicsResponse{}, err\n\t}\n\n\trequest.v = version\n\tresponse := createTopicsResponse{v: version}\n\n\terr = c.writeOperation(\n\t\tfunc(deadline time.Time, id int32) error {\n\t\t\tif request.Timeout == 0 {\n\t\t\t\tnow := time.Now()\n\t\t\t\tdeadline = adjustDeadlineForRTT(deadline, now, defaultRTT)\n\t\t\t\trequest.Timeout = milliseconds(deadlineToTimeout(deadline, now))\n\t\t\t}\n\t\t\treturn c.writeRequest(createTopics, version, id, request)\n\t\t},\n\t\tfunc(deadline time.Time, size int) error {\n\t\t\treturn expectZeroSize(func() (remain int, err error) {\n\t\t\t\treturn (&response).readFrom(&c.rbuf, size)\n\t\t\t}())\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn response, err\n\t}\n\tfor _, tr := range response.TopicErrors {\n\t\tif tr.ErrorCode == int16(TopicAlreadyExists) {\n\t\t\tcontinue\n\t\t}\n\t\tif tr.ErrorCode != 0 {\n\t\t\treturn response, Error(tr.ErrorCode)\n\t\t}\n\t}\n\n\treturn response, nil\n}\n\n// CreateTopics creates one topic per provided configuration with idempotent\n// operational semantics. In other words, if CreateTopics is invoked with a\n// configuration for an existing topic, it will have no effect.\nfunc (c *Conn) CreateTopics(topics ...TopicConfig) error {\n\trequestV0Topics := make([]createTopicsRequestV0Topic, 0, len(topics))\n\tfor _, t := range topics {\n\t\trequestV0Topics = append(\n\t\t\trequestV0Topics,\n\t\t\tt.toCreateTopicsRequestV0Topic())\n\t}\n\n\t_, err := c.createTopics(createTopicsRequest{\n\t\tTopics: requestV0Topics,\n\t})\n\treturn err\n}\n" }, { "path": "createtopics_test.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"net\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"testing\"\n)\n\nfunc TestConnCreateTopics(t *testing.T) {\n\ttopic1 := makeTopic()\n\ttopic2 := makeTopic()\n\n\tconn, err := DialContext(context.Background(), \"tcp\", \"localhost:9092\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdefer func() {\n\t\terr := conn.Close()\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"failed to close connection: %v\", err)\n\t\t}\n\t}()\n\n\tcontroller, _ := conn.Controller()\n\n\tcontrollerConn, err := Dial(\"tcp\", net.JoinHostPort(controller.Host, strconv.Itoa(controller.Port)))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer controllerConn.Close()\n\n\terr = controllerConn.CreateTopics(TopicConfig{\n\t\tTopic: topic1,\n\t\tNumPartitions: 1,\n\t\tReplicationFactor: 1,\n\t})\n\tif err != nil {\n\t\tt.Fatalf(\"unexpected error creating topic: %s\", err.Error())\n\t}\n\n\terr = controllerConn.CreateTopics(TopicConfig{\n\t\tTopic: topic1,\n\t\tNumPartitions: 1,\n\t\tReplicationFactor: 1,\n\t})\n\n\t// Duplicate topic should not return an error\n\tif err != nil {\n\t\tt.Fatalf(\"unexpected error creating duplicate topic topic: %v\", err)\n\t}\n\n\terr = controllerConn.CreateTopics(\n\t\tTopicConfig{\n\t\t\tTopic: topic1,\n\t\t\tNumPartitions: 1,\n\t\t\tReplicationFactor: 1,\n\t\t},\n\t\tTopicConfig{\n\t\t\tTopic: topic2,\n\t\t\tNumPartitions: 1,\n\t\t\tReplicationFactor: 1,\n\t\t},\n\t\tTopicConfig{\n\t\t\tTopic: topic2,\n\t\t\tNumPartitions: 1,\n\t\t\tReplicationFactor: 1,\n\t\t},\n\t)\n\n\tif err == nil {\n\t\tt.Fatal(\"CreateTopics should have returned an error for invalid requests\")\n\t}\n\n\tif !errors.Is(err, InvalidRequest) {\n\t\tt.Fatalf(\"expected invalid request: %v\", err)\n\t}\n\n\tdeleteTopic(t, topic1)\n}\n\nfunc TestClientCreateTopics(t *testing.T) {\n\tconst (\n\t\ttopic1 = \"client-topic-1\"\n\t\ttopic2 = \"client-topic-2\"\n\t\ttopic3 = \"client-topic-3\"\n\t)\n\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\tconfig := []ConfigEntry{{\n\t\tConfigName: \"retention.ms\",\n\t\tConfigValue: \"3600000\",\n\t}}\n\n\tres, err := client.CreateTopics(context.Background(), &CreateTopicsRequest{\n\t\tTopics: []TopicConfig{\n\t\t\t{\n\t\t\t\tTopic: topic1,\n\t\t\t\tNumPartitions: -1,\n\t\t\t\tReplicationFactor: -1,\n\t\t\t\tReplicaAssignments: []ReplicaAssignment{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\tReplicas: []int{1},\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\tReplicas: []int{1},\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 2,\n\t\t\t\t\t\tReplicas: []int{1},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tConfigEntries: config,\n\t\t\t},\n\t\t\t{\n\t\t\t\tTopic: topic2,\n\t\t\t\tNumPartitions: 2,\n\t\t\t\tReplicationFactor: 1,\n\t\t\t\tConfigEntries: config,\n\t\t\t},\n\t\t\t{\n\t\t\t\tTopic: topic3,\n\t\t\t\tNumPartitions: 1,\n\t\t\t\tReplicationFactor: 1,\n\t\t\t\tConfigEntries: config,\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdefer deleteTopic(t, topic1, topic2, topic3)\n\n\texpectTopics := map[string]struct{}{\n\t\ttopic1: {},\n\t\ttopic2: {},\n\t\ttopic3: {},\n\t}\n\n\tfor topic, error := range res.Errors {\n\t\tdelete(expectTopics, topic)\n\n\t\tif error != nil {\n\t\t\tt.Errorf(\"%s => %s\", topic, error)\n\t\t}\n\t}\n\n\tfor topic := range expectTopics {\n\t\tt.Errorf(\"topic missing in response: %s\", topic)\n\t}\n}\n\nfunc TestCreateTopicsResponse(t *testing.T) {\n\tsupportedVersions := []apiVersion{v0, v1, v2}\n\tfor _, v := range supportedVersions {\n\t\titem := createTopicsResponse{\n\t\t\tv: v,\n\t\t\tTopicErrors: []createTopicsResponseTopicError{\n\t\t\t\t{\n\t\t\t\t\tv: v,\n\t\t\t\t\tTopic: \"topic\",\n\t\t\t\t\tErrorCode: 2,\n\t\t\t\t},\n\t\t\t},\n\t\t}\n\n\t\tb := bytes.NewBuffer(nil)\n\t\tw := &writeBuffer{w: b}\n\t\titem.writeTo(w)\n\n\t\tfound := createTopicsResponse{v: v}\n\t\tremain, err := (&found).readFrom(bufio.NewReader(b), b.Len())\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t\tt.FailNow()\n\t\t}\n\t\tif remain != 0 {\n\t\t\tt.Errorf(\"expected 0 remain, got %v\", remain)\n\t\t\tt.FailNow()\n\t\t}\n\t\tif !reflect.DeepEqual(item, found) {\n\t\t\tt.Error(\"expected item and found to be the same\")\n\t\t\tt.FailNow()\n\t\t}\n\t}\n}\n" }, { "path": "deleteacls.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/deleteacls\"\n)\n\n// DeleteACLsRequest represents a request sent to a kafka broker to delete\n// ACLs.\ntype DeleteACLsRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// List of ACL filters to use for deletion.\n\tFilters []DeleteACLsFilter\n}\n\ntype DeleteACLsFilter struct {\n\tResourceTypeFilter ResourceType\n\tResourceNameFilter string\n\tResourcePatternTypeFilter PatternType\n\tPrincipalFilter string\n\tHostFilter string\n\tOperation ACLOperationType\n\tPermissionType ACLPermissionType\n}\n\n// DeleteACLsResponse represents a response from a kafka broker to an ACL\n// deletion request.\ntype DeleteACLsResponse struct {\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// List of the results from the deletion request.\n\tResults []DeleteACLsResult\n}\n\ntype DeleteACLsResult struct {\n\tError error\n\tMatchingACLs []DeleteACLsMatchingACLs\n}\n\ntype DeleteACLsMatchingACLs struct {\n\tError error\n\tResourceType ResourceType\n\tResourceName string\n\tResourcePatternType PatternType\n\tPrincipal string\n\tHost string\n\tOperation ACLOperationType\n\tPermissionType ACLPermissionType\n}\n\n// DeleteACLs sends ACLs deletion request to a kafka broker and returns the\n// response.\nfunc (c *Client) DeleteACLs(ctx context.Context, req *DeleteACLsRequest) (*DeleteACLsResponse, error) {\n\tfilters := make([]deleteacls.RequestFilter, 0, len(req.Filters))\n\n\tfor _, filter := range req.Filters {\n\t\tfilters = append(filters, deleteacls.RequestFilter{\n\t\t\tResourceTypeFilter: int8(filter.ResourceTypeFilter),\n\t\t\tResourceNameFilter: filter.ResourceNameFilter,\n\t\t\tResourcePatternTypeFilter: int8(filter.ResourcePatternTypeFilter),\n\t\t\tPrincipalFilter: filter.PrincipalFilter,\n\t\t\tHostFilter: filter.HostFilter,\n\t\t\tOperation: int8(filter.Operation),\n\t\t\tPermissionType: int8(filter.PermissionType),\n\t\t})\n\t}\n\n\tm, err := c.roundTrip(ctx, req.Addr, &deleteacls.Request{\n\t\tFilters: filters,\n\t})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).DeleteACLs: %w\", err)\n\t}\n\n\tres := m.(*deleteacls.Response)\n\n\tresults := make([]DeleteACLsResult, 0, len(res.FilterResults))\n\n\tfor _, result := range res.FilterResults {\n\t\tmatchingACLs := make([]DeleteACLsMatchingACLs, 0, len(result.MatchingACLs))\n\n\t\tfor _, matchingACL := range result.MatchingACLs {\n\t\t\tmatchingACLs = append(matchingACLs, DeleteACLsMatchingACLs{\n\t\t\t\tError: makeError(matchingACL.ErrorCode, matchingACL.ErrorMessage),\n\t\t\t\tResourceType: ResourceType(matchingACL.ResourceType),\n\t\t\t\tResourceName: matchingACL.ResourceName,\n\t\t\t\tResourcePatternType: PatternType(matchingACL.ResourcePatternType),\n\t\t\t\tPrincipal: matchingACL.Principal,\n\t\t\t\tHost: matchingACL.Host,\n\t\t\t\tOperation: ACLOperationType(matchingACL.Operation),\n\t\t\t\tPermissionType: ACLPermissionType(matchingACL.PermissionType),\n\t\t\t})\n\t\t}\n\n\t\tresults = append(results, DeleteACLsResult{\n\t\t\tError: makeError(result.ErrorCode, result.ErrorMessage),\n\t\t\tMatchingACLs: matchingACLs,\n\t\t})\n\t}\n\n\tret := &DeleteACLsResponse{\n\t\tThrottle: makeDuration(res.ThrottleTimeMs),\n\t\tResults: results,\n\t}\n\n\treturn ret, nil\n}\n" }, { "path": "deleteacls_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestClientDeleteACLs(t *testing.T) {\n\tif !ktesting.KafkaIsAtLeast(\"2.0.1\") {\n\t\treturn\n\t}\n\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\ttopic := makeTopic()\n\tgroup := makeGroupID()\n\n\tcreateRes, err := client.CreateACLs(context.Background(), &CreateACLsRequest{\n\t\tACLs: []ACLEntry{\n\t\t\t{\n\t\t\t\tPrincipal: \"User:alice\",\n\t\t\t\tPermissionType: ACLPermissionTypeAllow,\n\t\t\t\tOperation: ACLOperationTypeRead,\n\t\t\t\tResourceType: ResourceTypeTopic,\n\t\t\t\tResourcePatternType: PatternTypeLiteral,\n\t\t\t\tResourceName: topic,\n\t\t\t\tHost: \"*\",\n\t\t\t},\n\t\t\t{\n\t\t\t\tPrincipal: \"User:bob\",\n\t\t\t\tPermissionType: ACLPermissionTypeAllow,\n\t\t\t\tOperation: ACLOperationTypeRead,\n\t\t\t\tResourceType: ResourceTypeGroup,\n\t\t\t\tResourcePatternType: PatternTypeLiteral,\n\t\t\t\tResourceName: group,\n\t\t\t\tHost: \"*\",\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfor _, err := range createRes.Errors {\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}\n\n\tdeleteResp, err := client.DeleteACLs(context.Background(), &DeleteACLsRequest{\n\t\tFilters: []DeleteACLsFilter{\n\t\t\t{\n\t\t\t\tResourceTypeFilter: ResourceTypeTopic,\n\t\t\t\tResourceNameFilter: topic,\n\t\t\t\tResourcePatternTypeFilter: PatternTypeLiteral,\n\t\t\t\tOperation: ACLOperationTypeRead,\n\t\t\t\tPermissionType: ACLPermissionTypeAllow,\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\texpectedDeleteResp := DeleteACLsResponse{\n\t\tThrottle: 0,\n\t\tResults: []DeleteACLsResult{\n\t\t\t{\n\t\t\t\tError: makeError(0, \"\"),\n\t\t\t\tMatchingACLs: []DeleteACLsMatchingACLs{\n\t\t\t\t\t{\n\t\t\t\t\t\tError: makeError(0, \"\"),\n\t\t\t\t\t\tResourceType: ResourceTypeTopic,\n\t\t\t\t\t\tResourceName: topic,\n\t\t\t\t\t\tResourcePatternType: PatternTypeLiteral,\n\t\t\t\t\t\tPrincipal: \"User:alice\",\n\t\t\t\t\t\tHost: \"*\",\n\t\t\t\t\t\tOperation: ACLOperationTypeRead,\n\t\t\t\t\t\tPermissionType: ACLPermissionTypeAllow,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tassert.Equal(t, expectedDeleteResp, *deleteResp)\n\n\tdescribeResp, err := client.DescribeACLs(context.Background(), &DescribeACLsRequest{\n\t\tFilter: ACLFilter{\n\t\t\tResourceTypeFilter: ResourceTypeTopic,\n\t\t\tResourceNameFilter: topic,\n\t\t\tResourcePatternTypeFilter: PatternTypeLiteral,\n\t\t\tOperation: ACLOperationTypeRead,\n\t\t\tPermissionType: ACLPermissionTypeAllow,\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\texpectedDescribeResp := DescribeACLsResponse{\n\t\tThrottle: 0,\n\t\tError: makeError(0, \"\"),\n\t\tResources: []ACLResource{},\n\t}\n\n\tassert.Equal(t, expectedDescribeResp, *describeResp)\n}\n" }, { "path": "deletegroups.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/deletegroups\"\n)\n\n// DeleteGroupsRequest represents a request sent to a kafka broker to delete\n// consumer groups.\ntype DeleteGroupsRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// Identifiers of groups to delete.\n\tGroupIDs []string\n}\n\n// DeleteGroupsResponse represents a response from a kafka broker to a consumer group\n// deletion request.\ntype DeleteGroupsResponse struct {\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// Mapping of group ids to errors that occurred while attempting to delete those groups.\n\t//\n\t// The errors contain the kafka error code. Programs may use the standard\n\t// errors.Is function to test the error against kafka error codes.\n\tErrors map[string]error\n}\n\n// DeleteGroups sends a delete groups request and returns the response. The request is sent to the group coordinator of the first group\n// of the request. All deleted groups must be managed by the same group coordinator.\nfunc (c *Client) DeleteGroups(\n\tctx context.Context,\n\treq *DeleteGroupsRequest,\n) (*DeleteGroupsResponse, error) {\n\tm, err := c.roundTrip(ctx, req.Addr, &deletegroups.Request{\n\t\tGroupIDs: req.GroupIDs,\n\t})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).DeleteGroups: %w\", err)\n\t}\n\n\tr := m.(*deletegroups.Response)\n\n\tret := &DeleteGroupsResponse{\n\t\tThrottle: makeDuration(r.ThrottleTimeMs),\n\t\tErrors: make(map[string]error, len(r.Responses)),\n\t}\n\n\tfor _, t := range r.Responses {\n\t\tret.Errors[t.GroupID] = makeError(t.ErrorCode, \"\")\n\t}\n\n\treturn ret, nil\n}\n" }, { "path": "deletegroups_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"testing\"\n\t\"time\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc TestClientDeleteGroups(t *testing.T) {\n\tif !ktesting.KafkaIsAtLeast(\"1.1.0\") {\n\t\tt.Skip(\"Skipping test because kafka version is not high enough.\")\n\t}\n\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\ttopic := makeTopic()\n\tcreateTopic(t, topic, 1)\n\n\tgroupID := makeGroupID()\n\n\tgroup, err := NewConsumerGroup(ConsumerGroupConfig{\n\t\tID: groupID,\n\t\tTopics: []string{topic},\n\t\tBrokers: []string{\"localhost:9092\"},\n\t\tHeartbeatInterval: 2 * time.Second,\n\t\tRebalanceTimeout: 2 * time.Second,\n\t\tRetentionTime: time.Hour,\n\t\tLogger: &testKafkaLogger{T: t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer group.Close()\n\n\tctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)\n\tdefer cancel()\n\n\tgen, err := group.Next(ctx)\n\tif gen == nil {\n\t\tt.Fatalf(\"expected generation 1 not to be nil\")\n\t}\n\tif err != nil {\n\t\tt.Fatalf(\"expected no error, but got %+v\", err)\n\t}\n\n\t// delete not empty group\n\tres, err := client.DeleteGroups(ctx, &DeleteGroupsRequest{\n\t\tGroupIDs: []string{groupID},\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif !errors.Is(res.Errors[groupID], NonEmptyGroup) {\n\t\tt.Fatalf(\"expected NonEmptyGroup error, but got %+v\", res.Errors[groupID])\n\t}\n\n\terr = group.Close()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// delete empty group\n\tres, err = client.DeleteGroups(ctx, &DeleteGroupsRequest{\n\t\tGroupIDs: []string{groupID},\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif err = res.Errors[groupID]; err != nil {\n\t\tt.Error(err)\n\t}\n}\n" }, { "path": "deletetopics.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/deletetopics\"\n)\n\n// DeleteTopicsRequest represents a request sent to a kafka broker to delete\n// topics.\ntype DeleteTopicsRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// Names of topics to delete.\n\tTopics []string\n}\n\n// DeleteTopicsResponse represents a response from a kafka broker to a topic\n// deletion request.\ntype DeleteTopicsResponse struct {\n\t// The amount of time that the broker throttled the request.\n\t//\n\t// This field will be zero if the kafka broker did not support the\n\t// DeleteTopics API in version 1 or above.\n\tThrottle time.Duration\n\n\t// Mapping of topic names to errors that occurred while attempting to delete\n\t// the topics.\n\t//\n\t// The errors contain the kafka error code. Programs may use the standard\n\t// errors.Is function to test the error against kafka error codes.\n\tErrors map[string]error\n}\n\n// DeleteTopics sends a topic deletion request to a kafka broker and returns the\n// response.\nfunc (c *Client) DeleteTopics(ctx context.Context, req *DeleteTopicsRequest) (*DeleteTopicsResponse, error) {\n\tm, err := c.roundTrip(ctx, req.Addr, &deletetopics.Request{\n\t\tTopicNames: req.Topics,\n\t\tTimeoutMs: c.timeoutMs(ctx, defaultDeleteTopicsTimeout),\n\t})\n\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).DeleteTopics: %w\", err)\n\t}\n\n\tres := m.(*deletetopics.Response)\n\tret := &DeleteTopicsResponse{\n\t\tThrottle: makeDuration(res.ThrottleTimeMs),\n\t\tErrors: make(map[string]error, len(res.Responses)),\n\t}\n\n\tfor _, t := range res.Responses {\n\t\tif t.ErrorCode == 0 {\n\t\t\tret.Errors[t.Name] = nil\n\t\t} else {\n\t\t\tret.Errors[t.Name] = Error(t.ErrorCode)\n\t\t}\n\t}\n\n\treturn ret, nil\n}\n\n// See http://kafka.apache.org/protocol.html#The_Messages_DeleteTopics\ntype deleteTopicsRequest struct {\n\t// Topics holds the topic names\n\tTopics []string\n\n\t// Timeout holds the time in ms to wait for a topic to be completely deleted\n\t// on the controller node. Values <= 0 will trigger topic deletion and return\n\t// immediately.\n\tTimeout int32\n}\n\nfunc (t deleteTopicsRequest) size() int32 {\n\treturn sizeofStringArray(t.Topics) +\n\t\tsizeofInt32(t.Timeout)\n}\n\nfunc (t deleteTopicsRequest) writeTo(wb *writeBuffer) {\n\twb.writeStringArray(t.Topics)\n\twb.writeInt32(t.Timeout)\n}\n\ntype deleteTopicsResponse struct {\n\tv apiVersion // v0, v1\n\n\tThrottleTime int32\n\t// TopicErrorCodes holds per topic error codes\n\tTopicErrorCodes []deleteTopicsResponseV0TopicErrorCode\n}\n\nfunc (t deleteTopicsResponse) size() int32 {\n\tsz := sizeofArray(len(t.TopicErrorCodes), func(i int) int32 { return t.TopicErrorCodes[i].size() })\n\tif t.v >= v1 {\n\t\tsz += sizeofInt32(t.ThrottleTime)\n\t}\n\treturn sz\n}\n\nfunc (t *deleteTopicsResponse) readFrom(r *bufio.Reader, size int) (remain int, err error) {\n\tfn := func(withReader *bufio.Reader, withSize int) (fnRemain int, fnErr error) {\n\t\tvar item deleteTopicsResponseV0TopicErrorCode\n\t\tif fnRemain, fnErr = (&item).readFrom(withReader, withSize); fnErr != nil {\n\t\t\treturn\n\t\t}\n\t\tt.TopicErrorCodes = append(t.TopicErrorCodes, item)\n\t\treturn\n\t}\n\tremain = size\n\tif t.v >= v1 {\n\t\tif remain, err = readInt32(r, size, &t.ThrottleTime); err != nil {\n\t\t\treturn\n\t\t}\n\t}\n\tif remain, err = readArrayWith(r, remain, fn); err != nil {\n\t\treturn\n\t}\n\treturn\n}\n\nfunc (t deleteTopicsResponse) writeTo(wb *writeBuffer) {\n\tif t.v >= v1 {\n\t\twb.writeInt32(t.ThrottleTime)\n\t}\n\twb.writeArray(len(t.TopicErrorCodes), func(i int) { t.TopicErrorCodes[i].writeTo(wb) })\n}\n\ntype deleteTopicsResponseV0TopicErrorCode struct {\n\t// Topic holds the topic name\n\tTopic string\n\n\t// ErrorCode holds the error code\n\tErrorCode int16\n}\n\nfunc (t deleteTopicsResponseV0TopicErrorCode) size() int32 {\n\treturn sizeofString(t.Topic) +\n\t\tsizeofInt16(t.ErrorCode)\n}\n\nfunc (t *deleteTopicsResponseV0TopicErrorCode) readFrom(r *bufio.Reader, size int) (remain int, err error) {\n\tif remain, err = readString(r, size, &t.Topic); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readInt16(r, remain, &t.ErrorCode); err != nil {\n\t\treturn\n\t}\n\treturn\n}\n\nfunc (t deleteTopicsResponseV0TopicErrorCode) writeTo(wb *writeBuffer) {\n\twb.writeString(t.Topic)\n\twb.writeInt16(t.ErrorCode)\n}\n\n// deleteTopics deletes the specified topics.\n//\n// See http://kafka.apache.org/protocol.html#The_Messages_DeleteTopics\nfunc (c *Conn) deleteTopics(request deleteTopicsRequest) (deleteTopicsResponse, error) {\n\tversion, err := c.negotiateVersion(deleteTopics, v0, v1)\n\tif err != nil {\n\t\treturn deleteTopicsResponse{}, err\n\t}\n\n\tresponse := deleteTopicsResponse{\n\t\tv: version,\n\t}\n\n\terr = c.writeOperation(\n\t\tfunc(deadline time.Time, id int32) error {\n\t\t\tif request.Timeout == 0 {\n\t\t\t\tnow := time.Now()\n\t\t\t\tdeadline = adjustDeadlineForRTT(deadline, now, defaultRTT)\n\t\t\t\trequest.Timeout = milliseconds(deadlineToTimeout(deadline, now))\n\t\t\t}\n\t\t\treturn c.writeRequest(deleteTopics, version, id, request)\n\t\t},\n\t\tfunc(deadline time.Time, size int) error {\n\t\t\treturn expectZeroSize(func() (remain int, err error) {\n\t\t\t\treturn (&response).readFrom(&c.rbuf, size)\n\t\t\t}())\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn deleteTopicsResponse{}, err\n\t}\n\tfor _, c := range response.TopicErrorCodes {\n\t\tif c.ErrorCode != 0 {\n\t\t\treturn response, Error(c.ErrorCode)\n\t\t}\n\t}\n\treturn response, nil\n}\n" }, { "path": "deletetopics_test.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"reflect\"\n\t\"testing\"\n)\n\nfunc TestClientDeleteTopics(t *testing.T) {\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\ttopic := makeTopic()\n\tcreateTopic(t, topic, 1)\n\n\tres, err := client.DeleteTopics(context.Background(), &DeleteTopicsRequest{\n\t\tTopics: []string{topic},\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif err := res.Errors[topic]; err != nil {\n\t\tt.Error(err)\n\t}\n}\n\nfunc TestDeleteTopicsResponseV1(t *testing.T) {\n\titem := deleteTopicsResponse{\n\t\tTopicErrorCodes: []deleteTopicsResponseV0TopicErrorCode{\n\t\t\t{\n\t\t\t\tTopic: \"a\",\n\t\t\t\tErrorCode: 7,\n\t\t\t},\n\t\t},\n\t}\n\n\tb := bytes.NewBuffer(nil)\n\tw := &writeBuffer{w: b}\n\titem.writeTo(w)\n\n\tvar found deleteTopicsResponse\n\tremain, err := (&found).readFrom(bufio.NewReader(b), b.Len())\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif remain != 0 {\n\t\tt.Fatalf(\"expected 0 remain, got %v\", remain)\n\t}\n\tif !reflect.DeepEqual(item, found) {\n\t\tt.Fatal(\"expected item and found to be the same\")\n\t}\n}\n" }, { "path": "describeacls.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/describeacls\"\n)\n\n// DescribeACLsRequest represents a request sent to a kafka broker to describe\n// existing ACLs.\ntype DescribeACLsRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// Filter to filter ACLs on.\n\tFilter ACLFilter\n}\n\ntype ACLFilter struct {\n\tResourceTypeFilter ResourceType\n\tResourceNameFilter string\n\t// ResourcePatternTypeFilter was added in v1 and is not available prior to that.\n\tResourcePatternTypeFilter PatternType\n\tPrincipalFilter string\n\tHostFilter string\n\tOperation ACLOperationType\n\tPermissionType ACLPermissionType\n}\n\n// DescribeACLsResponse represents a response from a kafka broker to an ACL\n// describe request.\ntype DescribeACLsResponse struct {\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// Error that occurred while attempting to describe\n\t// the ACLs.\n\tError error\n\n\t// ACL resources returned from the describe request.\n\tResources []ACLResource\n}\n\ntype ACLResource struct {\n\tResourceType ResourceType\n\tResourceName string\n\tPatternType PatternType\n\tACLs []ACLDescription\n}\n\ntype ACLDescription struct {\n\tPrincipal string\n\tHost string\n\tOperation ACLOperationType\n\tPermissionType ACLPermissionType\n}\n\nfunc (c *Client) DescribeACLs(ctx context.Context, req *DescribeACLsRequest) (*DescribeACLsResponse, error) {\n\tm, err := c.roundTrip(ctx, req.Addr, &describeacls.Request{\n\t\tFilter: describeacls.ACLFilter{\n\t\t\tResourceTypeFilter: int8(req.Filter.ResourceTypeFilter),\n\t\t\tResourceNameFilter: req.Filter.ResourceNameFilter,\n\t\t\tResourcePatternTypeFilter: int8(req.Filter.ResourcePatternTypeFilter),\n\t\t\tPrincipalFilter: req.Filter.PrincipalFilter,\n\t\t\tHostFilter: req.Filter.HostFilter,\n\t\t\tOperation: int8(req.Filter.Operation),\n\t\t\tPermissionType: int8(req.Filter.PermissionType),\n\t\t},\n\t})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).DescribeACLs: %w\", err)\n\t}\n\n\tres := m.(*describeacls.Response)\n\tresources := make([]ACLResource, len(res.Resources))\n\n\tfor resourceIdx, respResource := range res.Resources {\n\t\tdescriptions := make([]ACLDescription, len(respResource.ACLs))\n\n\t\tfor descriptionIdx, respDescription := range respResource.ACLs {\n\t\t\tdescriptions[descriptionIdx] = ACLDescription{\n\t\t\t\tPrincipal: respDescription.Principal,\n\t\t\t\tHost: respDescription.Host,\n\t\t\t\tOperation: ACLOperationType(respDescription.Operation),\n\t\t\t\tPermissionType: ACLPermissionType(respDescription.PermissionType),\n\t\t\t}\n\t\t}\n\n\t\tresources[resourceIdx] = ACLResource{\n\t\t\tResourceType: ResourceType(respResource.ResourceType),\n\t\t\tResourceName: respResource.ResourceName,\n\t\t\tPatternType: PatternType(respResource.PatternType),\n\t\t\tACLs: descriptions,\n\t\t}\n\t}\n\n\tret := &DescribeACLsResponse{\n\t\tThrottle: makeDuration(res.ThrottleTimeMs),\n\t\tError: makeError(res.ErrorCode, res.ErrorMessage),\n\t\tResources: resources,\n\t}\n\n\treturn ret, nil\n}\n" }, { "path": "describeacls_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestClientDescribeACLs(t *testing.T) {\n\tif !ktesting.KafkaIsAtLeast(\"2.0.1\") {\n\t\treturn\n\t}\n\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\ttopic := makeTopic()\n\tgroup := makeGroupID()\n\n\tcreateRes, err := client.CreateACLs(context.Background(), &CreateACLsRequest{\n\t\tACLs: []ACLEntry{\n\t\t\t{\n\t\t\t\tPrincipal: \"User:alice\",\n\t\t\t\tPermissionType: ACLPermissionTypeAllow,\n\t\t\t\tOperation: ACLOperationTypeRead,\n\t\t\t\tResourceType: ResourceTypeTopic,\n\t\t\t\tResourcePatternType: PatternTypeLiteral,\n\t\t\t\tResourceName: topic,\n\t\t\t\tHost: \"*\",\n\t\t\t},\n\t\t\t{\n\t\t\t\tPrincipal: \"User:bob\",\n\t\t\t\tPermissionType: ACLPermissionTypeAllow,\n\t\t\t\tOperation: ACLOperationTypeRead,\n\t\t\t\tResourceType: ResourceTypeGroup,\n\t\t\t\tResourcePatternType: PatternTypeLiteral,\n\t\t\t\tResourceName: group,\n\t\t\t\tHost: \"*\",\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfor _, err := range createRes.Errors {\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}\n\n\tdescribeResp, err := client.DescribeACLs(context.Background(), &DescribeACLsRequest{\n\t\tFilter: ACLFilter{\n\t\t\tResourceTypeFilter: ResourceTypeTopic,\n\t\t\tResourceNameFilter: topic,\n\t\t\tResourcePatternTypeFilter: PatternTypeLiteral,\n\t\t\tOperation: ACLOperationTypeRead,\n\t\t\tPermissionType: ACLPermissionTypeAllow,\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\texpectedDescribeResp := DescribeACLsResponse{\n\t\tThrottle: 0,\n\t\tError: makeError(0, \"\"),\n\t\tResources: []ACLResource{\n\t\t\t{\n\t\t\t\tResourceType: ResourceTypeTopic,\n\t\t\t\tResourceName: topic,\n\t\t\t\tPatternType: PatternTypeLiteral,\n\t\t\t\tACLs: []ACLDescription{\n\t\t\t\t\t{\n\t\t\t\t\t\tPrincipal: \"User:alice\",\n\t\t\t\t\t\tHost: \"*\",\n\t\t\t\t\t\tOperation: ACLOperationTypeRead,\n\t\t\t\t\t\tPermissionType: ACLPermissionTypeAllow,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tassert.Equal(t, expectedDescribeResp, *describeResp)\n}\n" }, { "path": "describeclientquotas.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/describeclientquotas\"\n)\n\n// DescribeClientQuotasRequest represents a request sent to a kafka broker to\n// describe client quotas.\ntype DescribeClientQuotasRequest struct {\n\t// Address of the kafka broker to send the request to\n\tAddr net.Addr\n\n\t// List of quota components to describe.\n\tComponents []DescribeClientQuotasRequestComponent\n\n\t// Whether the match is strict, i.e. should exclude entities with\n\t// unspecified entity types.\n\tStrict bool\n}\n\ntype DescribeClientQuotasRequestComponent struct {\n\t// The entity type that the filter component applies to.\n\tEntityType string\n\n\t// How to match the entity (0 = exact name, 1 = default name,\n\t// 2 = any specified name).\n\tMatchType int8\n\n\t// The string to match against, or null if unused for the match type.\n\tMatch string\n}\n\n// DescribeClientQuotasResponse represents a response from a kafka broker to a describe client quota request.\ntype DescribeClientQuotasResponse struct {\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// Error is set to a non-nil value including the code and message if a top-level\n\t// error was encountered when doing the update.\n\tError error\n\n\t// List of describe client quota responses.\n\tEntries []DescribeClientQuotasResponseQuotas\n}\n\ntype DescribeClientQuotasEntity struct {\n\t// The quota entity type.\n\tEntityType string\n\n\t// The name of the quota entity, or null if the default.\n\tEntityName string\n}\n\ntype DescribeClientQuotasValue struct {\n\t// The quota configuration key.\n\tKey string\n\n\t// The quota configuration value.\n\tValue float64\n}\n\ntype DescribeClientQuotasResponseQuotas struct {\n\t// List of client quota entities and their descriptions.\n\tEntities []DescribeClientQuotasEntity\n\n\t// The client quota configuration values.\n\tValues []DescribeClientQuotasValue\n}\n\n// DescribeClientQuotas sends a describe client quotas request to a kafka broker and returns\n// the response.\nfunc (c *Client) DescribeClientQuotas(ctx context.Context, req *DescribeClientQuotasRequest) (*DescribeClientQuotasResponse, error) {\n\tcomponents := make([]describeclientquotas.Component, len(req.Components))\n\n\tfor componentIdx, component := range req.Components {\n\t\tcomponents[componentIdx] = describeclientquotas.Component{\n\t\t\tEntityType: component.EntityType,\n\t\t\tMatchType: component.MatchType,\n\t\t\tMatch: component.Match,\n\t\t}\n\t}\n\n\tm, err := c.roundTrip(ctx, req.Addr, &describeclientquotas.Request{\n\t\tComponents: components,\n\t\tStrict: req.Strict,\n\t})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).DescribeClientQuotas: %w\", err)\n\t}\n\n\tres := m.(*describeclientquotas.Response)\n\tresponseEntries := make([]DescribeClientQuotasResponseQuotas, len(res.Entries))\n\n\tfor responseEntryIdx, responseEntry := range res.Entries {\n\t\tresponseEntities := make([]DescribeClientQuotasEntity, len(responseEntry.Entities))\n\t\tfor responseEntityIdx, responseEntity := range responseEntry.Entities {\n\t\t\tresponseEntities[responseEntityIdx] = DescribeClientQuotasEntity{\n\t\t\t\tEntityType: responseEntity.EntityType,\n\t\t\t\tEntityName: responseEntity.EntityName,\n\t\t\t}\n\t\t}\n\n\t\tresponseValues := make([]DescribeClientQuotasValue, len(responseEntry.Values))\n\t\tfor responseValueIdx, responseValue := range responseEntry.Values {\n\t\t\tresponseValues[responseValueIdx] = DescribeClientQuotasValue{\n\t\t\t\tKey: responseValue.Key,\n\t\t\t\tValue: responseValue.Value,\n\t\t\t}\n\t\t}\n\t\tresponseEntries[responseEntryIdx] = DescribeClientQuotasResponseQuotas{\n\t\t\tEntities: responseEntities,\n\t\t\tValues: responseValues,\n\t\t}\n\t}\n\tret := &DescribeClientQuotasResponse{\n\t\tThrottle: time.Duration(res.ThrottleTimeMs),\n\t\tEntries: responseEntries,\n\t}\n\n\treturn ret, nil\n}\n" }, { "path": "describeconfigs.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/describeconfigs\"\n)\n\n// DescribeConfigsRequest represents a request sent to a kafka broker to describe configs.\ntype DescribeConfigsRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// List of resources to get details for.\n\tResources []DescribeConfigRequestResource\n\n\t// Ignored if API version is less than v1\n\tIncludeSynonyms bool\n\n\t// Ignored if API version is less than v3\n\tIncludeDocumentation bool\n}\n\ntype DescribeConfigRequestResource struct {\n\t// Resource Type\n\tResourceType ResourceType\n\n\t// Resource Name\n\tResourceName string\n\n\t// ConfigNames is a list of configurations to update.\n\tConfigNames []string\n}\n\n// DescribeConfigsResponse represents a response from a kafka broker to a describe config request.\ntype DescribeConfigsResponse struct {\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// Resources\n\tResources []DescribeConfigResponseResource\n}\n\n// DescribeConfigResponseResource.\ntype DescribeConfigResponseResource struct {\n\t// Resource Type\n\tResourceType int8\n\n\t// Resource Name\n\tResourceName string\n\n\t// Error\n\tError error\n\n\t// ConfigEntries\n\tConfigEntries []DescribeConfigResponseConfigEntry\n}\n\n// DescribeConfigResponseConfigEntry.\ntype DescribeConfigResponseConfigEntry struct {\n\tConfigName string\n\tConfigValue string\n\tReadOnly bool\n\n\t// Ignored if API version is greater than v0\n\tIsDefault bool\n\n\t// Ignored if API version is less than v1\n\tConfigSource int8\n\n\tIsSensitive bool\n\n\t// Ignored if API version is less than v1\n\tConfigSynonyms []DescribeConfigResponseConfigSynonym\n\n\t// Ignored if API version is less than v3\n\tConfigType int8\n\n\t// Ignored if API version is less than v3\n\tConfigDocumentation string\n}\n\n// DescribeConfigResponseConfigSynonym.\ntype DescribeConfigResponseConfigSynonym struct {\n\t// Ignored if API version is less than v1\n\tConfigName string\n\n\t// Ignored if API version is less than v1\n\tConfigValue string\n\n\t// Ignored if API version is less than v1\n\tConfigSource int8\n}\n\n// DescribeConfigs sends a config altering request to a kafka broker and returns the\n// response.\nfunc (c *Client) DescribeConfigs(ctx context.Context, req *DescribeConfigsRequest) (*DescribeConfigsResponse, error) {\n\tresources := make([]describeconfigs.RequestResource, len(req.Resources))\n\n\tfor i, t := range req.Resources {\n\t\tresources[i] = describeconfigs.RequestResource{\n\t\t\tResourceType: int8(t.ResourceType),\n\t\t\tResourceName: t.ResourceName,\n\t\t\tConfigNames: t.ConfigNames,\n\t\t}\n\t}\n\n\tm, err := c.roundTrip(ctx, req.Addr, &describeconfigs.Request{\n\t\tResources: resources,\n\t\tIncludeSynonyms: req.IncludeSynonyms,\n\t\tIncludeDocumentation: req.IncludeDocumentation,\n\t})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).DescribeConfigs: %w\", err)\n\t}\n\n\tres := m.(*describeconfigs.Response)\n\tret := &DescribeConfigsResponse{\n\t\tThrottle: makeDuration(res.ThrottleTimeMs),\n\t\tResources: make([]DescribeConfigResponseResource, len(res.Resources)),\n\t}\n\n\tfor i, t := range res.Resources {\n\n\t\tconfigEntries := make([]DescribeConfigResponseConfigEntry, len(t.ConfigEntries))\n\t\tfor j, v := range t.ConfigEntries {\n\n\t\t\tconfigSynonyms := make([]DescribeConfigResponseConfigSynonym, len(v.ConfigSynonyms))\n\t\t\tfor k, cs := range v.ConfigSynonyms {\n\t\t\t\tconfigSynonyms[k] = DescribeConfigResponseConfigSynonym{\n\t\t\t\t\tConfigName: cs.ConfigName,\n\t\t\t\t\tConfigValue: cs.ConfigValue,\n\t\t\t\t\tConfigSource: cs.ConfigSource,\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tconfigEntries[j] = DescribeConfigResponseConfigEntry{\n\t\t\t\tConfigName: v.ConfigName,\n\t\t\t\tConfigValue: v.ConfigValue,\n\t\t\t\tReadOnly: v.ReadOnly,\n\t\t\t\tConfigSource: v.ConfigSource,\n\t\t\t\tIsDefault: v.IsDefault,\n\t\t\t\tIsSensitive: v.IsSensitive,\n\t\t\t\tConfigSynonyms: configSynonyms,\n\t\t\t\tConfigType: v.ConfigType,\n\t\t\t\tConfigDocumentation: v.ConfigDocumentation,\n\t\t\t}\n\t\t}\n\n\t\tret.Resources[i] = DescribeConfigResponseResource{\n\t\t\tResourceType: t.ResourceType,\n\t\t\tResourceName: t.ResourceName,\n\t\t\tError: makeError(t.ErrorCode, t.ErrorMessage),\n\t\t\tConfigEntries: configEntries,\n\t\t}\n\t}\n\n\treturn ret, nil\n}\n" }, { "path": "describeconfigs_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestClientDescribeConfigs(t *testing.T) {\n\tif !ktesting.KafkaIsAtLeast(\"0.11.0\") {\n\t\treturn\n\t}\n\n\tconst (\n\t\tMaxMessageBytes = \"max.message.bytes\"\n\t\tMaxMessageBytesValue = \"200000\"\n\t)\n\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\ttopic := makeTopic()\n\tcreateTopic(t, topic, 1)\n\tdefer deleteTopic(t, topic)\n\n\t_, err := client.AlterConfigs(context.Background(), &AlterConfigsRequest{\n\t\tResources: []AlterConfigRequestResource{{\n\t\t\tResourceType: ResourceTypeTopic,\n\t\t\tResourceName: topic,\n\t\t\tConfigs: []AlterConfigRequestConfig{{\n\t\t\t\tName: MaxMessageBytes,\n\t\t\t\tValue: MaxMessageBytesValue,\n\t\t\t},\n\t\t\t},\n\t\t}},\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tdescribeResp, err := client.DescribeConfigs(context.Background(), &DescribeConfigsRequest{\n\t\tResources: []DescribeConfigRequestResource{{\n\t\t\tResourceType: ResourceTypeTopic,\n\t\t\tResourceName: topic,\n\t\t\tConfigNames: []string{MaxMessageBytes},\n\t\t}},\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmaxMessageBytesValue := \"0\"\n\tfor _, resource := range describeResp.Resources {\n\t\tif resource.ResourceType == int8(ResourceTypeTopic) && resource.ResourceName == topic {\n\t\t\tfor _, entry := range resource.ConfigEntries {\n\t\t\t\tif entry.ConfigName == MaxMessageBytes {\n\t\t\t\t\tmaxMessageBytesValue = entry.ConfigValue\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\tassert.Equal(t, maxMessageBytesValue, MaxMessageBytesValue)\n}\n" }, { "path": "describegroups.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\n\t\"github.com/segmentio/kafka-go/protocol/describegroups\"\n)\n\n// DescribeGroupsRequest is a request to the DescribeGroups API.\ntype DescribeGroupsRequest struct {\n\t// Addr is the address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// GroupIDs is a slice of groups to get details for.\n\tGroupIDs []string\n}\n\n// DescribeGroupsResponse is a response from the DescribeGroups API.\ntype DescribeGroupsResponse struct {\n\t// Groups is a slice of details for the requested groups.\n\tGroups []DescribeGroupsResponseGroup\n}\n\n// DescribeGroupsResponseGroup contains the response details for a single group.\ntype DescribeGroupsResponseGroup struct {\n\t// Error is set to a non-nil value if there was an error fetching the details\n\t// for this group.\n\tError error\n\n\t// GroupID is the ID of the group.\n\tGroupID string\n\n\t// GroupState is a description of the group state.\n\tGroupState string\n\n\t// Members contains details about each member of the group.\n\tMembers []DescribeGroupsResponseMember\n}\n\n// MemberInfo represents the membership information for a single group member.\ntype DescribeGroupsResponseMember struct {\n\t// MemberID is the ID of the group member.\n\tMemberID string\n\n\t// ClientID is the ID of the client that the group member is using.\n\tClientID string\n\n\t// ClientHost is the host of the client that the group member is connecting from.\n\tClientHost string\n\n\t// MemberMetadata contains metadata about this group member.\n\tMemberMetadata DescribeGroupsResponseMemberMetadata\n\n\t// MemberAssignments contains the topic partitions that this member is assigned to.\n\tMemberAssignments DescribeGroupsResponseAssignments\n}\n\n// GroupMemberMetadata stores metadata associated with a group member.\ntype DescribeGroupsResponseMemberMetadata struct {\n\t// Version is the version of the metadata.\n\tVersion int\n\n\t// Topics is the list of topics that the member is assigned to.\n\tTopics []string\n\n\t// UserData is the user data for the member.\n\tUserData []byte\n\n\t// OwnedPartitions contains the partitions owned by this group member; only set if\n\t// consumers are using a cooperative rebalancing assignor protocol.\n\tOwnedPartitions []DescribeGroupsResponseMemberMetadataOwnedPartition\n}\n\ntype DescribeGroupsResponseMemberMetadataOwnedPartition struct {\n\t// Topic is the name of the topic.\n\tTopic string\n\n\t// Partitions is the partitions that are owned by the group in the topic.\n\tPartitions []int\n}\n\n// GroupMemberAssignmentsInfo stores the topic partition assignment data for a group member.\ntype DescribeGroupsResponseAssignments struct {\n\t// Version is the version of the assignments data.\n\tVersion int\n\n\t// Topics contains the details of the partition assignments for each topic.\n\tTopics []GroupMemberTopic\n\n\t// UserData is the user data for the member.\n\tUserData []byte\n}\n\n// GroupMemberTopic is a mapping from a topic to a list of partitions in the topic. It is used\n// to represent the topic partitions that have been assigned to a group member.\ntype GroupMemberTopic struct {\n\t// Topic is the name of the topic.\n\tTopic string\n\n\t// Partitions is a slice of partition IDs that this member is assigned to in the topic.\n\tPartitions []int\n}\n\n// DescribeGroups calls the Kafka DescribeGroups API to get information about one or more\n// consumer groups. See https://kafka.apache.org/protocol#The_Messages_DescribeGroups for\n// more information.\nfunc (c *Client) DescribeGroups(\n\tctx context.Context,\n\treq *DescribeGroupsRequest,\n) (*DescribeGroupsResponse, error) {\n\tprotoResp, err := c.roundTrip(\n\t\tctx,\n\t\treq.Addr,\n\t\t&describegroups.Request{\n\t\t\tGroups: req.GroupIDs,\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tapiResp := protoResp.(*describegroups.Response)\n\tresp := &DescribeGroupsResponse{}\n\n\tfor _, apiGroup := range apiResp.Groups {\n\t\tgroup := DescribeGroupsResponseGroup{\n\t\t\tError: makeError(apiGroup.ErrorCode, \"\"),\n\t\t\tGroupID: apiGroup.GroupID,\n\t\t\tGroupState: apiGroup.GroupState,\n\t\t}\n\n\t\tfor _, member := range apiGroup.Members {\n\t\t\tdecodedMetadata, err := decodeMemberMetadata(member.MemberMetadata)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tdecodedAssignments, err := decodeMemberAssignments(member.MemberAssignment)\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\n\t\t\tgroup.Members = append(group.Members, DescribeGroupsResponseMember{\n\t\t\t\tMemberID: member.MemberID,\n\t\t\t\tClientID: member.ClientID,\n\t\t\t\tClientHost: member.ClientHost,\n\t\t\t\tMemberAssignments: decodedAssignments,\n\t\t\t\tMemberMetadata: decodedMetadata,\n\t\t\t})\n\t\t}\n\t\tresp.Groups = append(resp.Groups, group)\n\t}\n\n\treturn resp, nil\n}\n\n// decodeMemberMetadata converts raw metadata bytes to a\n// DescribeGroupsResponseMemberMetadata struct.\n//\n// See https://github.com/apache/kafka/blob/2.4/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java#L49\n// for protocol details.\nfunc decodeMemberMetadata(rawMetadata []byte) (DescribeGroupsResponseMemberMetadata, error) {\n\tmm := DescribeGroupsResponseMemberMetadata{}\n\n\tif len(rawMetadata) == 0 {\n\t\treturn mm, nil\n\t}\n\n\tbuf := bytes.NewBuffer(rawMetadata)\n\tbufReader := bufio.NewReader(buf)\n\tremain := len(rawMetadata)\n\n\tvar err error\n\tvar version16 int16\n\n\tif remain, err = readInt16(bufReader, remain, &version16); err != nil {\n\t\treturn mm, err\n\t}\n\tmm.Version = int(version16)\n\n\tif remain, err = readStringArray(bufReader, remain, &mm.Topics); err != nil {\n\t\treturn mm, err\n\t}\n\tif remain, err = readBytes(bufReader, remain, &mm.UserData); err != nil {\n\t\treturn mm, err\n\t}\n\n\tif mm.Version == 1 && remain > 0 {\n\t\tfn := func(r *bufio.Reader, size int) (fnRemain int, fnErr error) {\n\t\t\top := DescribeGroupsResponseMemberMetadataOwnedPartition{}\n\t\t\tif fnRemain, fnErr = readString(r, size, &op.Topic); fnErr != nil {\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tps := []int32{}\n\t\t\tif fnRemain, fnErr = readInt32Array(r, fnRemain, &ps); fnErr != nil {\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tfor _, p := range ps {\n\t\t\t\top.Partitions = append(op.Partitions, int(p))\n\t\t\t}\n\n\t\t\tmm.OwnedPartitions = append(mm.OwnedPartitions, op)\n\t\t\treturn\n\t\t}\n\n\t\tif remain, err = readArrayWith(bufReader, remain, fn); err != nil {\n\t\t\treturn mm, err\n\t\t}\n\t}\n\n\tif remain != 0 {\n\t\treturn mm, fmt.Errorf(\"Got non-zero number of bytes remaining: %d\", remain)\n\t}\n\n\treturn mm, nil\n}\n\n// decodeMemberAssignments converts raw assignment bytes to a DescribeGroupsResponseAssignments\n// struct.\n//\n// See https://github.com/apache/kafka/blob/2.4/clients/src/main/java/org/apache/kafka/clients/consumer/internals/ConsumerProtocol.java#L49\n// for protocol details.\nfunc decodeMemberAssignments(rawAssignments []byte) (DescribeGroupsResponseAssignments, error) {\n\tma := DescribeGroupsResponseAssignments{}\n\n\tif len(rawAssignments) == 0 {\n\t\treturn ma, nil\n\t}\n\n\tbuf := bytes.NewBuffer(rawAssignments)\n\tbufReader := bufio.NewReader(buf)\n\tremain := len(rawAssignments)\n\n\tvar err error\n\tvar version16 int16\n\n\tif remain, err = readInt16(bufReader, remain, &version16); err != nil {\n\t\treturn ma, err\n\t}\n\tma.Version = int(version16)\n\n\tfn := func(r *bufio.Reader, size int) (fnRemain int, fnErr error) {\n\t\titem := GroupMemberTopic{}\n\n\t\tif fnRemain, fnErr = readString(r, size, &item.Topic); fnErr != nil {\n\t\t\treturn\n\t\t}\n\n\t\tpartitions := []int32{}\n\n\t\tif fnRemain, fnErr = readInt32Array(r, fnRemain, &partitions); fnErr != nil {\n\t\t\treturn\n\t\t}\n\t\tfor _, partition := range partitions {\n\t\t\titem.Partitions = append(item.Partitions, int(partition))\n\t\t}\n\n\t\tma.Topics = append(ma.Topics, item)\n\t\treturn\n\t}\n\tif remain, err = readArrayWith(bufReader, remain, fn); err != nil {\n\t\treturn ma, err\n\t}\n\n\tif remain, err = readBytes(bufReader, remain, &ma.UserData); err != nil {\n\t\treturn ma, err\n\t}\n\n\tif remain != 0 {\n\t\treturn ma, fmt.Errorf(\"Got non-zero number of bytes remaining: %d\", remain)\n\t}\n\n\treturn ma, nil\n}\n\n// readInt32Array reads an array of int32s. It's adapted from the implementation of\n// readStringArray.\nfunc readInt32Array(r *bufio.Reader, sz int, v *[]int32) (remain int, err error) {\n\tvar content []int32\n\tfn := func(r *bufio.Reader, size int) (fnRemain int, fnErr error) {\n\t\tvar value int32\n\t\tif fnRemain, fnErr = readInt32(r, size, &value); fnErr != nil {\n\t\t\treturn\n\t\t}\n\t\tcontent = append(content, value)\n\t\treturn\n\t}\n\tif remain, err = readArrayWith(r, sz, fn); err != nil {\n\t\treturn\n\t}\n\n\t*v = content\n\treturn\n}\n" }, { "path": "describegroups_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"os\"\n\t\"reflect\"\n\t\"sort\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestClientDescribeGroups(t *testing.T) {\n\tif os.Getenv(\"KAFKA_VERSION\") == \"2.3.1\" {\n\t\t// There's a bug in 2.3.1 that causes the MemberMetadata to be in the wrong format and thus\n\t\t// leads to an error when decoding the DescribeGroupsResponse.\n\t\t//\n\t\t// See https://issues.apache.org/jira/browse/KAFKA-9150 for details.\n\t\tt.Skip(\"Skipping because kafka version is 2.3.1\")\n\t}\n\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\ttopic := makeTopic()\n\tgid := fmt.Sprintf(\"%s-test-group\", topic)\n\n\tcreateTopic(t, topic, 2)\n\tdefer deleteTopic(t, topic)\n\n\tw := newTestWriter(WriterConfig{\n\t\tTopic: topic,\n\t})\n\n\tctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)\n\tdefer cancel()\n\n\terr := w.WriteMessages(\n\t\tctx,\n\t\tMessage{\n\t\t\tKey: []byte(\"key\"),\n\t\t\tValue: []byte(\"value\"),\n\t\t},\n\t)\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tr := NewReader(ReaderConfig{\n\t\tBrokers: []string{\"localhost:9092\"},\n\t\tTopic: topic,\n\t\tGroupID: gid,\n\t\tMinBytes: 10,\n\t\tMaxBytes: 1000,\n\t})\n\t_, err = r.ReadMessage(ctx)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tresp, err := client.DescribeGroups(\n\t\tctx,\n\t\t&DescribeGroupsRequest{\n\t\t\tGroupIDs: []string{gid},\n\t\t},\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(resp.Groups) != 1 {\n\t\tt.Fatal(\n\t\t\t\"Unexpected number of groups returned\",\n\t\t\t\"expected\", 1,\n\t\t\t\"got\", len(resp.Groups),\n\t\t)\n\t}\n\tg := resp.Groups[0]\n\tif g.Error != nil {\n\t\tt.Error(\n\t\t\t\"Wrong error in group response\",\n\t\t\t\"expected\", nil,\n\t\t\t\"got\", g.Error,\n\t\t)\n\t}\n\n\tif g.GroupID != gid {\n\t\tt.Error(\n\t\t\t\"Wrong groupID\",\n\t\t\t\"expected\", gid,\n\t\t\t\"got\", g.GroupID,\n\t\t)\n\t}\n\n\tif len(g.Members) != 1 {\n\t\tt.Fatal(\n\t\t\t\"Wrong group members length\",\n\t\t\t\"expected\", 1,\n\t\t\t\"got\", len(g.Members),\n\t\t)\n\t}\n\tif len(g.Members[0].MemberAssignments.Topics) != 1 {\n\t\tt.Fatal(\n\t\t\t\"Wrong topics length\",\n\t\t\t\"expected\", 1,\n\t\t\t\"got\", len(g.Members[0].MemberAssignments.Topics),\n\t\t)\n\t}\n\tmt := g.Members[0].MemberAssignments.Topics[0]\n\tif mt.Topic != topic {\n\t\tt.Error(\n\t\t\t\"Wrong member assignment topic\",\n\t\t\t\"expected\", topic,\n\t\t\t\"got\", mt.Topic,\n\t\t)\n\t}\n\n\t// Partitions can be in any order, sort them\n\tsort.Slice(mt.Partitions, func(a, b int) bool {\n\t\treturn mt.Partitions[a] < mt.Partitions[b]\n\t})\n\n\tif !reflect.DeepEqual([]int{0, 1}, mt.Partitions) {\n\t\tt.Error(\n\t\t\t\"Wrong member assignment partitions\",\n\t\t\t\"expected\", []int{0, 1},\n\t\t\t\"got\", mt.Partitions,\n\t\t)\n\t}\n}\n" }, { "path": "describeuserscramcredentials.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/describeuserscramcredentials\"\n)\n\n// DescribeUserScramCredentialsRequest represents a request sent to a kafka broker to\n// describe user scram credentials.\ntype DescribeUserScramCredentialsRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// List of Scram users to describe\n\tUsers []UserScramCredentialsUser\n}\n\ntype UserScramCredentialsUser struct {\n\tName string\n}\n\n// DescribeUserScramCredentialsResponse represents a response from a kafka broker to a describe user\n// credentials request.\ntype DescribeUserScramCredentialsResponse struct {\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// Top level error that occurred while attempting to describe\n\t// the user scram credentials.\n\t//\n\t// The errors contain the kafka error code. Programs may use the standard\n\t// errors.Is function to test the error against kafka error codes.\n\tError error\n\n\t// List of described user scram credentials.\n\tResults []DescribeUserScramCredentialsResponseResult\n}\n\ntype DescribeUserScramCredentialsResponseResult struct {\n\tUser string\n\tCredentialInfos []DescribeUserScramCredentialsCredentialInfo\n\tError error\n}\n\ntype DescribeUserScramCredentialsCredentialInfo struct {\n\tMechanism ScramMechanism\n\tIterations int\n}\n\n// DescribeUserScramCredentials sends a user scram credentials describe request to a kafka broker and returns\n// the response.\nfunc (c *Client) DescribeUserScramCredentials(ctx context.Context, req *DescribeUserScramCredentialsRequest) (*DescribeUserScramCredentialsResponse, error) {\n\tusers := make([]describeuserscramcredentials.RequestUser, len(req.Users))\n\n\tfor userIdx, user := range req.Users {\n\t\tusers[userIdx] = describeuserscramcredentials.RequestUser{\n\t\t\tName: user.Name,\n\t\t}\n\t}\n\n\tm, err := c.roundTrip(ctx, req.Addr, &describeuserscramcredentials.Request{\n\t\tUsers: users,\n\t})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).DescribeUserScramCredentials: %w\", err)\n\t}\n\n\tres := m.(*describeuserscramcredentials.Response)\n\tresponseResults := make([]DescribeUserScramCredentialsResponseResult, len(res.Results))\n\n\tfor responseIdx, responseResult := range res.Results {\n\t\tcredentialInfos := make([]DescribeUserScramCredentialsCredentialInfo, len(responseResult.CredentialInfos))\n\n\t\tfor credentialInfoIdx, credentialInfo := range responseResult.CredentialInfos {\n\t\t\tcredentialInfos[credentialInfoIdx] = DescribeUserScramCredentialsCredentialInfo{\n\t\t\t\tMechanism: ScramMechanism(credentialInfo.Mechanism),\n\t\t\t\tIterations: int(credentialInfo.Iterations),\n\t\t\t}\n\t\t}\n\t\tresponseResults[responseIdx] = DescribeUserScramCredentialsResponseResult{\n\t\t\tUser: responseResult.User,\n\t\t\tCredentialInfos: credentialInfos,\n\t\t\tError: makeError(responseResult.ErrorCode, responseResult.ErrorMessage),\n\t\t}\n\t}\n\tret := &DescribeUserScramCredentialsResponse{\n\t\tThrottle: makeDuration(res.ThrottleTimeMs),\n\t\tError: makeError(res.ErrorCode, res.ErrorMessage),\n\t\tResults: responseResults,\n\t}\n\n\treturn ret, nil\n}\n" }, { "path": "describeuserscramcredentials_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"testing\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestDescribeUserScramCredentials(t *testing.T) {\n\t// https://issues.apache.org/jira/browse/KAFKA-10259\n\tif !ktesting.KafkaIsAtLeast(\"2.7.0\") {\n\t\treturn\n\t}\n\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\tname := makeTopic()\n\n\tcreateRes, err := client.AlterUserScramCredentials(context.Background(), &AlterUserScramCredentialsRequest{\n\t\tUpsertions: []UserScramCredentialsUpsertion{\n\t\t\t{\n\t\t\t\tName: name,\n\t\t\t\tMechanism: ScramMechanismSha512,\n\t\t\t\tIterations: 15000,\n\t\t\t\tSalt: []byte(\"my-salt\"),\n\t\t\t\tSaltedPassword: []byte(\"my-salted-password\"),\n\t\t\t},\n\t\t},\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(createRes.Results) != 1 {\n\t\tt.Fatalf(\"expected 1 createResult; got %d\", len(createRes.Results))\n\t}\n\n\tif createRes.Results[0].User != name {\n\t\tt.Fatalf(\"expected createResult with user: %s, got %s\", name, createRes.Results[0].User)\n\t}\n\n\tif createRes.Results[0].Error != nil {\n\t\tt.Fatalf(\"didn't expect an error in createResult, got %v\", createRes.Results[0].Error)\n\t}\n\n\tdescribeCreationRes, err := client.DescribeUserScramCredentials(context.Background(), &DescribeUserScramCredentialsRequest{\n\t\tUsers: []UserScramCredentialsUser{\n\t\t\t{\n\t\t\t\tName: name,\n\t\t\t},\n\t\t},\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\texpectedCreation := DescribeUserScramCredentialsResponse{\n\t\tThrottle: makeDuration(0),\n\t\tError: makeError(0, \"\"),\n\t\tResults: []DescribeUserScramCredentialsResponseResult{\n\t\t\t{\n\t\t\t\tUser: name,\n\t\t\t\tCredentialInfos: []DescribeUserScramCredentialsCredentialInfo{\n\t\t\t\t\t{\n\t\t\t\t\t\tMechanism: ScramMechanismSha512,\n\t\t\t\t\t\tIterations: 15000,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tError: makeError(0, \"\"),\n\t\t\t},\n\t\t},\n\t}\n\n\tassert.Equal(t, expectedCreation, *describeCreationRes)\n\n\tdeleteRes, err := client.AlterUserScramCredentials(context.Background(), &AlterUserScramCredentialsRequest{\n\t\tDeletions: []UserScramCredentialsDeletion{\n\t\t\t{\n\t\t\t\tName: name,\n\t\t\t\tMechanism: ScramMechanismSha512,\n\t\t\t},\n\t\t},\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(deleteRes.Results) != 1 {\n\t\tt.Fatalf(\"expected 1 deleteResult; got %d\", len(deleteRes.Results))\n\t}\n\n\tif deleteRes.Results[0].User != name {\n\t\tt.Fatalf(\"expected deleteResult with user: %s, got %s\", name, deleteRes.Results[0].User)\n\t}\n\n\tif deleteRes.Results[0].Error != nil {\n\t\tt.Fatalf(\"didn't expect an error in deleteResult, got %v\", deleteRes.Results[0].Error)\n\t}\n\n\tdescribeDeletionRes, err := client.DescribeUserScramCredentials(context.Background(), &DescribeUserScramCredentialsRequest{\n\t\tUsers: []UserScramCredentialsUser{\n\t\t\t{\n\t\t\t\tName: name,\n\t\t\t},\n\t\t},\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif !errors.Is(describeDeletionRes.Error, makeError(0, \"\")) {\n\t\tt.Fatalf(\"didn't expect a top level error on describe results after deletion, got %v\", describeDeletionRes.Error)\n\t}\n\n\tif len(describeDeletionRes.Results) != 1 {\n\t\tt.Fatalf(\"expected one describe results after deletion, got %d describe results\", len(describeDeletionRes.Results))\n\t}\n\n\tresult := describeDeletionRes.Results[0]\n\n\tif result.User != name {\n\t\tt.Fatalf(\"expected describeResult with user: %s, got %s\", name, result.User)\n\t}\n\n\tif len(result.CredentialInfos) != 0 {\n\t\tt.Fatalf(\"didn't expect describeResult credential infos, got %v\", result.CredentialInfos)\n\t}\n\n\tif !errors.Is(result.Error, ResourceNotFound) {\n\t\tt.Fatalf(\"expected describeResult resourcenotfound error, got %s\", result.Error)\n\t}\n}\n" }, { "path": "dialer.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/sasl\"\n)\n\n// The Dialer type mirrors the net.Dialer API but is designed to open kafka\n// connections instead of raw network connections.\ntype Dialer struct {\n\t// Unique identifier for client connections established by this Dialer.\n\tClientID string\n\n\t// Optionally specifies the function that the dialer uses to establish\n\t// network connections. If nil, net.(*Dialer).DialContext is used instead.\n\t//\n\t// When DialFunc is set, LocalAddr, DualStack, FallbackDelay, and KeepAlive\n\t// are ignored.\n\tDialFunc func(ctx context.Context, network string, address string) (net.Conn, error)\n\n\t// Timeout is the maximum amount of time a dial will wait for a connect to\n\t// complete. If Deadline is also set, it may fail earlier.\n\t//\n\t// The default is no timeout.\n\t//\n\t// When dialing a name with multiple IP addresses, the timeout may be\n\t// divided between them.\n\t//\n\t// With or without a timeout, the operating system may impose its own\n\t// earlier timeout. For instance, TCP timeouts are often around 3 minutes.\n\tTimeout time.Duration\n\n\t// Deadline is the absolute point in time after which dials will fail.\n\t// If Timeout is set, it may fail earlier.\n\t// Zero means no deadline, or dependent on the operating system as with the\n\t// Timeout option.\n\tDeadline time.Time\n\n\t// LocalAddr is the local address to use when dialing an address.\n\t// The address must be of a compatible type for the network being dialed.\n\t// If nil, a local address is automatically chosen.\n\tLocalAddr net.Addr\n\n\t// DualStack enables RFC 6555-compliant \"Happy Eyeballs\" dialing when the\n\t// network is \"tcp\" and the destination is a host name with both IPv4 and\n\t// IPv6 addresses. This allows a client to tolerate networks where one\n\t// address family is silently broken.\n\tDualStack bool\n\n\t// FallbackDelay specifies the length of time to wait before spawning a\n\t// fallback connection, when DualStack is enabled.\n\t// If zero, a default delay of 300ms is used.\n\tFallbackDelay time.Duration\n\n\t// KeepAlive specifies the keep-alive period for an active network\n\t// connection.\n\t// If zero, keep-alives are not enabled. Network protocols that do not\n\t// support keep-alives ignore this field.\n\tKeepAlive time.Duration\n\n\t// Resolver optionally gives a hook to convert the broker address into an\n\t// alternate host or IP address which is useful for custom service discovery.\n\t// If a custom resolver returns any possible hosts, the first one will be\n\t// used and the original discarded. If a port number is included with the\n\t// resolved host, it will only be used if a port number was not previously\n\t// specified. If no port is specified or resolved, the default of 9092 will be\n\t// used.\n\tResolver Resolver\n\n\t// TLS enables Dialer to open secure connections. If nil, standard net.Conn\n\t// will be used.\n\tTLS *tls.Config\n\n\t// SASLMechanism configures the Dialer to use SASL authentication. If nil,\n\t// no authentication will be performed.\n\tSASLMechanism sasl.Mechanism\n\n\t// The transactional id to use for transactional delivery. Idempotent\n\t// deliver should be enabled if transactional id is configured.\n\t// For more details look at transactional.id description here: http://kafka.apache.org/documentation.html#producerconfigs\n\t// Empty string means that the connection will be non-transactional.\n\tTransactionalID string\n}\n\n// Dial connects to the address on the named network.\nfunc (d *Dialer) Dial(network string, address string) (*Conn, error) {\n\treturn d.DialContext(context.Background(), network, address)\n}\n\n// DialContext connects to the address on the named network using the provided\n// context.\n//\n// The provided Context must be non-nil. If the context expires before the\n// connection is complete, an error is returned. Once successfully connected,\n// any expiration of the context will not affect the connection.\n//\n// When using TCP, and the host in the address parameter resolves to multiple\n// network addresses, any dial timeout (from d.Timeout or ctx) is spread over\n// each consecutive dial, such that each is given an appropriate fraction of the\n// time to connect. For example, if a host has 4 IP addresses and the timeout is\n// 1 minute, the connect to each single address will be given 15 seconds to\n// complete before trying the next one.\nfunc (d *Dialer) DialContext(ctx context.Context, network string, address string) (*Conn, error) {\n\treturn d.connect(\n\t\tctx,\n\t\tnetwork,\n\t\taddress,\n\t\tConnConfig{\n\t\t\tClientID: d.ClientID,\n\t\t\tTransactionalID: d.TransactionalID,\n\t\t},\n\t)\n}\n\n// DialLeader opens a connection to the leader of the partition for a given\n// topic.\n//\n// The address given to the DialContext method may not be the one that the\n// connection will end up being established to, because the dialer will lookup\n// the partition leader for the topic and return a connection to that server.\n// The original address is only used as a mechanism to discover the\n// configuration of the kafka cluster that we're connecting to.\nfunc (d *Dialer) DialLeader(ctx context.Context, network string, address string, topic string, partition int) (*Conn, error) {\n\tp, err := d.LookupPartition(ctx, network, address, topic, partition)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn d.DialPartition(ctx, network, address, p)\n}\n\n// DialPartition opens a connection to the leader of the partition specified by partition\n// descriptor. It's strongly advised to use descriptor of the partition that comes out of\n// functions LookupPartition or LookupPartitions.\nfunc (d *Dialer) DialPartition(ctx context.Context, network string, address string, partition Partition) (*Conn, error) {\n\treturn d.connect(ctx, network, net.JoinHostPort(partition.Leader.Host, strconv.Itoa(partition.Leader.Port)), ConnConfig{\n\t\tClientID: d.ClientID,\n\t\tTopic: partition.Topic,\n\t\tPartition: partition.ID,\n\t\tBroker: partition.Leader.ID,\n\t\tRack: partition.Leader.Rack,\n\t\tTransactionalID: d.TransactionalID,\n\t})\n}\n\n// LookupLeader searches for the kafka broker that is the leader of the\n// partition for a given topic, returning a Broker value representing it.\nfunc (d *Dialer) LookupLeader(ctx context.Context, network string, address string, topic string, partition int) (Broker, error) {\n\tp, err := d.LookupPartition(ctx, network, address, topic, partition)\n\treturn p.Leader, err\n}\n\n// LookupPartition searches for the description of specified partition id.\nfunc (d *Dialer) LookupPartition(ctx context.Context, network string, address string, topic string, partition int) (Partition, error) {\n\tc, err := d.DialContext(ctx, network, address)\n\tif err != nil {\n\t\treturn Partition{}, err\n\t}\n\tdefer c.Close()\n\n\tbrkch := make(chan Partition, 1)\n\terrch := make(chan error, 1)\n\n\tgo func() {\n\t\tfor attempt := 0; true; attempt++ {\n\t\t\tif attempt != 0 {\n\t\t\t\tif !sleep(ctx, backoff(attempt, 100*time.Millisecond, 10*time.Second)) {\n\t\t\t\t\terrch <- ctx.Err()\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tpartitions, err := c.ReadPartitions(topic)\n\t\t\tif err != nil {\n\t\t\t\tif isTemporary(err) {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\terrch <- err\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tfor _, p := range partitions {\n\t\t\t\tif p.ID == partition {\n\t\t\t\t\tbrkch <- p\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\terrch <- UnknownTopicOrPartition\n\t}()\n\n\tvar prt Partition\n\tselect {\n\tcase prt = <-brkch:\n\tcase err = <-errch:\n\tcase <-ctx.Done():\n\t\terr = ctx.Err()\n\t}\n\treturn prt, err\n}\n\n// LookupPartitions returns the list of partitions that exist for the given topic.\nfunc (d *Dialer) LookupPartitions(ctx context.Context, network string, address string, topic string) ([]Partition, error) {\n\tconn, err := d.DialContext(ctx, network, address)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tdefer conn.Close()\n\n\tprtch := make(chan []Partition, 1)\n\terrch := make(chan error, 1)\n\n\tgo func() {\n\t\tif prt, err := conn.ReadPartitions(topic); err != nil {\n\t\t\terrch <- err\n\t\t} else {\n\t\t\tprtch <- prt\n\t\t}\n\t}()\n\n\tvar prt []Partition\n\tselect {\n\tcase prt = <-prtch:\n\tcase err = <-errch:\n\tcase <-ctx.Done():\n\t\terr = ctx.Err()\n\t}\n\treturn prt, err\n}\n\n// connectTLS returns a tls.Conn that has already completed the Handshake.\nfunc (d *Dialer) connectTLS(ctx context.Context, conn net.Conn, config *tls.Config) (tlsConn *tls.Conn, err error) {\n\ttlsConn = tls.Client(conn, config)\n\terrch := make(chan error)\n\n\tgo func() {\n\t\tdefer close(errch)\n\t\terrch <- tlsConn.Handshake()\n\t}()\n\n\tselect {\n\tcase <-ctx.Done():\n\t\tconn.Close()\n\t\ttlsConn.Close()\n\t\t<-errch // ignore possible error from Handshake\n\t\terr = ctx.Err()\n\n\tcase err = <-errch:\n\t}\n\n\treturn\n}\n\n// connect opens a socket connection to the broker, wraps it to create a\n// kafka connection, and performs SASL authentication if configured to do so.\nfunc (d *Dialer) connect(ctx context.Context, network, address string, connCfg ConnConfig) (*Conn, error) {\n\tif d.Timeout != 0 {\n\t\tvar cancel context.CancelFunc\n\t\tctx, cancel = context.WithTimeout(ctx, d.Timeout)\n\t\tdefer cancel()\n\t}\n\n\tif !d.Deadline.IsZero() {\n\t\tvar cancel context.CancelFunc\n\t\tctx, cancel = context.WithDeadline(ctx, d.Deadline)\n\t\tdefer cancel()\n\t}\n\n\tc, err := d.dialContext(ctx, network, address)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to dial: %w\", err)\n\t}\n\n\tconn := NewConnWith(c, connCfg)\n\n\tif d.SASLMechanism != nil {\n\t\thost, port, err := splitHostPortNumber(address)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"could not determine host/port for SASL authentication: %w\", err)\n\t\t}\n\t\tmetadata := &sasl.Metadata{\n\t\t\tHost: host,\n\t\t\tPort: port,\n\t\t}\n\t\tif err := d.authenticateSASL(sasl.WithMetadata(ctx, metadata), conn); err != nil {\n\t\t\t_ = conn.Close()\n\t\t\treturn nil, fmt.Errorf(\"could not successfully authenticate to %s:%d with SASL: %w\", host, port, err)\n\t\t}\n\t}\n\n\treturn conn, nil\n}\n\n// authenticateSASL performs all of the required requests to authenticate this\n// connection. If any step fails, this function returns with an error. A nil\n// error indicates successful authentication.\n//\n// In case of error, this function *does not* close the connection. That is the\n// responsibility of the caller.\nfunc (d *Dialer) authenticateSASL(ctx context.Context, conn *Conn) error {\n\tif err := conn.saslHandshake(d.SASLMechanism.Name()); err != nil {\n\t\treturn fmt.Errorf(\"SASL handshake failed: %w\", err)\n\t}\n\n\tsess, state, err := d.SASLMechanism.Start(ctx)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"SASL authentication process could not be started: %w\", err)\n\t}\n\n\tfor completed := false; !completed; {\n\t\tchallenge, err := conn.saslAuthenticate(state)\n\t\tswitch {\n\t\tcase err == nil:\n\t\tcase errors.Is(err, io.EOF):\n\t\t\t// the broker may communicate a failed exchange by closing the\n\t\t\t// connection (esp. in the case where we're passing opaque sasl\n\t\t\t// data over the wire since there's no protocol info).\n\t\t\treturn SASLAuthenticationFailed\n\t\tdefault:\n\t\t\treturn err\n\t\t}\n\n\t\tcompleted, state, err = sess.Next(ctx, challenge)\n\t\tif err != nil {\n\t\t\treturn fmt.Errorf(\"SASL authentication process has failed: %w\", err)\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (d *Dialer) dialContext(ctx context.Context, network string, addr string) (net.Conn, error) {\n\taddress, err := lookupHost(ctx, addr, d.Resolver)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to resolve host: %w\", err)\n\t}\n\n\tdial := d.DialFunc\n\tif dial == nil {\n\t\tdial = (&net.Dialer{\n\t\t\tLocalAddr: d.LocalAddr,\n\t\t\tDualStack: d.DualStack,\n\t\t\tFallbackDelay: d.FallbackDelay,\n\t\t\tKeepAlive: d.KeepAlive,\n\t\t}).DialContext\n\t}\n\n\tconn, err := dial(ctx, network, address)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"failed to open connection to %s: %w\", address, err)\n\t}\n\n\tif d.TLS != nil {\n\t\tc := d.TLS\n\t\t// If no ServerName is set, infer the ServerName\n\t\t// from the hostname we're connecting to.\n\t\tif c.ServerName == \"\" {\n\t\t\tc = d.TLS.Clone()\n\t\t\t// Copied from tls.go in the standard library.\n\t\t\tcolonPos := strings.LastIndex(address, \":\")\n\t\t\tif colonPos == -1 {\n\t\t\t\tcolonPos = len(address)\n\t\t\t}\n\t\t\thostname := address[:colonPos]\n\t\t\tc.ServerName = hostname\n\t\t}\n\t\treturn d.connectTLS(ctx, conn, c)\n\t}\n\n\treturn conn, nil\n}\n\n// DefaultDialer is the default dialer used when none is specified.\nvar DefaultDialer = &Dialer{\n\tTimeout: 10 * time.Second,\n\tDualStack: true,\n}\n\n// Dial is a convenience wrapper for DefaultDialer.Dial.\nfunc Dial(network string, address string) (*Conn, error) {\n\treturn DefaultDialer.Dial(network, address)\n}\n\n// DialContext is a convenience wrapper for DefaultDialer.DialContext.\nfunc DialContext(ctx context.Context, network string, address string) (*Conn, error) {\n\treturn DefaultDialer.DialContext(ctx, network, address)\n}\n\n// DialLeader is a convenience wrapper for DefaultDialer.DialLeader.\nfunc DialLeader(ctx context.Context, network string, address string, topic string, partition int) (*Conn, error) {\n\treturn DefaultDialer.DialLeader(ctx, network, address, topic, partition)\n}\n\n// DialPartition is a convenience wrapper for DefaultDialer.DialPartition.\nfunc DialPartition(ctx context.Context, network string, address string, partition Partition) (*Conn, error) {\n\treturn DefaultDialer.DialPartition(ctx, network, address, partition)\n}\n\n// LookupPartition is a convenience wrapper for DefaultDialer.LookupPartition.\nfunc LookupPartition(ctx context.Context, network string, address string, topic string, partition int) (Partition, error) {\n\treturn DefaultDialer.LookupPartition(ctx, network, address, topic, partition)\n}\n\n// LookupPartitions is a convenience wrapper for DefaultDialer.LookupPartitions.\nfunc LookupPartitions(ctx context.Context, network string, address string, topic string) ([]Partition, error) {\n\treturn DefaultDialer.LookupPartitions(ctx, network, address, topic)\n}\n\nfunc sleep(ctx context.Context, duration time.Duration) bool {\n\tif duration == 0 {\n\t\tselect {\n\t\tdefault:\n\t\t\treturn true\n\t\tcase <-ctx.Done():\n\t\t\treturn false\n\t\t}\n\t}\n\ttimer := time.NewTimer(duration)\n\tdefer timer.Stop()\n\tselect {\n\tcase <-timer.C:\n\t\treturn true\n\tcase <-ctx.Done():\n\t\treturn false\n\t}\n}\n\nfunc backoff(attempt int, min time.Duration, max time.Duration) time.Duration {\n\td := time.Duration(attempt*attempt) * min\n\tif d > max {\n\t\td = max\n\t}\n\treturn d\n}\n\nfunc canonicalAddress(s string) string {\n\treturn net.JoinHostPort(splitHostPort(s))\n}\n\nfunc splitHostPort(s string) (host string, port string) {\n\thost, port, _ = net.SplitHostPort(s)\n\tif len(host) == 0 && len(port) == 0 {\n\t\thost = s\n\t\tport = \"9092\"\n\t}\n\treturn\n}\n\nfunc splitHostPortNumber(s string) (host string, portNumber int, err error) {\n\thost, port := splitHostPort(s)\n\tportNumber, err = strconv.Atoi(port)\n\tif err != nil {\n\t\treturn host, 0, fmt.Errorf(\"%s: %w\", s, err)\n\t}\n\treturn host, portNumber, nil\n}\n\nfunc lookupHost(ctx context.Context, address string, resolver Resolver) (string, error) {\n\thost, port := splitHostPort(address)\n\n\tif resolver != nil {\n\t\tresolved, err := resolver.LookupHost(ctx, host)\n\t\tif err != nil {\n\t\t\treturn \"\", fmt.Errorf(\"failed to resolve host %s: %w\", host, err)\n\t\t}\n\n\t\t// if the resolver doesn't return anything, we'll fall back on the provided\n\t\t// address instead\n\t\tif len(resolved) > 0 {\n\t\t\tresolvedHost, resolvedPort := splitHostPort(resolved[0])\n\n\t\t\t// we'll always prefer the resolved host\n\t\t\thost = resolvedHost\n\n\t\t\t// in the case of port though, the provided address takes priority, and we\n\t\t\t// only use the resolved address to set the port when not specified\n\t\t\tif port == \"\" {\n\t\t\t\tport = resolvedPort\n\t\t\t}\n\t\t}\n\t}\n\n\treturn net.JoinHostPort(host, port), nil\n}\n" }, { "path": "dialer_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"crypto/x509\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"reflect\"\n\t\"sort\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestDialer(t *testing.T) {\n\ttests := []struct {\n\t\tscenario string\n\t\tfunction func(*testing.T, context.Context, *Dialer)\n\t}{\n\t\t{\n\t\t\tscenario: \"looking up partitions returns the list of available partitions for a topic\",\n\t\t\tfunction: testDialerLookupPartitions,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\ttestFunc := test.function\n\t\tt.Run(test.scenario, func(t *testing.T) {\n\t\t\tctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)\n\t\t\tdefer cancel()\n\n\t\t\ttestFunc(t, ctx, &Dialer{})\n\t\t})\n\t}\n}\n\nfunc testDialerLookupPartitions(t *testing.T, ctx context.Context, d *Dialer) {\n\tclient, topic, shutdown := newLocalClientAndTopic()\n\tdefer shutdown()\n\n\t// Write a message to ensure the partition gets created.\n\tw := &Writer{\n\t\tAddr: TCP(\"localhost:9092\"),\n\t\tTopic: topic,\n\t\tTransport: client.Transport,\n\t}\n\tw.WriteMessages(ctx, Message{})\n\tw.Close()\n\n\tpartitions, err := d.LookupPartitions(ctx, \"tcp\", \"localhost:9092\", topic)\n\tif err != nil {\n\t\tt.Error(err)\n\t\treturn\n\t}\n\n\tsort.Slice(partitions, func(i int, j int) bool {\n\t\treturn partitions[i].ID < partitions[j].ID\n\t})\n\n\twant := []Partition{\n\t\t{\n\t\t\tTopic: topic,\n\t\t\tLeader: Broker{Host: \"localhost\", Port: 9092, ID: 1},\n\t\t\tReplicas: []Broker{{Host: \"localhost\", Port: 9092, ID: 1}},\n\t\t\tIsr: []Broker{{Host: \"localhost\", Port: 9092, ID: 1}},\n\t\t\tOfflineReplicas: []Broker{},\n\t\t\tID: 0,\n\t\t},\n\t}\n\tif !reflect.DeepEqual(partitions, want) {\n\t\tt.Errorf(\"bad partitions:\\ngot: %+v\\nwant: %+v\", partitions, want)\n\t}\n}\n\nfunc tlsConfig(t *testing.T) *tls.Config {\n\tconst (\n\t\tcertPEM = `-----BEGIN CERTIFICATE-----\nMIID2zCCAsOgAwIBAgIJAMSqbewCgw4xMA0GCSqGSIb3DQEBCwUAMGAxCzAJBgNV\nBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRYwFAYDVQQHDA1TYW4gRnJhbmNp\nc2NvMRAwDgYDVQQKDAdTZWdtZW50MRIwEAYDVQQDDAlsb2NhbGhvc3QwHhcNMTcx\nMjIzMTU1NzAxWhcNMjcxMjIxMTU1NzAxWjBgMQswCQYDVQQGEwJVUzETMBEGA1UE\nCAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwNU2FuIEZyYW5jaXNjbzEQMA4GA1UECgwH\nU2VnbWVudDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOC\nAQ8AMIIBCgKCAQEAtda9OWKYNtINe/BKAoB+/zLg2qbaTeHN7L722Ug7YoY6zMVB\naQEHrUmshw/TOrT7GLN/6e6rFN74UuNg72C1tsflZvxqkGdrup3I3jxMh2ApAxLi\nzem/M6Eke2OAqt+SzRPqc5GXH/nrWVd3wqg48DZOAR0jVTY2e0fWy+Er/cPJI1lc\nL6ZMIRJikHTXkaiFj2Jct1iWvgizx5HZJBxXJn2Awix5nvc+zmXM0ZhoedbJRoBC\ndGkRXd3xv2F4lqgVHtP3Ydjc/wYoPiGudSAkhyl9tnkHjvIjA/LeRNshWHbCIaQX\nyemnXIcyyf+W+7EK0gXio7uiP+QSoM5v/oeVMQIDAQABo4GXMIGUMHoGA1UdIwRz\nMHGhZKRiMGAxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRYwFAYD\nVQQHDA1TYW4gRnJhbmNpc2NvMRAwDgYDVQQKDAdTZWdtZW50MRIwEAYDVQQDDAls\nb2NhbGhvc3SCCQCBYUuEuypDMTAJBgNVHRMEAjAAMAsGA1UdDwQEAwIE8DANBgkq\nhkiG9w0BAQsFAAOCAQEATk6IlVsXtNp4C1yeegaM+jE8qgKJfNm1sV27zKx8HPiO\nF7LvTGYIG7zd+bf3pDSwRxfBhsLEwmN9TUN1d6Aa9zeu95qOnR76POfHILgttu2w\nIzegO8I7BycnLjU9o/l9gCpusnN95tIYQhfD08ygUpYTQRuI0cmZ/Dp3xb0S9f5N\nmiYTuUoStYSA4RWbDWo+Is9YWPu7rwieziOZ96oguGz3mtqvkjxVAQH1xZr3bKHr\nHU9LpQh0i6oTK0UCqnDwlhJl1c7A3UooxFpc3NGxyjogzTfI/gnBKfPo7eeswwsV\n77rjIkhBW49L35KOo1uyblgK1vTT7VPtzJnuDq3ORg==\n-----END CERTIFICATE-----`\n\n\t\tkeyPEM = `-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEAtda9OWKYNtINe/BKAoB+/zLg2qbaTeHN7L722Ug7YoY6zMVB\naQEHrUmshw/TOrT7GLN/6e6rFN74UuNg72C1tsflZvxqkGdrup3I3jxMh2ApAxLi\nzem/M6Eke2OAqt+SzRPqc5GXH/nrWVd3wqg48DZOAR0jVTY2e0fWy+Er/cPJI1lc\nL6ZMIRJikHTXkaiFj2Jct1iWvgizx5HZJBxXJn2Awix5nvc+zmXM0ZhoedbJRoBC\ndGkRXd3xv2F4lqgVHtP3Ydjc/wYoPiGudSAkhyl9tnkHjvIjA/LeRNshWHbCIaQX\nyemnXIcyyf+W+7EK0gXio7uiP+QSoM5v/oeVMQIDAQABAoIBAQCa6roHW8JGYipu\nvsau3v5TOOtsHN67n3arDf6MGwfM5oLN1ffmF6SMs8myv36781hBMRv3FwjWHSf+\npgz9o6zsbd05Ii8/m3yiXq609zZT107ZeYuU1mG5AL5uCNWjvhn5cdA6aX0RFwC0\n+tnjEyJ/NCS8ujBR9n/wA8IxrEKoTGcxRb6qFPPKWYoBevu34td1Szf0kH8AKjtQ\nrdPK0Of/ZEiAUxNMLTBEOmC0ZabxJV/YGWcUU4DpmEDZSgQSr4yLT4BFUwF2VC8t\n8VXn5dBP3RMo4h7JlteulcKYsMQZXD6KvUwY2LaEpFM/b14r+TZTUQGhwS+Ha11m\nxa4eNwFhAoGBANshGlpR9cUUq8vNex0Wb63P9BTRTXwg1yEJVMSua+DlaaqaX/hS\nhOxl3K4y2V5OCK31C+SOAqqbrGtMXVym5c5pX8YyC11HupFJwdFLUEc74uF3CtWY\nGMMvEvItCK5ZvYvS5I2CQGcp1fhEMle/Uz+hFi1eeWepMqgHbVx5vkdtAoGBANRv\nXYQsTAGSkhcHB++/ASDskAew5EoHfwtJzSX0BZC6DCACF/U4dCKzBVndOrELOPXs\n2CZXCG4ptWzNgt6YTlMX9U7nLei5pPjoivIJsMudnc22DrDS7C94rCk++M3JeLOM\nKSN0ou9+1iEdE7rQdMgZMryaY71OBonCIDsWgJZVAoGAB+k0CFq5IrpSUXNDpJMw\nyPee+jlsMLUGzzyFAOzDHEVsASq9mDtybQ5oXymay1rJ2W3lVgUCd6JTITSKklO8\nLC2FtaQM4Ps78w7Unne3mDrDQByKGZf6HOHQL0oM7C51N10Pv0Qaix7piKL9pklT\n+hIYuN6WR3XGTGaoPhRvGCkCgYBqaQ5y8q1v7Dd5iXAUS50JHPZYo+b2niKpSOKW\nLFHNWSRRtDrD/u9Nolb/2K1ZmcGCjo0HR3lVlVbnlVoEnk49mTaru2lntfZJKFLR\nQsFofR9at+NL95uPe+bhEkYW7uCjL4Y72GT1ipdAJwyG+3xD7ztW9g8X+EmWH8N9\nVZw7sQKBgGxp820jbjWhG1O9RnYLwflcZzUlSkhWJDg9tKJXBjD+hFX98Okuf0gu\nDUpdbxbJHSi0xAjOjLVswNws4pVwzgtZVK8R7k8j3Z5TtYTJTSQLfgVowuyEdAaI\nC8OxVJ/At/IJGnWSIz8z+/YCUf7p4jd2LJgmZVVzXeDsOFcH62gu\n-----END RSA PRIVATE KEY-----`\n\n\t\tcaPEM = `-----BEGIN CERTIFICATE-----\nMIIDPDCCAiQCCQCBYUuEuypDMTANBgkqhkiG9w0BAQsFADBgMQswCQYDVQQGEwJV\nUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwNU2FuIEZyYW5jaXNjbzEQ\nMA4GA1UECgwHU2VnbWVudDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE3MTIyMzE1\nNTMxOVoXDTI3MTIyMTE1NTMxOVowYDELMAkGA1UEBhMCVVMxEzARBgNVBAgMCkNh\nbGlmb3JuaWExFjAUBgNVBAcMDVNhbiBGcmFuY2lzY28xEDAOBgNVBAoMB1NlZ21l\nbnQxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC\nAQoCggEBAJwB+Yp6MyUepgtaRDxVjpMI2RmlAaV1qApMWu60LWGKJs4KWoIoLl6p\noSEqnWrpMmb38pyGP99X1+t3uZjiK9L8nFhuKZ581tsTKLxaSl+YVg7JbH5LVCS6\nopsfB5ON1gJxf1HA9YyMqKHkBFh8/hdOGR0T6Bll9TPO1NQB/UqMy/tKr3sA3KZm\nXVDbRKSuUAQWz5J9/hLPmVMU41F/uD7mvyDY+x8GymInZjUXG4e0oq2RJgU6SYZ8\nmkscM6qhKY3mL487w/kHVFtFlMkOhvI7LIh3zVvWwgGSAoAv9yai9BDZNFSk0cEb\nbb/IK7BQW9sNI3lcnGirdbnjV94X9/sCAwEAATANBgkqhkiG9w0BAQsFAAOCAQEA\nMJLeGdYO3dpsPx2R39Bw0qa5cUh42huPf8n7rp4a4Ca5jJjcAlCYV8HzqOzpiKYy\nZNuHy8LnNVYYh5Qoh8EO45bplMV1wnHfi6hW6DY5j3SQdcxkoVsW5R7rBF7a7SDg\n6uChVRPHgsnALUUc7Wvvd3sAs/NKHzHu86mgD3EefkdqWAaCapzcqT9mo9KXkWJM\nDhSJS+/iIaroc8umDnbPfhhgnlMf0/D4q0TjiLSSqyLzVifxnv9yHz56TrhHG/QP\nE/8+FEGCHYKM4JLr5smGlzv72Kfx9E1CkG6TgFNIHjipVv1AtYDvaNMdPF2533+F\nwE3YmpC3Q0g9r44nEbz4Bw==\n-----END CERTIFICATE-----`\n\t)\n\n\t// Define TLS configuration\n\tcertificate, err := tls.X509KeyPair([]byte(certPEM), []byte(keyPEM))\n\tif err != nil {\n\t\tt.Error(err)\n\t\tt.FailNow()\n\t}\n\n\tcaCertPool := x509.NewCertPool()\n\tif ok := caCertPool.AppendCertsFromPEM([]byte(caPEM)); !ok {\n\t\tt.Error(err)\n\t\tt.FailNow()\n\t}\n\n\treturn &tls.Config{\n\t\tCertificates: []tls.Certificate{certificate},\n\t\tRootCAs: caCertPool,\n\t\tInsecureSkipVerify: true,\n\t}\n}\n\nfunc TestDialerTLS(t *testing.T) {\n\tclient, topic, shutdown := newLocalClientAndTopic()\n\tdefer shutdown()\n\n\t// Write a message to ensure the partition gets created.\n\tw := &Writer{\n\t\tAddr: TCP(\"localhost:9092\"),\n\t\tTopic: topic,\n\t\tTransport: client.Transport,\n\t}\n\tw.WriteMessages(context.Background(), Message{})\n\tw.Close()\n\n\t// Create an SSL proxy using the tls.Config that connects to the\n\t// docker-composed kafka\n\tconfig := tlsConfig(t)\n\tl, err := tls.Listen(\"tcp\", \"127.0.0.1:\", config)\n\tif err != nil {\n\t\tt.Error(err)\n\t\treturn\n\t}\n\tdefer l.Close()\n\n\tgo func() {\n\t\tfor {\n\t\t\tconn, err := l.Accept()\n\t\t\tif err != nil {\n\t\t\t\treturn // intentionally ignored\n\t\t\t}\n\n\t\t\tgo func(in net.Conn) {\n\t\t\t\tout, err := net.Dial(\"tcp\", \"localhost:9092\")\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Error(err)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\tdefer out.Close()\n\n\t\t\t\tgo io.Copy(in, out)\n\t\t\t\tio.Copy(out, in)\n\t\t\t}(conn)\n\t\t}\n\t}()\n\n\t// Use the tls.Config and connect to the SSL proxy\n\td := &Dialer{\n\t\tTLS: config,\n\t}\n\tpartitions, err := d.LookupPartitions(context.Background(), \"tcp\", l.Addr().String(), topic)\n\tif err != nil {\n\t\tt.Error(err)\n\t\treturn\n\t}\n\n\t// Verify returned partition data is what we expect\n\tsort.Slice(partitions, func(i int, j int) bool {\n\t\treturn partitions[i].ID < partitions[j].ID\n\t})\n\n\twant := []Partition{\n\t\t{\n\t\t\tTopic: topic,\n\t\t\tLeader: Broker{Host: \"localhost\", Port: 9092, ID: 1},\n\t\t\tReplicas: []Broker{{Host: \"localhost\", Port: 9092, ID: 1}},\n\t\t\tIsr: []Broker{{Host: \"localhost\", Port: 9092, ID: 1}},\n\t\t\tOfflineReplicas: []Broker{},\n\t\t\tID: 0,\n\t\t},\n\t}\n\tif !reflect.DeepEqual(partitions, want) {\n\t\tt.Errorf(\"bad partitions:\\ngot: %+v\\nwant: %+v\", partitions, want)\n\t}\n}\n\ntype MockConn struct {\n\tnet.Conn\n\tdone chan struct{}\n\tpartitions []Partition\n}\n\nfunc (m *MockConn) Read(b []byte) (n int, err error) {\n\tselect {\n\tcase <-time.After(time.Minute):\n\tcase <-m.done:\n\t\treturn 0, context.Canceled\n\t}\n\n\treturn 0, io.EOF\n}\n\nfunc (m *MockConn) Write(b []byte) (n int, err error) {\n\tselect {\n\tcase <-time.After(time.Minute):\n\tcase <-m.done:\n\t\treturn 0, context.Canceled\n\t}\n\n\treturn 0, io.EOF\n}\n\nfunc (m *MockConn) Close() error {\n\tselect {\n\tcase <-m.done:\n\tdefault:\n\t\tclose(m.done)\n\t}\n\treturn nil\n}\n\nfunc (m *MockConn) ReadPartitions(topics ...string) (partitions []Partition, err error) {\n\treturn m.partitions, err\n}\n\nfunc TestDialerConnectTLSHonorsContext(t *testing.T) {\n\tconfig := tlsConfig(t)\n\td := &Dialer{\n\t\tTLS: config,\n\t}\n\n\tconn := &MockConn{\n\t\tdone: make(chan struct{}),\n\t}\n\n\tctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*25)\n\tdefer cancel()\n\n\t_, err := d.connectTLS(ctx, conn, d.TLS)\n\tif !errors.Is(err, context.DeadlineExceeded) {\n\t\tt.Errorf(\"expected err to be %v; got %v\", context.DeadlineExceeded, err)\n\t\tt.FailNow()\n\t}\n}\n\nfunc TestDialerResolver(t *testing.T) {\n\tctx := context.TODO()\n\n\ttests := []struct {\n\t\tscenario string\n\t\taddress string\n\t\tresolver map[string][]string\n\t}{\n\t\t{\n\t\t\tscenario: \"resolve domain to ip\",\n\t\t\taddress: \"example.com\",\n\t\t\tresolver: map[string][]string{\n\t\t\t\t\"example.com\": {\"127.0.0.1\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscenario: \"resolve domain to ip and port\",\n\t\t\taddress: \"example.com\",\n\t\t\tresolver: map[string][]string{\n\t\t\t\t\"example.com\": {\"127.0.0.1:9092\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscenario: \"resolve domain with port to ip\",\n\t\t\taddress: \"example.com:9092\",\n\t\t\tresolver: map[string][]string{\n\t\t\t\t\"example.com\": {\"127.0.0.1:9092\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscenario: \"resolve domain with port to ip with different port\",\n\t\t\taddress: \"example.com:9092\",\n\t\t\tresolver: map[string][]string{\n\t\t\t\t\"example.com\": {\"127.0.0.1:80\"},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscenario: \"resolve domain with port to ip\",\n\t\t\taddress: \"example.com:9092\",\n\t\t\tresolver: map[string][]string{\n\t\t\t\t\"example.com\": {\"127.0.0.1\"},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.scenario, func(t *testing.T) {\n\t\t\ttopic := makeTopic()\n\t\t\tcreateTopic(t, topic, 1)\n\t\t\tdefer deleteTopic(t, topic)\n\n\t\t\td := Dialer{\n\t\t\t\tResolver: &mockResolver{addrs: test.resolver},\n\t\t\t}\n\n\t\t\t// Write a message to ensure the partition gets created.\n\t\t\tw := NewWriter(WriterConfig{\n\t\t\t\tBrokers: []string{\"localhost:9092\"},\n\t\t\t\tTopic: topic,\n\t\t\t\tDialer: &d,\n\t\t\t})\n\t\t\tw.WriteMessages(context.Background(), Message{})\n\t\t\tw.Close()\n\n\t\t\tpartitions, err := d.LookupPartitions(ctx, \"tcp\", test.address, topic)\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tsort.Slice(partitions, func(i int, j int) bool {\n\t\t\t\treturn partitions[i].ID < partitions[j].ID\n\t\t\t})\n\n\t\t\twant := []Partition{\n\t\t\t\t{\n\t\t\t\t\tTopic: topic,\n\t\t\t\t\tLeader: Broker{Host: \"localhost\", Port: 9092, ID: 1},\n\t\t\t\t\tReplicas: []Broker{{Host: \"localhost\", Port: 9092, ID: 1}},\n\t\t\t\t\tIsr: []Broker{{Host: \"localhost\", Port: 9092, ID: 1}},\n\t\t\t\t\tOfflineReplicas: []Broker{},\n\t\t\t\t\tID: 0,\n\t\t\t\t},\n\t\t\t}\n\t\t\tif !reflect.DeepEqual(partitions, want) {\n\t\t\t\tt.Errorf(\"bad partitions:\\ngot: %+v\\nwant: %+v\", partitions, want)\n\t\t\t}\n\t\t})\n\t}\n}\n\ntype mockResolver struct {\n\taddrs map[string][]string\n}\n\nfunc (mr *mockResolver) LookupHost(ctx context.Context, host string) ([]string, error) {\n\tif addrs, ok := mr.addrs[host]; !ok {\n\t\treturn nil, fmt.Errorf(\"unrecognized host %s\", host)\n\t} else {\n\t\treturn addrs, nil\n\t}\n}\n" }, { "path": "discard.go", "content": "package kafka\n\nimport \"bufio\"\n\nfunc discardN(r *bufio.Reader, sz int, n int) (int, error) {\n\tvar err error\n\tif n <= sz {\n\t\tn, err = r.Discard(n)\n\t} else {\n\t\tn, err = r.Discard(sz)\n\t\tif err == nil {\n\t\t\terr = errShortRead\n\t\t}\n\t}\n\treturn sz - n, err\n}\n\nfunc discardInt32(r *bufio.Reader, sz int) (int, error) {\n\treturn discardN(r, sz, 4)\n}\n\nfunc discardString(r *bufio.Reader, sz int) (int, error) {\n\treturn readStringWith(r, sz, func(r *bufio.Reader, sz int, n int) (int, error) {\n\t\tif n < 0 {\n\t\t\treturn sz, nil\n\t\t}\n\t\treturn discardN(r, sz, n)\n\t})\n}\n\nfunc discardBytes(r *bufio.Reader, sz int) (int, error) {\n\treturn readBytesWith(r, sz, func(r *bufio.Reader, sz int, n int) (int, error) {\n\t\tif n < 0 {\n\t\t\treturn sz, nil\n\t\t}\n\t\treturn discardN(r, sz, n)\n\t})\n}\n" }, { "path": "discard_test.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"errors\"\n\t\"io\"\n\t\"testing\"\n)\n\nfunc TestDiscardN(t *testing.T) {\n\ttests := []struct {\n\t\tscenario string\n\t\tfunction func(*testing.T, *bufio.Reader, int)\n\t}{\n\t\t{\n\t\t\tscenario: \"discard nothing\",\n\t\t\tfunction: func(t *testing.T, r *bufio.Reader, sz int) {\n\t\t\t\tremain, err := discardN(r, sz, 0)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Errorf(\"Expected no error, got %v\", err)\n\t\t\t\t}\n\t\t\t\tif remain != sz {\n\t\t\t\t\tt.Errorf(\"Expected all bytes remaining, got %d\", remain)\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscenario: \"discard fewer than available\",\n\t\t\tfunction: func(t *testing.T, r *bufio.Reader, sz int) {\n\t\t\t\tremain, err := discardN(r, sz, sz-1)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Errorf(\"Expected no error, got %v\", err)\n\t\t\t\t}\n\t\t\t\tif remain != 1 {\n\t\t\t\t\tt.Errorf(\"Expected single remaining byte, got %d\", remain)\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscenario: \"discard all available\",\n\t\t\tfunction: func(t *testing.T, r *bufio.Reader, sz int) {\n\t\t\t\tremain, err := discardN(r, sz, sz)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Errorf(\"Expected no error, got %v\", err)\n\t\t\t\t}\n\t\t\t\tif remain != 0 {\n\t\t\t\t\tt.Errorf(\"Expected no remaining bytes, got %d\", remain)\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscenario: \"discard more than available\",\n\t\t\tfunction: func(t *testing.T, r *bufio.Reader, sz int) {\n\t\t\t\tremain, err := discardN(r, sz, sz+1)\n\t\t\t\tif !errors.Is(err, errShortRead) {\n\t\t\t\t\tt.Errorf(\"Expected errShortRead, got %v\", err)\n\t\t\t\t}\n\t\t\t\tif remain != 0 {\n\t\t\t\t\tt.Errorf(\"Expected no remaining bytes, got %d\", remain)\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscenario: \"discard returns error\",\n\t\t\tfunction: func(t *testing.T, r *bufio.Reader, sz int) {\n\t\t\t\tremain, err := discardN(r, sz+2, sz+1)\n\t\t\t\tif !errors.Is(err, io.EOF) {\n\t\t\t\t\tt.Errorf(\"Expected EOF, got %v\", err)\n\t\t\t\t}\n\t\t\t\tif remain != 2 {\n\t\t\t\t\tt.Errorf(\"Expected single remaining bytes, got %d\", remain)\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tscenario: \"errShortRead doesn't mask error\",\n\t\t\tfunction: func(t *testing.T, r *bufio.Reader, sz int) {\n\t\t\t\tremain, err := discardN(r, sz+1, sz+2)\n\t\t\t\tif !errors.Is(err, io.EOF) {\n\t\t\t\t\tt.Errorf(\"Expected EOF, got %v\", err)\n\t\t\t\t}\n\t\t\t\tif remain != 1 {\n\t\t\t\t\tt.Errorf(\"Expected single remaining bytes, got %d\", remain)\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t}\n\tfor _, test := range tests {\n\t\tt.Run(test.scenario, func(t *testing.T) {\n\t\t\tmsg := []byte(\"test message\")\n\t\t\tr := bufio.NewReader(bytes.NewReader(msg))\n\t\t\ttest.function(t, r, len(msg))\n\t\t})\n\t}\n}\n" }, { "path": "docker-compose.yml", "content": "# See https://hub.docker.com/r/bitnami/kafka/tags for the complete list.\nversion: '3'\nservices:\n zookeeper:\n container_name: zookeeper\n hostname: zookeeper\n image: bitnamilegacy/zookeeper:latest\n ports:\n - 2181:2181\n environment:\n ALLOW_ANONYMOUS_LOGIN: yes\n kafka:\n container_name: kafka\n image: bitnamilegacy/kafka:3.7.0\n restart: on-failure:3\n links:\n - zookeeper\n ports:\n - 9092:9092\n - 9093:9093\n environment:\n KAFKA_CFG_BROKER_ID: 1\n KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'\n KAFKA_CFG_ADVERTISED_HOST_NAME: 'localhost'\n KAFKA_CFG_ADVERTISED_PORT: '9092'\n KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181\n KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'true'\n KAFKA_CFG_MESSAGE_MAX_BYTES: '200000000'\n KAFKA_CFG_LISTENERS: 'PLAINTEXT://:9092,SASL_PLAINTEXT://:9093'\n KAFKA_CFG_ADVERTISED_LISTENERS: 'PLAINTEXT://localhost:9092,SASL_PLAINTEXT://localhost:9093'\n KAFKA_CFG_SASL_ENABLED_MECHANISMS: 'PLAIN,SCRAM-SHA-256,SCRAM-SHA-512'\n KAFKA_CFG_AUTHORIZER_CLASS_NAME: 'kafka.security.authorizer.AclAuthorizer'\n KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true'\n KAFKA_OPTS: \"-Djava.security.auth.login.config=/opt/bitnami/kafka/config/kafka_jaas.conf\"\n ALLOW_PLAINTEXT_LISTENER: yes\n entrypoint:\n - \"/bin/bash\"\n - \"-c\"\n - echo -e 'KafkaServer {\\norg.apache.kafka.common.security.scram.ScramLoginModule required\\n username=\"adminscram\"\\n password=\"admin-secret\";\\n org.apache.kafka.common.security.plain.PlainLoginModule required\\n username=\"adminplain\"\\n password=\"admin-secret\"\\n user_adminplain=\"admin-secret\";\\n };' > /opt/bitnami/kafka/config/kafka_jaas.conf; /opt/bitnami/kafka/bin/kafka-configs.sh --zookeeper zookeeper:2181 --alter --add-config \"SCRAM-SHA-256=[password=admin-secret-256],SCRAM-SHA-512=[password=admin-secret-512]\" --entity-type users --entity-name adminscram; exec /entrypoint.sh /run.sh\n" }, { "path": "docker_compose_versions/README.md", "content": "# Bitnami Kafka\n\nThis document outlines how to create a docker-compose file for a specific Bitnami Kafka version.\n\n\n## Steps to create docker-compose\n\n- Refer to [docker-hub Bitnami Kafka tags](https://hub.docker.com/r/bitnamilegacy/kafka/tags) and sort by NEWEST to locate the image preferred, for example: `2.7.0`\n- There is documentation in the (main branch)[https://github.com/bitnami/containers/blob/main/bitnami/kafka/README.md] for environment config setup information. Refer to the `Notable Changes` section.\n- Sometimes there is a need to understand how the set up is being done. To locate the appropriate Kafka release in the repo [bitnami/containers](https://github.com/bitnami/containers), go through the [kafka commit history](https://github.com/bitnami/containers/commits/main/bitnami/kafka).\n- Once a commit is located, Refer to README.md, Dockerfile, entrypoint and various init scripts to understand the environment variables to config server.properties mapping conventions. Alternatively, you can spin up the required Kafka image and refer the mapping inside the container.\n- Ensure you follow the environment variable conventions in your docker-compose. Without proper environment variables, the Kafka cluster cannot start or can start with undesired configs. For example, Since Kafka version 2.3, all server.properties docker-compose environment configs start with `KAFKA_CFG_`\n- Older versions of Bitnami Kafka have different conventions and limited docker-compose environment variables exposed for configs needed in server.properties\n\n\nIn kafka-go, for all the test cases to succeed, Kafka cluster should have following server.properties along with a relevant kafka_jaas.conf mentioned in the KAFKA_OPTS. Goal is to ensure that the docker-compose file generates below server.properties.\n\n\nserver.properties\n```\nadvertised.host.name=localhost\nadvertised.listeners=PLAINTEXT://localhost:9092,SASL_PLAINTEXT://localhost:9093\nadvertised.port=9092\nauto.create.topics.enable=true\nbroker.id=1\ndelete.topic.enable=true\ngroup.initial.rebalance.delay.ms=0\nlisteners=PLAINTEXT://:9092,SASL_PLAINTEXT://:9093\nlog.dirs=/kafka/kafka-logs-1d5951569d78\nlog.retention.check.interval.ms=300000\nlog.retention.hours=168\nlog.segment.bytes=1073741824\nmessage.max.bytes=200000000\nnum.io.threads=8\nnum.network.threads=3\nnum.partitions=1\nnum.recovery.threads.per.data.dir=1\noffsets.topic.replication.factor=1\nport=9092\nsasl.enabled.mechanisms=PLAIN,SCRAM-SHA-256,SCRAM-SHA-512\nsocket.receive.buffer.bytes=102400\nsocket.request.max.bytes=104857600\nsocket.send.buffer.bytes=102400\ntransaction.state.log.min.isr=1\ntransaction.state.log.replication.factor=1\nzookeeper.connect=zookeeper:2181\nzookeeper.connection.timeout.ms=6000\n```\n\n\n## run docker-compose and test cases\n\nrun docker-compose\n```\n# docker-compose -f ./docker_compose_versions/docker-compose-.yml up -d\n```\n\n\nrun test cases\n```\n# go clean -cache; KAFKA_SKIP_NETTEST=1 KAFKA_VERSION= go test -race -cover ./...;\n```\n\n\n## Various Bitnami Kafka version issues observed in circleci\n\n\n### Kafka v101, v111, v201, v211 and v221\n\n\nIn kafka-go repo, all the tests require sasl.enabled.mechanisms as PLAIN,SCRAM-SHA-256,SCRAM-SHA-512 for the Kafka cluster.\n\n\nIt has been observed for Kafka v101, v111, v201, v211 and v221 which are used in the circleci for build have issues with SCRAM.\n\n\nThere is no way to override the config sasl.enabled.mechanisms causing Kafka cluster to start up as PLAIN.\n\n\nThere has been some attempts made to override sasl.enabled.mechanisms \n- Modified entrypoint in docker-compose to append the server.properties with relevant configs sasl.enabled.mechanisms before running entrypoint.sh. This resulted in failures for Kafka v101, v111, v201, v211 and v221. Once Kafka server starts, server.properties gets appended with default value of sasl.enabled.mechanisms there by cluster to start with out PLAIN,SCRAM-SHA-256,SCRAM-SHA-512\n- Mounted a docker-compose volume for server.propeties. However, This also resulted in failures for Kafka v101, v111, v201, v211 and v221. Once Kafka server starts, server.properties gets appended with default value of sasl.enabled.mechanisms there by cluster to start with out PLAIN,SCRAM-SHA-256,SCRAM-SHA-512\n\n\nNOTE: \n- Kafka v101, v111, v201, v211 and v221 have no docker-compose files since we need SCRAM for kafka-go test cases to succeed. \n- There is no Bitnami Kafka image for v222 hence testing has been performed on v221\n\n\n### Kafka v231\n\nIn Bitnami Kafka v2.3, all server.properties docker-compose environment configs start with `KAFKA_CFG_`. However, it is not picking the custom populated kafka_jaas.conf.\n\n\nAfter a lot of debugging, it has been noticed that there aren't enough privileges to create the kafka_jaas.conf. Hence the environment variables below need to be added in docker-compose to generate the kafka_jaas.conf. This issue is not noticed after kafka v2.3\n\n\n```\nKAFKA_INTER_BROKER_USER: adminplain\nKAFKA_INTER_BROKER_PASSWORD: admin-secret\nKAFKA_BROKER_USER: adminplain\nKAFKA_BROKER_PASSWORD: admin-secret\n```\n\nThere is a docker-compose file `docker-compose-231.yml` in the folder `kafka-go/docker_compose_versions` for reference.\n\n\n## References\n\n\nFor user reference, please find the some of the older kafka versions commits from the [kafka commit history](https://github.com/bitnami/containers/commits/main/bitnami/kafka). For Kafka versions with no commit history, data is populated with the latest version available for the tag.\n\n\n### Kafka v010: docker-compose reference: `kafka-go/docker_compose_versions/docker-compose-010.yml`\n- [tag](https://hub.docker.com/r/bitnamilegacy/kafka/tags?page=1&ordering=last_updated&name=0.10.2.1)\n- [kafka commit](https://github.com/bitnami/containers/tree/c4240f0525916a418245c7ef46d9534a7a212c92/bitnami/kafka)\n\n\n### Kafka v011: docker-compose reference: `kafka-go/docker_compose_versions/docker-compose-011.yml`\n- [tag](https://hub.docker.com/r/bitnamilegacy/kafka/tags?page=1&ordering=last_updated&name=0.11.0)\n- [kafka commit](https://github.com/bitnami/containers/tree/7724adf655e4ca9aac69d606d41ad329ef31eeca/bitnami/kafka)\n\n\n### Kafka v101: docker-compose reference: N/A\n- [tag](https://hub.docker.com/r/bitnamilegacy/kafka/tags?page=1&ordering=last_updated&name=1.0.1)\n- [kafka commit](https://github.com/bitnami/containers/tree/44cc8f4c43ead6edebd3758c8df878f4f9da82c2/bitnami/kafka)\n\n\n### Kafka v111: docker-compose reference: N/A\n- [tag](https://hub.docker.com/r/bitnamilegacy/kafka/tags?page=1&ordering=last_updated&name=1.1.1)\n- [kafka commit](https://github.com/bitnami/containers/tree/cb593dc98c2eb7a39f2792641e741d395dbe50e7/bitnami/kafka)\n\n\n### Kafka v201: docker-compose reference: N/A\n- [tag](https://hub.docker.com/r/bitnamilegacy/kafka/tags?page=1&ordering=last_updated&name=2.0.1)\n- [kafka commit](https://github.com/bitnami/containers/tree/9ff8763df265c87c8b59f8d7ff0cf69299d636c9/bitnami/kafka)\n\n\n### Kafka v211: docker-compose reference: N/A\n- [tag](https://hub.docker.com/r/bitnamilegacy/kafka/tags?page=1&ordering=last_updated&name=2.1.1)\n- [kafka commit](https://github.com/bitnami/containers/tree/d3a9d40afc2b7e7de53486538a63084c1a565d43/bitnami/kafka)\n\n\n### Kafka v221: docker-compose reference: N/A\n- [tag](https://hub.docker.com/r/bitnamilegacy/kafka/tags?page=1&ordering=last_updated&name=2.2.1)\n- [kafka commit](https://github.com/bitnami/containers/tree/f132ef830d1ba9b78392ec4619174b4640c276c9/bitnami/kafka)\n\n\n### Kafka v231: docker-compose reference: `kafka-go/docker_compose_versions/docker-compose-231.yml`\n- [tag](https://hub.docker.com/r/bitnamilegacy/kafka/tags?page=1&ordering=last_updated&name=2.3.1)\n- [kafka commit](https://github.com/bitnami/containers/tree/ae572036b5281456b0086345fec0bdb74f7cf3a3/bitnami/kafka)\n\n" }, { "path": "docker_compose_versions/docker-compose-270.yml", "content": "# See https://hub.docker.com/r/bitnamilegacy/kafka/tags for the complete list.\nversion: '3'\nservices:\n zookeeper:\n container_name: zookeeper\n hostname: zookeeper\n image: bitnamilegacy/zookeeper:latest\n ports:\n - 2181:2181\n environment:\n ALLOW_ANONYMOUS_LOGIN: yes\n kafka:\n container_name: kafka\n image: bitnamilegacy/kafka:2.7.0\n restart: on-failure:3\n links:\n - zookeeper\n ports:\n - 9092:9092\n - 9093:9093\n environment:\n KAFKA_CFG_BROKER_ID: 1\n KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'\n KAFKA_CFG_ADVERTISED_HOST_NAME: 'localhost'\n KAFKA_CFG_ADVERTISED_PORT: '9092'\n KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181\n KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'true'\n KAFKA_CFG_MESSAGE_MAX_BYTES: '200000000'\n KAFKA_CFG_LISTENERS: 'PLAINTEXT://:9092,SASL_PLAINTEXT://:9093'\n KAFKA_CFG_ADVERTISED_LISTENERS: 'PLAINTEXT://localhost:9092,SASL_PLAINTEXT://localhost:9093'\n KAFKA_CFG_SASL_ENABLED_MECHANISMS: 'PLAIN,SCRAM-SHA-256,SCRAM-SHA-512'\n KAFKA_CFG_AUTHORIZER_CLASS_NAME: 'kafka.security.auth.SimpleAclAuthorizer'\n KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true'\n KAFKA_OPTS: \"-Djava.security.auth.login.config=/opt/bitnami/kafka/config/kafka_jaas.conf\"\n ALLOW_PLAINTEXT_LISTENER: yes\n entrypoint:\n - \"/bin/bash\"\n - \"-c\"\n - echo -e 'KafkaServer {\\norg.apache.kafka.common.security.scram.ScramLoginModule required\\n username=\"adminscram\"\\n password=\"admin-secret\";\\n org.apache.kafka.common.security.plain.PlainLoginModule required\\n username=\"adminplain\"\\n password=\"admin-secret\"\\n user_adminplain=\"admin-secret\";\\n };' > /opt/bitnami/kafka/config/kafka_jaas.conf; /opt/bitnami/kafka/bin/kafka-configs.sh --zookeeper zookeeper:2181 --alter --add-config \"SCRAM-SHA-256=[password=admin-secret-256],SCRAM-SHA-512=[password=admin-secret-512]\" --entity-type users --entity-name adminscram; exec /entrypoint.sh /run.sh\n" }, { "path": "docker_compose_versions/docker-compose-370.yml", "content": "# See https://hub.docker.com/r/bitnamilegacy/kafka/tags for the complete list.\nversion: '3'\nservices:\n zookeeper:\n container_name: zookeeper\n hostname: zookeeper\n image: bitnamilegacy/zookeeper:latest\n ports:\n - 2181:2181\n environment:\n ALLOW_ANONYMOUS_LOGIN: yes\n kafka:\n container_name: kafka\n image: bitnamilegacy/kafka:3.7.0\n restart: on-failure:3\n links:\n - zookeeper\n ports:\n - 9092:9092\n - 9093:9093\n environment:\n KAFKA_CFG_BROKER_ID: 1\n KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'\n KAFKA_CFG_ADVERTISED_HOST_NAME: 'localhost'\n KAFKA_CFG_ADVERTISED_PORT: '9092'\n KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181\n KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'true'\n KAFKA_CFG_MESSAGE_MAX_BYTES: '200000000'\n KAFKA_CFG_LISTENERS: 'PLAINTEXT://:9092,SASL_PLAINTEXT://:9093'\n KAFKA_CFG_ADVERTISED_LISTENERS: 'PLAINTEXT://localhost:9092,SASL_PLAINTEXT://localhost:9093'\n KAFKA_CFG_SASL_ENABLED_MECHANISMS: 'PLAIN,SCRAM-SHA-256,SCRAM-SHA-512'\n KAFKA_CFG_AUTHORIZER_CLASS_NAME: 'kafka.security.authorizer.AclAuthorizer'\n KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true'\n KAFKA_OPTS: \"-Djava.security.auth.login.config=/opt/bitnami/kafka/config/kafka_jaas.conf\"\n ALLOW_PLAINTEXT_LISTENER: yes\n entrypoint:\n - \"/bin/bash\"\n - \"-c\"\n - echo -e 'KafkaServer {\\norg.apache.kafka.common.security.scram.ScramLoginModule required\\n username=\"adminscram\"\\n password=\"admin-secret\";\\n org.apache.kafka.common.security.plain.PlainLoginModule required\\n username=\"adminplain\"\\n password=\"admin-secret\"\\n user_adminplain=\"admin-secret\";\\n };' > /opt/bitnami/kafka/config/kafka_jaas.conf; /opt/bitnami/kafka/bin/kafka-configs.sh --zookeeper zookeeper:2181 --alter --add-config \"SCRAM-SHA-256=[password=admin-secret-256],SCRAM-SHA-512=[password=admin-secret-512]\" --entity-type users --entity-name adminscram; exec /entrypoint.sh /run.sh\n" }, { "path": "docker_compose_versions/docker-compose-400.yml", "content": "# See https://hub.docker.com/r/bitnamilegacy/kafka/tags for the complete list.\nversion: '3'\nservices:\n kafka:\n container_name: kafka\n image: bitnamilegacy/kafka:4.0.0\n restart: on-failure:3\n ports:\n - 9092:9092\n - 9093:9093\n environment:\n KAFKA_CFG_NODE_ID: 1\n KAFKA_CFG_BROKER_ID: 1\n KAFKA_CFG_PROCESS_ROLES: broker,controller\n KAFKA_CFG_ADVERTISED_HOST_NAME: 'localhost'\n KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER\n KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAIN:PLAINTEXT,SASL:SASL_PLAINTEXT\n KAFKA_CFG_LISTENERS: CONTROLLER://:9094,PLAIN://:9092,SASL://:9093\n KAFKA_CFG_ADVERTISED_LISTENERS: PLAIN://localhost:9092,SASL://localhost:9093\n KAFKA_CFG_INTER_BROKER_LISTENER_NAME: PLAIN\n KAFKA_CFG_SASL_ENABLED_MECHANISMS: 'PLAIN,SCRAM-SHA-256,SCRAM-SHA-512'\n KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 1@localhost:9094\n ALLOW_PLAINTEXT_LISTENER: yes\n KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true'\n KAFKA_OPTS: \"-Djava.security.auth.login.config=/opt/bitnami/kafka/config/kafka_jaas.conf\"\n KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'true'\n KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'\n KAFKA_CFG_MESSAGE_MAX_BYTES: '200000000'\n KAFKA_CFG_AUTHORIZER_CLASS_NAME: 'org.apache.kafka.metadata.authorizer.StandardAuthorizer'\n KAFKA_CFG_SUPER_USERS: User:adminscram256;User:adminscram512;User:adminplain\n KAFKA_CLIENT_USERS: adminscram256,adminscram512,adminplain\n KAFKA_CLIENT_PASSWORDS: admin-secret-256,admin-secret-512,admin-secret\n KAFKA_CLIENT_SASL_MECHANISMS: SCRAM-SHA-256,SCRAM-SHA-512,PLAIN\n KAFKA_INTER_BROKER_USER: adminscram512\n KAFKA_INTER_BROKER_PASSWORD: admin-secret-512\n KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL: SCRAM-SHA-512\n # Note you will need to increase this to at least 4GB of memory for the tests to pass\n # https://github.com/segmentio/kafka-go/issues/1360#issuecomment-2858935900\n KAFKA_HEAP_OPTS: '-Xmx1000m -Xms1000m'\n KAFKA_JVM_OPTS: '-XX:+UseG1GC'" }, { "path": "electleaders.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/electleaders\"\n)\n\n// ElectLeadersRequest is a request to the ElectLeaders API.\ntype ElectLeadersRequest struct {\n\t// Addr is the address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// Topic is the name of the topic to do the leader elections in.\n\tTopic string\n\n\t// Partitions is the list of partitions to run leader elections for.\n\tPartitions []int\n\n\t// Timeout is the amount of time to wait for the election to run.\n\tTimeout time.Duration\n}\n\n// ElectLeadersResponse is a response from the ElectLeaders API.\ntype ElectLeadersResponse struct {\n\t// ErrorCode is set to a non-nil value if a top-level error occurred.\n\tError error\n\n\t// PartitionResults contains the results for each partition leader election.\n\tPartitionResults []ElectLeadersResponsePartitionResult\n}\n\n// ElectLeadersResponsePartitionResult contains the response details for a single partition.\ntype ElectLeadersResponsePartitionResult struct {\n\t// Partition is the ID of the partition.\n\tPartition int\n\n\t// Error is set to a non-nil value if an error occurred electing leaders\n\t// for this partition.\n\tError error\n}\n\nfunc (c *Client) ElectLeaders(\n\tctx context.Context,\n\treq *ElectLeadersRequest,\n) (*ElectLeadersResponse, error) {\n\tpartitions32 := []int32{}\n\tfor _, partition := range req.Partitions {\n\t\tpartitions32 = append(partitions32, int32(partition))\n\t}\n\n\tprotoResp, err := c.roundTrip(\n\t\tctx,\n\t\treq.Addr,\n\t\t&electleaders.Request{\n\t\t\tTopicPartitions: []electleaders.RequestTopicPartitions{\n\t\t\t\t{\n\t\t\t\t\tTopic: req.Topic,\n\t\t\t\t\tPartitionIDs: partitions32,\n\t\t\t\t},\n\t\t\t},\n\t\t\tTimeoutMs: int32(req.Timeout.Milliseconds()),\n\t\t},\n\t)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tapiResp := protoResp.(*electleaders.Response)\n\n\tresp := &ElectLeadersResponse{\n\t\tError: makeError(apiResp.ErrorCode, \"\"),\n\t}\n\n\tfor _, topicResult := range apiResp.ReplicaElectionResults {\n\t\tfor _, partitionResult := range topicResult.PartitionResults {\n\t\t\tresp.PartitionResults = append(\n\t\t\t\tresp.PartitionResults,\n\t\t\t\tElectLeadersResponsePartitionResult{\n\t\t\t\t\tPartition: int(partitionResult.PartitionID),\n\t\t\t\t\tError: makeError(partitionResult.ErrorCode, partitionResult.ErrorMessage),\n\t\t\t\t},\n\t\t\t)\n\t\t}\n\t}\n\n\treturn resp, nil\n}\n" }, { "path": "electleaders_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc TestClientElectLeaders(t *testing.T) {\n\tif !ktesting.KafkaIsAtLeast(\"2.4.0\") {\n\t\treturn\n\t}\n\n\tctx := context.Background()\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\ttopic := makeTopic()\n\tcreateTopic(t, topic, 2)\n\tdefer deleteTopic(t, topic)\n\n\t// Local kafka only has 1 broker, so leader elections are no-ops.\n\tresp, err := client.ElectLeaders(\n\t\tctx,\n\t\t&ElectLeadersRequest{\n\t\t\tTopic: topic,\n\t\t\tPartitions: []int{0, 1},\n\t\t},\n\t)\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif resp.Error != nil {\n\t\tt.Error(\n\t\t\t\"Unexpected error in response\",\n\t\t\t\"expected\", nil,\n\t\t\t\"got\", resp.Error,\n\t\t)\n\t}\n\tif len(resp.PartitionResults) != 2 {\n\t\tt.Error(\n\t\t\t\"Unexpected length of partition results\",\n\t\t\t\"expected\", 2,\n\t\t\t\"got\", len(resp.PartitionResults),\n\t\t)\n\t}\n}\n" }, { "path": "endtxn.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/endtxn\"\n)\n\n// EndTxnRequest represets a request sent to a kafka broker to end a transaction.\ntype EndTxnRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// The transactional id key.\n\tTransactionalID string\n\n\t// The Producer ID (PID) for the current producer session\n\tProducerID int\n\n\t// The epoch associated with the current producer session for the given PID\n\tProducerEpoch int\n\n\t// Committed should be set to true if the transaction was committed, false otherwise.\n\tCommitted bool\n}\n\n// EndTxnResponse represents a resposne from a kafka broker to an end transaction request.\ntype EndTxnResponse struct {\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// Error is non-nil if an error occureda and contains the kafka error code.\n\t// Programs may use the standard errors.Is function to test the error\n\t// against kafka error codes.\n\tError error\n}\n\n// EndTxn sends an EndTxn request to a kafka broker and returns its response.\nfunc (c *Client) EndTxn(ctx context.Context, req *EndTxnRequest) (*EndTxnResponse, error) {\n\tm, err := c.roundTrip(ctx, req.Addr, &endtxn.Request{\n\t\tTransactionalID: req.TransactionalID,\n\t\tProducerID: int64(req.ProducerID),\n\t\tProducerEpoch: int16(req.ProducerEpoch),\n\t\tCommitted: req.Committed,\n\t})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).EndTxn: %w\", err)\n\t}\n\n\tr := m.(*endtxn.Response)\n\n\tres := &EndTxnResponse{\n\t\tThrottle: makeDuration(r.ThrottleTimeMs),\n\t\tError: makeError(r.ErrorCode, \"\"),\n\t}\n\n\treturn res, nil\n}\n" }, { "path": "error.go", "content": "package kafka\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"syscall\"\n)\n\n// Error represents the different error codes that may be returned by kafka.\n// https://kafka.apache.org/protocol#protocol_error_codes\ntype Error int\n\nconst (\n\tUnknown Error = -1\n\tOffsetOutOfRange Error = 1\n\tInvalidMessage Error = 2\n\tUnknownTopicOrPartition Error = 3\n\tInvalidMessageSize Error = 4\n\tLeaderNotAvailable Error = 5\n\tNotLeaderForPartition Error = 6\n\tRequestTimedOut Error = 7\n\tBrokerNotAvailable Error = 8\n\tReplicaNotAvailable Error = 9\n\tMessageSizeTooLarge Error = 10\n\tStaleControllerEpoch Error = 11\n\tOffsetMetadataTooLarge Error = 12\n\tNetworkException Error = 13\n\tGroupLoadInProgress Error = 14\n\tGroupCoordinatorNotAvailable Error = 15\n\tNotCoordinatorForGroup Error = 16\n\tInvalidTopic Error = 17\n\tRecordListTooLarge Error = 18\n\tNotEnoughReplicas Error = 19\n\tNotEnoughReplicasAfterAppend Error = 20\n\tInvalidRequiredAcks Error = 21\n\tIllegalGeneration Error = 22\n\tInconsistentGroupProtocol Error = 23\n\tInvalidGroupId Error = 24\n\tUnknownMemberId Error = 25\n\tInvalidSessionTimeout Error = 26\n\tRebalanceInProgress Error = 27\n\tInvalidCommitOffsetSize Error = 28\n\tTopicAuthorizationFailed Error = 29\n\tGroupAuthorizationFailed Error = 30\n\tClusterAuthorizationFailed Error = 31\n\tInvalidTimestamp Error = 32\n\tUnsupportedSASLMechanism Error = 33\n\tIllegalSASLState Error = 34\n\tUnsupportedVersion Error = 35\n\tTopicAlreadyExists Error = 36\n\tInvalidPartitionNumber Error = 37\n\tInvalidReplicationFactor Error = 38\n\tInvalidReplicaAssignment Error = 39\n\tInvalidConfiguration Error = 40\n\tNotController Error = 41\n\tInvalidRequest Error = 42\n\tUnsupportedForMessageFormat Error = 43\n\tPolicyViolation Error = 44\n\tOutOfOrderSequenceNumber Error = 45\n\tDuplicateSequenceNumber Error = 46\n\tInvalidProducerEpoch Error = 47\n\tInvalidTransactionState Error = 48\n\tInvalidProducerIDMapping Error = 49\n\tInvalidTransactionTimeout Error = 50\n\tConcurrentTransactions Error = 51\n\tTransactionCoordinatorFenced Error = 52\n\tTransactionalIDAuthorizationFailed Error = 53\n\tSecurityDisabled Error = 54\n\tBrokerAuthorizationFailed Error = 55\n\tKafkaStorageError Error = 56\n\tLogDirNotFound Error = 57\n\tSASLAuthenticationFailed Error = 58\n\tUnknownProducerId Error = 59\n\tReassignmentInProgress Error = 60\n\tDelegationTokenAuthDisabled Error = 61\n\tDelegationTokenNotFound Error = 62\n\tDelegationTokenOwnerMismatch Error = 63\n\tDelegationTokenRequestNotAllowed Error = 64\n\tDelegationTokenAuthorizationFailed Error = 65\n\tDelegationTokenExpired Error = 66\n\tInvalidPrincipalType Error = 67\n\tNonEmptyGroup Error = 68\n\tGroupIdNotFound Error = 69\n\tFetchSessionIDNotFound Error = 70\n\tInvalidFetchSessionEpoch Error = 71\n\tListenerNotFound Error = 72\n\tTopicDeletionDisabled Error = 73\n\tFencedLeaderEpoch Error = 74\n\tUnknownLeaderEpoch Error = 75\n\tUnsupportedCompressionType Error = 76\n\tStaleBrokerEpoch Error = 77\n\tOffsetNotAvailable Error = 78\n\tMemberIDRequired Error = 79\n\tPreferredLeaderNotAvailable Error = 80\n\tGroupMaxSizeReached Error = 81\n\tFencedInstanceID Error = 82\n\tEligibleLeadersNotAvailable Error = 83\n\tElectionNotNeeded Error = 84\n\tNoReassignmentInProgress Error = 85\n\tGroupSubscribedToTopic Error = 86\n\tInvalidRecord Error = 87\n\tUnstableOffsetCommit Error = 88\n\tThrottlingQuotaExceeded Error = 89\n\tProducerFenced Error = 90\n\tResourceNotFound Error = 91\n\tDuplicateResource Error = 92\n\tUnacceptableCredential Error = 93\n\tInconsistentVoterSet Error = 94\n\tInvalidUpdateVersion Error = 95\n\tFeatureUpdateFailed Error = 96\n\tPrincipalDeserializationFailure Error = 97\n\tSnapshotNotFound Error = 98\n\tPositionOutOfRange Error = 99\n\tUnknownTopicID Error = 100\n\tDuplicateBrokerRegistration Error = 101\n\tBrokerIDNotRegistered Error = 102\n\tInconsistentTopicID Error = 103\n\tInconsistentClusterID Error = 104\n\tTransactionalIDNotFound Error = 105\n\tFetchSessionTopicIDError Error = 106\n)\n\n// Error satisfies the error interface.\nfunc (e Error) Error() string {\n\treturn fmt.Sprintf(\"[%d] %s: %s\", e, e.Title(), e.Description())\n}\n\n// Timeout returns true if the error was due to a timeout.\nfunc (e Error) Timeout() bool {\n\treturn e == RequestTimedOut\n}\n\n// Temporary returns true if the operation that generated the error may succeed\n// if retried at a later time.\n// Kafka error documentation specifies these as \"retriable\"\n// https://kafka.apache.org/protocol#protocol_error_codes\nfunc (e Error) Temporary() bool {\n\tswitch e {\n\tcase InvalidMessage,\n\t\tUnknownTopicOrPartition,\n\t\tLeaderNotAvailable,\n\t\tNotLeaderForPartition,\n\t\tRequestTimedOut,\n\t\tNetworkException,\n\t\tGroupLoadInProgress,\n\t\tGroupCoordinatorNotAvailable,\n\t\tNotCoordinatorForGroup,\n\t\tNotEnoughReplicas,\n\t\tNotEnoughReplicasAfterAppend,\n\t\tNotController,\n\t\tKafkaStorageError,\n\t\tFetchSessionIDNotFound,\n\t\tInvalidFetchSessionEpoch,\n\t\tListenerNotFound,\n\t\tFencedLeaderEpoch,\n\t\tUnknownLeaderEpoch,\n\t\tOffsetNotAvailable,\n\t\tPreferredLeaderNotAvailable,\n\t\tEligibleLeadersNotAvailable,\n\t\tElectionNotNeeded,\n\t\tNoReassignmentInProgress,\n\t\tGroupSubscribedToTopic,\n\t\tUnstableOffsetCommit,\n\t\tThrottlingQuotaExceeded,\n\t\tUnknownTopicID,\n\t\tInconsistentTopicID,\n\t\tFetchSessionTopicIDError:\n\t\treturn true\n\tdefault:\n\t\treturn false\n\t}\n}\n\n// Title returns a human readable title for the error.\nfunc (e Error) Title() string {\n\tswitch e {\n\tcase Unknown:\n\t\treturn \"Unknown\"\n\tcase OffsetOutOfRange:\n\t\treturn \"Offset Out Of Range\"\n\tcase InvalidMessage:\n\t\treturn \"Invalid Message\"\n\tcase UnknownTopicOrPartition:\n\t\treturn \"Unknown Topic Or Partition\"\n\tcase InvalidMessageSize:\n\t\treturn \"Invalid Message Size\"\n\tcase LeaderNotAvailable:\n\t\treturn \"Leader Not Available\"\n\tcase NotLeaderForPartition:\n\t\treturn \"Not Leader For Partition\"\n\tcase RequestTimedOut:\n\t\treturn \"Request Timed Out\"\n\tcase BrokerNotAvailable:\n\t\treturn \"Broker Not Available\"\n\tcase ReplicaNotAvailable:\n\t\treturn \"Replica Not Available\"\n\tcase MessageSizeTooLarge:\n\t\treturn \"Message Size Too Large\"\n\tcase StaleControllerEpoch:\n\t\treturn \"Stale Controller Epoch\"\n\tcase OffsetMetadataTooLarge:\n\t\treturn \"Offset Metadata Too Large\"\n\tcase GroupLoadInProgress:\n\t\treturn \"Group Load In Progress\"\n\tcase GroupCoordinatorNotAvailable:\n\t\treturn \"Group Coordinator Not Available\"\n\tcase NotCoordinatorForGroup:\n\t\treturn \"Not Coordinator For Group\"\n\tcase InvalidTopic:\n\t\treturn \"Invalid Topic\"\n\tcase RecordListTooLarge:\n\t\treturn \"Record List Too Large\"\n\tcase NotEnoughReplicas:\n\t\treturn \"Not Enough Replicas\"\n\tcase NotEnoughReplicasAfterAppend:\n\t\treturn \"Not Enough Replicas After Append\"\n\tcase InvalidRequiredAcks:\n\t\treturn \"Invalid Required Acks\"\n\tcase IllegalGeneration:\n\t\treturn \"Illegal Generation\"\n\tcase InconsistentGroupProtocol:\n\t\treturn \"Inconsistent Group Protocol\"\n\tcase InvalidGroupId:\n\t\treturn \"Invalid Group ID\"\n\tcase UnknownMemberId:\n\t\treturn \"Unknown Member ID\"\n\tcase InvalidSessionTimeout:\n\t\treturn \"Invalid Session Timeout\"\n\tcase RebalanceInProgress:\n\t\treturn \"Rebalance In Progress\"\n\tcase InvalidCommitOffsetSize:\n\t\treturn \"Invalid Commit Offset Size\"\n\tcase TopicAuthorizationFailed:\n\t\treturn \"Topic Authorization Failed\"\n\tcase GroupAuthorizationFailed:\n\t\treturn \"Group Authorization Failed\"\n\tcase ClusterAuthorizationFailed:\n\t\treturn \"Cluster Authorization Failed\"\n\tcase InvalidTimestamp:\n\t\treturn \"Invalid Timestamp\"\n\tcase UnsupportedSASLMechanism:\n\t\treturn \"Unsupported SASL Mechanism\"\n\tcase IllegalSASLState:\n\t\treturn \"Illegal SASL State\"\n\tcase UnsupportedVersion:\n\t\treturn \"Unsupported Version\"\n\tcase TopicAlreadyExists:\n\t\treturn \"Topic Already Exists\"\n\tcase InvalidPartitionNumber:\n\t\treturn \"Invalid Partition Number\"\n\tcase InvalidReplicationFactor:\n\t\treturn \"Invalid Replication Factor\"\n\tcase InvalidReplicaAssignment:\n\t\treturn \"Invalid Replica Assignment\"\n\tcase InvalidConfiguration:\n\t\treturn \"Invalid Configuration\"\n\tcase NotController:\n\t\treturn \"Not Controller\"\n\tcase InvalidRequest:\n\t\treturn \"Invalid Request\"\n\tcase UnsupportedForMessageFormat:\n\t\treturn \"Unsupported For Message Format\"\n\tcase PolicyViolation:\n\t\treturn \"Policy Violation\"\n\tcase OutOfOrderSequenceNumber:\n\t\treturn \"Out Of Order Sequence Number\"\n\tcase DuplicateSequenceNumber:\n\t\treturn \"Duplicate Sequence Number\"\n\tcase InvalidProducerEpoch:\n\t\treturn \"Invalid Producer Epoch\"\n\tcase InvalidTransactionState:\n\t\treturn \"Invalid Transaction State\"\n\tcase InvalidProducerIDMapping:\n\t\treturn \"Invalid Producer ID Mapping\"\n\tcase InvalidTransactionTimeout:\n\t\treturn \"Invalid Transaction Timeout\"\n\tcase ConcurrentTransactions:\n\t\treturn \"Concurrent Transactions\"\n\tcase TransactionCoordinatorFenced:\n\t\treturn \"Transaction Coordinator Fenced\"\n\tcase TransactionalIDAuthorizationFailed:\n\t\treturn \"Transactional ID Authorization Failed\"\n\tcase SecurityDisabled:\n\t\treturn \"Security Disabled\"\n\tcase BrokerAuthorizationFailed:\n\t\treturn \"Broker Authorization Failed\"\n\tcase KafkaStorageError:\n\t\treturn \"Kafka Storage Error\"\n\tcase LogDirNotFound:\n\t\treturn \"Log Dir Not Found\"\n\tcase SASLAuthenticationFailed:\n\t\treturn \"SASL Authentication Failed\"\n\tcase UnknownProducerId:\n\t\treturn \"Unknown Producer ID\"\n\tcase ReassignmentInProgress:\n\t\treturn \"Reassignment In Progress\"\n\tcase DelegationTokenAuthDisabled:\n\t\treturn \"Delegation Token Auth Disabled\"\n\tcase DelegationTokenNotFound:\n\t\treturn \"Delegation Token Not Found\"\n\tcase DelegationTokenOwnerMismatch:\n\t\treturn \"Delegation Token Owner Mismatch\"\n\tcase DelegationTokenRequestNotAllowed:\n\t\treturn \"Delegation Token Request Not Allowed\"\n\tcase DelegationTokenAuthorizationFailed:\n\t\treturn \"Delegation Token Authorization Failed\"\n\tcase DelegationTokenExpired:\n\t\treturn \"Delegation Token Expired\"\n\tcase InvalidPrincipalType:\n\t\treturn \"Invalid Principal Type\"\n\tcase NonEmptyGroup:\n\t\treturn \"Non Empty Group\"\n\tcase GroupIdNotFound:\n\t\treturn \"Group ID Not Found\"\n\tcase FetchSessionIDNotFound:\n\t\treturn \"Fetch Session ID Not Found\"\n\tcase InvalidFetchSessionEpoch:\n\t\treturn \"Invalid Fetch Session Epoch\"\n\tcase ListenerNotFound:\n\t\treturn \"Listener Not Found\"\n\tcase TopicDeletionDisabled:\n\t\treturn \"Topic Deletion Disabled\"\n\tcase FencedLeaderEpoch:\n\t\treturn \"Fenced Leader Epoch\"\n\tcase UnknownLeaderEpoch:\n\t\treturn \"Unknown Leader Epoch\"\n\tcase UnsupportedCompressionType:\n\t\treturn \"Unsupported Compression Type\"\n\tcase MemberIDRequired:\n\t\treturn \"Member ID Required\"\n\tcase FencedInstanceID:\n\t\treturn \"Fenced Instance ID\"\n\tcase EligibleLeadersNotAvailable:\n\t\treturn \"Eligible Leader Not Available\"\n\tcase ElectionNotNeeded:\n\t\treturn \"Election Not Needed\"\n\tcase NoReassignmentInProgress:\n\t\treturn \"No Reassignment In Progress\"\n\tcase GroupSubscribedToTopic:\n\t\treturn \"Group Subscribed To Topic\"\n\tcase InvalidRecord:\n\t\treturn \"Invalid Record\"\n\tcase UnstableOffsetCommit:\n\t\treturn \"Unstable Offset Commit\"\n\tcase ThrottlingQuotaExceeded:\n\t\treturn \"Throttling Quota Exceeded\"\n\tcase ProducerFenced:\n\t\treturn \"Producer Fenced\"\n\tcase ResourceNotFound:\n\t\treturn \"Resource Not Found\"\n\tcase DuplicateResource:\n\t\treturn \"Duplicate Resource\"\n\tcase UnacceptableCredential:\n\t\treturn \"Unacceptable Credential\"\n\tcase InconsistentVoterSet:\n\t\treturn \"Inconsistent Voter Set\"\n\tcase InvalidUpdateVersion:\n\t\treturn \"Invalid Update Version\"\n\tcase FeatureUpdateFailed:\n\t\treturn \"Feature Update Failed\"\n\tcase PrincipalDeserializationFailure:\n\t\treturn \"Principal Deserialization Failure\"\n\tcase SnapshotNotFound:\n\t\treturn \"Snapshot Not Found\"\n\tcase PositionOutOfRange:\n\t\treturn \"Position Out Of Range\"\n\tcase UnknownTopicID:\n\t\treturn \"Unknown Topic ID\"\n\tcase DuplicateBrokerRegistration:\n\t\treturn \"Duplicate Broker Registration\"\n\tcase BrokerIDNotRegistered:\n\t\treturn \"Broker ID Not Registered\"\n\tcase InconsistentTopicID:\n\t\treturn \"Inconsistent Topic ID\"\n\tcase InconsistentClusterID:\n\t\treturn \"Inconsistent Cluster ID\"\n\tcase TransactionalIDNotFound:\n\t\treturn \"Transactional ID Not Found\"\n\tcase FetchSessionTopicIDError:\n\t\treturn \"Fetch Session Topic ID Error\"\n\t}\n\treturn \"\"\n}\n\n// Description returns a human readable description of cause of the error.\nfunc (e Error) Description() string {\n\tswitch e {\n\tcase Unknown:\n\t\treturn \"an unexpected server error occurred\"\n\tcase OffsetOutOfRange:\n\t\treturn \"the requested offset is outside the range of offsets maintained by the server for the given topic/partition\"\n\tcase InvalidMessage:\n\t\treturn \"the message contents does not match its CRC\"\n\tcase UnknownTopicOrPartition:\n\t\treturn \"the request is for a topic or partition that does not exist on this broker\"\n\tcase InvalidMessageSize:\n\t\treturn \"the message has a negative size\"\n\tcase LeaderNotAvailable:\n\t\treturn \"the cluster is in the middle of a leadership election and there is currently no leader for this partition and hence it is unavailable for writes\"\n\tcase NotLeaderForPartition:\n\t\treturn \"the client attempted to send messages to a replica that is not the leader for some partition, the client's metadata are likely out of date\"\n\tcase RequestTimedOut:\n\t\treturn \"the request exceeded the user-specified time limit in the request\"\n\tcase BrokerNotAvailable:\n\t\treturn \"not a client facing error and is used mostly by tools when a broker is not alive\"\n\tcase ReplicaNotAvailable:\n\t\treturn \"a replica is expected on a broker, but is not (this can be safely ignored)\"\n\tcase MessageSizeTooLarge:\n\t\treturn \"the server has a configurable maximum message size to avoid unbounded memory allocation and the client attempted to produce a message larger than this maximum\"\n\tcase StaleControllerEpoch:\n\t\treturn \"internal error code for broker-to-broker communication\"\n\tcase OffsetMetadataTooLarge:\n\t\treturn \"the client specified a string larger than configured maximum for offset metadata\"\n\tcase GroupLoadInProgress:\n\t\treturn \"the broker returns this error code for an offset fetch request if it is still loading offsets (after a leader change for that offsets topic partition), or in response to group membership requests (such as heartbeats) when group metadata is being loaded by the coordinator\"\n\tcase GroupCoordinatorNotAvailable:\n\t\treturn \"the broker returns this error code for group coordinator requests, offset commits, and most group management requests if the offsets topic has not yet been created, or if the group coordinator is not active\"\n\tcase NotCoordinatorForGroup:\n\t\treturn \"the broker returns this error code if it receives an offset fetch or commit request for a group that it is not a coordinator for\"\n\tcase InvalidTopic:\n\t\treturn \"a request which attempted to access an invalid topic (e.g. one which has an illegal name), or if an attempt was made to write to an internal topic (such as the consumer offsets topic)\"\n\tcase RecordListTooLarge:\n\t\treturn \"a message batch in a produce request exceeds the maximum configured segment size\"\n\tcase NotEnoughReplicas:\n\t\treturn \"the number of in-sync replicas is lower than the configured minimum and requiredAcks is -1\"\n\tcase NotEnoughReplicasAfterAppend:\n\t\treturn \"the message was written to the log, but with fewer in-sync replicas than required.\"\n\tcase InvalidRequiredAcks:\n\t\treturn \"the requested requiredAcks is invalid (anything other than -1, 1, or 0)\"\n\tcase IllegalGeneration:\n\t\treturn \"the generation id provided in the request is not the current generation\"\n\tcase InconsistentGroupProtocol:\n\t\treturn \"the member provided a protocol type or set of protocols which is not compatible with the current group\"\n\tcase InvalidGroupId:\n\t\treturn \"the group id is empty or null\"\n\tcase UnknownMemberId:\n\t\treturn \"the member id is not in the current generation\"\n\tcase InvalidSessionTimeout:\n\t\treturn \"the requested session timeout is outside of the allowed range on the broker\"\n\tcase RebalanceInProgress:\n\t\treturn \"the coordinator has begun rebalancing the group, the client should rejoin the group\"\n\tcase InvalidCommitOffsetSize:\n\t\treturn \"an offset commit was rejected because of oversize metadata\"\n\tcase TopicAuthorizationFailed:\n\t\treturn \"the client is not authorized to access the requested topic\"\n\tcase GroupAuthorizationFailed:\n\t\treturn \"the client is not authorized to access a particular group id\"\n\tcase ClusterAuthorizationFailed:\n\t\treturn \"the client is not authorized to use an inter-broker or administrative API\"\n\tcase InvalidTimestamp:\n\t\treturn \"the timestamp of the message is out of acceptable range\"\n\tcase UnsupportedSASLMechanism:\n\t\treturn \"the broker does not support the requested SASL mechanism\"\n\tcase IllegalSASLState:\n\t\treturn \"the request is not valid given the current SASL state\"\n\tcase UnsupportedVersion:\n\t\treturn \"the version of API is not supported\"\n\tcase TopicAlreadyExists:\n\t\treturn \"a topic with this name already exists\"\n\tcase InvalidPartitionNumber:\n\t\treturn \"the number of partitions is invalid\"\n\tcase InvalidReplicationFactor:\n\t\treturn \"the replication-factor is invalid\"\n\tcase InvalidReplicaAssignment:\n\t\treturn \"the replica assignment is invalid\"\n\tcase InvalidConfiguration:\n\t\treturn \"the configuration is invalid\"\n\tcase NotController:\n\t\treturn \"this is not the correct controller for this cluster\"\n\tcase InvalidRequest:\n\t\treturn \"this most likely occurs because of a request being malformed by the client library or the message was sent to an incompatible broker, se the broker logs for more details\"\n\tcase UnsupportedForMessageFormat:\n\t\treturn \"the message format version on the broker does not support the request\"\n\tcase PolicyViolation:\n\t\treturn \"the request parameters do not satisfy the configured policy\"\n\tcase OutOfOrderSequenceNumber:\n\t\treturn \"the broker received an out of order sequence number\"\n\tcase DuplicateSequenceNumber:\n\t\treturn \"the broker received a duplicate sequence number\"\n\tcase InvalidProducerEpoch:\n\t\treturn \"the producer attempted an operation with an old epoch, either there is a newer producer with the same transactional ID, or the producer's transaction has been expired by the broker\"\n\tcase InvalidTransactionState:\n\t\treturn \"the producer attempted a transactional operation in an invalid state\"\n\tcase InvalidProducerIDMapping:\n\t\treturn \"the producer attempted to use a producer id which is not currently assigned to its transactional ID\"\n\tcase InvalidTransactionTimeout:\n\t\treturn \"the transaction timeout is larger than the maximum value allowed by the broker (as configured by max.transaction.timeout.ms)\"\n\tcase ConcurrentTransactions:\n\t\treturn \"the producer attempted to update a transaction while another concurrent operation on the same transaction was ongoing\"\n\tcase TransactionCoordinatorFenced:\n\t\treturn \"the transaction coordinator sending a WriteTxnMarker is no longer the current coordinator for a given producer\"\n\tcase TransactionalIDAuthorizationFailed:\n\t\treturn \"the transactional ID authorization failed\"\n\tcase SecurityDisabled:\n\t\treturn \"the security features are disabled\"\n\tcase BrokerAuthorizationFailed:\n\t\treturn \"the broker authorization failed\"\n\tcase KafkaStorageError:\n\t\treturn \"disk error when trying to access log file on the disk\"\n\tcase LogDirNotFound:\n\t\treturn \"the user-specified log directory is not found in the broker config\"\n\tcase SASLAuthenticationFailed:\n\t\treturn \"SASL Authentication failed\"\n\tcase UnknownProducerId:\n\t\treturn \"the broker could not locate the producer metadata associated with the producer ID\"\n\tcase ReassignmentInProgress:\n\t\treturn \"a partition reassignment is in progress\"\n\tcase DelegationTokenAuthDisabled:\n\t\treturn \"delegation token feature is not enabled\"\n\tcase DelegationTokenNotFound:\n\t\treturn \"delegation token is not found on server\"\n\tcase DelegationTokenOwnerMismatch:\n\t\treturn \"specified principal is not valid owner/renewer\"\n\tcase DelegationTokenRequestNotAllowed:\n\t\treturn \"delegation token requests are not allowed on plaintext/1-way ssl channels and on delegation token authenticated channels\"\n\tcase DelegationTokenAuthorizationFailed:\n\t\treturn \"delegation token authorization failed\"\n\tcase DelegationTokenExpired:\n\t\treturn \"delegation token is expired\"\n\tcase InvalidPrincipalType:\n\t\treturn \"supplied principaltype is not supported\"\n\tcase NonEmptyGroup:\n\t\treturn \"the group is not empty\"\n\tcase GroupIdNotFound:\n\t\treturn \"the group ID does not exist\"\n\tcase FetchSessionIDNotFound:\n\t\treturn \"the fetch session ID was not found\"\n\tcase InvalidFetchSessionEpoch:\n\t\treturn \"the fetch session epoch is invalid\"\n\tcase ListenerNotFound:\n\t\treturn \"there is no listener on the leader broker that matches the listener on which metadata request was processed\"\n\tcase TopicDeletionDisabled:\n\t\treturn \"topic deletion is disabled\"\n\tcase FencedLeaderEpoch:\n\t\treturn \"the leader epoch in the request is older than the epoch on the broker\"\n\tcase UnknownLeaderEpoch:\n\t\treturn \"the leader epoch in the request is newer than the epoch on the broker\"\n\tcase UnsupportedCompressionType:\n\t\treturn \"the requesting client does not support the compression type of given partition\"\n\tcase MemberIDRequired:\n\t\treturn \"the group member needs to have a valid member id before actually entering a consumer group\"\n\tcase FencedInstanceID:\n\t\treturn \"the broker rejected this static consumer since another consumer with the same group.instance.id has registered with a different member.id\"\n\tcase EligibleLeadersNotAvailable:\n\t\treturn \"eligible topic partition leaders are not available\"\n\tcase ElectionNotNeeded:\n\t\treturn \"leader election not needed for topic partition\"\n\tcase NoReassignmentInProgress:\n\t\treturn \"no partition reassignment is in progress\"\n\tcase GroupSubscribedToTopic:\n\t\treturn \"deleting offsets of a topic is forbidden while the consumer group is actively subscribed to it\"\n\tcase InvalidRecord:\n\t\treturn \"this record has failed the validation on broker and hence be rejected\"\n\tcase UnstableOffsetCommit:\n\t\treturn \"there are unstable offsets that need to be cleared\"\n\tcase ThrottlingQuotaExceeded:\n\t\treturn \"The throttling quota has been exceeded\"\n\tcase ProducerFenced:\n\t\treturn \"There is a newer producer with the same transactionalId which fences the current one\"\n\tcase ResourceNotFound:\n\t\treturn \"A request illegally referred to a resource that does not exist\"\n\tcase DuplicateResource:\n\t\treturn \"A request illegally referred to the same resource twice\"\n\tcase UnacceptableCredential:\n\t\treturn \"Requested credential would not meet criteria for acceptability\"\n\tcase InconsistentVoterSet:\n\t\treturn \"Indicates that the either the sender or recipient of a voter-only request is not one of the expected voters\"\n\tcase InvalidUpdateVersion:\n\t\treturn \"The given update version was invalid\"\n\tcase FeatureUpdateFailed:\n\t\treturn \"Unable to update finalized features due to an unexpected server error\"\n\tcase PrincipalDeserializationFailure:\n\t\treturn \"Request principal deserialization failed during forwarding. This indicates an internal error on the broker cluster security setup\"\n\tcase SnapshotNotFound:\n\t\treturn \"Requested snapshot was not found\"\n\tcase PositionOutOfRange:\n\t\treturn \"Requested position is not greater than or equal to zero, and less than the size of the snapshot\"\n\tcase UnknownTopicID:\n\t\treturn \"This server does not host this topic ID\"\n\tcase DuplicateBrokerRegistration:\n\t\treturn \"This broker ID is already in use\"\n\tcase BrokerIDNotRegistered:\n\t\treturn \"The given broker ID was not registered\"\n\tcase InconsistentTopicID:\n\t\treturn \"The log's topic ID did not match the topic ID in the request\"\n\tcase InconsistentClusterID:\n\t\treturn \"The clusterId in the request does not match that found on the server\"\n\tcase TransactionalIDNotFound:\n\t\treturn \"The transactionalId could not be found\"\n\tcase FetchSessionTopicIDError:\n\t\treturn \"The fetch session encountered inconsistent topic ID usage\"\n\t}\n\treturn \"\"\n}\n\nfunc isTimeout(err error) bool {\n\tvar timeoutError interface{ Timeout() bool }\n\tif errors.As(err, &timeoutError) {\n\t\treturn timeoutError.Timeout()\n\t}\n\treturn false\n}\n\nfunc isTemporary(err error) bool {\n\tvar tempError interface{ Temporary() bool }\n\tif errors.As(err, &tempError) {\n\t\treturn tempError.Temporary()\n\t}\n\treturn false\n}\n\nfunc isTransientNetworkError(err error) bool {\n\treturn errors.Is(err, io.ErrUnexpectedEOF) ||\n\t\terrors.Is(err, syscall.ECONNREFUSED) ||\n\t\terrors.Is(err, syscall.ECONNRESET) ||\n\t\terrors.Is(err, syscall.EPIPE)\n}\n\nfunc silentEOF(err error) error {\n\tif errors.Is(err, io.EOF) {\n\t\terr = nil\n\t}\n\treturn err\n}\n\nfunc dontExpectEOF(err error) error {\n\tif errors.Is(err, io.EOF) {\n\t\treturn io.ErrUnexpectedEOF\n\t}\n\treturn err\n}\n\nfunc coalesceErrors(errs ...error) error {\n\tfor _, err := range errs {\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\n// MessageTooLargeError is returned when a message is too large to fit within the allowed size.\ntype MessageTooLargeError struct {\n\tMessage Message\n\tRemaining []Message\n}\n\nfunc messageTooLarge(msgs []Message, i int) MessageTooLargeError {\n\tremain := make([]Message, 0, len(msgs)-1)\n\tremain = append(remain, msgs[:i]...)\n\tremain = append(remain, msgs[i+1:]...)\n\treturn MessageTooLargeError{\n\t\tMessage: msgs[i],\n\t\tRemaining: remain,\n\t}\n}\n\nfunc (e MessageTooLargeError) Error() string {\n\treturn MessageSizeTooLarge.Error()\n}\n\nfunc (e MessageTooLargeError) Unwrap() error {\n\treturn MessageSizeTooLarge\n}\n\nfunc makeError(code int16, message string) error {\n\tif code == 0 {\n\t\treturn nil\n\t}\n\tif message == \"\" {\n\t\treturn Error(code)\n\t}\n\treturn fmt.Errorf(\"%w: %s\", Error(code), message)\n}\n\n// WriteError is returned by kafka.(*Writer).WriteMessages when the writer is\n// not configured to write messages asynchronously. WriteError values contain\n// a list of errors where each entry matches the position of a message in the\n// WriteMessages call. The program can determine the status of each message by\n// looping over the error:\n//\n//\tswitch err := w.WriteMessages(ctx, msgs...).(type) {\n//\tcase nil:\n//\tcase kafka.WriteErrors:\n//\t\tfor i := range msgs {\n//\t\t\tif err[i] != nil {\n//\t\t\t\t// handle the error writing msgs[i]\n//\t\t\t\t...\n//\t\t\t}\n//\t\t}\n//\tdefault:\n//\t\t// handle other errors\n//\t\t...\n//\t}\ntype WriteErrors []error\n\n// Count counts the number of non-nil errors in err.\nfunc (err WriteErrors) Count() int {\n\tn := 0\n\n\tfor _, e := range err {\n\t\tif e != nil {\n\t\t\tn++\n\t\t}\n\t}\n\n\treturn n\n}\n\nfunc (err WriteErrors) Error() string {\n\terrCount := err.Count()\n\terrors := make([]string, 0, errCount)\n\tfor _, writeError := range err {\n\t\tif writeError == nil {\n\t\t\tcontinue\n\t\t}\n\t\terrors = append(errors, writeError.Error())\n\t}\n\treturn fmt.Sprintf(\"Kafka write errors (%d/%d), errors: %v\", errCount, len(err), errors)\n}\n" }, { "path": "error_test.go", "content": "package kafka\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestError(t *testing.T) {\n\terrorCodes := []Error{\n\t\tUnknown,\n\t\tOffsetOutOfRange,\n\t\tInvalidMessage,\n\t\tUnknownTopicOrPartition,\n\t\tInvalidMessageSize,\n\t\tLeaderNotAvailable,\n\t\tNotLeaderForPartition,\n\t\tRequestTimedOut,\n\t\tBrokerNotAvailable,\n\t\tReplicaNotAvailable,\n\t\tMessageSizeTooLarge,\n\t\tStaleControllerEpoch,\n\t\tOffsetMetadataTooLarge,\n\t\tGroupLoadInProgress,\n\t\tGroupCoordinatorNotAvailable,\n\t\tNotCoordinatorForGroup,\n\t\tInvalidTopic,\n\t\tRecordListTooLarge,\n\t\tNotEnoughReplicas,\n\t\tNotEnoughReplicasAfterAppend,\n\t\tInvalidRequiredAcks,\n\t\tIllegalGeneration,\n\t\tInconsistentGroupProtocol,\n\t\tInvalidGroupId,\n\t\tUnknownMemberId,\n\t\tInvalidSessionTimeout,\n\t\tRebalanceInProgress,\n\t\tInvalidCommitOffsetSize,\n\t\tTopicAuthorizationFailed,\n\t\tGroupAuthorizationFailed,\n\t\tClusterAuthorizationFailed,\n\t\tInvalidTimestamp,\n\t\tUnsupportedSASLMechanism,\n\t\tIllegalSASLState,\n\t\tUnsupportedVersion,\n\t\tTopicAlreadyExists,\n\t\tInvalidPartitionNumber,\n\t\tInvalidReplicationFactor,\n\t\tInvalidReplicaAssignment,\n\t\tInvalidConfiguration,\n\t\tNotController,\n\t\tInvalidRequest,\n\t\tUnsupportedForMessageFormat,\n\t\tPolicyViolation,\n\t\tOutOfOrderSequenceNumber,\n\t\tDuplicateSequenceNumber,\n\t\tInvalidProducerEpoch,\n\t\tInvalidTransactionState,\n\t\tInvalidProducerIDMapping,\n\t\tInvalidTransactionTimeout,\n\t\tConcurrentTransactions,\n\t\tTransactionCoordinatorFenced,\n\t\tTransactionalIDAuthorizationFailed,\n\t\tSecurityDisabled,\n\t\tBrokerAuthorizationFailed,\n\t\tKafkaStorageError,\n\t\tLogDirNotFound,\n\t\tSASLAuthenticationFailed,\n\t\tUnknownProducerId,\n\t\tReassignmentInProgress,\n\t\tDelegationTokenAuthDisabled,\n\t\tDelegationTokenNotFound,\n\t\tDelegationTokenOwnerMismatch,\n\t\tDelegationTokenRequestNotAllowed,\n\t\tDelegationTokenAuthorizationFailed,\n\t\tDelegationTokenExpired,\n\t\tInvalidPrincipalType,\n\t\tNonEmptyGroup,\n\t\tGroupIdNotFound,\n\t\tFetchSessionIDNotFound,\n\t\tInvalidFetchSessionEpoch,\n\t\tListenerNotFound,\n\t\tTopicDeletionDisabled,\n\t\tFencedLeaderEpoch,\n\t\tUnknownLeaderEpoch,\n\t\tUnsupportedCompressionType,\n\t}\n\n\tfor _, err := range errorCodes {\n\t\tt.Run(fmt.Sprintf(\"verify that error %d has a non-empty title, description, and error message\", err), func(t *testing.T) {\n\t\t\tif len(err.Title()) == 0 {\n\t\t\t\tt.Error(\"empty title\")\n\t\t\t}\n\t\t\tif len(err.Description()) == 0 {\n\t\t\t\tt.Error(\"empty description\")\n\t\t\t}\n\t\t\tif len(err.Error()) == 0 {\n\t\t\t\tt.Error(\"empty error message\")\n\t\t\t}\n\t\t})\n\t}\n\n\tt.Run(\"verify that an invalid error code has an empty title and description\", func(t *testing.T) {\n\t\terr := Error(-2)\n\n\t\tif s := err.Title(); len(s) != 0 {\n\t\t\tt.Error(\"non-empty title:\", s)\n\t\t}\n\n\t\tif s := err.Description(); len(s) != 0 {\n\t\t\tt.Error(\"non-empty description:\", s)\n\t\t}\n\t})\n\n\tt.Run(\"MessageTooLargeError error.Is satisfaction\", func(t *testing.T) {\n\t\terr := MessageSizeTooLarge\n\t\tmsg := []Message{\n\t\t\t{Key: []byte(\"key\"), Value: []byte(\"value\")},\n\t\t\t{Key: []byte(\"key\"), Value: make([]byte, 8)},\n\t\t}\n\t\tmsgTooLarge := messageTooLarge(msg, 1)\n\t\tassert.NotErrorIs(t, err, msgTooLarge)\n\t\tassert.Contains(t, msgTooLarge.Error(), MessageSizeTooLarge.Error())\n\t\tassert.ErrorIs(t, msgTooLarge, MessageSizeTooLarge)\n\t})\n}\n" }, { "path": "example_consumergroup_test.go", "content": "package kafka_test\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\n\t\"github.com/segmentio/kafka-go\"\n)\n\nfunc ExampleGeneration_Start_consumerGroupParallelReaders() {\n\tgroup, err := kafka.NewConsumerGroup(kafka.ConsumerGroupConfig{\n\t\tID: \"my-group\",\n\t\tBrokers: []string{\"kafka:9092\"},\n\t\tTopics: []string{\"my-topic\"},\n\t})\n\tif err != nil {\n\t\tfmt.Printf(\"error creating consumer group: %+v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\tdefer group.Close()\n\n\tfor {\n\t\tgen, err := group.Next(context.TODO())\n\t\tif err != nil {\n\t\t\tbreak\n\t\t}\n\n\t\tassignments := gen.Assignments[\"my-topic\"]\n\t\tfor _, assignment := range assignments {\n\t\t\tpartition, offset := assignment.ID, assignment.Offset\n\t\t\tgen.Start(func(ctx context.Context) {\n\t\t\t\t// create reader for this partition.\n\t\t\t\treader := kafka.NewReader(kafka.ReaderConfig{\n\t\t\t\t\tBrokers: []string{\"127.0.0.1:9092\"},\n\t\t\t\t\tTopic: \"my-topic\",\n\t\t\t\t\tPartition: partition,\n\t\t\t\t})\n\t\t\t\tdefer reader.Close()\n\n\t\t\t\t// seek to the last committed offset for this partition.\n\t\t\t\treader.SetOffset(offset)\n\t\t\t\tfor {\n\t\t\t\t\tmsg, err := reader.ReadMessage(ctx)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tif errors.Is(err, kafka.ErrGenerationEnded) {\n\t\t\t\t\t\t\t// generation has ended. commit offsets. in a real app,\n\t\t\t\t\t\t\t// offsets would be committed periodically.\n\t\t\t\t\t\t\tgen.CommitOffsets(map[string]map[int]int64{\"my-topic\": {partition: offset + 1}})\n\t\t\t\t\t\t\treturn\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tfmt.Printf(\"error reading message: %+v\\n\", err)\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\n\t\t\t\t\tfmt.Printf(\"received message %s/%d/%d : %s\\n\", msg.Topic, msg.Partition, msg.Offset, string(msg.Value))\n\t\t\t\t\toffset = msg.Offset\n\t\t\t\t}\n\t\t\t})\n\t\t}\n\t}\n}\n\nfunc ExampleGeneration_CommitOffsets_overwriteOffsets() {\n\tgroup, err := kafka.NewConsumerGroup(kafka.ConsumerGroupConfig{\n\t\tID: \"my-group\",\n\t\tBrokers: []string{\"kafka:9092\"},\n\t\tTopics: []string{\"my-topic\"},\n\t})\n\tif err != nil {\n\t\tfmt.Printf(\"error creating consumer group: %+v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\tdefer group.Close()\n\n\tgen, err := group.Next(context.TODO())\n\tif err != nil {\n\t\tfmt.Printf(\"error getting next generation: %+v\\n\", err)\n\t\tos.Exit(1)\n\t}\n\terr = gen.CommitOffsets(map[string]map[int]int64{\n\t\t\"my-topic\": {\n\t\t\t0: 123,\n\t\t\t1: 456,\n\t\t\t3: 789,\n\t\t},\n\t})\n\tif err != nil {\n\t\tfmt.Printf(\"error committing offsets next generation: %+v\\n\", err)\n\t\tos.Exit(1)\n\t}\n}\n" }, { "path": "example_groupbalancer_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"io/ioutil\"\n\t\"net/http\"\n\t\"os\"\n\t\"strings\"\n\t\"time\"\n)\n\n// ExampleNewReader_rackAffinity shows how the RackAffinityGroupBalancer can be\n// used to pair up consumers with brokers in the same AWS availability zone.\n// This code assumes that each brokers' rack is configured to be the name of the\n// AZ in which it is running.\nfunc ExampleNewReader_rackAffinity() {\n\tr := NewReader(ReaderConfig{\n\t\tBrokers: []string{\"kafka:9092\"},\n\t\tGroupID: \"my-group\",\n\t\tTopic: \"my-topic\",\n\t\tGroupBalancers: []GroupBalancer{\n\t\t\tRackAffinityGroupBalancer{Rack: findRack()},\n\t\t\tRangeGroupBalancer{},\n\t\t},\n\t})\n\n\tr.ReadMessage(context.Background())\n\n\tr.Close()\n}\n\n// findRack is the basic rack resolver strategy for use in AWS. It supports\n// * ECS with the task metadata endpoint enabled (returns the container\n// instance's availability zone)\n// * Linux EC2 (returns the instance's availability zone)\nfunc findRack() string {\n\tswitch whereAmI() {\n\tcase \"ecs\":\n\t\treturn ecsAvailabilityZone()\n\tcase \"ec2\":\n\t\treturn ec2AvailabilityZone()\n\t}\n\treturn \"\"\n}\n\nconst ecsContainerMetadataURI = \"ECS_CONTAINER_METADATA_URI\"\n\n// whereAmI determines which strategy the rack resolver should use.\nfunc whereAmI() string {\n\t// https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-metadata-endpoint.html\n\tif os.Getenv(ecsContainerMetadataURI) != \"\" {\n\t\treturn \"ecs\"\n\t}\n\t// https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/identify_ec2_instances.html\n\tfor _, path := range [...]string{\n\t\t\"/sys/devices/virtual/dmi/id/product_uuid\",\n\t\t\"/sys/hypervisor/uuid\",\n\t} {\n\t\tb, err := ioutil.ReadFile(path)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\ts := string(b)\n\t\tswitch {\n\t\tcase strings.HasPrefix(s, \"EC2\"), strings.HasPrefix(s, \"ec2\"):\n\t\t\treturn \"ec2\"\n\t\t}\n\t}\n\treturn \"somewhere\"\n}\n\n// ecsAvailabilityZone queries the task endpoint for the metadata URI that ECS\n// injects into the ECS_CONTAINER_METADATA_URI variable in order to retrieve\n// the availability zone where the task is running.\nfunc ecsAvailabilityZone() string {\n\tclient := http.Client{\n\t\tTimeout: time.Second,\n\t\tTransport: &http.Transport{\n\t\t\tDisableCompression: true,\n\t\t\tDisableKeepAlives: true,\n\t\t},\n\t}\n\tr, err := client.Get(os.Getenv(ecsContainerMetadataURI) + \"/task\")\n\tif err != nil {\n\t\treturn \"\"\n\t}\n\tdefer r.Body.Close()\n\n\tvar md struct {\n\t\tAvailabilityZone string\n\t}\n\tif err := json.NewDecoder(r.Body).Decode(&md); err != nil {\n\t\treturn \"\"\n\t}\n\treturn md.AvailabilityZone\n}\n\n// ec2AvailabilityZone queries the metadata endpoint to discover the\n// availability zone where this code is running. we avoid calling this function\n// unless we know we're in EC2. Otherwise, in other environments, we would need\n// to wait for the request to 169.254.169.254 to timeout before proceeding.\nfunc ec2AvailabilityZone() string {\n\tclient := http.Client{\n\t\tTimeout: time.Second,\n\t\tTransport: &http.Transport{\n\t\t\tDisableCompression: true,\n\t\t\tDisableKeepAlives: true,\n\t\t},\n\t}\n\tr, err := client.Get(\"http://169.254.169.254/latest/meta-data/placement/availability-zone\")\n\tif err != nil {\n\t\treturn \"\"\n\t}\n\tdefer r.Body.Close()\n\tb, err := ioutil.ReadAll(r.Body)\n\tif err != nil {\n\t\treturn \"\"\n\t}\n\treturn string(b)\n}\n" }, { "path": "example_writer_test.go", "content": "package kafka_test\n\nimport (\n\t\"context\"\n\n\t\"github.com/segmentio/kafka-go\"\n)\n\nfunc ExampleWriter() {\n\tw := &kafka.Writer{\n\t\tAddr: kafka.TCP(\"localhost:9092\"),\n\t\tTopic: \"Topic-1\",\n\t}\n\n\tw.WriteMessages(context.Background(),\n\t\tkafka.Message{\n\t\t\tKey: []byte(\"Key-A\"),\n\t\t\tValue: []byte(\"Hello World!\"),\n\t\t},\n\t)\n\n\tw.Close()\n}\n" }, { "path": "examples/.gitignore", "content": "kafka-go" }, { "path": "examples/consumer-logger/Dockerfile", "content": "#####################################\n# STEP 1 build executable binary #\n#####################################\nFROM golang:alpine AS builder\n\n# Install git.\n# Git is required for fetching the dependencies.\nRUN apk update && apk add --no-cache git\n\nWORKDIR /app\n\nCOPY go.mod .\nCOPY go.sum .\n\nRUN go mod download\n\nCOPY . .\n\n# Build the binary.\nRUN CGO_ENABLED=0 GOOS=linux go build -o main\n\n#####################################\n# STEP 2 build a small image #\n#####################################\nFROM scratch\n\n# Copy our static executable.\nCOPY --from=builder /app/main /app/main\n\n# Run the hello binary.\nENTRYPOINT [\"/app/main\"]" }, { "path": "examples/consumer-logger/go.mod", "content": "module github.com/segmentio/kafka-go/example/consumer-logger\n\ngo 1.15\n\nrequire (\n\tgithub.com/klauspost/compress v1.12.2 // indirect\n\tgithub.com/segmentio/kafka-go v0.4.28\n)\n" }, { "path": "examples/consumer-logger/go.sum", "content": "github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=\ngithub.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=\ngithub.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=\ngithub.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY=\ngithub.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=\ngithub.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=\ngithub.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA=\ngithub.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=\ngithub.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=\ngithub.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=\ngithub.com/klauspost/compress v1.12.2 h1:2KCfW3I9M7nSc5wOqXAlW2v2U6v+w6cbjvbfp+OykW8=\ngithub.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=\ngithub.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=\ngithub.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=\ngithub.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=\ngithub.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=\ngithub.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=\ngithub.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A=\ngithub.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=\ngithub.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=\ngithub.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/segmentio/kafka-go v0.4.28 h1:ATYbyenAlsoFxnV+VpIJMF87bvRuRsX7fezHNfpwkdM=\ngithub.com/segmentio/kafka-go v0.4.28/go.mod h1:XzMcoMjSzDGHcIwpWUI7GB43iKZ2fTVmryPSGLf/MPg=\ngithub.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=\ngithub.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=\ngithub.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=\ngithub.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0=\ngithub.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=\ngolang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=\ngolang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284 h1:rlLehGeYg6jfoyz/eDqDU1iRXLKfR42nnNh57ytKEWo=\ngolang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ=\ngolang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=\ngolang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=\ngolang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\n" }, { "path": "examples/consumer-logger/main.go", "content": "package main\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\t\"strings\"\n\n\tkafka \"github.com/segmentio/kafka-go\"\n)\n\nfunc getKafkaReader(kafkaURL, topic, groupID string) *kafka.Reader {\n\tbrokers := strings.Split(kafkaURL, \",\")\n\treturn kafka.NewReader(kafka.ReaderConfig{\n\t\tBrokers: brokers,\n\t\tGroupID: groupID,\n\t\tTopic: topic,\n\t\tMinBytes: 10e3, // 10KB\n\t\tMaxBytes: 10e6, // 10MB\n\t})\n}\n\nfunc main() {\n\t// get kafka reader using environment variables.\n\tkafkaURL := os.Getenv(\"kafkaURL\")\n\ttopic := os.Getenv(\"topic\")\n\tgroupID := os.Getenv(\"groupID\")\n\n\treader := getKafkaReader(kafkaURL, topic, groupID)\n\n\tdefer reader.Close()\n\n\tfmt.Println(\"start consuming ... !!\")\n\tfor {\n\t\tm, err := reader.ReadMessage(context.Background())\n\t\tif err != nil {\n\t\t\tlog.Fatalln(err)\n\t\t}\n\t\tfmt.Printf(\"message at topic:%v partition:%v offset:%v\t%s = %s\\n\", m.Topic, m.Partition, m.Offset, string(m.Key), string(m.Value))\n\t}\n}\n" }, { "path": "examples/consumer-mongo-db/Dockerfile", "content": "#####################################\n# STEP 1 build executable binary #\n#####################################\nFROM golang:alpine AS builder\n\n# Install git.\n# Git is required for fetching the dependencies.\nRUN apk update && apk add --no-cache git\n\nWORKDIR /app\n\nCOPY go.mod .\nCOPY go.sum .\n\nRUN go mod download\n\nCOPY . .\n\n# Build the binary.\nRUN CGO_ENABLED=0 GOOS=linux go build -o main\n\n#####################################\n# STEP 2 build a small image #\n#####################################\nFROM scratch\n\n# Copy our static executable.\nCOPY --from=builder /app/main /app/main\n\n# Run the hello binary.\nENTRYPOINT [\"/app/main\"]" }, { "path": "examples/consumer-mongo-db/go.mod", "content": "module github.com/segmentio/kafka-go/example/consumer-mongo-db\n\ngo 1.15\n\nrequire (\n\tgithub.com/go-stack/stack v1.8.0 // indirect\n\tgithub.com/klauspost/compress v1.12.2 // indirect\n\tgithub.com/mongodb/mongo-go-driver v0.3.0\n\tgithub.com/segmentio/kafka-go v0.4.28\n\tgithub.com/tidwall/pretty v1.1.0 // indirect\n\tgolang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 // indirect\n)\n" }, { "path": "examples/consumer-mongo-db/go.sum", "content": "github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=\ngithub.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=\ngithub.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=\ngithub.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY=\ngithub.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=\ngithub.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=\ngithub.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=\ngithub.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=\ngithub.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA=\ngithub.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=\ngithub.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=\ngithub.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=\ngithub.com/klauspost/compress v1.12.2 h1:2KCfW3I9M7nSc5wOqXAlW2v2U6v+w6cbjvbfp+OykW8=\ngithub.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=\ngithub.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=\ngithub.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=\ngithub.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=\ngithub.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=\ngithub.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=\ngithub.com/mongodb/mongo-go-driver v0.3.0 h1:00tKWMrabkVU1e57/TTP4ZBIfhn/wmjlSiRnIM9d0T8=\ngithub.com/mongodb/mongo-go-driver v0.3.0/go.mod h1:NK/HWDIIZkaYsnYa0hmtP443T5ELr0KDecmIioVuuyU=\ngithub.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A=\ngithub.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=\ngithub.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=\ngithub.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/segmentio/kafka-go v0.4.28 h1:ATYbyenAlsoFxnV+VpIJMF87bvRuRsX7fezHNfpwkdM=\ngithub.com/segmentio/kafka-go v0.4.28/go.mod h1:XzMcoMjSzDGHcIwpWUI7GB43iKZ2fTVmryPSGLf/MPg=\ngithub.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=\ngithub.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/tidwall/pretty v1.1.0 h1:K3hMW5epkdAVwibsQEfR/7Zj0Qgt4DxtNumTq/VloO8=\ngithub.com/tidwall/pretty v1.1.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=\ngithub.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=\ngithub.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=\ngithub.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0=\ngithub.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=\ngolang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=\ngolang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284 h1:rlLehGeYg6jfoyz/eDqDU1iRXLKfR42nnNh57ytKEWo=\ngolang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ=\ngolang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw=\ngolang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=\ngolang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=\ngolang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\n" }, { "path": "examples/consumer-mongo-db/main.go", "content": "package main\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\n\t\"github.com/mongodb/mongo-go-driver/mongo\"\n\tkafka \"github.com/segmentio/kafka-go\"\n)\n\nfunc getMongoCollection(mongoURL, dbName, collectionName string) *mongo.Collection {\n\tclient, err := mongo.Connect(context.Background(), mongoURL)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\t// Check the connection\n\terr = client.Ping(context.Background(), nil)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tfmt.Println(\"Connected to MongoDB ... !!\")\n\n\tdb := client.Database(dbName)\n\tcollection := db.Collection(collectionName)\n\treturn collection\n}\n\nfunc getKafkaReader(kafkaURL, topic, groupID string) *kafka.Reader {\n\treturn kafka.NewReader(kafka.ReaderConfig{\n\t\tBrokers: []string{kafkaURL},\n\t\tGroupID: groupID,\n\t\tTopic: topic,\n\t\tMinBytes: 10e3, // 10KB\n\t\tMaxBytes: 10e6, // 10MB\n\t})\n}\n\nfunc main() {\n\n\t// get Mongo db Collection using environment variables.\n\tmongoURL := os.Getenv(\"mongoURL\")\n\tdbName := os.Getenv(\"dbName\")\n\tcollectionName := os.Getenv(\"collectionName\")\n\tcollection := getMongoCollection(mongoURL, dbName, collectionName)\n\n\t// get kafka reader using environment variables.\n\tkafkaURL := os.Getenv(\"kafkaURL\")\n\ttopic := os.Getenv(\"topic\")\n\tgroupID := os.Getenv(\"groupID\")\n\treader := getKafkaReader(kafkaURL, topic, groupID)\n\n\tdefer reader.Close()\n\n\tfmt.Println(\"start consuming ... !!\")\n\n\tfor {\n\t\tmsg, err := reader.ReadMessage(context.Background())\n\t\tif err != nil {\n\t\t\tlog.Fatal(err)\n\t\t}\n\t\tinsertResult, err := collection.InsertOne(context.Background(), msg)\n\t\tif err != nil {\n\t\t\tlog.Fatal(err)\n\t\t}\n\n\t\tfmt.Println(\"Inserted a single document: \", insertResult.InsertedID)\n\t}\n}\n" }, { "path": "examples/docker-compose.yaml", "content": "version: '2.3'\nservices:\n\n zookeeper:\n hostname: zookeeper\n image: bitnamilegacy/zookeeper:latest\n restart: always\n expose:\n - \"2181\"\n ports:\n - \"2181:2181\"\n environment:\n ALLOW_ANONYMOUS_LOGIN: yes\n \n kafka:\n hostname: kafka\n image: bitnamilegacy/kafka:2.7.0\n restart: always\n env_file:\n - kafka/kafka-variables.env\n depends_on:\n - zookeeper\n expose:\n - \"9092\"\n - \"8082\"\n - \"8083\"\n ports:\n - '9092:9092'\n - '8082:8082'\n - '8083:8083'\n\n mongo-db:\n image: mongo:4.0\n restart: always\n expose:\n - \"27017\"\n ports:\n - \"27017:27017\"\n environment:\n MONGO_DATA_DIR: /data/db\n MONGO_LOG_DIR: /dev/null\n\n consumer-mongo-db:\n build:\n context: consumer-mongo-db\n environment:\n mongoURL: mongodb://mongo-db:27017\n dbName: example_db\n collectionName: example_coll\n kafkaURL: kafka:9092\n topic: topic1\n groupID: mongo-group\n depends_on: \n - kafka\n - mongo-db\n restart: always\n\n consumer-logger:\n build:\n context: consumer-logger\n environment:\n kafkaURL: kafka:9092\n topic: topic1\n groupID: logger-group\n depends_on: \n - kafka\n restart: always\n\n producer-random:\n build:\n context: producer-random\n environment:\n kafkaURL: kafka:9092\n topic: topic1\n depends_on: \n - kafka\n restart: always\n\n producer-api:\n build:\n context: producer-api\n environment:\n kafkaURL: kafka:9092\n topic: topic1\n expose:\n - \"8080\"\n ports:\n - \"8080:8080\"\n depends_on: \n - kafka\n restart: always\n" }, { "path": "examples/kafka/kafka-variables.env", "content": "\nKAFKA_CFG_ADVERTISED_HOST_NAME=kafka\nKAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181\nKAFKA_CFG_CONNECT_BOOTSTRAP_SERVERS=localhost:9092\n\nKAFKA_CFG_CONNECT_REST_PORT=8082\nKAFKA_CFG_CONNECT_REST_ADVERTISED_HOST_NAME=\"localhost\"\n\nKAFKA_CFG_CONNECT_KEY_CONVERTER=\"org.apache.kafka.connect.json.JsonConverter\"\nKAFKA_CFG_CONNECT_VALUE_CONVERTER=\"org.apache.kafka.connect.json.JsonConverter\"\nKAFKA_CFG_CONNECT_KEY_CONVERTER_SCHEMAS_ENABLE=0\nKAFKA_CFG_CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE=0\n\nKAFKA_CFG_CONNECT_INTERNAL_KEY_CONVERTER=\"org.apache.kafka.connect.json.JsonConverter\"\nKAFKA_CFG_CONNECT_INTERNAL_VALUE_CONVERTER=\"org.apache.kafka.connect.json.JsonConverter\"\nKAFKA_CFG_CONNECT_INTERNAL_KEY_CONVERTER_SCHEMAS_ENABLE=0\nKAFKA_CFG_CONNECT_INTERNAL_VALUE_CONVERTER_SCHEMAS_ENABLE=0\n\nKAFKA_CFG_CONNECT_OFFSET_STORAGE_FILE_FILENAME=\"/tmp/connect.offsets\"\n# Flush much faster than normal, which is useful for testing/debugging\nKAFKA_CFG_CONNECT_OFFSET_FLUSH_INTERVAL_MS=10000\n\nALLOW_PLAINTEXT_LISTENER: yes\n" }, { "path": "examples/producer-api/Dockerfile", "content": "#####################################\n# STEP 1 build executable binary #\n#####################################\nFROM golang:alpine AS builder\n\n# Install git.\n# Git is required for fetching the dependencies.\nRUN apk update && apk add --no-cache git\n\nWORKDIR /app\n\nCOPY go.mod .\nCOPY go.sum .\n\nRUN go mod download\n\nCOPY . .\n\n# Build the binary.\nRUN CGO_ENABLED=0 GOOS=linux go build -o main\n\n#####################################\n# STEP 2 build a small image #\n#####################################\nFROM scratch\n\n# Copy our static executable.\nCOPY --from=builder /app/main /app/main\n\n# Run the hello binary.\nENTRYPOINT [\"/app/main\"]" }, { "path": "examples/producer-api/go.mod", "content": "module github.com/segmentio/kafka-go/example/producer-api\n\ngo 1.15\n\nrequire (\n\tgithub.com/klauspost/compress v1.12.2 // indirect\n\tgithub.com/segmentio/kafka-go v0.4.28\n)\n" }, { "path": "examples/producer-api/go.sum", "content": "github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=\ngithub.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=\ngithub.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=\ngithub.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY=\ngithub.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=\ngithub.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=\ngithub.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA=\ngithub.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=\ngithub.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=\ngithub.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=\ngithub.com/klauspost/compress v1.12.2 h1:2KCfW3I9M7nSc5wOqXAlW2v2U6v+w6cbjvbfp+OykW8=\ngithub.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=\ngithub.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=\ngithub.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=\ngithub.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=\ngithub.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=\ngithub.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=\ngithub.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A=\ngithub.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=\ngithub.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=\ngithub.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/segmentio/kafka-go v0.4.28 h1:ATYbyenAlsoFxnV+VpIJMF87bvRuRsX7fezHNfpwkdM=\ngithub.com/segmentio/kafka-go v0.4.28/go.mod h1:XzMcoMjSzDGHcIwpWUI7GB43iKZ2fTVmryPSGLf/MPg=\ngithub.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=\ngithub.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=\ngithub.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=\ngithub.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0=\ngithub.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=\ngolang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=\ngolang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284 h1:rlLehGeYg6jfoyz/eDqDU1iRXLKfR42nnNh57ytKEWo=\ngolang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ=\ngolang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=\ngolang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=\ngolang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\n" }, { "path": "examples/producer-api/main.go", "content": "package main\n\nimport (\n\t\"fmt\"\n\t\"io/ioutil\"\n\t\"log\"\n\t\"net/http\"\n\t\"os\"\n\n\tkafka \"github.com/segmentio/kafka-go\"\n)\n\nfunc producerHandler(kafkaWriter *kafka.Writer) func(http.ResponseWriter, *http.Request) {\n\treturn http.HandlerFunc(func(wrt http.ResponseWriter, req *http.Request) {\n\t\tbody, err := ioutil.ReadAll(req.Body)\n\t\tif err != nil {\n\t\t\tlog.Fatalln(err)\n\t\t}\n\t\tmsg := kafka.Message{\n\t\t\tKey: []byte(fmt.Sprintf(\"address-%s\", req.RemoteAddr)),\n\t\t\tValue: body,\n\t\t}\n\t\terr = kafkaWriter.WriteMessages(req.Context(), msg)\n\n\t\tif err != nil {\n\t\t\twrt.Write([]byte(err.Error()))\n\t\t\tlog.Fatalln(err)\n\t\t}\n\t})\n}\n\nfunc getKafkaWriter(kafkaURL, topic string) *kafka.Writer {\n\treturn &kafka.Writer{\n\t\tAddr: kafka.TCP(kafkaURL),\n\t\tTopic: topic,\n\t\tBalancer: &kafka.LeastBytes{},\n\t}\n}\n\nfunc main() {\n\t// get kafka writer using environment variables.\n\tkafkaURL := os.Getenv(\"kafkaURL\")\n\ttopic := os.Getenv(\"topic\")\n\tkafkaWriter := getKafkaWriter(kafkaURL, topic)\n\n\tdefer kafkaWriter.Close()\n\n\t// Add handle func for producer.\n\thttp.HandleFunc(\"/\", producerHandler(kafkaWriter))\n\n\t// Run the web server.\n\tfmt.Println(\"start producer-api ... !!\")\n\tlog.Fatal(http.ListenAndServe(\":8080\", nil))\n}\n" }, { "path": "examples/producer-api/test.http", "content": "### send data text\nPOST http://localhost:8080\nContent-Type: text/plain\n\n\"Hello-api\"\n\n### send data json\nPOST http://localhost:8080\nContent-Type: application/json\n\n{\n \"data\":\"Hello-api\"\n}" }, { "path": "examples/producer-random/Dockerfile", "content": "#####################################\n# STEP 1 build executable binary #\n#####################################\nFROM golang:alpine AS builder\n\n# Install git.\n# Git is required for fetching the dependencies.\nRUN apk update && apk add --no-cache git\n\nWORKDIR /app\n\nCOPY go.mod .\nCOPY go.sum .\n\nRUN go mod download\n\nCOPY . .\n\n# Build the binary.\nRUN CGO_ENABLED=0 GOOS=linux go build -o main\n\n#####################################\n# STEP 2 build a small image #\n#####################################\nFROM scratch\n\n# Copy our static executable.\nCOPY --from=builder /app/main /app/main\n\n# Run the hello binary.\nENTRYPOINT [\"/app/main\"]" }, { "path": "examples/producer-random/go.mod", "content": "module github.com/segmentio/kafka-go/example/producer-random\n\ngo 1.15\n\nrequire (\n\tgithub.com/davecgh/go-spew v1.1.1 // indirect\n\tgithub.com/google/uuid v1.1.0\n\tgithub.com/klauspost/compress v1.12.2 // indirect\n\tgithub.com/segmentio/kafka-go v0.4.28\n)\n" }, { "path": "examples/producer-random/go.sum", "content": "github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=\ngithub.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=\ngithub.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=\ngithub.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY=\ngithub.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=\ngithub.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=\ngithub.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA=\ngithub.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=\ngithub.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=\ngithub.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/uuid v1.1.0 h1:Jf4mxPC/ziBnoPIdpQdPJ9OeiomAUHLvxmPRSPH9m4s=\ngithub.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=\ngithub.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=\ngithub.com/klauspost/compress v1.12.2 h1:2KCfW3I9M7nSc5wOqXAlW2v2U6v+w6cbjvbfp+OykW8=\ngithub.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=\ngithub.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=\ngithub.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=\ngithub.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=\ngithub.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=\ngithub.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=\ngithub.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A=\ngithub.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=\ngithub.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=\ngithub.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/segmentio/kafka-go v0.4.28 h1:ATYbyenAlsoFxnV+VpIJMF87bvRuRsX7fezHNfpwkdM=\ngithub.com/segmentio/kafka-go v0.4.28/go.mod h1:XzMcoMjSzDGHcIwpWUI7GB43iKZ2fTVmryPSGLf/MPg=\ngithub.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=\ngithub.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=\ngithub.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=\ngithub.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0=\ngithub.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=\ngolang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=\ngolang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284 h1:rlLehGeYg6jfoyz/eDqDU1iRXLKfR42nnNh57ytKEWo=\ngolang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ=\ngolang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=\ngolang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=\ngolang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\n" }, { "path": "examples/producer-random/main.go", "content": "package main\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"os\"\n\t\"time\"\n\n\t\"github.com/google/uuid\"\n\tkafka \"github.com/segmentio/kafka-go\"\n)\n\nfunc newKafkaWriter(kafkaURL, topic string) *kafka.Writer {\n\treturn &kafka.Writer{\n\t\tAddr: kafka.TCP(kafkaURL),\n\t\tTopic: topic,\n\t\tBalancer: &kafka.LeastBytes{},\n\t}\n}\n\nfunc main() {\n\t// get kafka writer using environment variables.\n\tkafkaURL := os.Getenv(\"kafkaURL\")\n\ttopic := os.Getenv(\"topic\")\n\twriter := newKafkaWriter(kafkaURL, topic)\n\tdefer writer.Close()\n\tfmt.Println(\"start producing ... !!\")\n\tfor i := 0; ; i++ {\n\t\tkey := fmt.Sprintf(\"Key-%d\", i)\n\t\tmsg := kafka.Message{\n\t\t\tKey: []byte(key),\n\t\t\tValue: []byte(fmt.Sprint(uuid.New())),\n\t\t}\n\t\terr := writer.WriteMessages(context.Background(), msg)\n\t\tif err != nil {\n\t\t\tfmt.Println(err)\n\t\t} else {\n\t\t\tfmt.Println(\"produced\", key)\n\t\t}\n\t\ttime.Sleep(1 * time.Second)\n\t}\n}\n" }, { "path": "fetch.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"math\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n\tfetchAPI \"github.com/segmentio/kafka-go/protocol/fetch\"\n)\n\n// FetchRequest represents a request sent to a kafka broker to retrieve records\n// from a topic partition.\ntype FetchRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// Topic, partition, and offset to retrieve records from. The offset may be\n\t// one of the special FirstOffset or LastOffset constants, in which case the\n\t// request will automatically discover the first or last offset of the\n\t// partition and submit the request for these.\n\tTopic string\n\tPartition int\n\tOffset int64\n\n\t// Size and time limits of the response returned by the broker.\n\tMinBytes int64\n\tMaxBytes int64\n\tMaxWait time.Duration\n\n\t// The isolation level for the request.\n\t//\n\t// Defaults to ReadUncommitted.\n\t//\n\t// This field requires the kafka broker to support the Fetch API in version\n\t// 4 or above (otherwise the value is ignored).\n\tIsolationLevel IsolationLevel\n}\n\n// FetchResponse represents a response from a kafka broker to a fetch request.\ntype FetchResponse struct {\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// The topic and partition that the response came for (will match the values\n\t// in the request).\n\tTopic string\n\tPartition int\n\n\t// Information about the topic partition layout returned from the broker.\n\t//\n\t// LastStableOffset requires the kafka broker to support the Fetch API in\n\t// version 4 or above (otherwise the value is zero).\n\t//\n\t/// LogStartOffset requires the kafka broker to support the Fetch API in\n\t// version 5 or above (otherwise the value is zero).\n\tHighWatermark int64\n\tLastStableOffset int64\n\tLogStartOffset int64\n\n\t// An error that may have occurred while attempting to fetch the records.\n\t//\n\t// The error contains both the kafka error code, and an error message\n\t// returned by the kafka broker. Programs may use the standard errors.Is\n\t// function to test the error against kafka error codes.\n\tError error\n\n\t// The set of records returned in the response.\n\t//\n\t// The program is expected to call the RecordSet's Close method when it\n\t// finished reading the records.\n\t//\n\t// Note that kafka may return record batches that start at an offset before\n\t// the one that was requested. It is the program's responsibility to skip\n\t// the offsets that it is not interested in.\n\tRecords RecordReader\n}\n\n// Fetch sends a fetch request to a kafka broker and returns the response.\n//\n// If the broker returned an invalid response with no topics, an error wrapping\n// protocol.ErrNoTopic is returned.\n//\n// If the broker returned an invalid response with no partitions, an error\n// wrapping ErrNoPartitions is returned.\nfunc (c *Client) Fetch(ctx context.Context, req *FetchRequest) (*FetchResponse, error) {\n\ttimeout := c.timeout(ctx, math.MaxInt64)\n\tmaxWait := req.maxWait()\n\n\tif maxWait < timeout {\n\t\ttimeout = maxWait\n\t}\n\n\toffset := req.Offset\n\tswitch offset {\n\tcase FirstOffset, LastOffset:\n\t\ttopic, partition := req.Topic, req.Partition\n\n\t\tr, err := c.ListOffsets(ctx, &ListOffsetsRequest{\n\t\t\tAddr: req.Addr,\n\t\t\tTopics: map[string][]OffsetRequest{\n\t\t\t\ttopic: {{\n\t\t\t\t\tPartition: partition,\n\t\t\t\t\tTimestamp: offset,\n\t\t\t\t}},\n\t\t\t},\n\t\t})\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"kafka.(*Client).Fetch: %w\", err)\n\t\t}\n\n\t\tfor _, p := range r.Topics[topic] {\n\t\t\tif p.Partition == partition {\n\t\t\t\tif p.Error != nil {\n\t\t\t\t\treturn nil, fmt.Errorf(\"kafka.(*Client).Fetch: %w\", p.Error)\n\t\t\t\t}\n\t\t\t\tswitch offset {\n\t\t\t\tcase FirstOffset:\n\t\t\t\t\toffset = p.FirstOffset\n\t\t\t\tcase LastOffset:\n\t\t\t\t\toffset = p.LastOffset\n\t\t\t\t}\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\n\tm, err := c.roundTrip(ctx, req.Addr, &fetchAPI.Request{\n\t\tReplicaID: -1,\n\t\tMaxWaitTime: milliseconds(timeout),\n\t\tMinBytes: int32(req.MinBytes),\n\t\tMaxBytes: int32(req.MaxBytes),\n\t\tIsolationLevel: int8(req.IsolationLevel),\n\t\tSessionID: -1,\n\t\tSessionEpoch: -1,\n\t\tTopics: []fetchAPI.RequestTopic{{\n\t\t\tTopic: req.Topic,\n\t\t\tPartitions: []fetchAPI.RequestPartition{{\n\t\t\t\tPartition: int32(req.Partition),\n\t\t\t\tCurrentLeaderEpoch: -1,\n\t\t\t\tFetchOffset: offset,\n\t\t\t\tLogStartOffset: -1,\n\t\t\t\tPartitionMaxBytes: int32(req.MaxBytes),\n\t\t\t}},\n\t\t}},\n\t})\n\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).Fetch: %w\", err)\n\t}\n\n\tres := m.(*fetchAPI.Response)\n\tif len(res.Topics) == 0 {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).Fetch: %w\", protocol.ErrNoTopic)\n\t}\n\ttopic := &res.Topics[0]\n\tif len(topic.Partitions) == 0 {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).Fetch: %w\", protocol.ErrNoPartition)\n\t}\n\tpartition := &topic.Partitions[0]\n\n\tret := &FetchResponse{\n\t\tThrottle: makeDuration(res.ThrottleTimeMs),\n\t\tTopic: topic.Topic,\n\t\tPartition: int(partition.Partition),\n\t\tError: makeError(res.ErrorCode, \"\"),\n\t\tHighWatermark: partition.HighWatermark,\n\t\tLastStableOffset: partition.LastStableOffset,\n\t\tLogStartOffset: partition.LogStartOffset,\n\t\tRecords: partition.RecordSet.Records,\n\t}\n\n\tif partition.ErrorCode != 0 {\n\t\tret.Error = makeError(partition.ErrorCode, \"\")\n\t}\n\n\tif ret.Records == nil {\n\t\tret.Records = NewRecordReader()\n\t}\n\n\treturn ret, nil\n}\n\nfunc (req *FetchRequest) maxWait() time.Duration {\n\tif req.MaxWait > 0 {\n\t\treturn req.MaxWait\n\t}\n\treturn defaultMaxWait\n}\n\ntype fetchRequestV2 struct {\n\tReplicaID int32\n\tMaxWaitTime int32\n\tMinBytes int32\n\tTopics []fetchRequestTopicV2\n}\n\nfunc (r fetchRequestV2) size() int32 {\n\treturn 4 + 4 + 4 + sizeofArray(len(r.Topics), func(i int) int32 { return r.Topics[i].size() })\n}\n\nfunc (r fetchRequestV2) writeTo(wb *writeBuffer) {\n\twb.writeInt32(r.ReplicaID)\n\twb.writeInt32(r.MaxWaitTime)\n\twb.writeInt32(r.MinBytes)\n\twb.writeArray(len(r.Topics), func(i int) { r.Topics[i].writeTo(wb) })\n}\n\ntype fetchRequestTopicV2 struct {\n\tTopicName string\n\tPartitions []fetchRequestPartitionV2\n}\n\nfunc (t fetchRequestTopicV2) size() int32 {\n\treturn sizeofString(t.TopicName) +\n\t\tsizeofArray(len(t.Partitions), func(i int) int32 { return t.Partitions[i].size() })\n}\n\nfunc (t fetchRequestTopicV2) writeTo(wb *writeBuffer) {\n\twb.writeString(t.TopicName)\n\twb.writeArray(len(t.Partitions), func(i int) { t.Partitions[i].writeTo(wb) })\n}\n\ntype fetchRequestPartitionV2 struct {\n\tPartition int32\n\tFetchOffset int64\n\tMaxBytes int32\n}\n\nfunc (p fetchRequestPartitionV2) size() int32 {\n\treturn 4 + 8 + 4\n}\n\nfunc (p fetchRequestPartitionV2) writeTo(wb *writeBuffer) {\n\twb.writeInt32(p.Partition)\n\twb.writeInt64(p.FetchOffset)\n\twb.writeInt32(p.MaxBytes)\n}\n\ntype fetchResponseV2 struct {\n\tThrottleTime int32\n\tTopics []fetchResponseTopicV2\n}\n\nfunc (r fetchResponseV2) size() int32 {\n\treturn 4 + sizeofArray(len(r.Topics), func(i int) int32 { return r.Topics[i].size() })\n}\n\nfunc (r fetchResponseV2) writeTo(wb *writeBuffer) {\n\twb.writeInt32(r.ThrottleTime)\n\twb.writeArray(len(r.Topics), func(i int) { r.Topics[i].writeTo(wb) })\n}\n\ntype fetchResponseTopicV2 struct {\n\tTopicName string\n\tPartitions []fetchResponsePartitionV2\n}\n\nfunc (t fetchResponseTopicV2) size() int32 {\n\treturn sizeofString(t.TopicName) +\n\t\tsizeofArray(len(t.Partitions), func(i int) int32 { return t.Partitions[i].size() })\n}\n\nfunc (t fetchResponseTopicV2) writeTo(wb *writeBuffer) {\n\twb.writeString(t.TopicName)\n\twb.writeArray(len(t.Partitions), func(i int) { t.Partitions[i].writeTo(wb) })\n}\n\ntype fetchResponsePartitionV2 struct {\n\tPartition int32\n\tErrorCode int16\n\tHighwaterMarkOffset int64\n\tMessageSetSize int32\n\tMessageSet messageSet\n}\n\nfunc (p fetchResponsePartitionV2) size() int32 {\n\treturn 4 + 2 + 8 + 4 + p.MessageSet.size()\n}\n\nfunc (p fetchResponsePartitionV2) writeTo(wb *writeBuffer) {\n\twb.writeInt32(p.Partition)\n\twb.writeInt16(p.ErrorCode)\n\twb.writeInt64(p.HighwaterMarkOffset)\n\twb.writeInt32(p.MessageSetSize)\n\tp.MessageSet.writeTo(wb)\n}\n" }, { "path": "fetch_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"io\"\n\t\"io/ioutil\"\n\t\"net\"\n\t\"reflect\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/compress\"\n)\n\nfunc produceRecords(t *testing.T, n int, addr net.Addr, topic string, compression compress.Codec) []Record {\n\tconn, err := (&Dialer{\n\t\tResolver: &net.Resolver{},\n\t}).DialLeader(context.Background(), addr.Network(), addr.String(), topic, 0)\n\n\tif err != nil {\n\t\tt.Fatal(\"failed to open a new kafka connection:\", err)\n\t}\n\tdefer conn.Close()\n\n\tmsgs := makeTestSequence(n)\n\tif compression == nil {\n\t\t_, err = conn.WriteMessages(msgs...)\n\t} else {\n\t\t_, err = conn.WriteCompressedMessages(compression, msgs...)\n\t}\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\trecords := make([]Record, len(msgs))\n\tfor offset, msg := range msgs {\n\t\trecords[offset] = Record{\n\t\t\tOffset: int64(offset),\n\t\t\tKey: NewBytes(msg.Key),\n\t\t\tValue: NewBytes(msg.Value),\n\t\t\tHeaders: msg.Headers,\n\t\t}\n\t}\n\n\treturn records\n}\n\nfunc TestClientFetch(t *testing.T) {\n\tclient, topic, shutdown := newLocalClientAndTopic()\n\tdefer shutdown()\n\n\trecords := produceRecords(t, 10, client.Addr, topic, nil)\n\n\tres, err := client.Fetch(context.Background(), &FetchRequest{\n\t\tTopic: topic,\n\t\tPartition: 0,\n\t\tOffset: 0,\n\t\tMinBytes: 1,\n\t\tMaxBytes: 64 * 1024,\n\t\tMaxWait: 100 * time.Millisecond,\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tassertFetchResponse(t, res, &FetchResponse{\n\t\tTopic: topic,\n\t\tPartition: 0,\n\t\tHighWatermark: 10,\n\t\tRecords: NewRecordReader(records...),\n\t})\n}\n\nfunc TestClientFetchCompressed(t *testing.T) {\n\tclient, topic, shutdown := newLocalClientAndTopic()\n\tdefer shutdown()\n\n\trecords := produceRecords(t, 10, client.Addr, topic, &compress.GzipCodec)\n\n\tres, err := client.Fetch(context.Background(), &FetchRequest{\n\t\tTopic: topic,\n\t\tPartition: 0,\n\t\tOffset: 0,\n\t\tMinBytes: 1,\n\t\tMaxBytes: 64 * 1024,\n\t\tMaxWait: 100 * time.Millisecond,\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tassertFetchResponse(t, res, &FetchResponse{\n\t\tTopic: topic,\n\t\tPartition: 0,\n\t\tHighWatermark: 10,\n\t\tRecords: NewRecordReader(records...),\n\t})\n}\n\nfunc assertFetchResponse(t *testing.T, found, expected *FetchResponse) {\n\tt.Helper()\n\n\tif found.Topic != expected.Topic {\n\t\tt.Error(\"invalid topic found in response:\", found.Topic)\n\t}\n\n\tif found.Partition != expected.Partition {\n\t\tt.Error(\"invalid partition found in response:\", found.Partition)\n\t}\n\n\tif found.HighWatermark != expected.HighWatermark {\n\t\tt.Error(\"invalid high watermark found in response:\", found.HighWatermark)\n\t}\n\n\tif found.Error != nil {\n\t\tt.Error(\"unexpected error found in response:\", found.Error)\n\t}\n\n\trecords1, err := readRecords(found.Records)\n\tif err != nil {\n\t\tt.Error(\"error reading records:\", err)\n\t}\n\n\trecords2, err := readRecords(expected.Records)\n\tif err != nil {\n\t\tt.Error(\"error reading records:\", err)\n\t}\n\n\tassertRecords(t, records1, records2)\n}\n\ntype memoryRecord struct {\n\toffset int64\n\tkey []byte\n\tvalue []byte\n\theaders []Header\n}\n\nfunc assertRecords(t *testing.T, found, expected []memoryRecord) {\n\tt.Helper()\n\ti := 0\n\n\tfor i < len(found) && i < len(expected) {\n\t\tr1 := found[i]\n\t\tr2 := expected[i]\n\n\t\tif !reflect.DeepEqual(r1, r2) {\n\t\t\tt.Errorf(\"records at index %d don't match\", i)\n\t\t\tt.Logf(\"expected:\\n%#v\", r2)\n\t\t\tt.Logf(\"found:\\n%#v\", r1)\n\t\t}\n\n\t\ti++\n\t}\n\n\tfor i < len(found) {\n\t\tt.Errorf(\"unexpected record at index %d:\\n%+v\", i, found[i])\n\t\ti++\n\t}\n\n\tfor i < len(expected) {\n\t\tt.Errorf(\"missing record at index %d:\\n%+v\", i, expected[i])\n\t\ti++\n\t}\n}\n\nfunc readRecords(records RecordReader) ([]memoryRecord, error) {\n\tlist := []memoryRecord{}\n\n\tfor {\n\t\trec, err := records.ReadRecord()\n\n\t\tif err != nil {\n\t\t\tif errors.Is(err, io.EOF) {\n\t\t\t\treturn list, nil\n\t\t\t}\n\t\t\treturn nil, err\n\t\t}\n\n\t\tvar (\n\t\t\toffset = rec.Offset\n\t\t\tkey = rec.Key\n\t\t\tvalue = rec.Value\n\t\t\theaders = rec.Headers\n\t\t\tbytesKey []byte\n\t\t\tbytesValues []byte\n\t\t)\n\n\t\tif key != nil {\n\t\t\tbytesKey, _ = ioutil.ReadAll(key)\n\t\t}\n\n\t\tif value != nil {\n\t\t\tbytesValues, _ = ioutil.ReadAll(value)\n\t\t}\n\n\t\tlist = append(list, memoryRecord{\n\t\t\toffset: offset,\n\t\t\tkey: bytesKey,\n\t\t\tvalue: bytesValues,\n\t\t\theaders: headers,\n\t\t})\n\t}\n}\n\nfunc TestClientPipeline(t *testing.T) {\n\tclient, topic, shutdown := newLocalClientAndTopic()\n\tdefer shutdown()\n\n\tconst numBatches = 100\n\tconst recordsPerBatch = 30\n\n\tunixEpoch := time.Unix(0, 0)\n\trecords := make([]Record, recordsPerBatch)\n\tcontent := []byte(\"1234567890\")\n\n\tfor i := 0; i < numBatches; i++ {\n\t\tfor j := range records {\n\t\t\trecords[j] = Record{Value: NewBytes(content)}\n\t\t}\n\n\t\t_, err := client.Produce(context.Background(), &ProduceRequest{\n\t\t\tTopic: topic,\n\t\t\tRequiredAcks: -1,\n\t\t\tRecords: NewRecordReader(records...),\n\t\t\tCompression: Snappy,\n\t\t})\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\toffset := int64(0)\n\n\tfor i := 0; i < (numBatches * recordsPerBatch); {\n\t\treq := &FetchRequest{\n\t\t\tTopic: topic,\n\t\t\tOffset: offset,\n\t\t\tMinBytes: 1,\n\t\t\tMaxBytes: 8192,\n\t\t\tMaxWait: 500 * time.Millisecond,\n\t\t}\n\n\t\tres, err := client.Fetch(context.Background(), req)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tif res.Error != nil {\n\t\t\tt.Fatal(res.Error)\n\t\t}\n\n\t\tfor {\n\t\t\tr, err := res.Records.ReadRecord()\n\t\t\tif err != nil {\n\t\t\t\tif errors.Is(err, io.EOF) {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\tif r.Key != nil {\n\t\t\t\tr.Key.Close()\n\t\t\t}\n\n\t\t\tif r.Value != nil {\n\t\t\t\tr.Value.Close()\n\t\t\t}\n\n\t\t\tif r.Offset != offset {\n\t\t\t\tt.Errorf(\"record at index %d has mismatching offset, want %d but got %d\", i, offset, r.Offset)\n\t\t\t}\n\n\t\t\tif r.Time.IsZero() || r.Time.Equal(unixEpoch) {\n\t\t\t\tt.Errorf(\"record at index %d with offset %d has not timestamp\", i, r.Offset)\n\t\t\t}\n\n\t\t\toffset = r.Offset + 1\n\t\t\ti++\n\t\t}\n\t}\n}\n" }, { "path": "findcoordinator.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/findcoordinator\"\n)\n\n// CoordinatorKeyType is used to specify the type of coordinator to look for.\ntype CoordinatorKeyType int8\n\nconst (\n\t// CoordinatorKeyTypeConsumer type is used when looking for a Group coordinator.\n\tCoordinatorKeyTypeConsumer CoordinatorKeyType = 0\n\n\t// CoordinatorKeyTypeTransaction type is used when looking for a Transaction coordinator.\n\tCoordinatorKeyTypeTransaction CoordinatorKeyType = 1\n)\n\n// FindCoordinatorRequest is the request structure for the FindCoordinator function.\ntype FindCoordinatorRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// The coordinator key.\n\tKey string\n\n\t// The coordinator key type. (Group, transaction, etc.)\n\tKeyType CoordinatorKeyType\n}\n\n// FindCoordinatorResponseCoordinator contains details about the found coordinator.\ntype FindCoordinatorResponseCoordinator struct {\n\t// NodeID holds the broker id.\n\tNodeID int\n\n\t// Host of the broker\n\tHost string\n\n\t// Port on which broker accepts requests\n\tPort int\n}\n\n// FindCoordinatorResponse is the response structure for the FindCoordinator function.\ntype FindCoordinatorResponse struct {\n\t// The Transaction/Group Coordinator details\n\tCoordinator *FindCoordinatorResponseCoordinator\n\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// An error that may have occurred while attempting to retrieve Coordinator\n\t//\n\t// The error contains both the kafka error code, and an error message\n\t// returned by the kafka broker.\n\tError error\n}\n\n// FindCoordinator sends a findCoordinator request to a kafka broker and returns the\n// response.\nfunc (c *Client) FindCoordinator(ctx context.Context, req *FindCoordinatorRequest) (*FindCoordinatorResponse, error) {\n\n\tm, err := c.roundTrip(ctx, req.Addr, &findcoordinator.Request{\n\t\tKey: req.Key,\n\t\tKeyType: int8(req.KeyType),\n\t})\n\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).FindCoordinator: %w\", err)\n\t}\n\n\tres := m.(*findcoordinator.Response)\n\tcoordinator := &FindCoordinatorResponseCoordinator{\n\t\tNodeID: int(res.NodeID),\n\t\tHost: res.Host,\n\t\tPort: int(res.Port),\n\t}\n\tret := &FindCoordinatorResponse{\n\t\tThrottle: makeDuration(res.ThrottleTimeMs),\n\t\tError: makeError(res.ErrorCode, res.ErrorMessage),\n\t\tCoordinator: coordinator,\n\t}\n\n\treturn ret, nil\n}\n\n// FindCoordinatorRequestV0 requests the coordinator for the specified group or transaction\n//\n// See http://kafka.apache.org/protocol.html#The_Messages_FindCoordinator\ntype findCoordinatorRequestV0 struct {\n\t// CoordinatorKey holds id to use for finding the coordinator (for groups, this is\n\t// the groupId, for transactional producers, this is the transactional id)\n\tCoordinatorKey string\n}\n\nfunc (t findCoordinatorRequestV0) size() int32 {\n\treturn sizeofString(t.CoordinatorKey)\n}\n\nfunc (t findCoordinatorRequestV0) writeTo(wb *writeBuffer) {\n\twb.writeString(t.CoordinatorKey)\n}\n\ntype findCoordinatorResponseCoordinatorV0 struct {\n\t// NodeID holds the broker id.\n\tNodeID int32\n\n\t// Host of the broker\n\tHost string\n\n\t// Port on which broker accepts requests\n\tPort int32\n}\n\nfunc (t findCoordinatorResponseCoordinatorV0) size() int32 {\n\treturn sizeofInt32(t.NodeID) +\n\t\tsizeofString(t.Host) +\n\t\tsizeofInt32(t.Port)\n}\n\nfunc (t findCoordinatorResponseCoordinatorV0) writeTo(wb *writeBuffer) {\n\twb.writeInt32(t.NodeID)\n\twb.writeString(t.Host)\n\twb.writeInt32(t.Port)\n}\n\nfunc (t *findCoordinatorResponseCoordinatorV0) readFrom(r *bufio.Reader, size int) (remain int, err error) {\n\tif remain, err = readInt32(r, size, &t.NodeID); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readString(r, remain, &t.Host); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readInt32(r, remain, &t.Port); err != nil {\n\t\treturn\n\t}\n\treturn\n}\n\ntype findCoordinatorResponseV0 struct {\n\t// ErrorCode holds response error code\n\tErrorCode int16\n\n\t// Coordinator holds host and port information for the coordinator\n\tCoordinator findCoordinatorResponseCoordinatorV0\n}\n\nfunc (t findCoordinatorResponseV0) size() int32 {\n\treturn sizeofInt16(t.ErrorCode) +\n\t\tt.Coordinator.size()\n}\n\nfunc (t findCoordinatorResponseV0) writeTo(wb *writeBuffer) {\n\twb.writeInt16(t.ErrorCode)\n\tt.Coordinator.writeTo(wb)\n}\n\nfunc (t *findCoordinatorResponseV0) readFrom(r *bufio.Reader, size int) (remain int, err error) {\n\tif remain, err = readInt16(r, size, &t.ErrorCode); err != nil {\n\t\treturn\n\t}\n\tif remain, err = (&t.Coordinator).readFrom(r, remain); err != nil {\n\t\treturn\n\t}\n\treturn\n}\n" }, { "path": "findcoordinator_test.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"reflect\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestFindCoordinatorResponseV0(t *testing.T) {\n\titem := findCoordinatorResponseV0{\n\t\tErrorCode: 2,\n\t\tCoordinator: findCoordinatorResponseCoordinatorV0{\n\t\t\tNodeID: 3,\n\t\t\tHost: \"b\",\n\t\t\tPort: 4,\n\t\t},\n\t}\n\n\tb := bytes.NewBuffer(nil)\n\tw := &writeBuffer{w: b}\n\titem.writeTo(w)\n\n\tvar found findCoordinatorResponseV0\n\tremain, err := (&found).readFrom(bufio.NewReader(b), b.Len())\n\tif err != nil {\n\t\tt.Error(err)\n\t\tt.FailNow()\n\t}\n\tif remain != 0 {\n\t\tt.Errorf(\"expected 0 remain, got %v\", remain)\n\t\tt.FailNow()\n\t}\n\tif !reflect.DeepEqual(item, found) {\n\t\tt.Error(\"expected item and found to be the same\")\n\t\tt.FailNow()\n\t}\n}\n\nfunc TestClientFindCoordinator(t *testing.T) {\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\tctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)\n\tdefer cancel()\n\tresp, err := waitForCoordinatorIndefinitely(ctx, client, &FindCoordinatorRequest{\n\t\tAddr: client.Addr,\n\t\tKey: \"TransactionalID-1\",\n\t\tKeyType: CoordinatorKeyTypeTransaction,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif resp.Coordinator.Host != \"localhost\" {\n\t\tt.Fatal(\"Coordinator should be found @ localhost\")\n\t}\n}\n\n// WaitForCoordinatorIndefinitely is a blocking call till a coordinator is found.\nfunc waitForCoordinatorIndefinitely(ctx context.Context, c *Client, req *FindCoordinatorRequest) (*FindCoordinatorResponse, error) {\n\tresp, err := c.FindCoordinator(ctx, req)\n\n\tfor shouldRetryfindingCoordinator(resp, err) && ctx.Err() == nil {\n\t\ttime.Sleep(1 * time.Second)\n\t\tresp, err = c.FindCoordinator(ctx, req)\n\t}\n\treturn resp, err\n}\n\n// Should retry looking for coordinator\n// Returns true when the test Kafka broker is still setting up.\nfunc shouldRetryfindingCoordinator(resp *FindCoordinatorResponse, err error) bool {\n\tbrokerSetupIncomplete := err != nil &&\n\t\tstrings.Contains(\n\t\t\tstrings.ToLower(err.Error()),\n\t\t\tstrings.ToLower(\"unexpected EOF\"))\n\tcoordinatorNotFound := resp != nil &&\n\t\tresp.Error != nil &&\n\t\terrors.Is(resp.Error, GroupCoordinatorNotAvailable)\n\treturn brokerSetupIncomplete || coordinatorNotFound\n}\n" }, { "path": "fixtures/v1-v1.hex", "content": "000001660000000a00000000000015c79861000000010009746573742d6564677900000001000000000000000000000000000400000000000000040000000000000000ffffffff0000011f00000000000000000000003ca293717501000000017c4f08dc7f00000005616c706861000000217b22636f756e74223a302c2266696c6c6572223a2261616161616161616161227d00000000000000010000003b3d4abab001000000017c4f08dc970000000462657461000000217b22636f756e74223a302c2266696c6c6572223a2262626262626262626262227d00000000000000020000003cbcad5cde01000000017c4f09b16d0000000567616d6d61000000217b22636f756e74223a302c2266696c6c6572223a2263636363636363636363227d00000000000000030000003c8585230b01000000017c4f09b6b20000000564656c7461000000217b22636f756e74223a302c2266696c6c6572223a2264646464646464646464227d" }, { "path": "fixtures/v1-v1c-v2-v2c-v2b-v2b-v2b-v2bc-v1b-v1bc.hex", "content": "000006b40000000a00000000000021f08796000000010007746573742d383800000001000000000000000000000000001400000000000000140000000000000000ffffffff0000066f00000000000000000000003c42f0d0f101000000017c477ab6a500000005616c706861000000217b22636f756e74223a302c2266696c6c6572223a2261616161616161616161227d00000000000000010000003bf4f7a99e01000000017c477abb610000000462657461000000217b22636f756e74223a302c2266696c6c6572223a2262626262626262626262227d00000000000000020000005fd3cf85ff01010000017c477b3bcbffffffff000000491f8b0800000000000000636080039bba2d51db18810cc61af76aebd340066b624e41462290a158ad949c5f9a57a26465a0a394969993935aa464a59408074ab5001b5f3ee14800000000000000000000030000005e5d1733a801010000017c477b408fffffffff000000481f8b080000000000000063608003eb95673d5f3002198c35eed50efd40064b526a49229056ac564ace2fcd2b51b232d0514acbccc9492d52b2524a8203a55a002737831e4700000000000000000000040000005e00000000020ab23c660000000000000000017c477d995f0000017c477d995fffffffffffffffffffffffffffff00000001580000000a67616d6d61427b22636f756e74223a302c2266696c6c6572223a2263636363636363636363227d0000000000000000050000005e000000000238c0553f0000000000000000017c477d9ec80000017c477d9ec8ffffffffffffffffffffffffffff00000001580000000a64656c7461427b22636f756e74223a302c2266696c6c6572223a2264646464646464646464227d0000000000000000060000006a0000000002188627120001000000000000017c477dd0b70000017c477dd0b7ffffffffffffffffffffffffffff000000011f8b08000000000000008b606060e04a4fcccd4d74aa564ace2fcd2b51b232d0514acbccc9492d52b2524a8603a55a0600788108a12d00000000000000000000070000006a0000000002b08e2b720001000000000000017c477dd7ef0000017c477dd7efffffffffffffffffffffffffffff000000011f8b08000000000000008b606060e04a49cd294974aa564ace2fcd2b51b232d0514acbccc9492d52b2524a8103a55a0600496dfe822d00000000000000000000080000008d00000000023cc016270000000000010000017c4784fe490000017c47850044ffffffffffffffffffffffffffff000000025c0000000e657073696c6f6e427b22636f756e74223a302c2266696c6c6572223a2265656565656565656565227d005800f60702087a657461427b22636f756e74223a302c2266696c6c6572223a2266666666666666666666227d00000000000000000a0000007d00000000026e844d550001000000010000017c4785514b0000017c47855423ffffffffffffffffffffffffffff000000021f8b08000000000000008b616060e04b2d28ceccc9cf73aa564ace2fcd2b51b232d0514acbccc9492d52b2524a8503a55a8608860ddc4c1c55a92589d815a7c1015031003ebb53a15c000000000000000000000c0000008a0000000002e5dfd9e20000000000010000017c4c8f1e1c0000017c4c8f20e8ffffffffffffffffffffffffffff000000025400000006657461427b22636f756e74223a302c2266696c6c6572223a2267676767676767676767227d005a00980b020a7468657461427b22636f756e74223a302c2266696c6c6572223a2268686868686868686868227d00000000000000000e0000007700000000020f80521f0001000000010000017c4c8f4da50000017c4c8f4fb8ffffffffffffffffffffffffffff000000021f8b08000000000000000b616060604b2d4974aa564ace2fcd2b51b232d0514acbccc9492d52b2524a8703a55a862886651c4c5c2519385567c001503500b01e3aa95900000000000000000000100000003a3b6d4cf601000000017c4cadbd7300000003657461000000217b22636f756e74223a302c2266696c6c6572223a2267676767676767676767227d00000000000000110000003c857f5cd501000000017c4cadbd99000000057468657461000000217b22636f756e74223a302c2266696c6c6572223a2268686868686868686868227d000000000000001300000076dbf0a20a01010000017c4cadf305ffffffff000000601f8b080000000000000063608003ab0d4959758c4006638dcfda8fcf810ce6d4924420a558ad949c5f9a57a26465a0a394969993935aa464a5940e074ab55013409a6d9412cf95c14cf9cc0a64b09664e03327030e946a01e34da7538e000000" }, { "path": "fixtures/v1c-v1-v1c.hex", "content": "000002350000000a0000000000003d15acfe00000001000b746573742d627265657a7900000001000000000000000000000000000600000000000000060000000000000000ffffffff000001ec000000000000000100000079779afa8b01010000017c4f11cdc9ffffffff000000631f8b0800000000000000636080039bf9617b7418810cc61a7fc1b32b810cd6c49c828c442043b15a2939bf34af44c9ca4047292d332727b548c94a29110e946aa16680b45b5b967f780937e72490c192945a82db98243850aa05001ea2107b8f00000000000000000000020000003cda0e410e01000000017c4f1212630000000567616d6d61000000217b22636f756e74223a302c2266696c6c6572223a2263636363636363636363227d00000000000000030000003c0470399301000000017c4f12154e0000000564656c7461000000217b22636f756e74223a302c2266696c6c6572223a2264646464646464646464227d0000000000000004000000613b0e4db601010000017c4f124947ffffffff0000004b1f8b080000000000000063608003bb67b39e743302198c35fe429eee40067b6a4171664e7e1e90a958ad949c5f9a57a26465a0a394969993935aa464a5940a074ab5007d95b7894a00000000000000000000050000005edb50180901010000017c4f124fd0ffffffff000000481f8b080000000000000063608003ebfbf2b32c18810cc61a7f21ff0b40064b556a49229056ac564ace2fcd2b51b232d0514acbccc9492d52b2524a8303a55a005ec594df47000000" }, { "path": "fixtures/v1c-v1c.hex", "content": "000001a20000000a0000000000001abffa5700000001000a746573742d677574737900000001000000000000000000000000000400000000000000040000000000000000ffffffff0000015a0000000000000001000000789125e5e201010000017c4f0ee474ffffffff000000621f8b0800000000000000636080039bfcfd51598c4006638d3fdf131f20833531a7202311c850ac564ace2fcd2b51b232d0514acbccc9492d52b2524a8403a55aa81920edd67a221c2e70734a800c96a4d412dcc624c181522d001d8564f48f00000000000000000000020000005f66e75d9b01010000017c4f0f55f5ffffffff000000491f8b0800000000000000636080039bfd2566fe8c4006638d3f7fe8572083353d31373711c850ac564ace2fcd2b51b232d0514acbccc9492d52b2524a8603a55a008ef7186d4800000000000000000000030000005f3cff26a901010000017c4f0f5d5cffffffff000000491f8b0800000000000000636080031b6f8db3d18c4006638d3f7f6c0c90c19a929a5392086428562b25e797e695285919e828a565e6e4a416295929a5c081522d00dd1f6ff148000000" }, { "path": "fixtures/v2-v2.hex", "content": "000001760000000a0000000000001163921100000001000a746573742d6c7563696400000001000000000000000000000000000400000000000000040000000000000000ffffffff0000012e00000000000000000000008a00000000023978fc3b0000000000010000017c4f173eb90000017c4f173ed2ffffffffffffffffffffffffffff00000002580000000a616c706861427b22636f756e74223a302c2266696c6c6572223a2261616161616161616161227d00560032020862657461427b22636f756e74223a302c2266696c6c6572223a2262626262626262626262227d0000000000000000020000008c0000000002fa7514ab0000000000010000017c4f175fa00000017c4f17631fffffffffffffffffffffffffffff00000002580000000a67616d6d61427b22636f756e74223a302c2266696c6c6572223a2263636363636363636363227d005a00fe0d020a64656c7461427b22636f756e74223a302c2266696c6c6572223a2264646464646464646464227d00" }, { "path": "fixtures/v2b-v1.hex", "content": "0000016e0000000a00000000000023f24a1a00000001000b746573742d66656973747900000001000000000000000000000000000400000000000000040000000000000000ffffffff0000012500000000000000000000008a000000000267762fd10000000000010000017c4e71efe10000017c4e71effdffffffffffffffffffffffffffff00000002580000000a616c706861427b22636f756e74223a302c2266696c6c6572223a2261616161616161616161227d00560038020862657461427b22636f756e74223a302c2266696c6c6572223a2262626262626262626262227d0000000000000000020000003c0d5ba69301000000017c4e743d2100000005616c706861000000217b22636f756e74223a302c2266696c6c6572223a2261616161616161616161227d00000000000000030000003be6e3d42501000000017c4e743d410000000462657461000000217b22636f756e74223a302c2266696c6c6572223a2262626262626262626262227d" }, { "path": "fixtures/v2bc-v1-v1c.hex", "content": "000001e60000000a000000000000530076a100000001000a746573742d686172647900000001000000000000000000000000000600000000000000060000000000000000ffffffff0000019e000000000000000000000079000000000214d2dc1d0001000000010000017c4ead43a90000017c4ead43c3ffffffffffffffffffffffffffff000000021f8b08000000000000008b606060e04acc29c84874aa564ace2fcd2b51b232d0514acbccc9492d52b2524a8403a55a86300613268ea4d4121c6a93e000a81600538562275900000000000000000000020000003c48deb52601000000017c4eae54050000000567616d6d61000000217b22636f756e74223a302c2266696c6c6572223a2263636363636363636363227d00000000000000030000003ca2ba5edc01000000017c4eae5cff0000000564656c7461000000217b22636f756e74223a302c2266696c6c6572223a2264646464646464646464227d00000000000000050000007d2e0ea95201010000017c4eb07250ffffffff000000671f8b080000000000000063608003bb5b69b1958c4006638ddf86fcef40067b6a4171664e7e1e90a958ad949c5f9a57a26465a0a394969993935aa464a5940a074ab550534006587fdc55d00633a92800c860a94a2d49c4694c1a1c28d5020087e0fa5d91000000" }, { "path": "fixtures/v2bc-v1.hex", "content": "0000015d0000000a0000000000006d36526200000001000a746573742d686172647900000001000000000000000000000000000400000000000000040000000000000000ffffffff00000115000000000000000000000079000000000214d2dc1d0001000000010000017c4ead43a90000017c4ead43c3ffffffffffffffffffffffffffff000000021f8b08000000000000008b606060e04acc29c84874aa564ace2fcd2b51b232d0514acbccc9492d52b2524a8403a55a86300613268ea4d4121c6a93e000a81600538562275900000000000000000000020000003c48deb52601000000017c4eae54050000000567616d6d61000000217b22636f756e74223a302c2266696c6c6572223a2263636363636363636363227d00000000000000030000003ca2ba5edc01000000017c4eae5cff0000000564656c7461000000217b22636f756e74223a302c2266696c6c6572223a2264646464646464646464227d" }, { "path": "fixtures/v2bc-v1c.hex", "content": "000001520000000a0000000000004aa4215500000001000b746573742d6b61726d696300000001000000000000000000000000000400000000000000040000000000000000ffffffff00000109000000000000000000000079000000000218f2e1220001000000010000017c4e8edde60000017c4e8eddffffffffffffffffffffffffffffff000000021f8b08000000000000008b606060e04acc29c84874aa564ace2fcd2b51b232d0514acbccc9492d52b2524a8403a55a86300623268ea4d4121c6a93e000a81600b5931557590000000000000000000003000000785a33562401010000017c4e8f57f1ffffffff000000621f8b0800000000000000636080031b93ef814f19810cc61abffef0ab40066b624e41462290a158ad949c5f9a57a26465a0a394969993935aa464a59408074ab5503340daad5d459b6ec0cdf90864b024a596e03626090e946a016143eac78f000000" }, { "path": "fixtures/v2c-v2-v2c.hex", "content": "000001ee0000000a000000000000670352ac00000001000a746573742d6e6174747900000001000000000000000000000000000600000000000000060000000000000000ffffffff000001a600000000000000000000007900000000025da9bf740001000000010000017c4f1eea730000017c4f1eea8dffffffffffffffffffffffffffff000000021f8b08000000000000008b606060e04acc29c84874aa564ace2fcd2b51b232d0514acbccc9492d52b2524a8403a55a86300613268ea4d4121c6a93e000a81600538562275900000000000000000000020000008c0000000002f53e2b600000000000010000017c4f1f1a600000017c4f1f1c65ffffffffffffffffffffffffffff00000002580000000a67616d6d61427b22636f756e74223a302c2266696c6c6572223a2263636363636363636363227d005a008a08020a64656c7461427b22636f756e74223a302c2266696c6c6572223a2264646464646464646464227d0000000000000000040000007d000000000268a8ca640001000000010000017c4f1f49f90000017c4f1f4db4ffffffffffffffffffffffffffff000000021f8b08000000000000008b616060e04b2d28ceccc9cf73aa564ace2fcd2b51b232d0514acbccc9492d52b2524a8503a55a8608866f7c4c1c55a92589d815a7c101503100ebf4f0655c000000" }, { "path": "fixtures/v2c-v2c.hex", "content": "000001560000000a0000000000005698dc5100000001000c746573742d6f6e656972696300000001000000000000000000000000000400000000000000040000000000000000ffffffff0000010c00000000000000000000007900000000021ad503db0001000000010000017c4f1a1f540000017c4f1a1f70ffffffffffffffffffffffffffff000000021f8b08000000000000008b606060e04acc29c84874aa564ace2fcd2b51b232d0514acbccc9492d52b2524a8403a55a8630060b268ea4d4121c6a93e000a816009fa88cc75900000000000000000000020000007b0000000002d346070a0001000000010000017c4f1a46110000017c4f1a48d0ffffffffffffffffffffffffffff000000021f8b08000000000000008b606060e04a4fcccd4d74aa564ace2fcd2b51b232d0514acbccc9492d52b2524a8603a55a8628867f5c4c5c29a939253854a7c0015035000f1406dd5b000000" }, { "path": "go.mod", "content": "module github.com/segmentio/kafka-go\n\ngo 1.23\n\nrequire (\n\tgithub.com/klauspost/compress v1.15.9\n\tgithub.com/pierrec/lz4/v4 v4.1.15\n\tgithub.com/stretchr/testify v1.8.0\n\tgithub.com/xdg-go/scram v1.1.2\n\tgolang.org/x/net v0.38.0\n)\n\nrequire (\n\tgithub.com/davecgh/go-spew v1.1.1 // indirect\n\tgithub.com/pmezard/go-difflib v1.0.0 // indirect\n\tgithub.com/xdg-go/pbkdf2 v1.0.0 // indirect\n\tgithub.com/xdg-go/stringprep v1.0.4 // indirect\n\tgolang.org/x/text v0.23.0 // indirect\n\tgopkg.in/yaml.v3 v3.0.1 // indirect\n)\n\nretract [v0.4.36, v0.4.37]\n" }, { "path": "go.sum", "content": "github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=\ngithub.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=\ngithub.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=\ngithub.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=\ngithub.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=\ngithub.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=\ngithub.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=\ngithub.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=\ngithub.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=\ngithub.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=\ngithub.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=\ngithub.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=\ngithub.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=\ngithub.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=\ngithub.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=\ngithub.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=\ngolang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=\ngolang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=\ngolang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=\ngolang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=\ngolang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=\ngolang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8=\ngolang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=\ngolang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=\ngolang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=\ngolang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=\ngolang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=\ngolang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=\ngolang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=\ngolang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=\ngolang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\ngopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=\ngopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\n" }, { "path": "groupbalancer.go", "content": "package kafka\n\nimport (\n\t\"sort\"\n)\n\n// GroupMember describes a single participant in a consumer group.\ntype GroupMember struct {\n\t// ID is the unique ID for this member as taken from the JoinGroup response.\n\tID string\n\n\t// Topics is a list of topics that this member is consuming.\n\tTopics []string\n\n\t// UserData contains any information that the GroupBalancer sent to the\n\t// consumer group coordinator.\n\tUserData []byte\n}\n\n// GroupMemberAssignments holds MemberID => topic => partitions.\ntype GroupMemberAssignments map[string]map[string][]int\n\n// GroupBalancer encapsulates the client side rebalancing logic.\ntype GroupBalancer interface {\n\t// ProtocolName of the GroupBalancer\n\tProtocolName() string\n\n\t// UserData provides the GroupBalancer an opportunity to embed custom\n\t// UserData into the metadata.\n\t//\n\t// Will be used by JoinGroup to begin the consumer group handshake.\n\t//\n\t// See https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-JoinGroupRequest\n\tUserData() ([]byte, error)\n\n\t// DefineMemberships returns which members will be consuming\n\t// which topic partitions\n\tAssignGroups(members []GroupMember, partitions []Partition) GroupMemberAssignments\n}\n\n// RangeGroupBalancer groups consumers by partition\n//\n// Example: 5 partitions, 2 consumers\n// \t\tC0: [0, 1, 2]\n// \t\tC1: [3, 4]\n//\n// Example: 6 partitions, 3 consumers\n// \t\tC0: [0, 1]\n// \t\tC1: [2, 3]\n// \t\tC2: [4, 5]\n//\ntype RangeGroupBalancer struct{}\n\nfunc (r RangeGroupBalancer) ProtocolName() string {\n\treturn \"range\"\n}\n\nfunc (r RangeGroupBalancer) UserData() ([]byte, error) {\n\treturn nil, nil\n}\n\nfunc (r RangeGroupBalancer) AssignGroups(members []GroupMember, topicPartitions []Partition) GroupMemberAssignments {\n\tgroupAssignments := GroupMemberAssignments{}\n\tmembersByTopic := findMembersByTopic(members)\n\n\tfor topic, members := range membersByTopic {\n\t\tpartitions := findPartitions(topic, topicPartitions)\n\t\tpartitionCount := len(partitions)\n\t\tmemberCount := len(members)\n\n\t\tfor memberIndex, member := range members {\n\t\t\tassignmentsByTopic, ok := groupAssignments[member.ID]\n\t\t\tif !ok {\n\t\t\t\tassignmentsByTopic = map[string][]int{}\n\t\t\t\tgroupAssignments[member.ID] = assignmentsByTopic\n\t\t\t}\n\n\t\t\tminIndex := memberIndex * partitionCount / memberCount\n\t\t\tmaxIndex := (memberIndex + 1) * partitionCount / memberCount\n\n\t\t\tfor partitionIndex, partition := range partitions {\n\t\t\t\tif partitionIndex >= minIndex && partitionIndex < maxIndex {\n\t\t\t\t\tassignmentsByTopic[topic] = append(assignmentsByTopic[topic], partition)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn groupAssignments\n}\n\n// RoundrobinGroupBalancer divides partitions evenly among consumers\n//\n// Example: 5 partitions, 2 consumers\n// \t\tC0: [0, 2, 4]\n// \t\tC1: [1, 3]\n//\n// Example: 6 partitions, 3 consumers\n// \t\tC0: [0, 3]\n// \t\tC1: [1, 4]\n// \t\tC2: [2, 5]\n//\ntype RoundRobinGroupBalancer struct{}\n\nfunc (r RoundRobinGroupBalancer) ProtocolName() string {\n\treturn \"roundrobin\"\n}\n\nfunc (r RoundRobinGroupBalancer) UserData() ([]byte, error) {\n\treturn nil, nil\n}\n\nfunc (r RoundRobinGroupBalancer) AssignGroups(members []GroupMember, topicPartitions []Partition) GroupMemberAssignments {\n\tgroupAssignments := GroupMemberAssignments{}\n\tmembersByTopic := findMembersByTopic(members)\n\tfor topic, members := range membersByTopic {\n\t\tpartitionIDs := findPartitions(topic, topicPartitions)\n\t\tmemberCount := len(members)\n\n\t\tfor memberIndex, member := range members {\n\t\t\tassignmentsByTopic, ok := groupAssignments[member.ID]\n\t\t\tif !ok {\n\t\t\t\tassignmentsByTopic = map[string][]int{}\n\t\t\t\tgroupAssignments[member.ID] = assignmentsByTopic\n\t\t\t}\n\n\t\t\tfor partitionIndex, partition := range partitionIDs {\n\t\t\t\tif (partitionIndex % memberCount) == memberIndex {\n\t\t\t\t\tassignmentsByTopic[topic] = append(assignmentsByTopic[topic], partition)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn groupAssignments\n}\n\n// RackAffinityGroupBalancer makes a best effort to pair up consumers with\n// partitions whose leader is in the same rack. This strategy can have\n// performance benefits by minimizing round trip latency between the consumer\n// and the broker. In environments where network traffic across racks incurs\n// charges (such as cross AZ data transfer in AWS), this strategy is also a cost\n// optimization measure because it keeps network traffic within the local rack\n// where possible.\n//\n// The primary objective is to spread partitions evenly across consumers with a\n// secondary focus on maximizing the number of partitions where the leader and\n// the consumer are in the same rack. For best affinity, it's recommended to\n// have a balanced spread of consumers and partition leaders across racks.\n//\n// This balancer requires Kafka version 0.10.0.0+ or later. Earlier versions do\n// not return the brokers' racks in the metadata request.\ntype RackAffinityGroupBalancer struct {\n\t// Rack is the name of the rack where this consumer is running. It will be\n\t// communicated to the consumer group leader via the UserData so that\n\t// assignments can be made with affinity to the partition leader.\n\tRack string\n}\n\nfunc (r RackAffinityGroupBalancer) ProtocolName() string {\n\treturn \"rack-affinity\"\n}\n\nfunc (r RackAffinityGroupBalancer) AssignGroups(members []GroupMember, partitions []Partition) GroupMemberAssignments {\n\tmembersByTopic := make(map[string][]GroupMember)\n\tfor _, m := range members {\n\t\tfor _, t := range m.Topics {\n\t\t\tmembersByTopic[t] = append(membersByTopic[t], m)\n\t\t}\n\t}\n\n\tpartitionsByTopic := make(map[string][]Partition)\n\tfor _, p := range partitions {\n\t\tpartitionsByTopic[p.Topic] = append(partitionsByTopic[p.Topic], p)\n\t}\n\n\tassignments := GroupMemberAssignments{}\n\tfor topic := range membersByTopic {\n\t\ttopicAssignments := r.assignTopic(membersByTopic[topic], partitionsByTopic[topic])\n\t\tfor member, parts := range topicAssignments {\n\t\t\tmemberAssignments, ok := assignments[member]\n\t\t\tif !ok {\n\t\t\t\tmemberAssignments = make(map[string][]int)\n\t\t\t\tassignments[member] = memberAssignments\n\t\t\t}\n\t\t\tmemberAssignments[topic] = parts\n\t\t}\n\t}\n\treturn assignments\n}\n\nfunc (r RackAffinityGroupBalancer) UserData() ([]byte, error) {\n\treturn []byte(r.Rack), nil\n}\n\nfunc (r *RackAffinityGroupBalancer) assignTopic(members []GroupMember, partitions []Partition) map[string][]int {\n\tzonedPartitions := make(map[string][]int)\n\tfor _, part := range partitions {\n\t\tzone := part.Leader.Rack\n\t\tzonedPartitions[zone] = append(zonedPartitions[zone], part.ID)\n\t}\n\n\tzonedConsumers := make(map[string][]string)\n\tfor _, member := range members {\n\t\tzone := string(member.UserData)\n\t\tzonedConsumers[zone] = append(zonedConsumers[zone], member.ID)\n\t}\n\n\ttargetPerMember := len(partitions) / len(members)\n\tremainder := len(partitions) % len(members)\n\tassignments := make(map[string][]int)\n\n\t// assign as many as possible in zone. this will assign up to partsPerMember\n\t// to each consumer. it will also prefer to allocate remainder partitions\n\t// in zone if possible.\n\tfor zone, parts := range zonedPartitions {\n\t\tconsumers := zonedConsumers[zone]\n\t\tif len(consumers) == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\t// don't over-allocate. cap partition assignments at the calculated\n\t\t// target.\n\t\tpartsPerMember := len(parts) / len(consumers)\n\t\tif partsPerMember > targetPerMember {\n\t\t\tpartsPerMember = targetPerMember\n\t\t}\n\n\t\tfor _, consumer := range consumers {\n\t\t\tassignments[consumer] = append(assignments[consumer], parts[:partsPerMember]...)\n\t\t\tparts = parts[partsPerMember:]\n\t\t}\n\n\t\t// if we had enough partitions for each consumer in this zone to hit its\n\t\t// target, attempt to use any leftover partitions to satisfy the total\n\t\t// remainder by adding at most 1 partition per consumer.\n\t\tleftover := len(parts)\n\t\tif partsPerMember == targetPerMember {\n\t\t\tif leftover > remainder {\n\t\t\t\tleftover = remainder\n\t\t\t}\n\t\t\tif leftover > len(consumers) {\n\t\t\t\tleftover = len(consumers)\n\t\t\t}\n\t\t\tremainder -= leftover\n\t\t}\n\n\t\t// this loop covers the case where we're assigning extra partitions or\n\t\t// if there weren't enough to satisfy the targetPerMember and the zoned\n\t\t// partitions didn't divide evenly.\n\t\tfor i := 0; i < leftover; i++ {\n\t\t\tassignments[consumers[i]] = append(assignments[consumers[i]], parts[i])\n\t\t}\n\t\tparts = parts[leftover:]\n\n\t\tif len(parts) == 0 {\n\t\t\tdelete(zonedPartitions, zone)\n\t\t} else {\n\t\t\tzonedPartitions[zone] = parts\n\t\t}\n\t}\n\n\t// assign out remainders regardless of zone.\n\tvar remaining []int\n\tfor _, partitions := range zonedPartitions {\n\t\tremaining = append(remaining, partitions...)\n\t}\n\n\tfor _, member := range members {\n\t\tassigned := assignments[member.ID]\n\t\tdelta := targetPerMember - len(assigned)\n\t\t// if it were possible to assign the remainder in zone, it's been taken\n\t\t// care of already. now we will portion out any remainder to a member\n\t\t// that can take it.\n\t\tif delta >= 0 && remainder > 0 {\n\t\t\tdelta++\n\t\t\tremainder--\n\t\t}\n\t\tif delta > 0 {\n\t\t\tassignments[member.ID] = append(assigned, remaining[:delta]...)\n\t\t\tremaining = remaining[delta:]\n\t\t}\n\t}\n\n\treturn assignments\n}\n\n// findPartitions extracts the partition ids associated with the topic from the\n// list of Partitions provided.\nfunc findPartitions(topic string, partitions []Partition) []int {\n\tvar ids []int\n\tfor _, partition := range partitions {\n\t\tif partition.Topic == topic {\n\t\t\tids = append(ids, partition.ID)\n\t\t}\n\t}\n\treturn ids\n}\n\n// findMembersByTopic groups the memberGroupMetadata by topic.\nfunc findMembersByTopic(members []GroupMember) map[string][]GroupMember {\n\tmembersByTopic := map[string][]GroupMember{}\n\tfor _, member := range members {\n\t\tfor _, topic := range member.Topics {\n\t\t\tmembersByTopic[topic] = append(membersByTopic[topic], member)\n\t\t}\n\t}\n\n\t// normalize ordering of members to enabling grouping across topics by partitions\n\t//\n\t// Want:\n\t// \t\tC0 [T0/P0, T1/P0]\n\t// \t\tC1 [T0/P1, T1/P1]\n\t//\n\t// Not:\n\t// \t\tC0 [T0/P0, T1/P1]\n\t// \t\tC1 [T0/P1, T1/P0]\n\t//\n\t// Even though the later is still round robin, the partitions are crossed\n\t//\n\tfor _, members := range membersByTopic {\n\t\tsort.Slice(members, func(i, j int) bool {\n\t\t\treturn members[i].ID < members[j].ID\n\t\t})\n\t}\n\n\treturn membersByTopic\n}\n\n// findGroupBalancer returns the GroupBalancer with the specified protocolName\n// from the slice provided.\nfunc findGroupBalancer(protocolName string, balancers []GroupBalancer) (GroupBalancer, bool) {\n\tfor _, balancer := range balancers {\n\t\tif balancer.ProtocolName() == protocolName {\n\t\t\treturn balancer, true\n\t\t}\n\t}\n\treturn nil, false\n}\n" }, { "path": "groupbalancer_test.go", "content": "package kafka\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"testing\"\n)\n\nfunc TestFindMembersByTopic(t *testing.T) {\n\ta1 := GroupMember{\n\t\tID: \"a\",\n\t\tTopics: []string{\"topic-1\"},\n\t}\n\ta12 := GroupMember{\n\t\tID: \"a\",\n\t\tTopics: []string{\"topic-1\", \"topic-2\"},\n\t}\n\tb23 := GroupMember{\n\t\tID: \"b\",\n\t\tTopics: []string{\"topic-2\", \"topic-3\"},\n\t}\n\n\ttests := map[string]struct {\n\t\tMembers []GroupMember\n\t\tExpected map[string][]GroupMember\n\t}{\n\t\t\"empty\": {\n\t\t\tExpected: map[string][]GroupMember{},\n\t\t},\n\t\t\"one member, one topic\": {\n\t\t\tMembers: []GroupMember{a1},\n\t\t\tExpected: map[string][]GroupMember{\n\t\t\t\t\"topic-1\": {\n\t\t\t\t\ta1,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t\"one member, multiple topics\": {\n\t\t\tMembers: []GroupMember{a12},\n\t\t\tExpected: map[string][]GroupMember{\n\t\t\t\t\"topic-1\": {\n\t\t\t\t\ta12,\n\t\t\t\t},\n\t\t\t\t\"topic-2\": {\n\t\t\t\t\ta12,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t\"multiple members, multiple topics\": {\n\t\t\tMembers: []GroupMember{a12, b23},\n\t\t\tExpected: map[string][]GroupMember{\n\t\t\t\t\"topic-1\": {\n\t\t\t\t\ta12,\n\t\t\t\t},\n\t\t\t\t\"topic-2\": {\n\t\t\t\t\ta12,\n\t\t\t\t\tb23,\n\t\t\t\t},\n\t\t\t\t\"topic-3\": {\n\t\t\t\t\tb23,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor label, test := range tests {\n\t\tt.Run(label, func(t *testing.T) {\n\t\t\tmembersByTopic := findMembersByTopic(test.Members)\n\t\t\tif !reflect.DeepEqual(test.Expected, membersByTopic) {\n\t\t\t\tt.Errorf(\"expected %#v; got %#v\", test.Expected, membersByTopic)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestRangeAssignGroups(t *testing.T) {\n\tnewMeta := func(memberID string, topics ...string) GroupMember {\n\t\treturn GroupMember{\n\t\t\tID: memberID,\n\t\t\tTopics: topics,\n\t\t}\n\t}\n\n\tnewPartitions := func(partitionCount int, topics ...string) []Partition {\n\t\tpartitions := make([]Partition, 0, len(topics)*partitionCount)\n\t\tfor _, topic := range topics {\n\t\t\tfor partition := 0; partition < partitionCount; partition++ {\n\t\t\t\tpartitions = append(partitions, Partition{\n\t\t\t\t\tTopic: topic,\n\t\t\t\t\tID: partition,\n\t\t\t\t})\n\t\t\t}\n\t\t}\n\t\treturn partitions\n\t}\n\n\ttests := map[string]struct {\n\t\tMembers []GroupMember\n\t\tPartitions []Partition\n\t\tExpected GroupMemberAssignments\n\t}{\n\t\t\"empty\": {\n\t\t\tExpected: GroupMemberAssignments{},\n\t\t},\n\t\t\"one member, one topic, one partition\": {\n\t\t\tMembers: []GroupMember{\n\t\t\t\tnewMeta(\"a\", \"topic-1\"),\n\t\t\t},\n\t\t\tPartitions: newPartitions(1, \"topic-1\"),\n\t\t\tExpected: GroupMemberAssignments{\n\t\t\t\t\"a\": map[string][]int{\n\t\t\t\t\t\"topic-1\": {0},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t\"one member, one topic, multiple partitions\": {\n\t\t\tMembers: []GroupMember{\n\t\t\t\tnewMeta(\"a\", \"topic-1\"),\n\t\t\t},\n\t\t\tPartitions: newPartitions(3, \"topic-1\"),\n\t\t\tExpected: GroupMemberAssignments{\n\t\t\t\t\"a\": map[string][]int{\n\t\t\t\t\t\"topic-1\": {0, 1, 2},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t\"multiple members, one topic, one partition\": {\n\t\t\tMembers: []GroupMember{\n\t\t\t\tnewMeta(\"a\", \"topic-1\"),\n\t\t\t\tnewMeta(\"b\", \"topic-1\"),\n\t\t\t},\n\t\t\tPartitions: newPartitions(1, \"topic-1\"),\n\t\t\tExpected: GroupMemberAssignments{\n\t\t\t\t\"a\": map[string][]int{},\n\t\t\t\t\"b\": map[string][]int{\n\t\t\t\t\t\"topic-1\": {0},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t\"multiple members, one topic, multiple partitions\": {\n\t\t\tMembers: []GroupMember{\n\t\t\t\tnewMeta(\"a\", \"topic-1\"),\n\t\t\t\tnewMeta(\"b\", \"topic-1\"),\n\t\t\t},\n\t\t\tPartitions: newPartitions(3, \"topic-1\"),\n\t\t\tExpected: GroupMemberAssignments{\n\t\t\t\t\"a\": map[string][]int{\n\t\t\t\t\t\"topic-1\": {0},\n\t\t\t\t},\n\t\t\t\t\"b\": map[string][]int{\n\t\t\t\t\t\"topic-1\": {1, 2},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t\"multiple members, multiple topics, multiple partitions\": {\n\t\t\tMembers: []GroupMember{\n\t\t\t\tnewMeta(\"a\", \"topic-1\", \"topic-2\"),\n\t\t\t\tnewMeta(\"b\", \"topic-2\", \"topic-3\"),\n\t\t\t},\n\t\t\tPartitions: newPartitions(3, \"topic-1\", \"topic-2\", \"topic-3\"),\n\t\t\tExpected: GroupMemberAssignments{\n\t\t\t\t\"a\": map[string][]int{\n\t\t\t\t\t\"topic-1\": {0, 1, 2},\n\t\t\t\t\t\"topic-2\": {0},\n\t\t\t\t},\n\t\t\t\t\"b\": map[string][]int{\n\t\t\t\t\t\"topic-2\": {1, 2},\n\t\t\t\t\t\"topic-3\": {0, 1, 2},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor label, test := range tests {\n\t\tt.Run(label, func(t *testing.T) {\n\t\t\tassignments := RangeGroupBalancer{}.AssignGroups(test.Members, test.Partitions)\n\t\t\tif !reflect.DeepEqual(test.Expected, assignments) {\n\t\t\t\tbuf := bytes.NewBuffer(nil)\n\t\t\t\tencoder := json.NewEncoder(buf)\n\t\t\t\tencoder.SetIndent(\"\", \" \")\n\n\t\t\t\tbuf.WriteString(\"expected: \")\n\t\t\t\tencoder.Encode(test.Expected)\n\t\t\t\tbuf.WriteString(\"got: \")\n\t\t\t\tencoder.Encode(assignments)\n\n\t\t\t\tt.Error(buf.String())\n\t\t\t}\n\t\t})\n\t}\n}\n\n// For 66 members, 213 partitions, each member should get 213/66 = 3.22 partitions.\n// This means that in practice, each member should get either 3 or 4 partitions\n// assigned to it. Any other number is a failure.\nfunc TestRangeAssignGroupsUnbalanced(t *testing.T) {\n\tmembers := []GroupMember{}\n\tfor i := 0; i < 66; i++ {\n\t\tmembers = append(members, GroupMember{\n\t\t\tID: strconv.Itoa(i),\n\t\t\tTopics: []string{\"topic-1\"},\n\t\t})\n\t}\n\tpartitions := []Partition{}\n\tfor i := 0; i < 213; i++ {\n\t\tpartitions = append(partitions, Partition{\n\t\t\tID: i,\n\t\t\tTopic: \"topic-1\",\n\t\t})\n\t}\n\n\tassignments := RangeGroupBalancer{}.AssignGroups(members, partitions)\n\tif len(assignments) != len(members) {\n\t\tt.Fatalf(\"Assignment count mismatch: %d != %d\", len(assignments), len(members))\n\t}\n\n\tfor _, m := range assignments {\n\t\tif len(m[\"topic-1\"]) < 3 || len(m[\"topic-1\"]) > 4 {\n\t\t\tt.Fatalf(\"Expected assignment of 3 or 4 partitions, got %d\", len(m[\"topic-1\"]))\n\t\t}\n\t}\n}\n\nfunc TestRoundRobinAssignGroups(t *testing.T) {\n\tnewPartitions := func(partitionCount int, topics ...string) []Partition {\n\t\tpartitions := make([]Partition, 0, len(topics)*partitionCount)\n\t\tfor _, topic := range topics {\n\t\t\tfor partition := 0; partition < partitionCount; partition++ {\n\t\t\t\tpartitions = append(partitions, Partition{\n\t\t\t\t\tTopic: topic,\n\t\t\t\t\tID: partition,\n\t\t\t\t})\n\t\t\t}\n\t\t}\n\t\treturn partitions\n\t}\n\n\ttests := map[string]struct {\n\t\tMembers []GroupMember\n\t\tPartitions []Partition\n\t\tExpected GroupMemberAssignments\n\t}{\n\t\t\"empty\": {\n\t\t\tExpected: GroupMemberAssignments{},\n\t\t},\n\t\t\"one member, one topic, one partition\": {\n\t\t\tMembers: []GroupMember{\n\t\t\t\t{\n\t\t\t\t\tID: \"a\",\n\t\t\t\t\tTopics: []string{\"topic-1\"},\n\t\t\t\t},\n\t\t\t},\n\t\t\tPartitions: newPartitions(1, \"topic-1\"),\n\t\t\tExpected: GroupMemberAssignments{\n\t\t\t\t\"a\": map[string][]int{\n\t\t\t\t\t\"topic-1\": {0},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t\"one member, one topic, multiple partitions\": {\n\t\t\tMembers: []GroupMember{\n\t\t\t\t{\n\t\t\t\t\tID: \"a\",\n\t\t\t\t\tTopics: []string{\"topic-1\"},\n\t\t\t\t},\n\t\t\t},\n\t\t\tPartitions: newPartitions(3, \"topic-1\"),\n\t\t\tExpected: GroupMemberAssignments{\n\t\t\t\t\"a\": map[string][]int{\n\t\t\t\t\t\"topic-1\": {0, 1, 2},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t\"multiple members, one topic, one partition\": {\n\t\t\tMembers: []GroupMember{\n\t\t\t\t{\n\t\t\t\t\tID: \"a\",\n\t\t\t\t\tTopics: []string{\"topic-1\"},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tID: \"b\",\n\t\t\t\t\tTopics: []string{\"topic-1\"},\n\t\t\t\t},\n\t\t\t},\n\t\t\tPartitions: newPartitions(1, \"topic-1\"),\n\t\t\tExpected: GroupMemberAssignments{\n\t\t\t\t\"a\": map[string][]int{\n\t\t\t\t\t\"topic-1\": {0},\n\t\t\t\t},\n\t\t\t\t\"b\": map[string][]int{},\n\t\t\t},\n\t\t},\n\t\t\"multiple members, multiple topics, multiple partitions\": {\n\t\t\tMembers: []GroupMember{\n\t\t\t\t{\n\t\t\t\t\tID: \"a\",\n\t\t\t\t\tTopics: []string{\"topic-1\", \"topic-2\"},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tID: \"b\",\n\t\t\t\t\tTopics: []string{\"topic-2\", \"topic-3\"},\n\t\t\t\t},\n\t\t\t},\n\t\t\tPartitions: newPartitions(3, \"topic-1\", \"topic-2\", \"topic-3\"),\n\t\t\tExpected: GroupMemberAssignments{\n\t\t\t\t\"a\": map[string][]int{\n\t\t\t\t\t\"topic-1\": {0, 1, 2},\n\t\t\t\t\t\"topic-2\": {0, 2},\n\t\t\t\t},\n\t\t\t\t\"b\": map[string][]int{\n\t\t\t\t\t\"topic-2\": {1},\n\t\t\t\t\t\"topic-3\": {0, 1, 2},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor label, test := range tests {\n\t\tt.Run(label, func(t *testing.T) {\n\t\t\tassignments := RoundRobinGroupBalancer{}.AssignGroups(test.Members, test.Partitions)\n\t\t\tif !reflect.DeepEqual(test.Expected, assignments) {\n\t\t\t\tbuf := bytes.NewBuffer(nil)\n\t\t\t\tencoder := json.NewEncoder(buf)\n\t\t\t\tencoder.SetIndent(\"\", \" \")\n\n\t\t\t\tbuf.WriteString(\"expected: \")\n\t\t\t\tencoder.Encode(test.Expected)\n\t\t\t\tbuf.WriteString(\"got: \")\n\t\t\t\tencoder.Encode(assignments)\n\n\t\t\t\tt.Error(buf.String())\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestFindMembersByTopicSortsByMemberID(t *testing.T) {\n\ttopic := \"topic-1\"\n\ta := GroupMember{\n\t\tID: \"a\",\n\t\tTopics: []string{topic},\n\t}\n\tb := GroupMember{\n\t\tID: \"b\",\n\t\tTopics: []string{topic},\n\t}\n\tc := GroupMember{\n\t\tID: \"c\",\n\t\tTopics: []string{topic},\n\t}\n\n\ttestCases := map[string]struct {\n\t\tData []GroupMember\n\t\tExpected []GroupMember\n\t}{\n\t\t\"in order\": {\n\t\t\tData: []GroupMember{a, b},\n\t\t\tExpected: []GroupMember{a, b},\n\t\t},\n\t\t\"out of order\": {\n\t\t\tData: []GroupMember{a, c, b},\n\t\t\tExpected: []GroupMember{a, b, c},\n\t\t},\n\t}\n\n\tfor label, test := range testCases {\n\t\tt.Run(label, func(t *testing.T) {\n\t\t\tmembersByTopic := findMembersByTopic(test.Data)\n\n\t\t\tif actual := membersByTopic[topic]; !reflect.DeepEqual(test.Expected, actual) {\n\t\t\t\tt.Errorf(\"expected %v; got %v\", test.Expected, actual)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestRackAffinityGroupBalancer(t *testing.T) {\n\tt.Run(\"User Data\", func(t *testing.T) {\n\t\tt.Run(\"unknown zone\", func(t *testing.T) {\n\t\t\tb := RackAffinityGroupBalancer{}\n\t\t\tzone, err := b.UserData()\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif string(zone) != \"\" {\n\t\t\t\tt.Fatalf(\"expected empty zone but got %s\", zone)\n\t\t\t}\n\t\t})\n\n\t\tt.Run(\"configure zone\", func(t *testing.T) {\n\t\t\tb := RackAffinityGroupBalancer{Rack: \"zone1\"}\n\t\t\tzone, err := b.UserData()\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif string(zone) != \"zone1\" {\n\t\t\t\tt.Fatalf(\"expected zone1 az but got %s\", zone)\n\t\t\t}\n\t\t})\n\t})\n\n\tt.Run(\"Balance\", func(t *testing.T) {\n\t\tb := RackAffinityGroupBalancer{}\n\n\t\tbrokers := map[string]Broker{\n\t\t\t\"z1\": {ID: 1, Rack: \"z1\"},\n\t\t\t\"z2\": {ID: 2, Rack: \"z2\"},\n\t\t\t\"z3\": {ID: 2, Rack: \"z3\"},\n\t\t\t\"\": {},\n\t\t}\n\n\t\ttests := []struct {\n\t\t\tname string\n\t\t\tmemberCounts map[string]int\n\t\t\tpartitionCounts map[string]int\n\t\t\tresult map[string]map[string]int\n\t\t}{\n\t\t\t{\n\t\t\t\tname: \"unknown and known zones\",\n\t\t\t\tmemberCounts: map[string]int{\n\t\t\t\t\t\"\": 1,\n\t\t\t\t\t\"z1\": 1,\n\t\t\t\t\t\"z2\": 1,\n\t\t\t\t},\n\t\t\t\tpartitionCounts: map[string]int{\n\t\t\t\t\t\"z1\": 5,\n\t\t\t\t\t\"z2\": 4,\n\t\t\t\t\t\"\": 9,\n\t\t\t\t},\n\t\t\t\tresult: map[string]map[string]int{\n\t\t\t\t\t\"z1\": {\"\": 1, \"z1\": 5},\n\t\t\t\t\t\"z2\": {\"\": 2, \"z2\": 4},\n\t\t\t\t\t\"\": {\"\": 6},\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tname: \"all unknown\",\n\t\t\t\tmemberCounts: map[string]int{\n\t\t\t\t\t\"\": 5,\n\t\t\t\t},\n\t\t\t\tpartitionCounts: map[string]int{\n\t\t\t\t\t\"\": 103,\n\t\t\t\t},\n\t\t\t\tresult: map[string]map[string]int{\n\t\t\t\t\t\"\": {\"\": 103},\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tname: \"remainder stays local\",\n\t\t\t\tmemberCounts: map[string]int{\n\t\t\t\t\t\"z1\": 3,\n\t\t\t\t\t\"z2\": 3,\n\t\t\t\t\t\"z3\": 3,\n\t\t\t\t},\n\t\t\t\tpartitionCounts: map[string]int{\n\t\t\t\t\t\"z1\": 20,\n\t\t\t\t\t\"z2\": 19,\n\t\t\t\t\t\"z3\": 20,\n\t\t\t\t},\n\t\t\t\tresult: map[string]map[string]int{\n\t\t\t\t\t\"z1\": {\"z1\": 20},\n\t\t\t\t\t\"z2\": {\"z2\": 19},\n\t\t\t\t\t\"z3\": {\"z3\": 20},\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tname: \"imbalanced partitions\",\n\t\t\t\tmemberCounts: map[string]int{\n\t\t\t\t\t\"z1\": 1,\n\t\t\t\t\t\"z2\": 1,\n\t\t\t\t\t\"z3\": 1,\n\t\t\t\t},\n\t\t\t\tpartitionCounts: map[string]int{\n\t\t\t\t\t\"z1\": 7,\n\t\t\t\t\t\"z2\": 0,\n\t\t\t\t\t\"z3\": 7,\n\t\t\t\t},\n\t\t\t\tresult: map[string]map[string]int{\n\t\t\t\t\t\"z1\": {\"z1\": 5},\n\t\t\t\t\t\"z2\": {\"z1\": 2, \"z3\": 2},\n\t\t\t\t\t\"z3\": {\"z3\": 5},\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tname: \"imbalanced members\",\n\t\t\t\tmemberCounts: map[string]int{\n\t\t\t\t\t\"z1\": 5,\n\t\t\t\t\t\"z2\": 3,\n\t\t\t\t\t\"z3\": 1,\n\t\t\t\t},\n\t\t\t\tpartitionCounts: map[string]int{\n\t\t\t\t\t\"z1\": 9,\n\t\t\t\t\t\"z2\": 9,\n\t\t\t\t\t\"z3\": 9,\n\t\t\t\t},\n\t\t\t\tresult: map[string]map[string]int{\n\t\t\t\t\t\"z1\": {\"z1\": 9, \"z3\": 6},\n\t\t\t\t\t\"z2\": {\"z2\": 9},\n\t\t\t\t\t\"z3\": {\"z3\": 3},\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tname: \"no consumers in zone\",\n\t\t\t\tmemberCounts: map[string]int{\n\t\t\t\t\t\"z2\": 10,\n\t\t\t\t},\n\t\t\t\tpartitionCounts: map[string]int{\n\t\t\t\t\t\"z1\": 20,\n\t\t\t\t\t\"z3\": 19,\n\t\t\t\t},\n\t\t\t\tresult: map[string]map[string]int{\n\t\t\t\t\t\"z2\": {\"z1\": 20, \"z3\": 19},\n\t\t\t\t},\n\t\t\t},\n\t\t}\n\n\t\tfor _, tt := range tests {\n\t\t\tt.Run(tt.name, func(t *testing.T) {\n\n\t\t\t\t// create members per the distribution in the test case.\n\t\t\t\tvar members []GroupMember\n\t\t\t\tfor zone, count := range tt.memberCounts {\n\t\t\t\t\tfor i := 0; i < count; i++ {\n\t\t\t\t\t\tmembers = append(members, GroupMember{\n\t\t\t\t\t\t\tID: zone + \":\" + strconv.Itoa(len(members)+1),\n\t\t\t\t\t\t\tTopics: []string{\"test\"},\n\t\t\t\t\t\t\tUserData: []byte(zone),\n\t\t\t\t\t\t})\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\t// create partitions per the distribution in the test case.\n\t\t\t\tvar partitions []Partition\n\t\t\t\tfor zone, count := range tt.partitionCounts {\n\t\t\t\t\tfor i := 0; i < count; i++ {\n\t\t\t\t\t\tpartitions = append(partitions, Partition{\n\t\t\t\t\t\t\tID: len(partitions),\n\t\t\t\t\t\t\tTopic: \"test\",\n\t\t\t\t\t\t\tLeader: brokers[zone],\n\t\t\t\t\t\t})\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tres := b.AssignGroups(members, partitions)\n\n\t\t\t\t// verification #1...all members must be assigned and with the\n\t\t\t\t// correct load.\n\t\t\t\tminLoad := len(partitions) / len(members)\n\t\t\t\tmaxLoad := minLoad\n\t\t\t\tif len(partitions)%len(members) != 0 {\n\t\t\t\t\tmaxLoad++\n\t\t\t\t}\n\t\t\t\tfor _, member := range members {\n\t\t\t\t\tassignments := res[member.ID][\"test\"]\n\t\t\t\t\tif len(assignments) < minLoad || len(assignments) > maxLoad {\n\t\t\t\t\t\tt.Errorf(\"expected between %d and %d partitions for member %s\", minLoad, maxLoad, member.ID)\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\t// verification #2...all partitions are assigned, and the distribution\n\t\t\t\t// per source zone matches.\n\t\t\t\tpartsPerZone := make(map[string]map[string]int)\n\t\t\t\tuniqueParts := make(map[int]struct{})\n\t\t\t\tfor id, topicToPartitions := range res {\n\n\t\t\t\t\tfor topic, assignments := range topicToPartitions {\n\t\t\t\t\t\tif topic != \"test\" {\n\t\t\t\t\t\t\tt.Fatalf(\"wrong topic...expected test but got %s\", topic)\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tvar member GroupMember\n\t\t\t\t\t\tfor _, m := range members {\n\t\t\t\t\t\t\tif id == m.ID {\n\t\t\t\t\t\t\t\tmember = m\n\t\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t}\n\t\t\t\t\t\tif member.ID == \"\" {\n\t\t\t\t\t\t\tt.Fatal(\"empty member ID returned\")\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\tvar partition Partition\n\t\t\t\t\t\tfor _, id := range assignments {\n\n\t\t\t\t\t\t\tuniqueParts[id] = struct{}{}\n\n\t\t\t\t\t\t\tfor _, p := range partitions {\n\t\t\t\t\t\t\t\tif p.ID == int(id) {\n\t\t\t\t\t\t\t\t\tpartition = p\n\t\t\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tif partition.Topic == \"\" {\n\t\t\t\t\t\t\t\tt.Fatal(\"empty topic ID returned\")\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tcounts, ok := partsPerZone[string(member.UserData)]\n\t\t\t\t\t\t\tif !ok {\n\t\t\t\t\t\t\t\tcounts = make(map[string]int)\n\t\t\t\t\t\t\t\tpartsPerZone[string(member.UserData)] = counts\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\tcounts[partition.Leader.Rack]++\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tif len(partitions) != len(uniqueParts) {\n\t\t\t\t\tt.Error(\"not all partitions were assigned\")\n\t\t\t\t}\n\t\t\t\tif !reflect.DeepEqual(tt.result, partsPerZone) {\n\t\t\t\t\tt.Errorf(\"wrong balanced zones. expected %v but got %v\", tt.result, partsPerZone)\n\t\t\t\t}\n\t\t\t})\n\t\t}\n\t})\n\n\tt.Run(\"Multi Topic\", func(t *testing.T) {\n\t\tb := RackAffinityGroupBalancer{}\n\n\t\tbrokers := map[string]Broker{\n\t\t\t\"z1\": {ID: 1, Rack: \"z1\"},\n\t\t\t\"z2\": {ID: 2, Rack: \"z2\"},\n\t\t\t\"z3\": {ID: 2, Rack: \"z3\"},\n\t\t\t\"\": {},\n\t\t}\n\n\t\tmembers := []GroupMember{\n\t\t\t{\n\t\t\t\tID: \"z1\",\n\t\t\t\tTopics: []string{\"topic1\", \"topic2\"},\n\t\t\t\tUserData: []byte(\"z1\"),\n\t\t\t},\n\t\t\t{\n\t\t\t\tID: \"z2\",\n\t\t\t\tTopics: []string{\"topic2\", \"topic3\"},\n\t\t\t\tUserData: []byte(\"z2\"),\n\t\t\t},\n\t\t\t{\n\t\t\t\tID: \"z3\",\n\t\t\t\tTopics: []string{\"topic3\", \"topic1\"},\n\t\t\t\tUserData: []byte(\"z3\"),\n\t\t\t},\n\t\t}\n\n\t\tpartitions := []Partition{\n\t\t\t{\n\t\t\t\tID: 1,\n\t\t\t\tTopic: \"topic1\",\n\t\t\t\tLeader: brokers[\"z1\"],\n\t\t\t},\n\t\t\t{\n\t\t\t\tID: 2,\n\t\t\t\tTopic: \"topic1\",\n\t\t\t\tLeader: brokers[\"z3\"],\n\t\t\t},\n\t\t\t{\n\t\t\t\tID: 1,\n\t\t\t\tTopic: \"topic2\",\n\t\t\t\tLeader: brokers[\"z1\"],\n\t\t\t},\n\t\t\t{\n\t\t\t\tID: 2,\n\t\t\t\tTopic: \"topic2\",\n\t\t\t\tLeader: brokers[\"z2\"],\n\t\t\t},\n\t\t\t{\n\t\t\t\tID: 1,\n\t\t\t\tTopic: \"topic3\",\n\t\t\t\tLeader: brokers[\"z3\"],\n\t\t\t},\n\t\t\t{\n\t\t\t\tID: 2,\n\t\t\t\tTopic: \"topic3\",\n\t\t\t\tLeader: brokers[\"z2\"],\n\t\t\t},\n\t\t}\n\n\t\texpected := GroupMemberAssignments{\n\t\t\t\"z1\": {\"topic1\": []int{1}, \"topic2\": []int{1}},\n\t\t\t\"z2\": {\"topic2\": []int{2}, \"topic3\": []int{2}},\n\t\t\t\"z3\": {\"topic3\": []int{1}, \"topic1\": []int{2}},\n\t\t}\n\n\t\tres := b.AssignGroups(members, partitions)\n\t\tif !reflect.DeepEqual(expected, res) {\n\t\t\tt.Fatalf(\"incorrect group assignment. expected %v but got %v\", expected, res)\n\t\t}\n\t})\n}\n" }, { "path": "gzip/gzip.go", "content": "// Package gzip does nothing, it's kept for backward compatibility to avoid\n// breaking the majority of programs that imported it to install the compression\n// codec, which is now always included.\npackage gzip\n\nimport (\n\tgz \"github.com/klauspost/compress/gzip\"\n\t\"github.com/segmentio/kafka-go/compress/gzip\"\n)\n\nconst (\n\tCode = 1\n\tDefaultCompressionLevel = gz.DefaultCompression\n)\n\ntype CompressionCodec = gzip.Codec\n\nfunc NewCompressionCodec() *CompressionCodec {\n\treturn NewCompressionCodecLevel(DefaultCompressionLevel)\n}\n\nfunc NewCompressionCodecLevel(level int) *CompressionCodec {\n\treturn &CompressionCodec{Level: level}\n}\n" }, { "path": "heartbeat.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\theartbeatAPI \"github.com/segmentio/kafka-go/protocol/heartbeat\"\n)\n\n// HeartbeatRequest represents a heartbeat sent to kafka to indicate consume liveness.\ntype HeartbeatRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// GroupID is the ID of the group.\n\tGroupID string\n\n\t// GenerationID is the current generation for the group.\n\tGenerationID int32\n\n\t// MemberID is the ID of the group member.\n\tMemberID string\n\n\t// GroupInstanceID is a unique identifier for the consumer.\n\tGroupInstanceID string\n}\n\n// HeartbeatResponse represents a response from a heartbeat request.\ntype HeartbeatResponse struct {\n\t// Error is set to non-nil if an error occurred.\n\tError error\n\n\t// The amount of time that the broker throttled the request.\n\t//\n\t// This field will be zero if the kafka broker did not support the\n\t// Heartbeat API in version 1 or above.\n\tThrottle time.Duration\n}\n\ntype heartbeatRequestV0 struct {\n\t// GroupID holds the unique group identifier\n\tGroupID string\n\n\t// GenerationID holds the generation of the group.\n\tGenerationID int32\n\n\t// MemberID assigned by the group coordinator\n\tMemberID string\n}\n\n// Heartbeat sends a heartbeat request to a kafka broker and returns the response.\nfunc (c *Client) Heartbeat(ctx context.Context, req *HeartbeatRequest) (*HeartbeatResponse, error) {\n\tm, err := c.roundTrip(ctx, req.Addr, &heartbeatAPI.Request{\n\t\tGroupID: req.GroupID,\n\t\tGenerationID: req.GenerationID,\n\t\tMemberID: req.MemberID,\n\t\tGroupInstanceID: req.GroupInstanceID,\n\t})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).Heartbeat: %w\", err)\n\t}\n\n\tres := m.(*heartbeatAPI.Response)\n\n\tret := &HeartbeatResponse{\n\t\tThrottle: makeDuration(res.ThrottleTimeMs),\n\t}\n\n\tif res.ErrorCode != 0 {\n\t\tret.Error = Error(res.ErrorCode)\n\t}\n\n\treturn ret, nil\n}\n\nfunc (t heartbeatRequestV0) size() int32 {\n\treturn sizeofString(t.GroupID) +\n\t\tsizeofInt32(t.GenerationID) +\n\t\tsizeofString(t.MemberID)\n}\n\nfunc (t heartbeatRequestV0) writeTo(wb *writeBuffer) {\n\twb.writeString(t.GroupID)\n\twb.writeInt32(t.GenerationID)\n\twb.writeString(t.MemberID)\n}\n\ntype heartbeatResponseV0 struct {\n\t// ErrorCode holds response error code\n\tErrorCode int16\n}\n\nfunc (t heartbeatResponseV0) size() int32 {\n\treturn sizeofInt16(t.ErrorCode)\n}\n\nfunc (t heartbeatResponseV0) writeTo(wb *writeBuffer) {\n\twb.writeInt16(t.ErrorCode)\n}\n\nfunc (t *heartbeatResponseV0) readFrom(r *bufio.Reader, sz int) (remain int, err error) {\n\tif remain, err = readInt16(r, sz, &t.ErrorCode); err != nil {\n\t\treturn\n\t}\n\treturn\n}\n" }, { "path": "heartbeat_test.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"log\"\n\t\"os\"\n\t\"reflect\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestClientHeartbeat(t *testing.T) {\n\tclient, topic, shutdown := newLocalClientAndTopic()\n\tdefer shutdown()\n\n\tctx, cancel := context.WithTimeout(context.Background(), time.Second*30)\n\tdefer cancel()\n\n\tgroupID := makeGroupID()\n\n\tgroup, err := NewConsumerGroup(ConsumerGroupConfig{\n\t\tID: groupID,\n\t\tTopics: []string{topic},\n\t\tBrokers: []string{\"localhost:9092\"},\n\t\tHeartbeatInterval: 2 * time.Second,\n\t\tRebalanceTimeout: 2 * time.Second,\n\t\tRetentionTime: time.Hour,\n\t\tLogger: log.New(os.Stdout, \"cg-test: \", 0),\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer group.Close()\n\n\tgen, err := group.Next(ctx)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tctx, cancel = context.WithTimeout(context.Background(), time.Second*30)\n\tdefer cancel()\n\n\tresp, err := client.Heartbeat(ctx, &HeartbeatRequest{\n\t\tGroupID: groupID,\n\t\tGenerationID: gen.ID,\n\t\tMemberID: gen.MemberID,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif resp.Error != nil {\n\t\tt.Error(resp.Error)\n\t}\n}\n\nfunc TestHeartbeatRequestV0(t *testing.T) {\n\titem := heartbeatResponseV0{\n\t\tErrorCode: 2,\n\t}\n\n\tb := bytes.NewBuffer(nil)\n\tw := &writeBuffer{w: b}\n\titem.writeTo(w)\n\n\tvar found heartbeatResponseV0\n\tremain, err := (&found).readFrom(bufio.NewReader(b), b.Len())\n\tif err != nil {\n\t\tt.Error(err)\n\t\tt.FailNow()\n\t}\n\tif remain != 0 {\n\t\tt.Errorf(\"expected 0 remain, got %v\", remain)\n\t\tt.FailNow()\n\t}\n\tif !reflect.DeepEqual(item, found) {\n\t\tt.Error(\"expected item and found to be the same\")\n\t\tt.FailNow()\n\t}\n}\n" }, { "path": "incrementalalterconfigs.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"net\"\n\n\t\"github.com/segmentio/kafka-go/protocol/incrementalalterconfigs\"\n)\n\ntype ConfigOperation int8\n\nconst (\n\tConfigOperationSet ConfigOperation = 0\n\tConfigOperationDelete ConfigOperation = 1\n\tConfigOperationAppend ConfigOperation = 2\n\tConfigOperationSubtract ConfigOperation = 3\n)\n\n// IncrementalAlterConfigsRequest is a request to the IncrementalAlterConfigs API.\ntype IncrementalAlterConfigsRequest struct {\n\t// Addr is the address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// Resources contains the list of resources to update configs for.\n\tResources []IncrementalAlterConfigsRequestResource\n\n\t// ValidateOnly indicates whether Kafka should validate the changes without actually\n\t// applying them.\n\tValidateOnly bool\n}\n\n// IncrementalAlterConfigsRequestResource contains the details of a single resource type whose\n// configs should be altered.\ntype IncrementalAlterConfigsRequestResource struct {\n\t// ResourceType is the type of resource to update.\n\tResourceType ResourceType\n\n\t// ResourceName is the name of the resource to update (i.e., topic name or broker ID).\n\tResourceName string\n\n\t// Configs contains the list of config key/values to update.\n\tConfigs []IncrementalAlterConfigsRequestConfig\n}\n\n// IncrementalAlterConfigsRequestConfig describes a single config key/value pair that should\n// be altered.\ntype IncrementalAlterConfigsRequestConfig struct {\n\t// Name is the name of the config.\n\tName string\n\n\t// Value is the value to set for this config.\n\tValue string\n\n\t// ConfigOperation indicates how this config should be updated (e.g., add, delete, etc.).\n\tConfigOperation ConfigOperation\n}\n\n// IncrementalAlterConfigsResponse is a response from the IncrementalAlterConfigs API.\ntype IncrementalAlterConfigsResponse struct {\n\t// Resources contains details of each resource config that was updated.\n\tResources []IncrementalAlterConfigsResponseResource\n}\n\n// IncrementalAlterConfigsResponseResource contains the response details for a single resource\n// whose configs were updated.\ntype IncrementalAlterConfigsResponseResource struct {\n\t// Error is set to a non-nil value if an error occurred while updating this specific\n\t// config.\n\tError error\n\n\t// ResourceType is the type of resource that was updated.\n\tResourceType ResourceType\n\n\t// ResourceName is the name of the resource that was updated.\n\tResourceName string\n}\n\nfunc (c *Client) IncrementalAlterConfigs(\n\tctx context.Context,\n\treq *IncrementalAlterConfigsRequest,\n) (*IncrementalAlterConfigsResponse, error) {\n\tapiReq := &incrementalalterconfigs.Request{\n\t\tValidateOnly: req.ValidateOnly,\n\t}\n\n\tfor _, res := range req.Resources {\n\t\tapiRes := incrementalalterconfigs.RequestResource{\n\t\t\tResourceType: int8(res.ResourceType),\n\t\t\tResourceName: res.ResourceName,\n\t\t}\n\n\t\tfor _, config := range res.Configs {\n\t\t\tapiRes.Configs = append(\n\t\t\t\tapiRes.Configs,\n\t\t\t\tincrementalalterconfigs.RequestConfig{\n\t\t\t\t\tName: config.Name,\n\t\t\t\t\tValue: config.Value,\n\t\t\t\t\tConfigOperation: int8(config.ConfigOperation),\n\t\t\t\t},\n\t\t\t)\n\t\t}\n\n\t\tapiReq.Resources = append(\n\t\t\tapiReq.Resources,\n\t\t\tapiRes,\n\t\t)\n\t}\n\n\tprotoResp, err := c.roundTrip(\n\t\tctx,\n\t\treq.Addr,\n\t\tapiReq,\n\t)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tresp := &IncrementalAlterConfigsResponse{}\n\n\tapiResp := protoResp.(*incrementalalterconfigs.Response)\n\tfor _, res := range apiResp.Responses {\n\t\tresp.Resources = append(\n\t\t\tresp.Resources,\n\t\t\tIncrementalAlterConfigsResponseResource{\n\t\t\t\tError: makeError(res.ErrorCode, res.ErrorMessage),\n\t\t\t\tResourceType: ResourceType(res.ResourceType),\n\t\t\t\tResourceName: res.ResourceName,\n\t\t\t},\n\t\t)\n\t}\n\n\treturn resp, nil\n}\n" }, { "path": "incrementalalterconfigs_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"reflect\"\n\t\"testing\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc TestClientIncrementalAlterConfigs(t *testing.T) {\n\tif !ktesting.KafkaIsAtLeast(\"2.4.0\") {\n\t\treturn\n\t}\n\n\tconst (\n\t\tconfigKey = \"max.message.bytes\"\n\t\tconfigValue = \"200000\"\n\t)\n\n\tctx := context.Background()\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\ttopic := makeTopic()\n\tcreateTopic(t, topic, 1)\n\tdefer deleteTopic(t, topic)\n\n\tresp, err := client.IncrementalAlterConfigs(\n\t\tctx,\n\t\t&IncrementalAlterConfigsRequest{\n\t\t\tResources: []IncrementalAlterConfigsRequestResource{\n\t\t\t\t{\n\t\t\t\t\tResourceName: topic,\n\t\t\t\t\tResourceType: ResourceTypeTopic,\n\t\t\t\t\tConfigs: []IncrementalAlterConfigsRequestConfig{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tName: configKey,\n\t\t\t\t\t\t\tValue: configValue,\n\t\t\t\t\t\t\tConfigOperation: ConfigOperationSet,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\texpRes := []IncrementalAlterConfigsResponseResource{\n\t\t{\n\t\t\tResourceType: ResourceTypeTopic,\n\t\t\tResourceName: topic,\n\t\t},\n\t}\n\tif !reflect.DeepEqual(expRes, resp.Resources) {\n\t\tt.Error(\n\t\t\t\"Wrong response resources\",\n\t\t\t\"expected\", expRes,\n\t\t\t\"got\", resp.Resources,\n\t\t)\n\t}\n\n\tdResp, err := client.DescribeConfigs(\n\t\tctx,\n\t\t&DescribeConfigsRequest{\n\t\t\tResources: []DescribeConfigRequestResource{\n\t\t\t\t{\n\t\t\t\t\tResourceType: ResourceTypeTopic,\n\t\t\t\t\tResourceName: topic,\n\t\t\t\t\tConfigNames: []string{\n\t\t\t\t\t\t\"max.message.bytes\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(dResp.Resources) != 1 || len(dResp.Resources[0].ConfigEntries) != 1 {\n\t\tt.Fatal(\"Invalid structure for DescribeResourcesResponse\")\n\t}\n\n\tv := dResp.Resources[0].ConfigEntries[0].ConfigValue\n\tif v != configValue {\n\t\tt.Error(\n\t\t\t\"Wrong altered value for max.message.bytes\",\n\t\t\t\"expected\", configValue,\n\t\t\t\"got\", v,\n\t\t)\n\t}\n}\n" }, { "path": "initproducerid.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/initproducerid\"\n)\n\n// InitProducerIDRequest is the request structure for the InitProducerId function.\ntype InitProducerIDRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// The transactional id key.\n\tTransactionalID string\n\n\t// Time after which a transaction should time out\n\tTransactionTimeoutMs int\n\n\t// The Producer ID (PID).\n\t// This is used to disambiguate requests if a transactional id is reused following its expiration.\n\t// Only supported in version >=3 of the request, will be ignore otherwise.\n\tProducerID int\n\n\t// The producer's current epoch.\n\t// This will be checked against the producer epoch on the broker,\n\t// and the request will return an error if they do not match.\n\t// Only supported in version >=3 of the request, will be ignore otherwise.\n\tProducerEpoch int\n}\n\n// ProducerSession contains useful information about the producer session from the broker's response.\ntype ProducerSession struct {\n\t// The Producer ID (PID) for the current producer session\n\tProducerID int\n\n\t// The epoch associated with the current producer session for the given PID\n\tProducerEpoch int\n}\n\n// InitProducerIDResponse is the response structure for the InitProducerId function.\ntype InitProducerIDResponse struct {\n\t// The Transaction/Group Coordinator details\n\tProducer *ProducerSession\n\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// An error that may have occurred while attempting to retrieve initProducerId\n\t//\n\t// The error contains both the kafka error code, and an error message\n\t// returned by the kafka broker.\n\tError error\n}\n\n// InitProducerID sends a initProducerId request to a kafka broker and returns the\n// response.\nfunc (c *Client) InitProducerID(ctx context.Context, req *InitProducerIDRequest) (*InitProducerIDResponse, error) {\n\tm, err := c.roundTrip(ctx, req.Addr, &initproducerid.Request{\n\t\tTransactionalID: req.TransactionalID,\n\t\tTransactionTimeoutMs: int32(req.TransactionTimeoutMs),\n\t\tProducerID: int64(req.ProducerID),\n\t\tProducerEpoch: int16(req.ProducerEpoch),\n\t})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).InitProducerId: %w\", err)\n\t}\n\n\tres := m.(*initproducerid.Response)\n\n\treturn &InitProducerIDResponse{\n\t\tProducer: &ProducerSession{\n\t\t\tProducerID: int(res.ProducerID),\n\t\t\tProducerEpoch: int(res.ProducerEpoch),\n\t\t},\n\t\tThrottle: makeDuration(res.ThrottleTimeMs),\n\t\tError: makeError(res.ErrorCode, \"\"),\n\t}, nil\n}\n" }, { "path": "initproducerid_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"net\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc TestClientInitProducerId(t *testing.T) {\n\tif !ktesting.KafkaIsAtLeast(\"0.11.0\") {\n\t\treturn\n\t}\n\n\t// TODO: look into why this test fails on Kafka 3.0.0 and higher when transactional support\n\t// work is revisited.\n\tif ktesting.KafkaIsAtLeast(\"3.0.0\") {\n\t\tt.Skip(\"Skipping test because it fails on Kafka version 3.0.0 or higher.\")\n\t}\n\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\ttid := makeTransactionalID()\n\t// Wait for kafka setup and Coordinator to be available.\n\tctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)\n\tdefer cancel()\n\trespc, err := waitForCoordinatorIndefinitely(ctx, client, &FindCoordinatorRequest{\n\t\tAddr: client.Addr,\n\t\tKey: tid,\n\t\tKeyType: CoordinatorKeyTypeTransaction,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// Now establish a connection with the transaction coordinator\n\ttransactionCoordinator := TCP(net.JoinHostPort(respc.Coordinator.Host, strconv.Itoa(int(respc.Coordinator.Port))))\n\tclient, shutdown = newClient(transactionCoordinator)\n\tdefer shutdown()\n\n\t// Check if producer epoch increases and PID remains the same when producer is\n\t// initialized again with the same transactionalID\n\tresp, err := client.InitProducerID(context.Background(), &InitProducerIDRequest{\n\t\tAddr: transactionCoordinator,\n\t\tTransactionalID: tid,\n\t\tTransactionTimeoutMs: 30000,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif resp.Error != nil {\n\t\tt.Fatal(resp.Error)\n\t}\n\n\tepoch1 := resp.Producer.ProducerEpoch\n\tpid1 := resp.Producer.ProducerID\n\n\tresp, err = client.InitProducerID(context.Background(), &InitProducerIDRequest{\n\t\tAddr: transactionCoordinator,\n\t\tTransactionalID: tid,\n\t\tTransactionTimeoutMs: 30000,\n\t\tProducerID: pid1,\n\t\tProducerEpoch: epoch1,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif resp.Error != nil {\n\t\tt.Fatal(resp.Error)\n\t}\n\n\tepoch2 := resp.Producer.ProducerEpoch\n\tpid2 := resp.Producer.ProducerID\n\n\tif pid1 != pid2 {\n\t\tt.Fatalf(\"PID should stay the same across producer sessions; expected: %v got: %v\", pid1, pid2)\n\t}\n\n\tif epoch2-epoch1 <= 0 {\n\t\tt.Fatal(\"Epoch should increase when producer is initialized again with the same transactionID\")\n\t}\n\n\t// Checks if transaction timeout is too high\n\t// Transaction timeout should never be higher than broker config `transaction.max.timeout.ms`\n\tresp, _ = client.InitProducerID(context.Background(), &InitProducerIDRequest{\n\t\tAddr: client.Addr,\n\t\tTransactionalID: tid,\n\t\tTransactionTimeoutMs: 30000000,\n\t})\n\tif !errors.Is(resp.Error, InvalidTransactionTimeout) {\n\t\tt.Fatal(\"Should have errored with: Transaction timeout specified is higher than `transaction.max.timeout.ms`\")\n\t}\n}\n" }, { "path": "joingroup.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n\t\"github.com/segmentio/kafka-go/protocol/consumer\"\n\t\"github.com/segmentio/kafka-go/protocol/joingroup\"\n)\n\n// JoinGroupRequest is the request structure for the JoinGroup function.\ntype JoinGroupRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// GroupID of the group to join.\n\tGroupID string\n\n\t// The duration after which the coordinator considers the consumer dead\n\t// if it has not received a heartbeat.\n\tSessionTimeout time.Duration\n\n\t// The duration the coordination will wait for each member to rejoin when rebalancing the group.\n\tRebalanceTimeout time.Duration\n\n\t// The ID assigned by the group coordinator.\n\tMemberID string\n\n\t// The unique identifier for the consumer instance.\n\tGroupInstanceID string\n\n\t// The name for the class of protocols implemented by the group being joined.\n\tProtocolType string\n\n\t// The list of protocols the member supports.\n\tProtocols []GroupProtocol\n}\n\n// GroupProtocol represents a consumer group protocol.\ntype GroupProtocol struct {\n\t// The protocol name.\n\tName string\n\n\t// The protocol metadata.\n\tMetadata GroupProtocolSubscription\n}\n\ntype GroupProtocolSubscription struct {\n\t// The Topics to subscribe to.\n\tTopics []string\n\n\t// UserData assosiated with the subscription for the given protocol\n\tUserData []byte\n\n\t// Partitions owned by this consumer.\n\tOwnedPartitions map[string][]int\n}\n\n// JoinGroupResponse is the response structure for the JoinGroup function.\ntype JoinGroupResponse struct {\n\t// An error that may have occurred when attempting to join the group.\n\t//\n\t// The errors contain the kafka error code. Programs may use the standard\n\t// errors.Is function to test the error against kafka error codes.\n\tError error\n\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// The generation ID of the group.\n\tGenerationID int\n\n\t// The group protocol selected by the coordinatior.\n\tProtocolName string\n\n\t// The group protocol name.\n\tProtocolType string\n\n\t// The leader of the group.\n\tLeaderID string\n\n\t// The group member ID.\n\tMemberID string\n\n\t// The members of the group.\n\tMembers []JoinGroupResponseMember\n}\n\n// JoinGroupResponseMember represents a group memmber in a reponse to a JoinGroup request.\ntype JoinGroupResponseMember struct {\n\t// The group memmber ID.\n\tID string\n\n\t// The unique identifier of the consumer instance.\n\tGroupInstanceID string\n\n\t// The group member metadata.\n\tMetadata GroupProtocolSubscription\n}\n\n// JoinGroup sends a join group request to the coordinator and returns the response.\nfunc (c *Client) JoinGroup(ctx context.Context, req *JoinGroupRequest) (*JoinGroupResponse, error) {\n\tjoinGroup := joingroup.Request{\n\t\tGroupID: req.GroupID,\n\t\tSessionTimeoutMS: int32(req.SessionTimeout.Milliseconds()),\n\t\tRebalanceTimeoutMS: int32(req.RebalanceTimeout.Milliseconds()),\n\t\tMemberID: req.MemberID,\n\t\tGroupInstanceID: req.GroupInstanceID,\n\t\tProtocolType: req.ProtocolType,\n\t\tProtocols: make([]joingroup.RequestProtocol, 0, len(req.Protocols)),\n\t}\n\n\tfor _, proto := range req.Protocols {\n\t\tprotoMeta := consumer.Subscription{\n\t\t\tVersion: consumer.MaxVersionSupported,\n\t\t\tTopics: proto.Metadata.Topics,\n\t\t\tUserData: proto.Metadata.UserData,\n\t\t\tOwnedPartitions: make([]consumer.TopicPartition, 0, len(proto.Metadata.OwnedPartitions)),\n\t\t}\n\t\tfor topic, partitions := range proto.Metadata.OwnedPartitions {\n\t\t\ttp := consumer.TopicPartition{\n\t\t\t\tTopic: topic,\n\t\t\t\tPartitions: make([]int32, 0, len(partitions)),\n\t\t\t}\n\t\t\tfor _, partition := range partitions {\n\t\t\t\ttp.Partitions = append(tp.Partitions, int32(partition))\n\t\t\t}\n\t\t\tprotoMeta.OwnedPartitions = append(protoMeta.OwnedPartitions, tp)\n\t\t}\n\n\t\tmetaBytes, err := protocol.Marshal(consumer.MaxVersionSupported, protoMeta)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"kafka.(*Client).JoinGroup: %w\", err)\n\t\t}\n\n\t\tjoinGroup.Protocols = append(joinGroup.Protocols, joingroup.RequestProtocol{\n\t\t\tName: proto.Name,\n\t\t\tMetadata: metaBytes,\n\t\t})\n\t}\n\n\tm, err := c.roundTrip(ctx, req.Addr, &joinGroup)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).JoinGroup: %w\", err)\n\t}\n\n\tr := m.(*joingroup.Response)\n\n\tres := &JoinGroupResponse{\n\t\tError: makeError(r.ErrorCode, \"\"),\n\t\tThrottle: makeDuration(r.ThrottleTimeMS),\n\t\tGenerationID: int(r.GenerationID),\n\t\tProtocolName: r.ProtocolName,\n\t\tProtocolType: r.ProtocolType,\n\t\tLeaderID: r.LeaderID,\n\t\tMemberID: r.MemberID,\n\t\tMembers: make([]JoinGroupResponseMember, 0, len(r.Members)),\n\t}\n\n\tfor _, member := range r.Members {\n\t\tvar meta consumer.Subscription\n\t\terr = protocol.Unmarshal(member.Metadata, consumer.MaxVersionSupported, &meta)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"kafka.(*Client).JoinGroup: %w\", err)\n\t\t}\n\t\tsubscription := GroupProtocolSubscription{\n\t\t\tTopics: meta.Topics,\n\t\t\tUserData: meta.UserData,\n\t\t\tOwnedPartitions: make(map[string][]int, len(meta.OwnedPartitions)),\n\t\t}\n\t\tfor _, owned := range meta.OwnedPartitions {\n\t\t\tsubscription.OwnedPartitions[owned.Topic] = make([]int, 0, len(owned.Partitions))\n\t\t\tfor _, partition := range owned.Partitions {\n\t\t\t\tsubscription.OwnedPartitions[owned.Topic] = append(subscription.OwnedPartitions[owned.Topic], int(partition))\n\t\t\t}\n\t\t}\n\t\tres.Members = append(res.Members, JoinGroupResponseMember{\n\t\t\tID: member.MemberID,\n\t\t\tGroupInstanceID: member.GroupInstanceID,\n\t\t\tMetadata: subscription,\n\t\t})\n\t}\n\n\treturn res, nil\n}\n\ntype groupMetadata struct {\n\tVersion int16\n\tTopics []string\n\tUserData []byte\n}\n\nfunc (t groupMetadata) size() int32 {\n\treturn sizeofInt16(t.Version) +\n\t\tsizeofStringArray(t.Topics) +\n\t\tsizeofBytes(t.UserData)\n}\n\nfunc (t groupMetadata) writeTo(wb *writeBuffer) {\n\twb.writeInt16(t.Version)\n\twb.writeStringArray(t.Topics)\n\twb.writeBytes(t.UserData)\n}\n\nfunc (t groupMetadata) bytes() []byte {\n\tbuf := bytes.NewBuffer(nil)\n\tt.writeTo(&writeBuffer{w: buf})\n\treturn buf.Bytes()\n}\n\nfunc (t *groupMetadata) readFrom(r *bufio.Reader, size int) (remain int, err error) {\n\tif remain, err = readInt16(r, size, &t.Version); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readStringArray(r, remain, &t.Topics); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readBytes(r, remain, &t.UserData); err != nil {\n\t\treturn\n\t}\n\treturn\n}\n\ntype joinGroupRequestGroupProtocolV1 struct {\n\tProtocolName string\n\tProtocolMetadata []byte\n}\n\nfunc (t joinGroupRequestGroupProtocolV1) size() int32 {\n\treturn sizeofString(t.ProtocolName) +\n\t\tsizeofBytes(t.ProtocolMetadata)\n}\n\nfunc (t joinGroupRequestGroupProtocolV1) writeTo(wb *writeBuffer) {\n\twb.writeString(t.ProtocolName)\n\twb.writeBytes(t.ProtocolMetadata)\n}\n\ntype joinGroupRequest struct {\n\t// GroupID holds the unique group identifier\n\tGroupID string\n\n\t// SessionTimeout holds the coordinator considers the consumer dead if it\n\t// receives no heartbeat after this timeout in ms.\n\tSessionTimeout int32\n\n\t// RebalanceTimeout holds the maximum time that the coordinator will wait\n\t// for each member to rejoin when rebalancing the group in ms\n\tRebalanceTimeout int32\n\n\t// MemberID assigned by the group coordinator or the zero string if joining\n\t// for the first time.\n\tMemberID string\n\n\t// ProtocolType holds the unique name for class of protocols implemented by group\n\tProtocolType string\n\n\t// GroupProtocols holds the list of protocols that the member supports\n\tGroupProtocols []joinGroupRequestGroupProtocolV1\n}\n\nfunc (t joinGroupRequest) size() int32 {\n\treturn sizeofString(t.GroupID) +\n\t\tsizeofInt32(t.SessionTimeout) +\n\t\tsizeofInt32(t.RebalanceTimeout) +\n\t\tsizeofString(t.MemberID) +\n\t\tsizeofString(t.ProtocolType) +\n\t\tsizeofArray(len(t.GroupProtocols), func(i int) int32 { return t.GroupProtocols[i].size() })\n}\n\nfunc (t joinGroupRequest) writeTo(wb *writeBuffer) {\n\twb.writeString(t.GroupID)\n\twb.writeInt32(t.SessionTimeout)\n\twb.writeInt32(t.RebalanceTimeout)\n\twb.writeString(t.MemberID)\n\twb.writeString(t.ProtocolType)\n\twb.writeArray(len(t.GroupProtocols), func(i int) { t.GroupProtocols[i].writeTo(wb) })\n}\n\ntype joinGroupResponseMember struct {\n\t// MemberID assigned by the group coordinator\n\tMemberID string\n\tMemberMetadata []byte\n}\n\nfunc (t joinGroupResponseMember) size() int32 {\n\treturn sizeofString(t.MemberID) +\n\t\tsizeofBytes(t.MemberMetadata)\n}\n\nfunc (t joinGroupResponseMember) writeTo(wb *writeBuffer) {\n\twb.writeString(t.MemberID)\n\twb.writeBytes(t.MemberMetadata)\n}\n\nfunc (t *joinGroupResponseMember) readFrom(r *bufio.Reader, size int) (remain int, err error) {\n\tif remain, err = readString(r, size, &t.MemberID); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readBytes(r, remain, &t.MemberMetadata); err != nil {\n\t\treturn\n\t}\n\treturn\n}\n\ntype joinGroupResponse struct {\n\tv apiVersion // v1, v2\n\n\tThrottleTime int32\n\n\t// ErrorCode holds response error code\n\tErrorCode int16\n\n\t// GenerationID holds the generation of the group.\n\tGenerationID int32\n\n\t// GroupProtocol holds the group protocol selected by the coordinator\n\tGroupProtocol string\n\n\t// LeaderID holds the leader of the group\n\tLeaderID string\n\n\t// MemberID assigned by the group coordinator\n\tMemberID string\n\tMembers []joinGroupResponseMember\n}\n\nfunc (t joinGroupResponse) size() int32 {\n\tsz := sizeofInt16(t.ErrorCode) +\n\t\tsizeofInt32(t.GenerationID) +\n\t\tsizeofString(t.GroupProtocol) +\n\t\tsizeofString(t.LeaderID) +\n\t\tsizeofString(t.MemberID) +\n\t\tsizeofArray(len(t.MemberID), func(i int) int32 { return t.Members[i].size() })\n\tif t.v >= v2 {\n\t\tsz += sizeofInt32(t.ThrottleTime)\n\t}\n\treturn sz\n}\n\nfunc (t joinGroupResponse) writeTo(wb *writeBuffer) {\n\tif t.v >= v2 {\n\t\twb.writeInt32(t.ThrottleTime)\n\t}\n\twb.writeInt16(t.ErrorCode)\n\twb.writeInt32(t.GenerationID)\n\twb.writeString(t.GroupProtocol)\n\twb.writeString(t.LeaderID)\n\twb.writeString(t.MemberID)\n\twb.writeArray(len(t.Members), func(i int) { t.Members[i].writeTo(wb) })\n}\n\nfunc (t *joinGroupResponse) readFrom(r *bufio.Reader, size int) (remain int, err error) {\n\tremain = size\n\tif t.v >= v2 {\n\t\tif remain, err = readInt32(r, remain, &t.ThrottleTime); err != nil {\n\t\t\treturn\n\t\t}\n\t}\n\tif remain, err = readInt16(r, remain, &t.ErrorCode); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readInt32(r, remain, &t.GenerationID); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readString(r, remain, &t.GroupProtocol); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readString(r, remain, &t.LeaderID); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readString(r, remain, &t.MemberID); err != nil {\n\t\treturn\n\t}\n\n\tfn := func(r *bufio.Reader, size int) (fnRemain int, fnErr error) {\n\t\tvar item joinGroupResponseMember\n\t\tif fnRemain, fnErr = (&item).readFrom(r, size); fnErr != nil {\n\t\t\treturn\n\t\t}\n\t\tt.Members = append(t.Members, item)\n\t\treturn\n\t}\n\tif remain, err = readArrayWith(r, remain, fn); err != nil {\n\t\treturn\n\t}\n\n\treturn\n}\n" }, { "path": "joingroup_test.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"reflect\"\n\t\"testing\"\n\t\"time\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc TestClientJoinGroup(t *testing.T) {\n\ttopic := makeTopic()\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\terr := clientCreateTopic(client, topic, 3)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tgroupID := makeGroupID()\n\n\tctx, cancel := context.WithTimeout(context.Background(), time.Second*30)\n\tdefer cancel()\n\trespc, err := waitForCoordinatorIndefinitely(ctx, client, &FindCoordinatorRequest{\n\t\tAddr: client.Addr,\n\t\tKey: groupID,\n\t\tKeyType: CoordinatorKeyTypeConsumer,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif respc.Error != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tgroupInstanceID := \"group-instance-id\"\n\tif !ktesting.KafkaIsAtLeast(\"2.4.1\") {\n\t\tgroupInstanceID = \"\"\n\t}\n\tconst userData = \"user-data\"\n\n\treq := &JoinGroupRequest{\n\t\tGroupID: groupID,\n\t\tGroupInstanceID: groupInstanceID,\n\t\tProtocolType: \"consumer\",\n\t\tSessionTimeout: time.Minute,\n\t\tRebalanceTimeout: time.Minute,\n\t\tProtocols: []GroupProtocol{\n\t\t\t{\n\t\t\t\tName: RoundRobinGroupBalancer{}.ProtocolName(),\n\t\t\t\tMetadata: GroupProtocolSubscription{\n\t\t\t\t\tTopics: []string{topic},\n\t\t\t\t\tUserData: []byte(userData),\n\t\t\t\t\tOwnedPartitions: map[string][]int{\n\t\t\t\t\t\ttopic: {0, 1, 2},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tvar resp *JoinGroupResponse\n\n\tfor {\n\t\tresp, err = client.JoinGroup(ctx, req)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tif errors.Is(resp.Error, MemberIDRequired) {\n\t\t\treq.MemberID = resp.MemberID\n\t\t\ttime.Sleep(time.Second)\n\t\t\tcontinue\n\t\t}\n\n\t\tif resp.Error != nil {\n\t\t\tt.Fatal(resp.Error)\n\t\t}\n\t\tbreak\n\t}\n\n\tif resp.GenerationID != 1 {\n\t\tt.Fatalf(\"expected generation ID to be 1 but got %v\", resp.GenerationID)\n\t}\n\n\tif resp.MemberID == \"\" {\n\t\tt.Fatal(\"expected a member ID in response\")\n\t}\n\n\tif resp.LeaderID != resp.MemberID {\n\t\tt.Fatalf(\"expected to be group leader but got %v\", resp.LeaderID)\n\t}\n\n\tif len(resp.Members) != 1 {\n\t\tt.Fatalf(\"expected 1 member got %v\", resp.Members)\n\t}\n\n\tmember := resp.Members[0]\n\n\tif member.ID != resp.MemberID {\n\t\tt.Fatal(\"expected to be the only group memmber\")\n\t}\n\n\tif member.GroupInstanceID != groupInstanceID {\n\t\tt.Fatalf(\"expected the group instance ID to be %v, got %v\", groupInstanceID, member.GroupInstanceID)\n\t}\n\n\texpectedMetadata := GroupProtocolSubscription{\n\t\tTopics: []string{topic},\n\t\tUserData: []byte(userData),\n\t\tOwnedPartitions: map[string][]int{\n\t\t\ttopic: {0, 1, 2},\n\t\t},\n\t}\n\n\tif !reflect.DeepEqual(member.Metadata, expectedMetadata) {\n\t\tt.Fatalf(\"\\nexpected assignment to be \\n%v\\nbut got\\n%v\", expectedMetadata, member.Metadata)\n\t}\n}\n\nfunc TestSaramaCompatibility(t *testing.T) {\n\tvar (\n\t\t// sample data from github.com/Shopify/sarama\n\t\t//\n\t\t// See consumer_group_members_test.go\n\t\t//\n\t\tgroupMemberMetadata = []byte{\n\t\t\t0, 1, // Version\n\t\t\t0, 0, 0, 2, // Topic array length\n\t\t\t0, 3, 'o', 'n', 'e', // Topic one\n\t\t\t0, 3, 't', 'w', 'o', // Topic two\n\t\t\t0, 0, 0, 3, 0x01, 0x02, 0x03, // Userdata\n\t\t}\n\t\tgroupMemberAssignment = []byte{\n\t\t\t0, 1, // Version\n\t\t\t0, 0, 0, 1, // Topic array length\n\t\t\t0, 3, 'o', 'n', 'e', // Topic one\n\t\t\t0, 0, 0, 3, // Topic one, partition array length\n\t\t\t0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 4, // 0, 2, 4\n\t\t\t0, 0, 0, 3, 0x01, 0x02, 0x03, // Userdata\n\t\t}\n\t)\n\n\tt.Run(\"verify metadata\", func(t *testing.T) {\n\t\tvar item groupMetadata\n\t\tremain, err := (&item).readFrom(bufio.NewReader(bytes.NewReader(groupMemberMetadata)), len(groupMemberMetadata))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"bad err: %v\", err)\n\t\t}\n\t\tif remain != 0 {\n\t\t\tt.Fatalf(\"expected 0; got %v\", remain)\n\t\t}\n\n\t\tif v := item.Version; v != 1 {\n\t\t\tt.Errorf(\"expected Version 1; got %v\", v)\n\t\t}\n\t\tif v := item.Topics; !reflect.DeepEqual([]string{\"one\", \"two\"}, v) {\n\t\t\tt.Errorf(`expected {\"one\", \"two\"}; got %v`, v)\n\t\t}\n\t\tif v := item.UserData; !reflect.DeepEqual([]byte{0x01, 0x02, 0x03}, v) {\n\t\t\tt.Errorf(\"expected []byte{0x01, 0x02, 0x03}; got %v\", v)\n\t\t}\n\t})\n\n\tt.Run(\"verify assignments\", func(t *testing.T) {\n\t\tvar item groupAssignment\n\t\tremain, err := (&item).readFrom(bufio.NewReader(bytes.NewReader(groupMemberAssignment)), len(groupMemberAssignment))\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"bad err: %v\", err)\n\t\t}\n\t\tif remain != 0 {\n\t\t\tt.Fatalf(\"expected 0; got %v\", remain)\n\t\t}\n\n\t\tif v := item.Version; v != 1 {\n\t\t\tt.Errorf(\"expected Version 1; got %v\", v)\n\t\t}\n\t\tif v := item.Topics; !reflect.DeepEqual(map[string][]int32{\"one\": {0, 2, 4}}, v) {\n\t\t\tt.Errorf(`expected map[string][]int32{\"one\": {0, 2, 4}}; got %v`, v)\n\t\t}\n\t\tif v := item.UserData; !reflect.DeepEqual([]byte{0x01, 0x02, 0x03}, v) {\n\t\t\tt.Errorf(\"expected []byte{0x01, 0x02, 0x03}; got %v\", v)\n\t\t}\n\t})\n}\n\nfunc TestMemberMetadata(t *testing.T) {\n\titem := groupMetadata{\n\t\tVersion: 1,\n\t\tTopics: []string{\"a\", \"b\"},\n\t\tUserData: []byte(`blah`),\n\t}\n\n\tb := bytes.NewBuffer(nil)\n\tw := &writeBuffer{w: b}\n\titem.writeTo(w)\n\n\tvar found groupMetadata\n\tremain, err := (&found).readFrom(bufio.NewReader(b), b.Len())\n\tif err != nil {\n\t\tt.Error(err)\n\t\tt.FailNow()\n\t}\n\tif remain != 0 {\n\t\tt.Errorf(\"expected 0 remain, got %v\", remain)\n\t\tt.FailNow()\n\t}\n\tif !reflect.DeepEqual(item, found) {\n\t\tt.Error(\"expected item and found to be the same\")\n\t\tt.FailNow()\n\t}\n}\n\nfunc TestJoinGroupResponse(t *testing.T) {\n\tsupportedVersions := []apiVersion{v1, v2}\n\tfor _, v := range supportedVersions {\n\t\titem := joinGroupResponse{\n\t\t\tv: v,\n\t\t\tErrorCode: 2,\n\t\t\tGenerationID: 3,\n\t\t\tGroupProtocol: \"a\",\n\t\t\tLeaderID: \"b\",\n\t\t\tMemberID: \"c\",\n\t\t\tMembers: []joinGroupResponseMember{\n\t\t\t\t{\n\t\t\t\t\tMemberID: \"d\",\n\t\t\t\t\tMemberMetadata: []byte(\"blah\"),\n\t\t\t\t},\n\t\t\t},\n\t\t}\n\n\t\tb := bytes.NewBuffer(nil)\n\t\tw := &writeBuffer{w: b}\n\t\titem.writeTo(w)\n\n\t\tfound := joinGroupResponse{v: v}\n\t\tremain, err := (&found).readFrom(bufio.NewReader(b), b.Len())\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t\tt.FailNow()\n\t\t}\n\t\tif remain != 0 {\n\t\t\tt.Errorf(\"expected 0 remain, got %v\", remain)\n\t\t\tt.FailNow()\n\t\t}\n\t\tif !reflect.DeepEqual(item, found) {\n\t\t\tt.Error(\"expected item and found to be the same\")\n\t\t\tt.FailNow()\n\t\t}\n\t}\n}\n" }, { "path": "kafka.go", "content": "package kafka\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\n// Broker represents a kafka broker in a kafka cluster.\ntype Broker struct {\n\tHost string\n\tPort int\n\tID int\n\tRack string\n}\n\n// Topic represents a topic in a kafka cluster.\ntype Topic struct {\n\t// Name of the topic.\n\tName string\n\n\t// True if the topic is internal.\n\tInternal bool\n\n\t// The list of partition currently available on this topic.\n\tPartitions []Partition\n\n\t// An error that may have occurred while attempting to read the topic\n\t// metadata.\n\t//\n\t// The error contains both the kafka error code, and an error message\n\t// returned by the kafka broker. Programs may use the standard errors.Is\n\t// function to test the error against kafka error codes.\n\tError error\n}\n\n// Partition carries the metadata associated with a kafka partition.\ntype Partition struct {\n\t// Name of the topic that the partition belongs to, and its index in the\n\t// topic.\n\tTopic string\n\tID int\n\n\t// Leader, replicas, and ISR for the partition.\n\t//\n\t// When no physical host is known to be running a broker, the Host and Port\n\t// fields will be set to the zero values. The logical broker ID is always\n\t// set to the value known to the kafka cluster, even if the broker is not\n\t// currently backed by a physical host.\n\tLeader Broker\n\tReplicas []Broker\n\tIsr []Broker\n\n\t// Available only with metadata API level >= 6:\n\tOfflineReplicas []Broker\n\n\t// An error that may have occurred while attempting to read the partition\n\t// metadata.\n\t//\n\t// The error contains both the kafka error code, and an error message\n\t// returned by the kafka broker. Programs may use the standard errors.Is\n\t// function to test the error against kafka error codes.\n\tError error\n}\n\n// Marshal encodes v into a binary representation of the value in the kafka data\n// format.\n//\n// If v is a, or contains struct types, the kafka struct fields are interpreted\n// and may contain one of these values:\n//\n//\tnullable valid on bytes and strings, encodes as a nullable value\n//\tcompact valid on strings, encodes as a compact string\n//\n// The kafka struct tags should not contain min and max versions. If you need to\n// encode types based on specific versions of kafka APIs, use the Version type\n// instead.\nfunc Marshal(v interface{}) ([]byte, error) {\n\treturn protocol.Marshal(-1, v)\n}\n\n// Unmarshal decodes a binary representation from b into v.\n//\n// See Marshal for details.\nfunc Unmarshal(b []byte, v interface{}) error {\n\treturn protocol.Unmarshal(b, -1, v)\n}\n\n// Version represents a version number for kafka APIs.\ntype Version int16\n\n// Marshal is like the top-level Marshal function, but will only encode struct\n// fields for which n falls within the min and max versions specified on the\n// struct tag.\nfunc (n Version) Marshal(v interface{}) ([]byte, error) {\n\treturn protocol.Marshal(int16(n), v)\n}\n\n// Unmarshal is like the top-level Unmarshal function, but will only decode\n// struct fields for which n falls within the min and max versions specified on\n// the struct tag.\nfunc (n Version) Unmarshal(b []byte, v interface{}) error {\n\treturn protocol.Unmarshal(b, int16(n), v)\n}\n" }, { "path": "kafka_test.go", "content": "package kafka\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"testing\"\n)\n\nfunc TestMarshalUnmarshal(t *testing.T) {\n\tvalues := []interface{}{\n\t\ttrue,\n\t\tfalse,\n\n\t\tint8(0),\n\t\tint8(1),\n\t\tint8(math.MinInt8),\n\t\tint8(math.MaxInt8),\n\n\t\tint16(0),\n\t\tint16(1),\n\t\tint16(math.MinInt16),\n\t\tint16(math.MaxInt16),\n\n\t\tint32(0),\n\t\tint32(1),\n\t\tint32(math.MinInt32),\n\t\tint32(math.MaxInt32),\n\n\t\tint64(0),\n\t\tint64(1),\n\t\tint64(math.MinInt64),\n\t\tint64(math.MaxInt64),\n\n\t\t\"\",\n\t\t\"hello world!\",\n\n\t\t([]byte)(nil),\n\t\t[]byte(\"\"),\n\t\t[]byte(\"hello world!\"),\n\n\t\t([]int32)(nil),\n\t\t[]int32{},\n\t\t[]int32{0, 1, 2, 3, 4},\n\n\t\tstruct{}{},\n\t\tstruct {\n\t\t\tA int32\n\t\t\tB string\n\t\t\tC []byte\n\t\t}{A: 1, B: \"42\", C: []byte{}},\n\t}\n\n\tfor _, v := range values {\n\t\tt.Run(fmt.Sprintf(\"%+v\", v), func(t *testing.T) {\n\t\t\tb, err := Marshal(v)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(\"marshal error:\", err)\n\t\t\t}\n\n\t\t\tx := reflect.New(reflect.TypeOf(v))\n\n\t\t\tif err := Unmarshal(b, x.Interface()); err != nil {\n\t\t\t\tt.Fatal(\"unmarshal error:\", err)\n\t\t\t}\n\n\t\t\tif !reflect.DeepEqual(v, x.Elem().Interface()) {\n\t\t\t\tt.Fatalf(\"values mismatch:\\nexpected: %#v\\nfound: %#v\\n\", v, x.Elem().Interface())\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestVersionMarshalUnmarshal(t *testing.T) {\n\ttype T struct {\n\t\tA int32 `kafka:\"min=v0,max=v1\"`\n\t\tB string `kafka:\"min=v1,max=v2\"`\n\t\tC []byte `kafka:\"min=v2,max=v2,nullable\"`\n\t}\n\n\ttests := []struct {\n\t\tout T\n\t\tver Version\n\t}{\n\t\t{\n\t\t\tout: T{A: 42},\n\t\t\tver: Version(0),\n\t\t},\n\t}\n\n\tin := T{\n\t\tA: 42,\n\t\tB: \"Hello World!\",\n\t\tC: []byte(\"question?\"),\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(strconv.Itoa(int(test.ver)), func(t *testing.T) {\n\t\t\tb, err := test.ver.Marshal(in)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(\"marshal error:\", err)\n\t\t\t}\n\n\t\t\tx1 := test.out\n\t\t\tx2 := T{}\n\n\t\t\tif err := test.ver.Unmarshal(b, &x2); err != nil {\n\t\t\t\tt.Fatal(\"unmarshal error:\", err)\n\t\t\t}\n\n\t\t\tif !reflect.DeepEqual(x1, x2) {\n\t\t\t\tt.Fatalf(\"values mismatch:\\nexpected: %#v\\nfound: %#v\\n\", x1, x2)\n\t\t\t}\n\t\t})\n\t}\n\n}\n\ntype Struct struct {\n\tA int32\n\tB int32\n\tC int32\n}\n\nvar benchmarkValues = []interface{}{\n\ttrue,\n\tint8(1),\n\tint16(1),\n\tint32(1),\n\tint64(1),\n\t\"Hello World!\",\n\t[]byte(\"Hello World!\"),\n\t[]int32{1, 2, 3},\n\tStruct{A: 1, B: 2, C: 3},\n}\n\nfunc BenchmarkMarshal(b *testing.B) {\n\tfor _, v := range benchmarkValues {\n\t\tb.Run(fmt.Sprintf(\"%T\", v), func(b *testing.B) {\n\t\t\tfor i := 0; i < b.N; i++ {\n\t\t\t\t_, err := Marshal(v)\n\t\t\t\tif err != nil {\n\t\t\t\t\tb.Fatal(err)\n\t\t\t\t}\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc BenchmarkUnmarshal(b *testing.B) {\n\tfor _, v := range benchmarkValues {\n\t\tb.Run(fmt.Sprintf(\"%T\", v), func(b *testing.B) {\n\t\t\tdata, err := Marshal(v)\n\n\t\t\tif err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\n\t\t\tvalue := reflect.New(reflect.TypeOf(v))\n\t\t\tptr := value.Interface()\n\t\t\telem := value.Elem()\n\t\t\tzero := reflect.Zero(reflect.TypeOf(v))\n\n\t\t\tfor i := 0; i < b.N; i++ {\n\t\t\t\tif err := Unmarshal(data, ptr); err != nil {\n\t\t\t\t\tb.Fatal(err)\n\t\t\t\t}\n\t\t\t\telem.Set(zero)\n\t\t\t}\n\t\t})\n\t}\n}\n\ntype testKafkaLogger struct {\n\tPrefix string\n\tT *testing.T\n}\n\nfunc newTestKafkaLogger(t *testing.T, prefix string) Logger {\n\treturn &testKafkaLogger{Prefix: prefix, T: t}\n}\n\nfunc (l *testKafkaLogger) Printf(msg string, args ...interface{}) {\n\tl.T.Helper()\n\tif l.Prefix != \"\" {\n\t\tl.T.Logf(l.Prefix+\" \"+msg, args...)\n\t} else {\n\t\tl.T.Logf(msg, args...)\n\t}\n}\n" }, { "path": "leavegroup.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/leavegroup\"\n)\n\n// LeaveGroupRequest is the request structure for the LeaveGroup function.\ntype LeaveGroupRequest struct {\n\t// Address of the kafka broker to sent he request to.\n\tAddr net.Addr\n\n\t// GroupID of the group to leave.\n\tGroupID string\n\n\t// List of leaving member identities.\n\tMembers []LeaveGroupRequestMember\n}\n\n// LeaveGroupRequestMember represents the indentify of a member leaving a group.\ntype LeaveGroupRequestMember struct {\n\t// The member ID to remove from the group.\n\tID string\n\n\t// The group instance ID to remove from the group.\n\tGroupInstanceID string\n}\n\n// LeaveGroupResponse is the response structure for the LeaveGroup function.\ntype LeaveGroupResponse struct {\n\t// An error that may have occurred when attempting to leave the group.\n\t//\n\t// The errors contain the kafka error code. Programs may use the standard\n\t// errors.Is function to test the error against kafka error codes.\n\tError error\n\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// List of leaving member responses.\n\tMembers []LeaveGroupResponseMember\n}\n\n// LeaveGroupResponseMember represents a member leaving the group.\ntype LeaveGroupResponseMember struct {\n\t// The member ID of the member leaving the group.\n\tID string\n\n\t// The group instance ID to remove from the group.\n\tGroupInstanceID string\n\n\t// An error that may have occured when attempting to remove the member from the group.\n\t//\n\t// The errors contain the kafka error code. Programs may use the standard\n\t// errors.Is function to test the error against kafka error codes.\n\tError error\n}\n\nfunc (c *Client) LeaveGroup(ctx context.Context, req *LeaveGroupRequest) (*LeaveGroupResponse, error) {\n\tleaveGroup := leavegroup.Request{\n\t\tGroupID: req.GroupID,\n\t\tMembers: make([]leavegroup.RequestMember, 0, len(req.Members)),\n\t}\n\n\tfor _, member := range req.Members {\n\t\tleaveGroup.Members = append(leaveGroup.Members, leavegroup.RequestMember{\n\t\t\tMemberID: member.ID,\n\t\t\tGroupInstanceID: member.GroupInstanceID,\n\t\t})\n\t}\n\n\tm, err := c.roundTrip(ctx, req.Addr, &leaveGroup)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).LeaveGroup: %w\", err)\n\t}\n\n\tr := m.(*leavegroup.Response)\n\n\tres := &LeaveGroupResponse{\n\t\tError: makeError(r.ErrorCode, \"\"),\n\t\tThrottle: makeDuration(r.ThrottleTimeMS),\n\t}\n\n\tif len(r.Members) == 0 {\n\t\t// If we're using a version of the api without the\n\t\t// members array in the response, just add a member\n\t\t// so the api is consistent across versions.\n\t\tr.Members = []leavegroup.ResponseMember{\n\t\t\t{\n\t\t\t\tMemberID: req.Members[0].ID,\n\t\t\t\tGroupInstanceID: req.Members[0].GroupInstanceID,\n\t\t\t},\n\t\t}\n\t}\n\n\tres.Members = make([]LeaveGroupResponseMember, 0, len(r.Members))\n\tfor _, member := range r.Members {\n\t\tres.Members = append(res.Members, LeaveGroupResponseMember{\n\t\t\tID: member.MemberID,\n\t\t\tGroupInstanceID: member.GroupInstanceID,\n\t\t\tError: makeError(member.ErrorCode, \"\"),\n\t\t})\n\t}\n\n\treturn res, nil\n}\n\ntype leaveGroupRequestV0 struct {\n\t// GroupID holds the unique group identifier\n\tGroupID string\n\n\t// MemberID assigned by the group coordinator or the zero string if joining\n\t// for the first time.\n\tMemberID string\n}\n\nfunc (t leaveGroupRequestV0) size() int32 {\n\treturn sizeofString(t.GroupID) + sizeofString(t.MemberID)\n}\n\nfunc (t leaveGroupRequestV0) writeTo(wb *writeBuffer) {\n\twb.writeString(t.GroupID)\n\twb.writeString(t.MemberID)\n}\n\ntype leaveGroupResponseV0 struct {\n\t// ErrorCode holds response error code\n\tErrorCode int16\n}\n\nfunc (t leaveGroupResponseV0) size() int32 {\n\treturn sizeofInt16(t.ErrorCode)\n}\n\nfunc (t leaveGroupResponseV0) writeTo(wb *writeBuffer) {\n\twb.writeInt16(t.ErrorCode)\n}\n\nfunc (t *leaveGroupResponseV0) readFrom(r *bufio.Reader, size int) (remain int, err error) {\n\tremain, err = readInt16(r, size, &t.ErrorCode)\n\treturn\n}\n" }, { "path": "leavegroup_test.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"reflect\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestClientLeaveGroup(t *testing.T) {\n\t// In order to get to a leave group call we need to first\n\t// join a group then sync the group.\n\ttopic := makeTopic()\n\tclient, shutdown := newLocalClient()\n\tclient.Timeout = time.Minute\n\t// Although at higher api versions ClientID is nullable\n\t// for some reason the SyncGroup API call errors\n\t// when ClientID is null.\n\t// The Java Kafka Consumer generates a ClientID if one is not\n\t// present or if the provided ClientID is empty.\n\tclient.Transport.(*Transport).ClientID = \"test-client\"\n\tdefer shutdown()\n\n\terr := clientCreateTopic(client, topic, 3)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tgroupID := makeGroupID()\n\n\tctx, cancel := context.WithTimeout(context.Background(), time.Minute)\n\tdefer cancel()\n\trespc, err := waitForCoordinatorIndefinitely(ctx, client, &FindCoordinatorRequest{\n\t\tAddr: client.Addr,\n\t\tKey: groupID,\n\t\tKeyType: CoordinatorKeyTypeConsumer,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif respc.Error != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tgroupInstanceID := \"group-instance-id\"\n\tuserData := \"user-data\"\n\n\tvar rrGroupBalancer RoundRobinGroupBalancer\n\n\treq := &JoinGroupRequest{\n\t\tGroupID: groupID,\n\t\tGroupInstanceID: groupInstanceID,\n\t\tProtocolType: \"consumer\",\n\t\tSessionTimeout: time.Minute,\n\t\tRebalanceTimeout: time.Minute,\n\t\tProtocols: []GroupProtocol{\n\t\t\t{\n\t\t\t\tName: rrGroupBalancer.ProtocolName(),\n\t\t\t\tMetadata: GroupProtocolSubscription{\n\t\t\t\t\tTopics: []string{topic},\n\t\t\t\t\tUserData: []byte(userData),\n\t\t\t\t\tOwnedPartitions: map[string][]int{\n\t\t\t\t\t\ttopic: {0, 1, 2},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tvar resp *JoinGroupResponse\n\n\tfor {\n\t\tresp, err = client.JoinGroup(ctx, req)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tif errors.Is(resp.Error, MemberIDRequired) {\n\t\t\treq.MemberID = resp.MemberID\n\t\t\ttime.Sleep(time.Second)\n\t\t\tcontinue\n\t\t}\n\n\t\tif resp.Error != nil {\n\t\t\tt.Fatal(resp.Error)\n\t\t}\n\t\tbreak\n\t}\n\n\tif resp.MemberID != resp.LeaderID {\n\t\tt.Fatal(\"expected to be group leader\")\n\t}\n\n\tgroupMembers := make([]GroupMember, 0, len(resp.Members))\n\tgroupUserDataLookup := make(map[string]GroupMember)\n\tfor _, member := range resp.Members {\n\t\tgm := GroupMember{\n\t\t\tID: member.ID,\n\t\t\tTopics: member.Metadata.Topics,\n\t\t\tUserData: member.Metadata.UserData,\n\t\t}\n\t\tgroupMembers = append(groupMembers, gm)\n\t\tgroupUserDataLookup[member.ID] = gm\n\t}\n\n\tmetaResp, err := client.Metadata(ctx, &MetadataRequest{\n\t\tTopics: []string{topic},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tassignments := rrGroupBalancer.AssignGroups(groupMembers, metaResp.Topics[0].Partitions)\n\n\tsgRequest := &SyncGroupRequest{\n\t\tGroupID: groupID,\n\t\tGenerationID: resp.GenerationID,\n\t\tMemberID: resp.MemberID,\n\t\tGroupInstanceID: groupInstanceID,\n\t\tProtocolType: \"consumer\",\n\t\tProtocolName: rrGroupBalancer.ProtocolName(),\n\t}\n\n\tfor member, assignment := range assignments {\n\t\tsgRequest.Assignments = append(sgRequest.Assignments, SyncGroupRequestAssignment{\n\t\t\tMemberID: member,\n\t\t\tAssignment: GroupProtocolAssignment{\n\t\t\t\tAssignedPartitions: assignment,\n\t\t\t\tUserData: groupUserDataLookup[member].UserData,\n\t\t\t},\n\t\t})\n\t}\n\tsgResp, err := client.SyncGroup(ctx, sgRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif sgResp.Error != nil {\n\t\tt.Fatal(sgResp.Error)\n\t}\n\n\texpectedAssignment := GroupProtocolAssignment{\n\t\tAssignedPartitions: map[string][]int{\n\t\t\ttopic: {0, 1, 2},\n\t\t},\n\t\tUserData: []byte(userData),\n\t}\n\n\tif !reflect.DeepEqual(sgResp.Assignment, expectedAssignment) {\n\t\tt.Fatalf(\"\\nexpected assignment to be \\n%#v \\ngot\\n%#v\", expectedAssignment, sgResp.Assignment)\n\t}\n\n\tlgResp, err := client.LeaveGroup(ctx, &LeaveGroupRequest{\n\t\tGroupID: groupID,\n\t\tMembers: []LeaveGroupRequestMember{\n\t\t\t{\n\t\t\t\tID: resp.MemberID,\n\t\t\t\tGroupInstanceID: groupInstanceID,\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif lgResp.Error != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(lgResp.Members) != 1 {\n\t\tt.Fatalf(\"expected 1 member in response, got %#v\", lgResp.Members)\n\t}\n\n\tmember := lgResp.Members[0]\n\n\tif member.Error != nil {\n\t\tt.Fatalf(\"unexpected member error %v\", member.Error)\n\t}\n\n\tif member.GroupInstanceID != groupInstanceID {\n\t\tt.Fatalf(\"expected group instance id to be %s got %s\", groupInstanceID, member.GroupInstanceID)\n\t}\n\n\tif member.ID != resp.MemberID {\n\t\tt.Fatalf(\"expected member id to be %s got %s\", resp.MemberID, member.ID)\n\t}\n}\n\nfunc TestLeaveGroupResponseV0(t *testing.T) {\n\titem := leaveGroupResponseV0{\n\t\tErrorCode: 2,\n\t}\n\n\tb := bytes.NewBuffer(nil)\n\tw := &writeBuffer{w: b}\n\titem.writeTo(w)\n\n\tvar found leaveGroupResponseV0\n\tremain, err := (&found).readFrom(bufio.NewReader(b), b.Len())\n\tif err != nil {\n\t\tt.Error(err)\n\t\tt.FailNow()\n\t}\n\tif remain != 0 {\n\t\tt.Errorf(\"expected 0 remain, got %v\", remain)\n\t\tt.FailNow()\n\t}\n\tif !reflect.DeepEqual(item, found) {\n\t\tt.Error(\"expected item and found to be the same\")\n\t\tt.FailNow()\n\t}\n}\n" }, { "path": "listgroups.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"net\"\n\n\t\"github.com/segmentio/kafka-go/protocol/listgroups\"\n)\n\n// ListGroupsRequest is a request to the ListGroups API.\ntype ListGroupsRequest struct {\n\t// Addr is the address of the kafka broker to send the request to.\n\tAddr net.Addr\n}\n\n// ListGroupsResponse is a response from the ListGroups API.\ntype ListGroupsResponse struct {\n\t// Error is set to a non-nil value if a top-level error occurred while fetching\n\t// groups.\n\tError error\n\n\t// Groups contains the list of groups.\n\tGroups []ListGroupsResponseGroup\n}\n\n// ListGroupsResponseGroup contains the response details for a single group.\ntype ListGroupsResponseGroup struct {\n\t// GroupID is the ID of the group.\n\tGroupID string\n\n\t// Coordinator is the ID of the coordinator broker for the group.\n\tCoordinator int\n\n\t// The group protocol type (eg \"consumer\", \"connect\")\n\tProtocolType string\n}\n\nfunc (c *Client) ListGroups(\n\tctx context.Context,\n\treq *ListGroupsRequest,\n) (*ListGroupsResponse, error) {\n\tprotoResp, err := c.roundTrip(ctx, req.Addr, &listgroups.Request{})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tapiResp := protoResp.(*listgroups.Response)\n\tresp := &ListGroupsResponse{\n\t\tError: makeError(apiResp.ErrorCode, \"\"),\n\t}\n\n\tfor _, apiGroupInfo := range apiResp.Groups {\n\t\tresp.Groups = append(resp.Groups, ListGroupsResponseGroup{\n\t\t\tGroupID: apiGroupInfo.GroupID,\n\t\t\tCoordinator: int(apiGroupInfo.BrokerID),\n\t\t\tProtocolType: apiGroupInfo.ProtocolType,\n\t\t})\n\t}\n\n\treturn resp, nil\n}\n\n// TODO: Remove everything below and use protocol-based version above everywhere.\ntype listGroupsRequestV1 struct {\n}\n\nfunc (t listGroupsRequestV1) size() int32 {\n\treturn 0\n}\n\nfunc (t listGroupsRequestV1) writeTo(wb *writeBuffer) {\n}\n\ntype listGroupsResponseGroupV1 struct {\n\t// GroupID holds the unique group identifier\n\tGroupID string\n\tProtocolType string\n}\n\nfunc (t listGroupsResponseGroupV1) size() int32 {\n\treturn sizeofString(t.GroupID) + sizeofString(t.ProtocolType)\n}\n\nfunc (t listGroupsResponseGroupV1) writeTo(wb *writeBuffer) {\n\twb.writeString(t.GroupID)\n\twb.writeString(t.ProtocolType)\n}\n\nfunc (t *listGroupsResponseGroupV1) readFrom(r *bufio.Reader, size int) (remain int, err error) {\n\tif remain, err = readString(r, size, &t.GroupID); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readString(r, remain, &t.ProtocolType); err != nil {\n\t\treturn\n\t}\n\treturn\n}\n\ntype listGroupsResponseV1 struct {\n\t// ThrottleTimeMS holds the duration in milliseconds for which the request\n\t// was throttled due to quota violation (Zero if the request did not violate\n\t// any quota)\n\tThrottleTimeMS int32\n\n\t// ErrorCode holds response error code\n\tErrorCode int16\n\tGroups []listGroupsResponseGroupV1\n}\n\nfunc (t listGroupsResponseV1) size() int32 {\n\treturn sizeofInt32(t.ThrottleTimeMS) +\n\t\tsizeofInt16(t.ErrorCode) +\n\t\tsizeofArray(len(t.Groups), func(i int) int32 { return t.Groups[i].size() })\n}\n\nfunc (t listGroupsResponseV1) writeTo(wb *writeBuffer) {\n\twb.writeInt32(t.ThrottleTimeMS)\n\twb.writeInt16(t.ErrorCode)\n\twb.writeArray(len(t.Groups), func(i int) { t.Groups[i].writeTo(wb) })\n}\n\nfunc (t *listGroupsResponseV1) readFrom(r *bufio.Reader, size int) (remain int, err error) {\n\tif remain, err = readInt32(r, size, &t.ThrottleTimeMS); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readInt16(r, remain, &t.ErrorCode); err != nil {\n\t\treturn\n\t}\n\n\tfn := func(withReader *bufio.Reader, withSize int) (fnRemain int, fnErr error) {\n\t\tvar item listGroupsResponseGroupV1\n\t\tif fnRemain, fnErr = (&item).readFrom(withReader, withSize); fnErr != nil {\n\t\t\treturn\n\t\t}\n\t\tt.Groups = append(t.Groups, item)\n\t\treturn\n\t}\n\tif remain, err = readArrayWith(r, remain, fn); err != nil {\n\t\treturn\n\t}\n\n\treturn\n}\n" }, { "path": "listgroups_test.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"reflect\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestListGroupsResponseV1(t *testing.T) {\n\titem := listGroupsResponseV1{\n\t\tErrorCode: 2,\n\t\tGroups: []listGroupsResponseGroupV1{\n\t\t\t{\n\t\t\t\tGroupID: \"a\",\n\t\t\t\tProtocolType: \"b\",\n\t\t\t},\n\t\t},\n\t}\n\n\tb := bytes.NewBuffer(nil)\n\tw := &writeBuffer{w: b}\n\titem.writeTo(w)\n\n\tvar found listGroupsResponseV1\n\tremain, err := (&found).readFrom(bufio.NewReader(b), b.Len())\n\tif err != nil {\n\t\tt.Error(err)\n\t\tt.FailNow()\n\t}\n\tif remain != 0 {\n\t\tt.Errorf(\"expected 0 remain, got %v\", remain)\n\t\tt.FailNow()\n\t}\n\tif !reflect.DeepEqual(item, found) {\n\t\tt.Error(\"expected item and found to be the same\")\n\t\tt.FailNow()\n\t}\n}\n\nfunc TestClientListGroups(t *testing.T) {\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\ttopic := makeTopic()\n\tgid := fmt.Sprintf(\"%s-test-group\", topic)\n\n\tcreateTopic(t, topic, 1)\n\tdefer deleteTopic(t, topic)\n\n\tw := newTestWriter(WriterConfig{\n\t\tTopic: topic,\n\t})\n\n\tctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)\n\tdefer cancel()\n\n\terr := w.WriteMessages(\n\t\tctx,\n\t\tMessage{\n\t\t\tKey: []byte(\"key\"),\n\t\t\tValue: []byte(\"value\"),\n\t\t},\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tr := NewReader(ReaderConfig{\n\t\tBrokers: []string{\"localhost:9092\"},\n\t\tTopic: topic,\n\t\tGroupID: gid,\n\t\tMinBytes: 10,\n\t\tMaxBytes: 1000,\n\t})\n\t_, err = r.ReadMessage(ctx)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tresp, err := client.ListGroups(\n\t\tctx,\n\t\t&ListGroupsRequest{},\n\t)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif resp.Error != nil {\n\t\tt.Error(\n\t\t\t\"Unexpected error in response\",\n\t\t\t\"expected\", nil,\n\t\t\t\"got\", resp.Error,\n\t\t)\n\t}\n\thasGroup := false\n\thasProtocol := false\n\tfor _, group := range resp.Groups {\n\t\tif group.GroupID == gid {\n\t\t\thasGroup = true\n\t\t\tif group.ProtocolType == \"consumer\" {\n\t\t\t\thasProtocol = true\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t}\n\n\tif !hasGroup {\n\t\tt.Error(\"Group not found in list\")\n\t}\n\tif !hasProtocol {\n\t\tt.Error(\"Group does not have expected protocol type\")\n\t}\n}\n" }, { "path": "listoffset.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/listoffsets\"\n)\n\n// OffsetRequest represents a request to retrieve a single partition offset.\ntype OffsetRequest struct {\n\tPartition int\n\tTimestamp int64\n}\n\n// FirstOffsetOf constructs an OffsetRequest which asks for the first offset of\n// the partition given as argument.\nfunc FirstOffsetOf(partition int) OffsetRequest {\n\treturn OffsetRequest{Partition: partition, Timestamp: FirstOffset}\n}\n\n// LastOffsetOf constructs an OffsetRequest which asks for the last offset of\n// the partition given as argument.\nfunc LastOffsetOf(partition int) OffsetRequest {\n\treturn OffsetRequest{Partition: partition, Timestamp: LastOffset}\n}\n\n// TimeOffsetOf constructs an OffsetRequest which asks for a partition offset\n// at a given time.\nfunc TimeOffsetOf(partition int, at time.Time) OffsetRequest {\n\treturn OffsetRequest{Partition: partition, Timestamp: timestamp(at)}\n}\n\n// PartitionOffsets carries information about offsets available in a topic\n// partition.\ntype PartitionOffsets struct {\n\tPartition int\n\tFirstOffset int64\n\tLastOffset int64\n\tOffsets map[int64]time.Time\n\tError error\n}\n\n// ListOffsetsRequest represents a request sent to a kafka broker to list of the\n// offsets of topic partitions.\ntype ListOffsetsRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// A mapping of topic names to list of partitions that the program wishes to\n\t// get the offsets for.\n\tTopics map[string][]OffsetRequest\n\n\t// The isolation level for the request.\n\t//\n\t// Defaults to ReadUncommitted.\n\t//\n\t// This field requires the kafka broker to support the ListOffsets API in\n\t// version 2 or above (otherwise the value is ignored).\n\tIsolationLevel IsolationLevel\n}\n\n// ListOffsetsResponse represents a response from a kafka broker to a offset\n// listing request.\ntype ListOffsetsResponse struct {\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// Mappings of topics names to partition offsets, there will be one entry\n\t// for each topic in the request.\n\tTopics map[string][]PartitionOffsets\n}\n\n// ListOffsets sends an offset request to a kafka broker and returns the\n// response.\nfunc (c *Client) ListOffsets(ctx context.Context, req *ListOffsetsRequest) (*ListOffsetsResponse, error) {\n\ttype topicPartition struct {\n\t\ttopic string\n\t\tpartition int\n\t}\n\n\tpartitionOffsets := make(map[topicPartition]PartitionOffsets)\n\n\tfor topicName, requests := range req.Topics {\n\t\tfor _, r := range requests {\n\t\t\tkey := topicPartition{\n\t\t\t\ttopic: topicName,\n\t\t\t\tpartition: r.Partition,\n\t\t\t}\n\n\t\t\tpartition, ok := partitionOffsets[key]\n\t\t\tif !ok {\n\t\t\t\tpartition = PartitionOffsets{\n\t\t\t\t\tPartition: r.Partition,\n\t\t\t\t\tFirstOffset: -1,\n\t\t\t\t\tLastOffset: -1,\n\t\t\t\t\tOffsets: make(map[int64]time.Time),\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tswitch r.Timestamp {\n\t\t\tcase FirstOffset:\n\t\t\t\tpartition.FirstOffset = 0\n\t\t\tcase LastOffset:\n\t\t\t\tpartition.LastOffset = 0\n\t\t\t}\n\n\t\t\tpartitionOffsets[topicPartition{\n\t\t\t\ttopic: topicName,\n\t\t\t\tpartition: r.Partition,\n\t\t\t}] = partition\n\t\t}\n\t}\n\n\ttopics := make([]listoffsets.RequestTopic, 0, len(req.Topics))\n\n\tfor topicName, requests := range req.Topics {\n\t\tpartitions := make([]listoffsets.RequestPartition, len(requests))\n\n\t\tfor i, r := range requests {\n\t\t\tpartitions[i] = listoffsets.RequestPartition{\n\t\t\t\tPartition: int32(r.Partition),\n\t\t\t\tCurrentLeaderEpoch: -1,\n\t\t\t\tTimestamp: r.Timestamp,\n\t\t\t}\n\t\t}\n\n\t\ttopics = append(topics, listoffsets.RequestTopic{\n\t\t\tTopic: topicName,\n\t\t\tPartitions: partitions,\n\t\t})\n\t}\n\n\tm, err := c.roundTrip(ctx, req.Addr, &listoffsets.Request{\n\t\tReplicaID: -1,\n\t\tIsolationLevel: int8(req.IsolationLevel),\n\t\tTopics: topics,\n\t})\n\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).ListOffsets: %w\", err)\n\t}\n\n\tres := m.(*listoffsets.Response)\n\tret := &ListOffsetsResponse{\n\t\tThrottle: makeDuration(res.ThrottleTimeMs),\n\t\tTopics: make(map[string][]PartitionOffsets, len(res.Topics)),\n\t}\n\n\tfor _, t := range res.Topics {\n\t\tfor _, p := range t.Partitions {\n\t\t\tkey := topicPartition{\n\t\t\t\ttopic: t.Topic,\n\t\t\t\tpartition: int(p.Partition),\n\t\t\t}\n\n\t\t\tpartition := partitionOffsets[key]\n\n\t\t\tswitch p.Timestamp {\n\t\t\tcase FirstOffset:\n\t\t\t\tpartition.FirstOffset = p.Offset\n\t\t\tcase LastOffset:\n\t\t\t\tpartition.LastOffset = p.Offset\n\t\t\tdefault:\n\t\t\t\tpartition.Offsets[p.Offset] = makeTime(p.Timestamp)\n\t\t\t}\n\n\t\t\tif p.ErrorCode != 0 {\n\t\t\t\tpartition.Error = Error(p.ErrorCode)\n\t\t\t}\n\n\t\t\tpartitionOffsets[key] = partition\n\t\t}\n\t}\n\n\tfor key, partition := range partitionOffsets {\n\t\tret.Topics[key.topic] = append(ret.Topics[key.topic], partition)\n\t}\n\n\treturn ret, nil\n}\n\ntype listOffsetRequestV1 struct {\n\tReplicaID int32\n\tTopics []listOffsetRequestTopicV1\n}\n\nfunc (r listOffsetRequestV1) size() int32 {\n\treturn 4 + sizeofArray(len(r.Topics), func(i int) int32 { return r.Topics[i].size() })\n}\n\nfunc (r listOffsetRequestV1) writeTo(wb *writeBuffer) {\n\twb.writeInt32(r.ReplicaID)\n\twb.writeArray(len(r.Topics), func(i int) { r.Topics[i].writeTo(wb) })\n}\n\ntype listOffsetRequestTopicV1 struct {\n\tTopicName string\n\tPartitions []listOffsetRequestPartitionV1\n}\n\nfunc (t listOffsetRequestTopicV1) size() int32 {\n\treturn sizeofString(t.TopicName) +\n\t\tsizeofArray(len(t.Partitions), func(i int) int32 { return t.Partitions[i].size() })\n}\n\nfunc (t listOffsetRequestTopicV1) writeTo(wb *writeBuffer) {\n\twb.writeString(t.TopicName)\n\twb.writeArray(len(t.Partitions), func(i int) { t.Partitions[i].writeTo(wb) })\n}\n\ntype listOffsetRequestPartitionV1 struct {\n\tPartition int32\n\tTime int64\n}\n\nfunc (p listOffsetRequestPartitionV1) size() int32 {\n\treturn 4 + 8\n}\n\nfunc (p listOffsetRequestPartitionV1) writeTo(wb *writeBuffer) {\n\twb.writeInt32(p.Partition)\n\twb.writeInt64(p.Time)\n}\n\ntype listOffsetResponseV1 []listOffsetResponseTopicV1\n\nfunc (r listOffsetResponseV1) size() int32 {\n\treturn sizeofArray(len(r), func(i int) int32 { return r[i].size() })\n}\n\nfunc (r listOffsetResponseV1) writeTo(wb *writeBuffer) {\n\twb.writeArray(len(r), func(i int) { r[i].writeTo(wb) })\n}\n\ntype listOffsetResponseTopicV1 struct {\n\tTopicName string\n\tPartitionOffsets []partitionOffsetV1\n}\n\nfunc (t listOffsetResponseTopicV1) size() int32 {\n\treturn sizeofString(t.TopicName) +\n\t\tsizeofArray(len(t.PartitionOffsets), func(i int) int32 { return t.PartitionOffsets[i].size() })\n}\n\nfunc (t listOffsetResponseTopicV1) writeTo(wb *writeBuffer) {\n\twb.writeString(t.TopicName)\n\twb.writeArray(len(t.PartitionOffsets), func(i int) { t.PartitionOffsets[i].writeTo(wb) })\n}\n\ntype partitionOffsetV1 struct {\n\tPartition int32\n\tErrorCode int16\n\tTimestamp int64\n\tOffset int64\n}\n\nfunc (p partitionOffsetV1) size() int32 {\n\treturn 4 + 2 + 8 + 8\n}\n\nfunc (p partitionOffsetV1) writeTo(wb *writeBuffer) {\n\twb.writeInt32(p.Partition)\n\twb.writeInt16(p.ErrorCode)\n\twb.writeInt64(p.Timestamp)\n\twb.writeInt64(p.Offset)\n}\n\nfunc (p *partitionOffsetV1) readFrom(r *bufio.Reader, sz int) (remain int, err error) {\n\tif remain, err = readInt32(r, sz, &p.Partition); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readInt16(r, remain, &p.ErrorCode); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readInt64(r, remain, &p.Timestamp); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readInt64(r, remain, &p.Offset); err != nil {\n\t\treturn\n\t}\n\treturn\n}\n" }, { "path": "listoffset_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestClientListOffsets(t *testing.T) {\n\tclient, topic, shutdown := newLocalClientAndTopic()\n\tdefer shutdown()\n\n\tnow := time.Now()\n\n\t_, err := client.Produce(context.Background(), &ProduceRequest{\n\t\tTopic: topic,\n\t\tPartition: 0,\n\t\tRequiredAcks: -1,\n\t\tRecords: NewRecordReader(\n\t\t\tRecord{Time: now, Value: NewBytes([]byte(`hello-1`))},\n\t\t\tRecord{Time: now, Value: NewBytes([]byte(`hello-2`))},\n\t\t\tRecord{Time: now, Value: NewBytes([]byte(`hello-3`))},\n\t\t),\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tres, err := client.ListOffsets(context.Background(), &ListOffsetsRequest{\n\t\tTopics: map[string][]OffsetRequest{\n\t\t\ttopic: {FirstOffsetOf(0), LastOffsetOf(0)},\n\t\t},\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(res.Topics) != 1 {\n\t\tt.Fatal(\"invalid number of topics found in list offsets response:\", len(res.Topics))\n\t}\n\n\tpartitions, ok := res.Topics[topic]\n\tif !ok {\n\t\tt.Fatal(\"missing topic in the list offsets response:\", topic)\n\t}\n\tif len(partitions) != 1 {\n\t\tt.Fatal(\"invalid number of partitions found in list offsets response:\", len(partitions))\n\t}\n\tpartition := partitions[0]\n\n\tif partition.Partition != 0 {\n\t\tt.Error(\"invalid partition id found in list offsets response:\", partition.Partition)\n\t}\n\n\tif partition.FirstOffset != 0 {\n\t\tt.Error(\"invalid first offset found in list offsets response:\", partition.FirstOffset)\n\t}\n\n\tif partition.LastOffset != 3 {\n\t\tt.Error(\"invalid last offset found in list offsets response:\", partition.LastOffset)\n\t}\n\n\tif firstOffsetTime := partition.Offsets[partition.FirstOffset]; !firstOffsetTime.IsZero() {\n\t\tt.Error(\"unexpected first offset time in list offsets response:\", partition.Offsets)\n\t}\n\n\tif lastOffsetTime := partition.Offsets[partition.LastOffset]; !lastOffsetTime.IsZero() {\n\t\tt.Error(\"unexpected last offset time in list offsets response:\", partition.Offsets)\n\t}\n\n\tif partition.Error != nil {\n\t\tt.Error(\"unexpected error in list offsets response:\", partition.Error)\n\t}\n}\n" }, { "path": "listpartitionreassignments.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/listpartitionreassignments\"\n)\n\n// ListPartitionReassignmentsRequest is a request to the ListPartitionReassignments API.\ntype ListPartitionReassignmentsRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// Topics we want reassignments for, mapped by their name, or nil to list everything.\n\tTopics map[string]ListPartitionReassignmentsRequestTopic\n\n\t// Timeout is the amount of time to wait for the request to complete.\n\tTimeout time.Duration\n}\n\n// ListPartitionReassignmentsRequestTopic contains the requested partitions for a single\n// topic.\ntype ListPartitionReassignmentsRequestTopic struct {\n\t// The partitions to list partition reassignments for.\n\tPartitionIndexes []int\n}\n\n// ListPartitionReassignmentsResponse is a response from the ListPartitionReassignments API.\ntype ListPartitionReassignmentsResponse struct {\n\t// Error is set to a non-nil value including the code and message if a top-level\n\t// error was encountered.\n\tError error\n\n\t// Topics contains results for each topic, mapped by their name.\n\tTopics map[string]ListPartitionReassignmentsResponseTopic\n}\n\n// ListPartitionReassignmentsResponseTopic contains the detailed result of\n// ongoing reassignments for a topic.\ntype ListPartitionReassignmentsResponseTopic struct {\n\t// Partitions contains result for topic partitions.\n\tPartitions []ListPartitionReassignmentsResponsePartition\n}\n\n// ListPartitionReassignmentsResponsePartition contains the detailed result of\n// ongoing reassignments for a single partition.\ntype ListPartitionReassignmentsResponsePartition struct {\n\t// PartitionIndex contains index of the partition.\n\tPartitionIndex int\n\n\t// Replicas contains the current replica set.\n\tReplicas []int\n\n\t// AddingReplicas contains the set of replicas we are currently adding.\n\tAddingReplicas []int\n\n\t// RemovingReplicas contains the set of replicas we are currently removing.\n\tRemovingReplicas []int\n}\n\nfunc (c *Client) ListPartitionReassignments(\n\tctx context.Context,\n\treq *ListPartitionReassignmentsRequest,\n) (*ListPartitionReassignmentsResponse, error) {\n\tapiReq := &listpartitionreassignments.Request{\n\t\tTimeoutMs: int32(req.Timeout.Milliseconds()),\n\t}\n\n\tfor topicName, topicReq := range req.Topics {\n\t\tapiReq.Topics = append(\n\t\t\tapiReq.Topics,\n\t\t\tlistpartitionreassignments.RequestTopic{\n\t\t\t\tName: topicName,\n\t\t\t\tPartitionIndexes: intToInt32Array(topicReq.PartitionIndexes),\n\t\t\t},\n\t\t)\n\t}\n\n\tprotoResp, err := c.roundTrip(\n\t\tctx,\n\t\treq.Addr,\n\t\tapiReq,\n\t)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tapiResp := protoResp.(*listpartitionreassignments.Response)\n\n\tresp := &ListPartitionReassignmentsResponse{\n\t\tError: makeError(apiResp.ErrorCode, apiResp.ErrorMessage),\n\t\tTopics: make(map[string]ListPartitionReassignmentsResponseTopic),\n\t}\n\n\tfor _, topicResult := range apiResp.Topics {\n\t\trespTopic := ListPartitionReassignmentsResponseTopic{}\n\t\tfor _, partitionResult := range topicResult.Partitions {\n\t\t\trespTopic.Partitions = append(\n\t\t\t\trespTopic.Partitions,\n\t\t\t\tListPartitionReassignmentsResponsePartition{\n\t\t\t\t\tPartitionIndex: int(partitionResult.PartitionIndex),\n\t\t\t\t\tReplicas: int32ToIntArray(partitionResult.Replicas),\n\t\t\t\t\tAddingReplicas: int32ToIntArray(partitionResult.AddingReplicas),\n\t\t\t\t\tRemovingReplicas: int32ToIntArray(partitionResult.RemovingReplicas),\n\t\t\t\t},\n\t\t\t)\n\t\t}\n\t\tresp.Topics[topicResult.Name] = respTopic\n\t}\n\n\treturn resp, nil\n}\n\nfunc intToInt32Array(arr []int) []int32 {\n\tif arr == nil {\n\t\treturn nil\n\t}\n\tres := make([]int32, len(arr))\n\tfor i := range arr {\n\t\tres[i] = int32(arr[i])\n\t}\n\treturn res\n}\n\nfunc int32ToIntArray(arr []int32) []int {\n\tif arr == nil {\n\t\treturn nil\n\t}\n\tres := make([]int, len(arr))\n\tfor i := range arr {\n\t\tres[i] = int(arr[i])\n\t}\n\treturn res\n}\n" }, { "path": "listpartitionreassignments_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc TestClientListPartitionReassignments(t *testing.T) {\n\tif !ktesting.KafkaIsAtLeast(\"2.4.0\") {\n\t\treturn\n\t}\n\n\tctx := context.Background()\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\ttopic := makeTopic()\n\tcreateTopic(t, topic, 2)\n\tdefer deleteTopic(t, topic)\n\n\t// Can't really get an ongoing partition reassignment with local Kafka, so just do a superficial test here.\n\tresp, err := client.ListPartitionReassignments(\n\t\tctx,\n\t\t&ListPartitionReassignmentsRequest{\n\t\t\tTopics: map[string]ListPartitionReassignmentsRequestTopic{\n\t\t\t\ttopic: {PartitionIndexes: []int{0, 1}},\n\t\t\t},\n\t\t},\n\t)\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif resp.Error != nil {\n\t\tt.Error(\n\t\t\t\"Unexpected error in response\",\n\t\t\t\"expected\", nil,\n\t\t\t\"got\", resp.Error,\n\t\t)\n\t}\n\tif len(resp.Topics) != 0 {\n\t\tt.Error(\n\t\t\t\"Unexpected length of topic results\",\n\t\t\t\"expected\", 0,\n\t\t\t\"got\", len(resp.Topics),\n\t\t)\n\t}\n}\n" }, { "path": "logger.go", "content": "package kafka\n\n// Logger interface API for log.Logger.\ntype Logger interface {\n\tPrintf(string, ...interface{})\n}\n\n// LoggerFunc is a bridge between Logger and any third party logger\n// Usage:\n// l := NewLogger() // some logger\n// r := kafka.NewReader(kafka.ReaderConfig{\n// Logger: kafka.LoggerFunc(l.Infof),\n// ErrorLogger: kafka.LoggerFunc(l.Errorf),\n// })\ntype LoggerFunc func(string, ...interface{})\n\nfunc (f LoggerFunc) Printf(msg string, args ...interface{}) { f(msg, args...) }\n" }, { "path": "lz4/lz4.go", "content": "// Package lz4 does nothing, it's kept for backward compatibility to avoid\n// breaking the majority of programs that imported it to install the compression\n// codec, which is now always included.\npackage lz4\n\nimport \"github.com/segmentio/kafka-go/compress/lz4\"\n\nconst (\n\tCode = 3\n)\n\ntype CompressionCodec = lz4.Codec\n\nfunc NewCompressionCodec() *CompressionCodec {\n\treturn &CompressionCodec{}\n}\n" }, { "path": "message.go", "content": "package kafka\n\nimport (\n\t\"time\"\n)\n\n// Message is a data structure representing kafka messages.\ntype Message struct {\n\t// Topic indicates which topic this message was consumed from via Reader.\n\t//\n\t// When being used with Writer, this can be used to configure the topic if\n\t// not already specified on the writer itself.\n\tTopic string\n\n\t// Partition is read-only and MUST NOT be set when writing messages\n\tPartition int\n\tOffset int64\n\tHighWaterMark int64\n\tKey []byte\n\tValue []byte\n\tHeaders []Header\n\n\t// This field is used to hold arbitrary data you wish to include, so it\n\t// will be available when handle it on the Writer's `Completion` method,\n\t// this support the application can do any post operation on each message.\n\tWriterData interface{}\n\n\t// If not set at the creation, Time will be automatically set when\n\t// writing the message.\n\tTime time.Time\n}\n\nfunc (msg Message) message(cw *crc32Writer) message {\n\tm := message{\n\t\tMagicByte: 1,\n\t\tKey: msg.Key,\n\t\tValue: msg.Value,\n\t\tTimestamp: timestamp(msg.Time),\n\t}\n\tif cw != nil {\n\t\tm.CRC = m.crc32(cw)\n\t}\n\treturn m\n}\n\nconst timestampSize = 8\n\nfunc (msg *Message) size() int32 {\n\treturn 4 + 1 + 1 + sizeofBytes(msg.Key) + sizeofBytes(msg.Value) + timestampSize\n}\n\nfunc (msg *Message) headerSize() int {\n\treturn varArrayLen(len(msg.Headers), func(i int) int {\n\t\th := &msg.Headers[i]\n\t\treturn varStringLen(h.Key) + varBytesLen(h.Value)\n\t})\n}\n\nfunc (msg *Message) totalSize() int32 {\n\treturn int32(msg.headerSize()) + msg.size()\n}\n\ntype message struct {\n\tCRC int32\n\tMagicByte int8\n\tAttributes int8\n\tTimestamp int64\n\tKey []byte\n\tValue []byte\n}\n\nfunc (m message) crc32(cw *crc32Writer) int32 {\n\tcw.crc32 = 0\n\tcw.writeInt8(m.MagicByte)\n\tcw.writeInt8(m.Attributes)\n\tif m.MagicByte != 0 {\n\t\tcw.writeInt64(m.Timestamp)\n\t}\n\tcw.writeBytes(m.Key)\n\tcw.writeBytes(m.Value)\n\treturn int32(cw.crc32)\n}\n\nfunc (m message) size() int32 {\n\tsize := 4 + 1 + 1 + sizeofBytes(m.Key) + sizeofBytes(m.Value)\n\tif m.MagicByte != 0 {\n\t\tsize += timestampSize\n\t}\n\treturn size\n}\n\nfunc (m message) writeTo(wb *writeBuffer) {\n\twb.writeInt32(m.CRC)\n\twb.writeInt8(m.MagicByte)\n\twb.writeInt8(m.Attributes)\n\tif m.MagicByte != 0 {\n\t\twb.writeInt64(m.Timestamp)\n\t}\n\twb.writeBytes(m.Key)\n\twb.writeBytes(m.Value)\n}\n\ntype messageSetItem struct {\n\tOffset int64\n\tMessageSize int32\n\tMessage message\n}\n\nfunc (m messageSetItem) size() int32 {\n\treturn 8 + 4 + m.Message.size()\n}\n\nfunc (m messageSetItem) writeTo(wb *writeBuffer) {\n\twb.writeInt64(m.Offset)\n\twb.writeInt32(m.MessageSize)\n\tm.Message.writeTo(wb)\n}\n\ntype messageSet []messageSetItem\n\nfunc (s messageSet) size() (size int32) {\n\tfor _, m := range s {\n\t\tsize += m.size()\n\t}\n\treturn\n}\n\nfunc (s messageSet) writeTo(wb *writeBuffer) {\n\tfor _, m := range s {\n\t\tm.writeTo(wb)\n\t}\n}\n" }, { "path": "message_reader.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n)\n\ntype readBytesFunc func(*bufio.Reader, int, int) (int, error)\n\n// messageSetReader processes the messages encoded into a fetch response.\n// The response may contain a mix of Record Batches (newer format) and Messages\n// (older format).\ntype messageSetReader struct {\n\t*readerStack // used for decompressing compressed messages and record batches\n\tempty bool // if true, short circuits messageSetReader methods\n\tdebug bool // enable debug log messages\n\t// How many bytes are expected to remain in the response.\n\t//\n\t// This is used to detect truncation of the response.\n\tlengthRemain int\n\n\tdecompressed *bytes.Buffer\n}\n\ntype readerStack struct {\n\treader *bufio.Reader\n\tremain int\n\tbase int64\n\tparent *readerStack\n\tcount int // how many messages left in the current message set\n\theader messagesHeader // the current header for a subset of messages within the set.\n}\n\n// messagesHeader describes a set of records. there may be many messagesHeader's in a message set.\ntype messagesHeader struct {\n\tfirstOffset int64\n\tlength int32\n\tcrc int32\n\tmagic int8\n\t// v1 composes attributes specific to v0 and v1 message headers\n\tv1 struct {\n\t\tattributes int8\n\t\ttimestamp int64\n\t}\n\t// v2 composes attributes specific to v2 message headers\n\tv2 struct {\n\t\tleaderEpoch int32\n\t\tattributes int16\n\t\tlastOffsetDelta int32\n\t\tfirstTimestamp int64\n\t\tlastTimestamp int64\n\t\tproducerID int64\n\t\tproducerEpoch int16\n\t\tbaseSequence int32\n\t\tcount int32\n\t}\n}\n\nfunc (h messagesHeader) compression() (codec CompressionCodec, err error) {\n\tconst compressionCodecMask = 0x07\n\tvar code int8\n\tswitch h.magic {\n\tcase 0, 1:\n\t\tcode = h.v1.attributes & compressionCodecMask\n\tcase 2:\n\t\tcode = int8(h.v2.attributes & compressionCodecMask)\n\tdefault:\n\t\terr = h.badMagic()\n\t\treturn\n\t}\n\tif code != 0 {\n\t\tcodec, err = resolveCodec(code)\n\t}\n\treturn\n}\n\nfunc (h messagesHeader) badMagic() error {\n\treturn fmt.Errorf(\"unsupported magic byte %d in header\", h.magic)\n}\n\nfunc newMessageSetReader(reader *bufio.Reader, remain int) (*messageSetReader, error) {\n\tres := &messageSetReader{\n\t\treaderStack: &readerStack{\n\t\t\treader: reader,\n\t\t\tremain: remain,\n\t\t},\n\t\tdecompressed: acquireBuffer(),\n\t}\n\terr := res.readHeader()\n\treturn res, err\n}\n\nfunc (r *messageSetReader) remaining() (remain int) {\n\tif r.empty {\n\t\treturn 0\n\t}\n\tfor s := r.readerStack; s != nil; s = s.parent {\n\t\tremain += s.remain\n\t}\n\treturn\n}\n\nfunc (r *messageSetReader) discard() (err error) {\n\tswitch {\n\tcase r.empty:\n\tcase r.readerStack == nil:\n\tdefault:\n\t\t// rewind up to the top-most reader b/c it's the only one that's doing\n\t\t// actual i/o. the rest are byte buffers that have been pushed on the stack\n\t\t// while reading compressed message sets.\n\t\tfor r.parent != nil {\n\t\t\tr.readerStack = r.parent\n\t\t}\n\t\terr = r.discardN(r.remain)\n\t}\n\treturn\n}\n\nfunc (r *messageSetReader) readMessage(min int64, key readBytesFunc, val readBytesFunc) (\n\toffset int64, lastOffset int64, timestamp int64, headers []Header, err error) {\n\n\tif r.empty {\n\t\terr = RequestTimedOut\n\t\treturn\n\t}\n\tif err = r.readHeader(); err != nil {\n\t\treturn\n\t}\n\tswitch r.header.magic {\n\tcase 0, 1:\n\t\toffset, timestamp, headers, err = r.readMessageV1(min, key, val)\n\t\t// Set an invalid value so that it can be ignored\n\t\tlastOffset = -1\n\tcase 2:\n\t\toffset, lastOffset, timestamp, headers, err = r.readMessageV2(min, key, val)\n\tdefault:\n\t\terr = r.header.badMagic()\n\t}\n\treturn\n}\n\nfunc (r *messageSetReader) readMessageV1(min int64, key readBytesFunc, val readBytesFunc) (\n\toffset int64, timestamp int64, headers []Header, err error) {\n\n\tfor r.readerStack != nil {\n\t\tif r.remain == 0 {\n\t\t\tr.readerStack = r.parent\n\t\t\tcontinue\n\t\t}\n\t\tif err = r.readHeader(); err != nil {\n\t\t\treturn\n\t\t}\n\t\toffset = r.header.firstOffset\n\t\ttimestamp = r.header.v1.timestamp\n\t\tvar codec CompressionCodec\n\t\tif codec, err = r.header.compression(); err != nil {\n\t\t\treturn\n\t\t}\n\t\tif r.debug {\n\t\t\tr.log(\"Reading with codec=%T\", codec)\n\t\t}\n\t\tif codec != nil {\n\t\t\t// discard next four bytes...will be -1 to indicate null key\n\t\t\tif err = r.discardN(4); err != nil {\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\t// read and decompress the contained message set.\n\t\t\tr.decompressed.Reset()\n\t\t\tif err = r.readBytesWith(func(br *bufio.Reader, sz int, n int) (remain int, err error) {\n\t\t\t\t// x4 as a guess that the average compression ratio is near 75%\n\t\t\t\tr.decompressed.Grow(4 * n)\n\t\t\t\tlimitReader := io.LimitedReader{R: br, N: int64(n)}\n\t\t\t\tcodecReader := codec.NewReader(&limitReader)\n\t\t\t\t_, err = r.decompressed.ReadFrom(codecReader)\n\t\t\t\tremain = sz - (n - int(limitReader.N))\n\t\t\t\tcodecReader.Close()\n\t\t\t\treturn\n\t\t\t}); err != nil {\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\t// the compressed message's offset will be equal to the offset of\n\t\t\t// the last message in the set. within the compressed set, the\n\t\t\t// offsets will be relative, so we have to scan through them to\n\t\t\t// get the base offset. for example, if there are four compressed\n\t\t\t// messages at offsets 10-13, then the container message will have\n\t\t\t// offset 13 and the contained messages will be 0,1,2,3. the base\n\t\t\t// offset for the container, then is 13-3=10.\n\t\t\tif offset, err = extractOffset(offset, r.decompressed.Bytes()); err != nil {\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\t// mark the outer message as being read\n\t\t\tr.markRead()\n\n\t\t\t// then push the decompressed bytes onto the stack.\n\t\t\tr.readerStack = &readerStack{\n\t\t\t\t// Allocate a buffer of size 0, which gets capped at 16 bytes\n\t\t\t\t// by the bufio package. We are already reading buffered data\n\t\t\t\t// here, no need to reserve another 4KB buffer.\n\t\t\t\treader: bufio.NewReaderSize(r.decompressed, 0),\n\t\t\t\tremain: r.decompressed.Len(),\n\t\t\t\tbase: offset,\n\t\t\t\tparent: r.readerStack,\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\t\t// adjust the offset in case we're reading compressed messages. the\n\t\t// base will be zero otherwise.\n\t\toffset += r.base\n\n\t\t// When the messages are compressed kafka may return messages at an\n\t\t// earlier offset than the one that was requested, it's the client's\n\t\t// responsibility to ignore those.\n\t\t//\n\t\t// At this point, the message header has been read, so discarding\n\t\t// the rest of the message means we have to discard the key, and then\n\t\t// the value. Each of those are preceded by a 4-byte length. Discarding\n\t\t// them is then reading that length variable and then discarding that\n\t\t// amount.\n\t\tif offset < min {\n\t\t\t// discard the key\n\t\t\tif err = r.discardBytes(); err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// discard the value\n\t\t\tif err = r.discardBytes(); err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\t// since we have fully consumed the message, mark as read\n\t\t\tr.markRead()\n\t\t\tcontinue\n\t\t}\n\t\tif err = r.readBytesWith(key); err != nil {\n\t\t\treturn\n\t\t}\n\t\tif err = r.readBytesWith(val); err != nil {\n\t\t\treturn\n\t\t}\n\t\tr.markRead()\n\t\treturn\n\t}\n\terr = errShortRead\n\treturn\n}\n\nfunc (r *messageSetReader) readMessageV2(_ int64, key readBytesFunc, val readBytesFunc) (\n\toffset int64, lastOffset int64, timestamp int64, headers []Header, err error) {\n\tif err = r.readHeader(); err != nil {\n\t\treturn\n\t}\n\tif r.count == int(r.header.v2.count) { // first time reading this set, so check for compression headers.\n\t\tvar codec CompressionCodec\n\t\tif codec, err = r.header.compression(); err != nil {\n\t\t\treturn\n\t\t}\n\t\tif codec != nil {\n\t\t\tbatchRemain := int(r.header.length - 49) // TODO: document this magic number\n\t\t\tif batchRemain > r.remain {\n\t\t\t\terr = errShortRead\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif batchRemain < 0 {\n\t\t\t\terr = fmt.Errorf(\"batch remain < 0 (%d)\", batchRemain)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tr.decompressed.Reset()\n\t\t\t// x4 as a guess that the average compression ratio is near 75%\n\t\t\tr.decompressed.Grow(4 * batchRemain)\n\t\t\tlimitReader := io.LimitedReader{R: r.reader, N: int64(batchRemain)}\n\t\t\tcodecReader := codec.NewReader(&limitReader)\n\t\t\t_, err = r.decompressed.ReadFrom(codecReader)\n\t\t\tcodecReader.Close()\n\t\t\tif err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tr.remain -= batchRemain - int(limitReader.N)\n\t\t\tr.readerStack = &readerStack{\n\t\t\t\treader: bufio.NewReaderSize(r.decompressed, 0), // the new stack reads from the decompressed buffer\n\t\t\t\tremain: r.decompressed.Len(),\n\t\t\t\tbase: -1, // base is unused here\n\t\t\t\tparent: r.readerStack,\n\t\t\t\theader: r.header,\n\t\t\t\tcount: r.count,\n\t\t\t}\n\t\t\t// all of the messages in this set are in the decompressed set just pushed onto the reader\n\t\t\t// stack. here we set the parent count to 0 so that when the child set is exhausted, the\n\t\t\t// reader will then try to read the header of the next message set\n\t\t\tr.readerStack.parent.count = 0\n\t\t}\n\t}\n\tremainBefore := r.remain\n\tvar length int64\n\tif err = r.readVarInt(&length); err != nil {\n\t\treturn\n\t}\n\tlengthOfLength := remainBefore - r.remain\n\tvar attrs int8\n\tif err = r.readInt8(&attrs); err != nil {\n\t\treturn\n\t}\n\tvar timestampDelta int64\n\tif err = r.readVarInt(×tampDelta); err != nil {\n\t\treturn\n\t}\n\ttimestamp = r.header.v2.firstTimestamp + timestampDelta\n\tvar offsetDelta int64\n\tif err = r.readVarInt(&offsetDelta); err != nil {\n\t\treturn\n\t}\n\toffset = r.header.firstOffset + offsetDelta\n\tif err = r.runFunc(key); err != nil {\n\t\treturn\n\t}\n\tif err = r.runFunc(val); err != nil {\n\t\treturn\n\t}\n\tvar headerCount int64\n\tif err = r.readVarInt(&headerCount); err != nil {\n\t\treturn\n\t}\n\tif headerCount > 0 {\n\t\theaders = make([]Header, headerCount)\n\t\tfor i := range headers {\n\t\t\tif err = r.readMessageHeader(&headers[i]); err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\tlastOffset = r.header.firstOffset + int64(r.header.v2.lastOffsetDelta)\n\tr.lengthRemain -= int(length) + lengthOfLength\n\tr.markRead()\n\treturn\n}\n\nfunc (r *messageSetReader) discardBytes() (err error) {\n\tr.remain, err = discardBytes(r.reader, r.remain)\n\treturn\n}\n\nfunc (r *messageSetReader) discardN(sz int) (err error) {\n\tr.remain, err = discardN(r.reader, r.remain, sz)\n\treturn\n}\n\nfunc (r *messageSetReader) markRead() {\n\tif r.count == 0 {\n\t\tpanic(\"markRead: negative count\")\n\t}\n\tr.count--\n\tr.unwindStack()\n\tif r.debug {\n\t\tr.log(\"Mark read remain=%d\", r.remain)\n\t}\n}\n\nfunc (r *messageSetReader) unwindStack() {\n\tfor r.count == 0 {\n\t\tif r.remain == 0 {\n\t\t\tif r.parent != nil {\n\t\t\t\tif r.debug {\n\t\t\t\t\tr.log(\"Popped reader stack\")\n\t\t\t\t}\n\t\t\t\tr.readerStack = r.parent\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t\tbreak\n\t}\n}\n\nfunc (r *messageSetReader) readMessageHeader(header *Header) (err error) {\n\tvar keyLen int64\n\tif err = r.readVarInt(&keyLen); err != nil {\n\t\treturn\n\t}\n\tif header.Key, err = r.readNewString(int(keyLen)); err != nil {\n\t\treturn\n\t}\n\tvar valLen int64\n\tif err = r.readVarInt(&valLen); err != nil {\n\t\treturn\n\t}\n\tif header.Value, err = r.readNewBytes(int(valLen)); err != nil {\n\t\treturn\n\t}\n\treturn nil\n}\n\nfunc (r *messageSetReader) runFunc(rbFunc readBytesFunc) (err error) {\n\tvar length int64\n\tif err = r.readVarInt(&length); err != nil {\n\t\treturn\n\t}\n\tif r.remain, err = rbFunc(r.reader, r.remain, int(length)); err != nil {\n\t\treturn\n\t}\n\treturn\n}\n\nfunc (r *messageSetReader) readHeader() (err error) {\n\tif r.count > 0 {\n\t\t// currently reading a set of messages, no need to read a header until they are exhausted.\n\t\treturn\n\t}\n\tr.header = messagesHeader{}\n\tif err = r.readInt64(&r.header.firstOffset); err != nil {\n\t\treturn\n\t}\n\tif err = r.readInt32(&r.header.length); err != nil {\n\t\treturn\n\t}\n\tvar crcOrLeaderEpoch int32\n\tif err = r.readInt32(&crcOrLeaderEpoch); err != nil {\n\t\treturn\n\t}\n\tif err = r.readInt8(&r.header.magic); err != nil {\n\t\treturn\n\t}\n\tswitch r.header.magic {\n\tcase 0:\n\t\tr.header.crc = crcOrLeaderEpoch\n\t\tif err = r.readInt8(&r.header.v1.attributes); err != nil {\n\t\t\treturn\n\t\t}\n\t\tr.count = 1\n\t\t// Set arbitrary non-zero length so that we always assume the\n\t\t// message is truncated since bytes remain.\n\t\tr.lengthRemain = 1\n\t\tif r.debug {\n\t\t\tr.log(\"Read v0 header with offset=%d len=%d magic=%d attributes=%d\", r.header.firstOffset, r.header.length, r.header.magic, r.header.v1.attributes)\n\t\t}\n\tcase 1:\n\t\tr.header.crc = crcOrLeaderEpoch\n\t\tif err = r.readInt8(&r.header.v1.attributes); err != nil {\n\t\t\treturn\n\t\t}\n\t\tif err = r.readInt64(&r.header.v1.timestamp); err != nil {\n\t\t\treturn\n\t\t}\n\t\tr.count = 1\n\t\t// Set arbitrary non-zero length so that we always assume the\n\t\t// message is truncated since bytes remain.\n\t\tr.lengthRemain = 1\n\t\tif r.debug {\n\t\t\tr.log(\"Read v1 header with remain=%d offset=%d magic=%d and attributes=%d\", r.remain, r.header.firstOffset, r.header.magic, r.header.v1.attributes)\n\t\t}\n\tcase 2:\n\t\tr.header.v2.leaderEpoch = crcOrLeaderEpoch\n\t\tif err = r.readInt32(&r.header.crc); err != nil {\n\t\t\treturn\n\t\t}\n\t\tif err = r.readInt16(&r.header.v2.attributes); err != nil {\n\t\t\treturn\n\t\t}\n\t\tif err = r.readInt32(&r.header.v2.lastOffsetDelta); err != nil {\n\t\t\treturn\n\t\t}\n\t\tif err = r.readInt64(&r.header.v2.firstTimestamp); err != nil {\n\t\t\treturn\n\t\t}\n\t\tif err = r.readInt64(&r.header.v2.lastTimestamp); err != nil {\n\t\t\treturn\n\t\t}\n\t\tif err = r.readInt64(&r.header.v2.producerID); err != nil {\n\t\t\treturn\n\t\t}\n\t\tif err = r.readInt16(&r.header.v2.producerEpoch); err != nil {\n\t\t\treturn\n\t\t}\n\t\tif err = r.readInt32(&r.header.v2.baseSequence); err != nil {\n\t\t\treturn\n\t\t}\n\t\tif err = r.readInt32(&r.header.v2.count); err != nil {\n\t\t\treturn\n\t\t}\n\t\tr.count = int(r.header.v2.count)\n\t\t// Subtracts the header bytes from the length\n\t\tr.lengthRemain = int(r.header.length) - 49\n\t\tif r.debug {\n\t\t\tr.log(\"Read v2 header with count=%d offset=%d len=%d magic=%d attributes=%d\", r.count, r.header.firstOffset, r.header.length, r.header.magic, r.header.v2.attributes)\n\t\t}\n\tdefault:\n\t\terr = r.header.badMagic()\n\t\treturn\n\t}\n\treturn\n}\n\nfunc (r *messageSetReader) readNewBytes(len int) (res []byte, err error) {\n\tres, r.remain, err = readNewBytes(r.reader, r.remain, len)\n\treturn\n}\n\nfunc (r *messageSetReader) readNewString(len int) (res string, err error) {\n\tres, r.remain, err = readNewString(r.reader, r.remain, len)\n\treturn\n}\n\nfunc (r *messageSetReader) readInt8(val *int8) (err error) {\n\tr.remain, err = readInt8(r.reader, r.remain, val)\n\treturn\n}\n\nfunc (r *messageSetReader) readInt16(val *int16) (err error) {\n\tr.remain, err = readInt16(r.reader, r.remain, val)\n\treturn\n}\n\nfunc (r *messageSetReader) readInt32(val *int32) (err error) {\n\tr.remain, err = readInt32(r.reader, r.remain, val)\n\treturn\n}\n\nfunc (r *messageSetReader) readInt64(val *int64) (err error) {\n\tr.remain, err = readInt64(r.reader, r.remain, val)\n\treturn\n}\n\nfunc (r *messageSetReader) readVarInt(val *int64) (err error) {\n\tr.remain, err = readVarInt(r.reader, r.remain, val)\n\treturn\n}\n\nfunc (r *messageSetReader) readBytesWith(fn readBytesFunc) (err error) {\n\tr.remain, err = readBytesWith(r.reader, r.remain, fn)\n\treturn\n}\n\nfunc (r *messageSetReader) log(msg string, args ...interface{}) {\n\tlog.Printf(\"[DEBUG] \"+msg, args...)\n}\n\nfunc extractOffset(base int64, msgSet []byte) (offset int64, err error) {\n\tr, remain := bufio.NewReader(bytes.NewReader(msgSet)), len(msgSet)\n\tfor remain > 0 {\n\t\tif remain, err = readInt64(r, remain, &offset); err != nil {\n\t\t\treturn\n\t\t}\n\t\tvar sz int32\n\t\tif remain, err = readInt32(r, remain, &sz); err != nil {\n\t\t\treturn\n\t\t}\n\t\tif remain, err = discardN(r, remain, int(sz)); err != nil {\n\t\t\treturn\n\t\t}\n\t}\n\toffset = base - offset\n\treturn\n}\n" }, { "path": "message_test.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"encoding/hex\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math/rand\"\n\t\"os\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/compress/gzip\"\n\t\"github.com/segmentio/kafka-go/compress/lz4\"\n\t\"github.com/segmentio/kafka-go/compress/snappy\"\n\t\"github.com/segmentio/kafka-go/compress/zstd\"\n\t\"github.com/stretchr/testify/require\"\n)\n\n// This regression test covers reading messages using offsets that\n// are at the beginning and in the middle of compressed and uncompressed\n// v1 message sets.\nfunc TestV1BatchOffsets(t *testing.T) {\n\tconst highWatermark = 5000\n\tconst topic = \"test-topic\"\n\tvar (\n\t\tmsg0 = Message{\n\t\t\tOffset: 0,\n\t\t\tKey: []byte(\"msg-0\"),\n\t\t\tValue: []byte(\"key-0\"),\n\t\t}\n\t\tmsg1 = Message{\n\t\t\tOffset: 1,\n\t\t\tKey: []byte(\"msg-1\"),\n\t\t\tValue: []byte(\"key-1\"),\n\t\t}\n\t\tmsg2 = Message{\n\t\t\tOffset: 2,\n\t\t\tKey: []byte(\"msg-2\"),\n\t\t\tValue: []byte(\"key-2\"),\n\t\t}\n\t)\n\n\tfor _, tc := range []struct {\n\t\tname string\n\t\tbuilder fetchResponseBuilder\n\t\toffset int64\n\t\texpected []Message\n\t\tdebug bool\n\t}{\n\t\t{\n\t\t\tname: \"num=1 off=0\",\n\t\t\toffset: 0,\n\t\t\tbuilder: fetchResponseBuilder{\n\t\t\t\theader: fetchResponseHeader{\n\t\t\t\t\thighWatermarkOffset: highWatermark,\n\t\t\t\t\tlastStableOffset: highWatermark,\n\t\t\t\t\ttopic: topic,\n\t\t\t\t},\n\t\t\t\tmsgSets: []messageSetBuilder{\n\t\t\t\t\tv1MessageSetBuilder{\n\t\t\t\t\t\tmsgs: []Message{msg0},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: []Message{msg0},\n\t\t},\n\t\t{\n\t\t\tname: \"num=1 off=0 compressed\",\n\t\t\toffset: 0,\n\t\t\tbuilder: fetchResponseBuilder{\n\t\t\t\theader: fetchResponseHeader{\n\t\t\t\t\thighWatermarkOffset: highWatermark,\n\t\t\t\t\tlastStableOffset: highWatermark,\n\t\t\t\t\ttopic: topic,\n\t\t\t\t},\n\t\t\t\tmsgSets: []messageSetBuilder{\n\t\t\t\t\tv1MessageSetBuilder{\n\t\t\t\t\t\tcodec: new(gzip.Codec),\n\t\t\t\t\t\tmsgs: []Message{msg0},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: []Message{msg0},\n\t\t},\n\t\t{\n\t\t\tname: \"num=1 off=1\",\n\t\t\toffset: 1,\n\t\t\tbuilder: fetchResponseBuilder{\n\t\t\t\theader: fetchResponseHeader{\n\t\t\t\t\thighWatermarkOffset: highWatermark,\n\t\t\t\t\tlastStableOffset: highWatermark,\n\t\t\t\t\ttopic: topic,\n\t\t\t\t},\n\t\t\t\tmsgSets: []messageSetBuilder{\n\t\t\t\t\tv1MessageSetBuilder{\n\t\t\t\t\t\tmsgs: []Message{msg1},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: []Message{msg1},\n\t\t},\n\t\t{\n\t\t\tname: \"num=1 off=1 compressed\",\n\t\t\toffset: 1,\n\t\t\tbuilder: fetchResponseBuilder{\n\t\t\t\theader: fetchResponseHeader{\n\t\t\t\t\thighWatermarkOffset: highWatermark,\n\t\t\t\t\tlastStableOffset: highWatermark,\n\t\t\t\t\ttopic: topic,\n\t\t\t\t},\n\t\t\t\tmsgSets: []messageSetBuilder{\n\t\t\t\t\tv1MessageSetBuilder{\n\t\t\t\t\t\tcodec: new(gzip.Codec),\n\t\t\t\t\t\tmsgs: []Message{msg1},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: []Message{msg1},\n\t\t},\n\t\t{\n\t\t\tname: \"num=3 off=0\",\n\t\t\toffset: 0,\n\t\t\tbuilder: fetchResponseBuilder{\n\t\t\t\theader: fetchResponseHeader{\n\t\t\t\t\thighWatermarkOffset: highWatermark,\n\t\t\t\t\tlastStableOffset: highWatermark,\n\t\t\t\t\ttopic: topic,\n\t\t\t\t},\n\t\t\t\tmsgSets: []messageSetBuilder{\n\t\t\t\t\tv1MessageSetBuilder{\n\t\t\t\t\t\tmsgs: []Message{msg0, msg1, msg2},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: []Message{msg0, msg1, msg2},\n\t\t},\n\t\t{\n\t\t\tname: \"num=3 off=0 compressed\",\n\t\t\toffset: 0,\n\t\t\tbuilder: fetchResponseBuilder{\n\t\t\t\theader: fetchResponseHeader{\n\t\t\t\t\thighWatermarkOffset: highWatermark,\n\t\t\t\t\tlastStableOffset: highWatermark,\n\t\t\t\t\ttopic: topic,\n\t\t\t\t},\n\t\t\t\tmsgSets: []messageSetBuilder{\n\t\t\t\t\tv1MessageSetBuilder{\n\t\t\t\t\t\tcodec: new(gzip.Codec),\n\t\t\t\t\t\tmsgs: []Message{msg0, msg1, msg2},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: []Message{msg0, msg1, msg2},\n\t\t},\n\t\t{\n\t\t\tname: \"num=3 off=1\",\n\t\t\toffset: 1,\n\t\t\tdebug: true,\n\t\t\tbuilder: fetchResponseBuilder{\n\t\t\t\theader: fetchResponseHeader{\n\t\t\t\t\thighWatermarkOffset: highWatermark,\n\t\t\t\t\tlastStableOffset: highWatermark,\n\t\t\t\t\ttopic: topic,\n\t\t\t\t},\n\t\t\t\tmsgSets: []messageSetBuilder{\n\t\t\t\t\tv1MessageSetBuilder{\n\t\t\t\t\t\tmsgs: []Message{msg0, msg1, msg2},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: []Message{msg1, msg2},\n\t\t},\n\t\t{\n\t\t\tname: \"num=3 off=1 compressed\",\n\t\t\toffset: 1,\n\t\t\tdebug: true,\n\t\t\tbuilder: fetchResponseBuilder{\n\t\t\t\theader: fetchResponseHeader{\n\t\t\t\t\thighWatermarkOffset: highWatermark,\n\t\t\t\t\tlastStableOffset: highWatermark,\n\t\t\t\t\ttopic: topic,\n\t\t\t\t},\n\t\t\t\tmsgSets: []messageSetBuilder{\n\t\t\t\t\tv1MessageSetBuilder{\n\t\t\t\t\t\tcodec: new(gzip.Codec),\n\t\t\t\t\t\tmsgs: []Message{msg0, msg1, msg2},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: []Message{msg1, msg2},\n\t\t},\n\t\t{\n\t\t\tname: \"num=3 off=2 compressed\",\n\t\t\toffset: 2,\n\t\t\tdebug: true,\n\t\t\tbuilder: fetchResponseBuilder{\n\t\t\t\theader: fetchResponseHeader{\n\t\t\t\t\thighWatermarkOffset: highWatermark,\n\t\t\t\t\tlastStableOffset: highWatermark,\n\t\t\t\t\ttopic: topic,\n\t\t\t\t},\n\t\t\t\tmsgSets: []messageSetBuilder{\n\t\t\t\t\tv1MessageSetBuilder{\n\t\t\t\t\t\tcodec: new(gzip.Codec),\n\t\t\t\t\t\tmsgs: []Message{msg0, msg1, msg2},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\texpected: []Message{msg2},\n\t\t},\n\t} {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tbs := tc.builder.bytes()\n\t\t\tr, err := newReaderHelper(t, bs)\n\t\t\trequire.NoError(t, err)\n\t\t\tr.offset = tc.offset\n\t\t\tr.debug = tc.debug\n\t\t\tfilter := func(msg Message) (res Message) {\n\t\t\t\tres.Offset = msg.Offset\n\t\t\t\tres.Key = msg.Key\n\t\t\t\tres.Value = msg.Value\n\t\t\t\treturn res\n\t\t\t}\n\t\t\tfor _, expected := range tc.expected {\n\t\t\t\tmsg := filter(r.readMessage())\n\t\t\t\trequire.EqualValues(t, expected, msg)\n\t\t\t}\n\t\t\t// finally, verify no more bytes remain\n\t\t\trequire.EqualValues(t, 0, r.remain)\n\t\t\t_, err = r.readMessageErr()\n\t\t\trequire.EqualError(t, err, errShortRead.Error())\n\t\t})\n\t}\n}\n\nfunc TestMessageSetReader(t *testing.T) {\n\tconst startOffset = 1000\n\tconst highWatermark = 5000\n\tconst topic = \"test-topic\"\n\tmsgs := make([]Message, 100)\n\tfor i := 0; i < 100; i++ {\n\t\tmsgs[i] = Message{\n\t\t\tTime: time.Now(),\n\t\t\tOffset: int64(i + startOffset),\n\t\t\tKey: []byte(fmt.Sprintf(\"key-%d\", i)),\n\t\t\tValue: []byte(fmt.Sprintf(\"val-%d\", i)),\n\t\t\tHeaders: []Header{\n\t\t\t\t{\n\t\t\t\t\tKey: fmt.Sprintf(\"header-key-%d\", i),\n\t\t\t\t\tValue: []byte(fmt.Sprintf(\"header-value-%d\", i)),\n\t\t\t\t},\n\t\t\t},\n\t\t}\n\t}\n\tdefaultHeader := fetchResponseHeader{\n\t\thighWatermarkOffset: highWatermark,\n\t\tlastStableOffset: highWatermark,\n\t\ttopic: topic,\n\t}\n\tfor _, tc := range []struct {\n\t\tname string\n\t\tbuilder fetchResponseBuilder\n\t\terr error\n\t\tdebug bool\n\t}{\n\t\t{\n\t\t\tname: \"empty\",\n\t\t\tbuilder: fetchResponseBuilder{\n\t\t\t\theader: defaultHeader,\n\t\t\t},\n\t\t\terr: errShortRead,\n\t\t},\n\t\t{\n\t\t\tname: \"v0\",\n\t\t\tbuilder: fetchResponseBuilder{\n\t\t\t\theader: defaultHeader,\n\t\t\t\tmsgSets: []messageSetBuilder{\n\t\t\t\t\tv0MessageSetBuilder{\n\t\t\t\t\t\tmsgs: []Message{msgs[0]},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"v0 compressed\",\n\t\t\tbuilder: fetchResponseBuilder{\n\t\t\t\theader: defaultHeader,\n\t\t\t\tmsgSets: []messageSetBuilder{\n\t\t\t\t\tv0MessageSetBuilder{\n\t\t\t\t\t\tcodec: new(gzip.Codec),\n\t\t\t\t\t\tmsgs: []Message{msgs[0]},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"v1\",\n\t\t\tbuilder: fetchResponseBuilder{\n\t\t\t\theader: defaultHeader,\n\t\t\t\tmsgSets: []messageSetBuilder{\n\t\t\t\t\tv1MessageSetBuilder{\n\t\t\t\t\t\tmsgs: []Message{msgs[0]},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"v1 compressed\",\n\t\t\tbuilder: fetchResponseBuilder{\n\t\t\t\theader: defaultHeader,\n\t\t\t\tmsgSets: []messageSetBuilder{\n\t\t\t\t\tv1MessageSetBuilder{\n\t\t\t\t\t\tcodec: new(gzip.Codec),\n\t\t\t\t\t\tmsgs: []Message{msgs[0]},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"v2\",\n\t\t\tbuilder: fetchResponseBuilder{\n\t\t\t\theader: defaultHeader,\n\t\t\t\tmsgSets: []messageSetBuilder{\n\t\t\t\t\tv2MessageSetBuilder{\n\t\t\t\t\t\tmsgs: []Message{msgs[0]},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"v2 compressed\",\n\t\t\tbuilder: fetchResponseBuilder{\n\t\t\t\theader: defaultHeader,\n\t\t\t\tmsgSets: []messageSetBuilder{\n\t\t\t\t\tv2MessageSetBuilder{\n\t\t\t\t\t\tcodec: new(zstd.Codec),\n\t\t\t\t\t\tmsgs: []Message{msgs[0]},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"v2 multiple messages\",\n\t\t\tbuilder: fetchResponseBuilder{\n\t\t\t\theader: defaultHeader,\n\t\t\t\tmsgSets: []messageSetBuilder{\n\t\t\t\t\tv2MessageSetBuilder{\n\t\t\t\t\t\tmsgs: []Message{msgs[0], msgs[1], msgs[2], msgs[3], msgs[4]},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"v2 multiple messages compressed\",\n\t\t\tbuilder: fetchResponseBuilder{\n\t\t\t\theader: defaultHeader,\n\t\t\t\tmsgSets: []messageSetBuilder{\n\t\t\t\t\tv2MessageSetBuilder{\n\t\t\t\t\t\tcodec: new(snappy.Codec),\n\t\t\t\t\t\tmsgs: []Message{msgs[0], msgs[1], msgs[2], msgs[3], msgs[4]},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"v2 mix of compressed and uncompressed message sets\",\n\t\t\tbuilder: fetchResponseBuilder{\n\t\t\t\theader: defaultHeader,\n\t\t\t\tmsgSets: []messageSetBuilder{\n\t\t\t\t\tv2MessageSetBuilder{\n\t\t\t\t\t\tcodec: new(snappy.Codec),\n\t\t\t\t\t\tmsgs: []Message{msgs[0], msgs[1], msgs[2], msgs[3], msgs[4]},\n\t\t\t\t\t},\n\t\t\t\t\tv2MessageSetBuilder{\n\t\t\t\t\t\tmsgs: []Message{msgs[5], msgs[6], msgs[7], msgs[8], msgs[9]},\n\t\t\t\t\t},\n\t\t\t\t\tv2MessageSetBuilder{\n\t\t\t\t\t\tcodec: new(snappy.Codec),\n\t\t\t\t\t\tmsgs: []Message{msgs[10], msgs[11], msgs[12], msgs[13], msgs[14]},\n\t\t\t\t\t},\n\t\t\t\t\tv2MessageSetBuilder{\n\t\t\t\t\t\tmsgs: []Message{msgs[15], msgs[16], msgs[17], msgs[18], msgs[19]},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"v0 v2 v1 v2 v1 v1 v0 v2\",\n\t\t\tbuilder: fetchResponseBuilder{\n\t\t\t\theader: defaultHeader,\n\t\t\t\tmsgSets: []messageSetBuilder{\n\t\t\t\t\tv0MessageSetBuilder{\n\t\t\t\t\t\tmsgs: []Message{msgs[0]},\n\t\t\t\t\t},\n\t\t\t\t\tv2MessageSetBuilder{\n\t\t\t\t\t\tmsgs: []Message{msgs[1], msgs[2]},\n\t\t\t\t\t},\n\t\t\t\t\tv1MessageSetBuilder{\n\t\t\t\t\t\tmsgs: []Message{msgs[3]},\n\t\t\t\t\t},\n\t\t\t\t\tv2MessageSetBuilder{\n\t\t\t\t\t\tmsgs: []Message{msgs[4], msgs[5]},\n\t\t\t\t\t},\n\t\t\t\t\tv1MessageSetBuilder{\n\t\t\t\t\t\tmsgs: []Message{msgs[6]},\n\t\t\t\t\t},\n\t\t\t\t\tv1MessageSetBuilder{\n\t\t\t\t\t\tmsgs: []Message{msgs[7]},\n\t\t\t\t\t},\n\t\t\t\t\tv0MessageSetBuilder{\n\t\t\t\t\t\tmsgs: []Message{msgs[8]},\n\t\t\t\t\t},\n\t\t\t\t\tv2MessageSetBuilder{\n\t\t\t\t\t\tmsgs: []Message{msgs[9], msgs[10]},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"v0 v2 v1 v2 v1 v1 v0 v2 mixed compression\",\n\t\t\tbuilder: fetchResponseBuilder{\n\t\t\t\theader: defaultHeader,\n\t\t\t\tmsgSets: []messageSetBuilder{\n\t\t\t\t\tv0MessageSetBuilder{\n\t\t\t\t\t\tcodec: new(gzip.Codec),\n\t\t\t\t\t\tmsgs: []Message{msgs[0]},\n\t\t\t\t\t},\n\t\t\t\t\tv2MessageSetBuilder{\n\t\t\t\t\t\tcodec: new(zstd.Codec),\n\t\t\t\t\t\tmsgs: []Message{msgs[1], msgs[2]},\n\t\t\t\t\t},\n\t\t\t\t\tv1MessageSetBuilder{\n\t\t\t\t\t\tcodec: new(snappy.Codec),\n\t\t\t\t\t\tmsgs: []Message{msgs[3]},\n\t\t\t\t\t},\n\t\t\t\t\tv2MessageSetBuilder{\n\t\t\t\t\t\tcodec: new(lz4.Codec),\n\t\t\t\t\t\tmsgs: []Message{msgs[4], msgs[5]},\n\t\t\t\t\t},\n\t\t\t\t\tv1MessageSetBuilder{\n\t\t\t\t\t\tcodec: new(gzip.Codec),\n\t\t\t\t\t\tmsgs: []Message{msgs[6]},\n\t\t\t\t\t},\n\t\t\t\t\tv1MessageSetBuilder{\n\t\t\t\t\t\tcodec: new(zstd.Codec),\n\t\t\t\t\t\tmsgs: []Message{msgs[7]},\n\t\t\t\t\t},\n\t\t\t\t\tv0MessageSetBuilder{\n\t\t\t\t\t\tcodec: new(snappy.Codec),\n\t\t\t\t\t\tmsgs: []Message{msgs[8]},\n\t\t\t\t\t},\n\t\t\t\t\tv2MessageSetBuilder{\n\t\t\t\t\t\tcodec: new(lz4.Codec),\n\t\t\t\t\t\tmsgs: []Message{msgs[9], msgs[10]},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tname: \"v0 v2 v1 v2 v1 v1 v0 v2 mixed compression with non-compressed\",\n\t\t\tbuilder: fetchResponseBuilder{\n\t\t\t\theader: defaultHeader,\n\t\t\t\tmsgSets: []messageSetBuilder{\n\t\t\t\t\tv0MessageSetBuilder{\n\t\t\t\t\t\tcodec: new(gzip.Codec),\n\t\t\t\t\t\tmsgs: []Message{msgs[0]},\n\t\t\t\t\t},\n\t\t\t\t\tv2MessageSetBuilder{\n\t\t\t\t\t\tmsgs: []Message{msgs[1], msgs[2]},\n\t\t\t\t\t},\n\t\t\t\t\tv1MessageSetBuilder{\n\t\t\t\t\t\tcodec: new(snappy.Codec),\n\t\t\t\t\t\tmsgs: []Message{msgs[3]},\n\t\t\t\t\t},\n\t\t\t\t\tv2MessageSetBuilder{\n\t\t\t\t\t\tmsgs: []Message{msgs[4], msgs[5]},\n\t\t\t\t\t},\n\t\t\t\t\tv1MessageSetBuilder{\n\t\t\t\t\t\tmsgs: []Message{msgs[6]},\n\t\t\t\t\t},\n\t\t\t\t\tv1MessageSetBuilder{\n\t\t\t\t\t\tcodec: new(zstd.Codec),\n\t\t\t\t\t\tmsgs: []Message{msgs[7]},\n\t\t\t\t\t},\n\t\t\t\t\tv0MessageSetBuilder{\n\t\t\t\t\t\tmsgs: []Message{msgs[8]},\n\t\t\t\t\t},\n\t\t\t\t\tv2MessageSetBuilder{\n\t\t\t\t\t\tcodec: new(lz4.Codec),\n\t\t\t\t\t\tmsgs: []Message{msgs[9], msgs[10]},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t} {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\trh, err := newReaderHelper(t, tc.builder.bytes())\n\t\t\trequire.Equal(t, tc.err, err)\n\t\t\tif tc.err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\trh.debug = tc.debug\n\t\t\tfor _, messageSet := range tc.builder.msgSets {\n\t\t\t\tfor _, expected := range messageSet.messages() {\n\t\t\t\t\tmsg := rh.readMessage()\n\t\t\t\t\trequire.Equal(t, expected.Offset, msg.Offset)\n\t\t\t\t\trequire.Equal(t, string(expected.Key), string(msg.Key))\n\t\t\t\t\trequire.Equal(t, string(expected.Value), string(msg.Value))\n\t\t\t\t\tswitch messageSet.(type) {\n\t\t\t\t\tcase v0MessageSetBuilder, v1MessageSetBuilder:\n\t\t\t\t\t\t// v0 and v1 message sets do not have headers\n\t\t\t\t\t\trequire.Len(t, msg.Headers, 0)\n\t\t\t\t\tcase v2MessageSetBuilder:\n\t\t\t\t\t\t// v2 message sets can have headers\n\t\t\t\t\t\trequire.EqualValues(t, expected.Headers, msg.Headers)\n\t\t\t\t\tdefault:\n\t\t\t\t\t\tt.Fatalf(\"unknown builder: %T\", messageSet)\n\t\t\t\t\t}\n\t\t\t\t\trequire.Equal(t, expected.Offset, msg.Offset)\n\t\t\t\t}\n\t\t\t}\n\t\t\t// verify the reader stack is empty\n\t\t\trequire.EqualValues(t, 0, rh.remain)\n\t\t\trequire.EqualValues(t, 0, rh.count)\n\t\t\trequire.EqualValues(t, 0, rh.remaining())\n\t\t\trequire.Nil(t, rh.readerStack.parent)\n\t\t\t// any further message is a short read\n\t\t\t_, err = rh.readMessageErr()\n\t\t\trequire.EqualError(t, err, errShortRead.Error())\n\t\t})\n\t}\n\n}\n\nfunc TestMessageSetReaderEmpty(t *testing.T) {\n\tm := messageSetReader{empty: true}\n\n\tnoop := func(*bufio.Reader, int, int) (int, error) {\n\t\treturn 0, nil\n\t}\n\n\toffset, _, timestamp, headers, err := m.readMessage(0, noop, noop)\n\tif offset != 0 {\n\t\tt.Errorf(\"expected offset of 0, get %d\", offset)\n\t}\n\tif timestamp != 0 {\n\t\tt.Errorf(\"expected timestamp of 0, get %d\", timestamp)\n\t}\n\tif headers != nil {\n\t\tt.Errorf(\"expected nil headers, got %v\", headers)\n\t}\n\tif !errors.Is(err, RequestTimedOut) {\n\t\tt.Errorf(\"expected RequestTimedOut, got %v\", err)\n\t}\n\n\tif m.remaining() != 0 {\n\t\tt.Errorf(\"expected 0 remaining, got %d\", m.remaining())\n\t}\n\n\tif m.discard() != nil {\n\t\tt.Errorf(\"unexpected error from discard(): %v\", m.discard())\n\t}\n}\n\nfunc TestMessageFixtures(t *testing.T) {\n\ttype fixtureMessage struct {\n\t\tkey string\n\t\tvalue string\n\t}\n\tvar fixtureMessages = map[string]fixtureMessage{\n\t\t\"a\": {key: \"alpha\", value: `{\"count\":0,\"filler\":\"aaaaaaaaaa\"}`},\n\t\t\"b\": {key: \"beta\", value: `{\"count\":0,\"filler\":\"bbbbbbbbbb\"}`},\n\t\t\"c\": {key: \"gamma\", value: `{\"count\":0,\"filler\":\"cccccccccc\"}`},\n\t\t\"d\": {key: \"delta\", value: `{\"count\":0,\"filler\":\"dddddddddd\"}`},\n\t\t\"e\": {key: \"epsilon\", value: `{\"count\":0,\"filler\":\"eeeeeeeeee\"}`},\n\t\t\"f\": {key: \"zeta\", value: `{\"count\":0,\"filler\":\"ffffffffff\"}`},\n\t\t\"g\": {key: \"eta\", value: `{\"count\":0,\"filler\":\"gggggggggg\"}`},\n\t\t\"h\": {key: \"theta\", value: `{\"count\":0,\"filler\":\"hhhhhhhhhh\"}`},\n\t}\n\n\tfor _, tc := range []struct {\n\t\tname string\n\t\tfile string\n\t\tmessages []string\n\t}{\n\t\t{\n\t\t\tname: \"v2 followed by v1\",\n\t\t\tfile: \"fixtures/v2b-v1.hex\",\n\t\t\tmessages: []string{\"a\", \"b\", \"a\", \"b\"},\n\t\t},\n\t\t{\n\t\t\tname: \"v2 compressed followed by v1 compressed\",\n\t\t\tfile: \"fixtures/v2bc-v1c.hex\",\n\t\t\tmessages: []string{\"a\", \"b\", \"a\", \"b\"},\n\t\t},\n\t\t{\n\t\t\tname: \"v2 compressed followed by v1 uncompressed\",\n\t\t\tfile: \"fixtures/v2bc-v1.hex\",\n\t\t\tmessages: []string{\"a\", \"b\", \"c\", \"d\"},\n\t\t},\n\t\t{\n\t\t\tname: \"v2 compressed followed by v1 uncompressed then v1 compressed\",\n\t\t\tfile: \"fixtures/v2bc-v1-v1c.hex\",\n\t\t\tmessages: []string{\"a\", \"b\", \"c\", \"d\", \"e\", \"f\"},\n\t\t},\n\t\t{\n\t\t\tname: \"v2 compressed followed by v1 uncompressed then v1 compressed\",\n\t\t\tfile: \"fixtures/v2bc-v1-v1c.hex\",\n\t\t\tmessages: []string{\"a\", \"b\", \"c\", \"d\", \"e\", \"f\"},\n\t\t},\n\t\t{\n\t\t\tname: \"v1 followed by v1\",\n\t\t\tfile: \"fixtures/v1-v1.hex\",\n\t\t\tmessages: []string{\"a\", \"b\", \"c\", \"d\"},\n\t\t},\n\t\t{\n\t\t\tname: \"v1 compressed followed by v1 compressed\",\n\t\t\tfile: \"fixtures/v1c-v1c.hex\",\n\t\t\tmessages: []string{\"a\", \"b\", \"c\", \"d\"},\n\t\t},\n\t\t{\n\t\t\tname: \"v1 compressed followed by v1 uncompressed then v1 compressed\",\n\t\t\tfile: \"fixtures/v1c-v1-v1c.hex\",\n\t\t\tmessages: []string{\"a\", \"b\", \"c\", \"d\", \"e\", \"f\"},\n\t\t},\n\t\t{\n\t\t\tname: \"v2 followed by v2\",\n\t\t\tfile: \"fixtures/v2-v2.hex\",\n\t\t\tmessages: []string{\"a\", \"b\", \"c\", \"d\"},\n\t\t},\n\t\t{\n\t\t\tname: \"v2 compressed followed by v2 compressed\",\n\t\t\tfile: \"fixtures/v2c-v2c.hex\",\n\t\t\tmessages: []string{\"a\", \"b\", \"c\", \"d\"},\n\t\t},\n\t\t{\n\t\t\tname: \"v2 compressed followed by v2 uncompressed then v2 compressed\",\n\t\t\tfile: \"fixtures/v2c-v2-v2c.hex\",\n\t\t\tmessages: []string{\"a\", \"b\", \"c\", \"d\", \"e\", \"f\"},\n\t\t},\n\t\t{\n\t\t\tname: \"v1 followed by v2 followed by v1 with mixture of compressed and uncompressed\",\n\t\t\tfile: \"fixtures/v1-v1c-v2-v2c-v2b-v2b-v2b-v2bc-v1b-v1bc.hex\",\n\t\t\tmessages: []string{\"a\", \"b\", \"a\", \"b\", \"c\", \"d\", \"c\", \"d\", \"e\", \"f\", \"e\", \"f\", \"g\", \"h\", \"g\", \"h\", \"g\", \"h\", \"g\", \"h\"},\n\t\t},\n\t} {\n\t\tt.Run(tc.name, func(t *testing.T) {\n\t\t\tbs, err := os.ReadFile(tc.file)\n\t\t\trequire.NoError(t, err)\n\t\t\tbuf := new(bytes.Buffer)\n\t\t\t_, err = io.Copy(buf, hex.NewDecoder(bytes.NewReader(bs)))\n\t\t\trequire.NoError(t, err)\n\n\t\t\t// discard 4 byte len and 4 byte correlation id\n\t\t\tbs = make([]byte, 8)\n\t\t\tbuf.Read(bs)\n\n\t\t\trh, err := newReaderHelper(t, buf.Bytes())\n\t\t\trequire.NoError(t, err)\n\t\t\tmessageCount := 0\n\t\t\texpectedMessageCount := len(tc.messages)\n\t\t\tfor _, expectedMessageId := range tc.messages {\n\t\t\t\texpectedMessage := fixtureMessages[expectedMessageId]\n\t\t\t\tmsg := rh.readMessage()\n\t\t\t\tmessageCount++\n\t\t\t\trequire.Equal(t, expectedMessage.key, string(msg.Key))\n\t\t\t\trequire.Equal(t, expectedMessage.value, string(msg.Value))\n\t\t\t\tt.Logf(\"Message %d key & value are what we expected: %s -> %s\\n\",\n\t\t\t\t\tmessageCount, string(msg.Key), string(msg.Value))\n\t\t\t}\n\t\t\trequire.Equal(t, expectedMessageCount, messageCount)\n\t\t})\n\t}\n}\n\nfunc TestMessageSize(t *testing.T) {\n\trand.Seed(time.Now().UnixNano())\n\tfor i := 0; i < 20; i++ {\n\t\tt.Run(\"Run\", func(t *testing.T) {\n\t\t\tmsg := Message{\n\t\t\t\tKey: make([]byte, rand.Intn(200)),\n\t\t\t\tValue: make([]byte, rand.Intn(200)),\n\t\t\t\tTime: randate(),\n\t\t\t}\n\t\t\texpSize := msg.message(nil).size()\n\t\t\tgotSize := msg.size()\n\t\t\tif expSize != gotSize {\n\t\t\t\tt.Errorf(\"Expected size %d, but got size %d\", expSize, gotSize)\n\t\t\t}\n\t\t})\n\t}\n\n}\n\n// https://stackoverflow.com/questions/43495745/how-to-generate-random-date-in-go-lang/43497333#43497333\nfunc randate() time.Time {\n\tmin := time.Date(1970, 1, 0, 0, 0, 0, 0, time.UTC).Unix()\n\tmax := time.Date(2070, 1, 0, 0, 0, 0, 0, time.UTC).Unix()\n\tdelta := max - min\n\n\tsec := rand.Int63n(delta) + min\n\treturn time.Unix(sec, 0)\n}\n\n// readerHelper composes a messageSetReader to provide convenience methods to read\n// messages.\ntype readerHelper struct {\n\tt *testing.T\n\t*messageSetReader\n\toffset int64\n}\n\nfunc newReaderHelper(t *testing.T, bs []byte) (r *readerHelper, err error) {\n\tbufReader := bufio.NewReader(bytes.NewReader(bs))\n\t_, _, remain, err := readFetchResponseHeaderV10(bufReader, len(bs))\n\trequire.NoError(t, err)\n\tvar msgs *messageSetReader\n\tmsgs, err = newMessageSetReader(bufReader, remain)\n\tif err != nil {\n\t\treturn\n\t}\n\tr = &readerHelper{t: t, messageSetReader: msgs}\n\trequire.Truef(t, msgs.remaining() > 0, \"remaining should be > 0 but was %d\", msgs.remaining())\n\treturn\n}\n\nfunc (r *readerHelper) readMessageErr() (msg Message, err error) {\n\tkeyFunc := func(r *bufio.Reader, size int, nbytes int) (remain int, err error) {\n\t\tmsg.Key, remain, err = readNewBytes(r, size, nbytes)\n\t\treturn\n\t}\n\tvalueFunc := func(r *bufio.Reader, size int, nbytes int) (remain int, err error) {\n\t\tmsg.Value, remain, err = readNewBytes(r, size, nbytes)\n\t\treturn\n\t}\n\tvar timestamp int64\n\tvar headers []Header\n\tr.offset, _, timestamp, headers, err = r.messageSetReader.readMessage(r.offset, keyFunc, valueFunc)\n\tif err != nil {\n\t\treturn\n\t}\n\tmsg.Offset = r.offset\n\tmsg.Time = time.Unix(timestamp/1000, (timestamp%1000)*1000000)\n\tmsg.Headers = headers\n\treturn\n}\n\nfunc (r *readerHelper) readMessage() (msg Message) {\n\tvar err error\n\tmsg, err = r.readMessageErr()\n\trequire.NoError(r.t, err)\n\treturn\n}\n" }, { "path": "metadata.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\tmetadataAPI \"github.com/segmentio/kafka-go/protocol/metadata\"\n)\n\n// MetadataRequest represents a request sent to a kafka broker to retrieve its\n// cluster metadata.\ntype MetadataRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// The list of topics to retrieve metadata for.\n\tTopics []string\n}\n\n// MetadataResponse represents a response from a kafka broker to a metadata\n// request.\ntype MetadataResponse struct {\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// Name of the kafka cluster that client retrieved metadata from.\n\tClusterID string\n\n\t// The broker which is currently the controller for the cluster.\n\tController Broker\n\n\t// The list of brokers registered to the cluster.\n\tBrokers []Broker\n\n\t// The list of topics available on the cluster.\n\tTopics []Topic\n}\n\n// Metadata sends a metadata request to a kafka broker and returns the response.\nfunc (c *Client) Metadata(ctx context.Context, req *MetadataRequest) (*MetadataResponse, error) {\n\tm, err := c.roundTrip(ctx, req.Addr, &metadataAPI.Request{\n\t\tTopicNames: req.Topics,\n\t})\n\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).Metadata: %w\", err)\n\t}\n\n\tres := m.(*metadataAPI.Response)\n\tret := &MetadataResponse{\n\t\tThrottle: makeDuration(res.ThrottleTimeMs),\n\t\tBrokers: make([]Broker, len(res.Brokers)),\n\t\tTopics: make([]Topic, len(res.Topics)),\n\t\tClusterID: res.ClusterID,\n\t}\n\n\tbrokers := make(map[int32]Broker, len(res.Brokers))\n\n\tfor i, b := range res.Brokers {\n\t\tbroker := Broker{\n\t\t\tHost: b.Host,\n\t\t\tPort: int(b.Port),\n\t\t\tID: int(b.NodeID),\n\t\t\tRack: b.Rack,\n\t\t}\n\n\t\tret.Brokers[i] = broker\n\t\tbrokers[b.NodeID] = broker\n\n\t\tif b.NodeID == res.ControllerID {\n\t\t\tret.Controller = broker\n\t\t}\n\t}\n\n\tfor i, t := range res.Topics {\n\t\tret.Topics[i] = Topic{\n\t\t\tName: t.Name,\n\t\t\tInternal: t.IsInternal,\n\t\t\tPartitions: make([]Partition, len(t.Partitions)),\n\t\t\tError: makeError(t.ErrorCode, \"\"),\n\t\t}\n\n\t\tfor j, p := range t.Partitions {\n\t\t\tpartition := Partition{\n\t\t\t\tTopic: t.Name,\n\t\t\t\tID: int(p.PartitionIndex),\n\t\t\t\tLeader: brokers[p.LeaderID],\n\t\t\t\tReplicas: make([]Broker, len(p.ReplicaNodes)),\n\t\t\t\tIsr: make([]Broker, len(p.IsrNodes)),\n\t\t\t\tError: makeError(p.ErrorCode, \"\"),\n\t\t\t}\n\n\t\t\tfor i, id := range p.ReplicaNodes {\n\t\t\t\tpartition.Replicas[i] = brokers[id]\n\t\t\t}\n\n\t\t\tfor i, id := range p.IsrNodes {\n\t\t\t\tpartition.Isr[i] = brokers[id]\n\t\t\t}\n\n\t\t\tret.Topics[i].Partitions[j] = partition\n\t\t}\n\t}\n\n\treturn ret, nil\n}\n\ntype topicMetadataRequestV1 []string\n\nfunc (r topicMetadataRequestV1) size() int32 {\n\treturn sizeofStringArray([]string(r))\n}\n\nfunc (r topicMetadataRequestV1) writeTo(wb *writeBuffer) {\n\t// communicate nil-ness to the broker by passing -1 as the array length.\n\t// for this particular request, the broker interpets a zero length array\n\t// as a request for no topics whereas a nil array is for all topics.\n\tif r == nil {\n\t\twb.writeArrayLen(-1)\n\t} else {\n\t\twb.writeStringArray([]string(r))\n\t}\n}\n\ntype metadataResponseV1 struct {\n\tBrokers []brokerMetadataV1\n\tControllerID int32\n\tTopics []topicMetadataV1\n}\n\nfunc (r metadataResponseV1) size() int32 {\n\tn1 := sizeofArray(len(r.Brokers), func(i int) int32 { return r.Brokers[i].size() })\n\tn2 := sizeofArray(len(r.Topics), func(i int) int32 { return r.Topics[i].size() })\n\treturn 4 + n1 + n2\n}\n\nfunc (r metadataResponseV1) writeTo(wb *writeBuffer) {\n\twb.writeArray(len(r.Brokers), func(i int) { r.Brokers[i].writeTo(wb) })\n\twb.writeInt32(r.ControllerID)\n\twb.writeArray(len(r.Topics), func(i int) { r.Topics[i].writeTo(wb) })\n}\n\ntype brokerMetadataV1 struct {\n\tNodeID int32\n\tHost string\n\tPort int32\n\tRack string\n}\n\nfunc (b brokerMetadataV1) size() int32 {\n\treturn 4 + 4 + sizeofString(b.Host) + sizeofString(b.Rack)\n}\n\nfunc (b brokerMetadataV1) writeTo(wb *writeBuffer) {\n\twb.writeInt32(b.NodeID)\n\twb.writeString(b.Host)\n\twb.writeInt32(b.Port)\n\twb.writeString(b.Rack)\n}\n\ntype topicMetadataV1 struct {\n\tTopicErrorCode int16\n\tTopicName string\n\tInternal bool\n\tPartitions []partitionMetadataV1\n}\n\nfunc (t topicMetadataV1) size() int32 {\n\treturn 2 + 1 +\n\t\tsizeofString(t.TopicName) +\n\t\tsizeofArray(len(t.Partitions), func(i int) int32 { return t.Partitions[i].size() })\n}\n\nfunc (t topicMetadataV1) writeTo(wb *writeBuffer) {\n\twb.writeInt16(t.TopicErrorCode)\n\twb.writeString(t.TopicName)\n\twb.writeBool(t.Internal)\n\twb.writeArray(len(t.Partitions), func(i int) { t.Partitions[i].writeTo(wb) })\n}\n\ntype partitionMetadataV1 struct {\n\tPartitionErrorCode int16\n\tPartitionID int32\n\tLeader int32\n\tReplicas []int32\n\tIsr []int32\n}\n\nfunc (p partitionMetadataV1) size() int32 {\n\treturn 2 + 4 + 4 + sizeofInt32Array(p.Replicas) + sizeofInt32Array(p.Isr)\n}\n\nfunc (p partitionMetadataV1) writeTo(wb *writeBuffer) {\n\twb.writeInt16(p.PartitionErrorCode)\n\twb.writeInt32(p.PartitionID)\n\twb.writeInt32(p.Leader)\n\twb.writeInt32Array(p.Replicas)\n\twb.writeInt32Array(p.Isr)\n}\n\ntype topicMetadataRequestV6 struct {\n\tTopics []string\n\tAllowAutoTopicCreation bool\n}\n\nfunc (r topicMetadataRequestV6) size() int32 {\n\treturn sizeofStringArray([]string(r.Topics)) + 1\n}\n\nfunc (r topicMetadataRequestV6) writeTo(wb *writeBuffer) {\n\t// communicate nil-ness to the broker by passing -1 as the array length.\n\t// for this particular request, the broker interpets a zero length array\n\t// as a request for no topics whereas a nil array is for all topics.\n\tif r.Topics == nil {\n\t\twb.writeArrayLen(-1)\n\t} else {\n\t\twb.writeStringArray([]string(r.Topics))\n\t}\n\twb.writeBool(r.AllowAutoTopicCreation)\n}\n\ntype metadataResponseV6 struct {\n\tThrottleTimeMs int32\n\tBrokers []brokerMetadataV1\n\tClusterId string\n\tControllerID int32\n\tTopics []topicMetadataV6\n}\n\nfunc (r metadataResponseV6) size() int32 {\n\tn1 := sizeofArray(len(r.Brokers), func(i int) int32 { return r.Brokers[i].size() })\n\tn2 := sizeofNullableString(&r.ClusterId)\n\tn3 := sizeofArray(len(r.Topics), func(i int) int32 { return r.Topics[i].size() })\n\treturn 4 + 4 + n1 + n2 + n3\n}\n\nfunc (r metadataResponseV6) writeTo(wb *writeBuffer) {\n\twb.writeInt32(r.ThrottleTimeMs)\n\twb.writeArray(len(r.Brokers), func(i int) { r.Brokers[i].writeTo(wb) })\n\twb.writeString(r.ClusterId)\n\twb.writeInt32(r.ControllerID)\n\twb.writeArray(len(r.Topics), func(i int) { r.Topics[i].writeTo(wb) })\n}\n\ntype topicMetadataV6 struct {\n\tTopicErrorCode int16\n\tTopicName string\n\tInternal bool\n\tPartitions []partitionMetadataV6\n}\n\nfunc (t topicMetadataV6) size() int32 {\n\treturn 2 + 1 +\n\t\tsizeofString(t.TopicName) +\n\t\tsizeofArray(len(t.Partitions), func(i int) int32 { return t.Partitions[i].size() })\n}\n\nfunc (t topicMetadataV6) writeTo(wb *writeBuffer) {\n\twb.writeInt16(t.TopicErrorCode)\n\twb.writeString(t.TopicName)\n\twb.writeBool(t.Internal)\n\twb.writeArray(len(t.Partitions), func(i int) { t.Partitions[i].writeTo(wb) })\n}\n\ntype partitionMetadataV6 struct {\n\tPartitionErrorCode int16\n\tPartitionID int32\n\tLeader int32\n\tReplicas []int32\n\tIsr []int32\n\tOfflineReplicas []int32\n}\n\nfunc (p partitionMetadataV6) size() int32 {\n\treturn 2 + 4 + 4 + sizeofInt32Array(p.Replicas) + sizeofInt32Array(p.Isr) + sizeofInt32Array(p.OfflineReplicas)\n}\n\nfunc (p partitionMetadataV6) writeTo(wb *writeBuffer) {\n\twb.writeInt16(p.PartitionErrorCode)\n\twb.writeInt32(p.PartitionID)\n\twb.writeInt32(p.Leader)\n\twb.writeInt32Array(p.Replicas)\n\twb.writeInt32Array(p.Isr)\n\twb.writeInt32Array(p.OfflineReplicas)\n}\n" }, { "path": "metadata_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n)\n\nfunc TestClientMetadata(t *testing.T) {\n\tclient, topic, shutdown := newLocalClientAndTopic()\n\tdefer shutdown()\n\n\tmetadata, err := client.Metadata(context.Background(), &MetadataRequest{\n\t\tTopics: []string{topic},\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif len(metadata.Brokers) == 0 {\n\t\tt.Error(\"no brokers were returned in the metadata response\")\n\t}\n\n\tfor _, b := range metadata.Brokers {\n\t\tif b == (Broker{}) {\n\t\t\tt.Error(\"unexpected broker with zero-value in metadata response\")\n\t\t}\n\t}\n\n\tif len(metadata.Topics) == 0 {\n\t\tt.Error(\"no topics were returned in the metadata response\")\n\t} else {\n\t\ttopicMetadata := metadata.Topics[0]\n\n\t\tif topicMetadata.Name != topic {\n\t\t\tt.Error(\"invalid topic name:\", topicMetadata.Name)\n\t\t}\n\n\t\tif len(topicMetadata.Partitions) == 0 {\n\t\t\tt.Error(\"no partitions were returned in the topic metadata response\")\n\t\t} else {\n\t\t\tpartitionMetadata := topicMetadata.Partitions[0]\n\n\t\t\tif partitionMetadata.Topic != topic {\n\t\t\t\tt.Error(\"invalid partition topic name:\", partitionMetadata.Topic)\n\t\t\t}\n\n\t\t\tif partitionMetadata.ID != 0 {\n\t\t\t\tt.Error(\"invalid partition index:\", partitionMetadata.ID)\n\t\t\t}\n\n\t\t\tif partitionMetadata.Leader == (Broker{}) {\n\t\t\t\tt.Error(\"no partition leader was returned in the partition metadata response\")\n\t\t\t}\n\n\t\t\tif partitionMetadata.Error != nil {\n\t\t\t\tt.Error(\"unexpected error found in the partition metadata response:\", partitionMetadata.Error)\n\t\t\t}\n\n\t\t\t// assume newLocalClientAndTopic creates the topic with one\n\t\t\t// partition\n\t\t\tif len(topicMetadata.Partitions) > 1 {\n\t\t\t\tt.Error(\"too many partitions were returned in the topic metadata response\")\n\t\t\t}\n\t\t}\n\n\t\tif topicMetadata.Error != nil {\n\t\t\tt.Error(\"unexpected error found in the topic metadata response:\", topicMetadata.Error)\n\t\t}\n\n\t\tif len(metadata.Topics) > 1 {\n\t\t\tt.Error(\"too many topics were returned in the metadata response\")\n\t\t}\n\t}\n}\n" }, { "path": "offsetcommit.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/offsetcommit\"\n)\n\n// OffsetCommit represent the commit of an offset to a partition.\n//\n// The extra metadata is opaque to the kafka protocol, it is intended to hold\n// information like an identifier for the process that committed the offset,\n// or the time at which the commit was made.\ntype OffsetCommit struct {\n\tPartition int\n\tOffset int64\n\tMetadata string\n}\n\n// OffsetCommitRequest represents a request sent to a kafka broker to commit\n// offsets for a partition.\ntype OffsetCommitRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// ID of the consumer group to publish the offsets for.\n\tGroupID string\n\n\t// ID of the consumer group generation.\n\tGenerationID int\n\n\t// ID of the group member submitting the offsets.\n\tMemberID string\n\n\t// ID of the group instance.\n\tInstanceID string\n\n\t// Set of topic partitions to publish the offsets for.\n\t//\n\t// Not that offset commits need to be submitted to the broker acting as the\n\t// group coordinator. This will be automatically resolved by the transport.\n\tTopics map[string][]OffsetCommit\n}\n\n// OffsetFetchResponse represents a response from a kafka broker to an offset\n// commit request.\ntype OffsetCommitResponse struct {\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// Set of topic partitions that the kafka broker has accepted offset commits\n\t// for.\n\tTopics map[string][]OffsetCommitPartition\n}\n\n// OffsetFetchPartition represents the state of a single partition in responses\n// to committing offsets.\ntype OffsetCommitPartition struct {\n\t// ID of the partition.\n\tPartition int\n\n\t// An error that may have occurred while attempting to publish consumer\n\t// group offsets for this partition.\n\t//\n\t// The error contains both the kafka error code, and an error message\n\t// returned by the kafka broker. Programs may use the standard errors.Is\n\t// function to test the error against kafka error codes.\n\tError error\n}\n\n// OffsetCommit sends an offset commit request to a kafka broker and returns the\n// response.\nfunc (c *Client) OffsetCommit(ctx context.Context, req *OffsetCommitRequest) (*OffsetCommitResponse, error) {\n\tnow := time.Now().UnixNano() / int64(time.Millisecond)\n\ttopics := make([]offsetcommit.RequestTopic, 0, len(req.Topics))\n\n\tfor topicName, commits := range req.Topics {\n\t\tpartitions := make([]offsetcommit.RequestPartition, len(commits))\n\n\t\tfor i, c := range commits {\n\t\t\tpartitions[i] = offsetcommit.RequestPartition{\n\t\t\t\tPartitionIndex: int32(c.Partition),\n\t\t\t\tCommittedOffset: c.Offset,\n\t\t\t\tCommittedMetadata: c.Metadata,\n\t\t\t\t// This field existed in v1 of the OffsetCommit API, setting it\n\t\t\t\t// to the current timestamp is probably a safe thing to do, but\n\t\t\t\t// it is hard to tell.\n\t\t\t\tCommitTimestamp: now,\n\t\t\t}\n\t\t}\n\n\t\ttopics = append(topics, offsetcommit.RequestTopic{\n\t\t\tName: topicName,\n\t\t\tPartitions: partitions,\n\t\t})\n\t}\n\n\tm, err := c.roundTrip(ctx, req.Addr, &offsetcommit.Request{\n\t\tGroupID: req.GroupID,\n\t\tGenerationID: int32(req.GenerationID),\n\t\tMemberID: req.MemberID,\n\t\tGroupInstanceID: req.InstanceID,\n\t\tTopics: topics,\n\t\t// Hardcoded retention; this field existed between v2 and v4 of the\n\t\t// OffsetCommit API, we would have to figure out a way to give the\n\t\t// client control over the API version being used to support configuring\n\t\t// it in the request object.\n\t\tRetentionTimeMs: int64((24 * time.Hour) / time.Millisecond),\n\t})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).OffsetCommit: %w\", err)\n\t}\n\tr := m.(*offsetcommit.Response)\n\n\tres := &OffsetCommitResponse{\n\t\tThrottle: makeDuration(r.ThrottleTimeMs),\n\t\tTopics: make(map[string][]OffsetCommitPartition, len(r.Topics)),\n\t}\n\n\tfor _, topic := range r.Topics {\n\t\tpartitions := make([]OffsetCommitPartition, len(topic.Partitions))\n\n\t\tfor i, p := range topic.Partitions {\n\t\t\tpartitions[i] = OffsetCommitPartition{\n\t\t\t\tPartition: int(p.PartitionIndex),\n\t\t\t\tError: makeError(p.ErrorCode, \"\"),\n\t\t\t}\n\t\t}\n\n\t\tres.Topics[topic.Name] = partitions\n\t}\n\n\treturn res, nil\n}\n\ntype offsetCommitRequestV2Partition struct {\n\t// Partition ID\n\tPartition int32\n\n\t// Offset to be committed\n\tOffset int64\n\n\t// Metadata holds any associated metadata the client wants to keep\n\tMetadata string\n}\n\nfunc (t offsetCommitRequestV2Partition) size() int32 {\n\treturn sizeofInt32(t.Partition) +\n\t\tsizeofInt64(t.Offset) +\n\t\tsizeofString(t.Metadata)\n}\n\nfunc (t offsetCommitRequestV2Partition) writeTo(wb *writeBuffer) {\n\twb.writeInt32(t.Partition)\n\twb.writeInt64(t.Offset)\n\twb.writeString(t.Metadata)\n}\n\ntype offsetCommitRequestV2Topic struct {\n\t// Topic name\n\tTopic string\n\n\t// Partitions to commit offsets\n\tPartitions []offsetCommitRequestV2Partition\n}\n\nfunc (t offsetCommitRequestV2Topic) size() int32 {\n\treturn sizeofString(t.Topic) +\n\t\tsizeofArray(len(t.Partitions), func(i int) int32 { return t.Partitions[i].size() })\n}\n\nfunc (t offsetCommitRequestV2Topic) writeTo(wb *writeBuffer) {\n\twb.writeString(t.Topic)\n\twb.writeArray(len(t.Partitions), func(i int) { t.Partitions[i].writeTo(wb) })\n}\n\ntype offsetCommitRequestV2 struct {\n\t// GroupID holds the unique group identifier\n\tGroupID string\n\n\t// GenerationID holds the generation of the group.\n\tGenerationID int32\n\n\t// MemberID assigned by the group coordinator\n\tMemberID string\n\n\t// RetentionTime holds the time period in ms to retain the offset.\n\tRetentionTime int64\n\n\t// Topics to commit offsets\n\tTopics []offsetCommitRequestV2Topic\n}\n\nfunc (t offsetCommitRequestV2) size() int32 {\n\treturn sizeofString(t.GroupID) +\n\t\tsizeofInt32(t.GenerationID) +\n\t\tsizeofString(t.MemberID) +\n\t\tsizeofInt64(t.RetentionTime) +\n\t\tsizeofArray(len(t.Topics), func(i int) int32 { return t.Topics[i].size() })\n}\n\nfunc (t offsetCommitRequestV2) writeTo(wb *writeBuffer) {\n\twb.writeString(t.GroupID)\n\twb.writeInt32(t.GenerationID)\n\twb.writeString(t.MemberID)\n\twb.writeInt64(t.RetentionTime)\n\twb.writeArray(len(t.Topics), func(i int) { t.Topics[i].writeTo(wb) })\n}\n\ntype offsetCommitResponseV2PartitionResponse struct {\n\tPartition int32\n\n\t// ErrorCode holds response error code\n\tErrorCode int16\n}\n\nfunc (t offsetCommitResponseV2PartitionResponse) size() int32 {\n\treturn sizeofInt32(t.Partition) +\n\t\tsizeofInt16(t.ErrorCode)\n}\n\nfunc (t offsetCommitResponseV2PartitionResponse) writeTo(wb *writeBuffer) {\n\twb.writeInt32(t.Partition)\n\twb.writeInt16(t.ErrorCode)\n}\n\nfunc (t *offsetCommitResponseV2PartitionResponse) readFrom(r *bufio.Reader, size int) (remain int, err error) {\n\tif remain, err = readInt32(r, size, &t.Partition); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readInt16(r, remain, &t.ErrorCode); err != nil {\n\t\treturn\n\t}\n\treturn\n}\n\ntype offsetCommitResponseV2Response struct {\n\tTopic string\n\tPartitionResponses []offsetCommitResponseV2PartitionResponse\n}\n\nfunc (t offsetCommitResponseV2Response) size() int32 {\n\treturn sizeofString(t.Topic) +\n\t\tsizeofArray(len(t.PartitionResponses), func(i int) int32 { return t.PartitionResponses[i].size() })\n}\n\nfunc (t offsetCommitResponseV2Response) writeTo(wb *writeBuffer) {\n\twb.writeString(t.Topic)\n\twb.writeArray(len(t.PartitionResponses), func(i int) { t.PartitionResponses[i].writeTo(wb) })\n}\n\nfunc (t *offsetCommitResponseV2Response) readFrom(r *bufio.Reader, size int) (remain int, err error) {\n\tif remain, err = readString(r, size, &t.Topic); err != nil {\n\t\treturn\n\t}\n\n\tfn := func(r *bufio.Reader, withSize int) (fnRemain int, fnErr error) {\n\t\titem := offsetCommitResponseV2PartitionResponse{}\n\t\tif fnRemain, fnErr = (&item).readFrom(r, withSize); fnErr != nil {\n\t\t\treturn\n\t\t}\n\t\tt.PartitionResponses = append(t.PartitionResponses, item)\n\t\treturn\n\t}\n\tif remain, err = readArrayWith(r, remain, fn); err != nil {\n\t\treturn\n\t}\n\n\treturn\n}\n\ntype offsetCommitResponseV2 struct {\n\tResponses []offsetCommitResponseV2Response\n}\n\nfunc (t offsetCommitResponseV2) size() int32 {\n\treturn sizeofArray(len(t.Responses), func(i int) int32 { return t.Responses[i].size() })\n}\n\nfunc (t offsetCommitResponseV2) writeTo(wb *writeBuffer) {\n\twb.writeArray(len(t.Responses), func(i int) { t.Responses[i].writeTo(wb) })\n}\n\nfunc (t *offsetCommitResponseV2) readFrom(r *bufio.Reader, size int) (remain int, err error) {\n\tfn := func(r *bufio.Reader, withSize int) (fnRemain int, fnErr error) {\n\t\titem := offsetCommitResponseV2Response{}\n\t\tif fnRemain, fnErr = (&item).readFrom(r, withSize); fnErr != nil {\n\t\t\treturn\n\t\t}\n\t\tt.Responses = append(t.Responses, item)\n\t\treturn\n\t}\n\tif remain, err = readArrayWith(r, size, fn); err != nil {\n\t\treturn\n\t}\n\n\treturn\n}\n" }, { "path": "offsetcommit_test.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"log\"\n\t\"os\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestOffsetCommitResponseV2(t *testing.T) {\n\titem := offsetCommitResponseV2{\n\t\tResponses: []offsetCommitResponseV2Response{\n\t\t\t{\n\t\t\t\tTopic: \"a\",\n\t\t\t\tPartitionResponses: []offsetCommitResponseV2PartitionResponse{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\tErrorCode: 2,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tb := bytes.NewBuffer(nil)\n\tw := &writeBuffer{w: b}\n\titem.writeTo(w)\n\n\tvar found offsetCommitResponseV2\n\tremain, err := (&found).readFrom(bufio.NewReader(b), b.Len())\n\tif err != nil {\n\t\tt.Error(err)\n\t\tt.FailNow()\n\t}\n\tif remain != 0 {\n\t\tt.Errorf(\"expected 0 remain, got %v\", remain)\n\t\tt.FailNow()\n\t}\n\tif !reflect.DeepEqual(item, found) {\n\t\tt.Error(\"expected item and found to be the same\")\n\t\tt.FailNow()\n\t}\n}\n\nfunc TestClientOffsetCommit(t *testing.T) {\n\ttopic := makeTopic()\n\tclient, shutdown := newLocalClientWithTopic(topic, 3)\n\tdefer shutdown()\n\tnow := time.Now()\n\n\tconst N = 10 * 3\n\trecords := make([]Record, 0, N)\n\tfor i := 0; i < N; i++ {\n\t\trecords = append(records, Record{\n\t\t\tTime: now,\n\t\t\tValue: NewBytes([]byte(\"test-message-\" + strconv.Itoa(i))),\n\t\t})\n\t}\n\tctx, cancel := context.WithTimeout(context.Background(), time.Second*30)\n\tdefer cancel()\n\tres, err := client.Produce(ctx, &ProduceRequest{\n\t\tTopic: topic,\n\t\tRequiredAcks: RequireAll,\n\t\tRecords: NewRecordReader(records...),\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif res.Error != nil {\n\t\tt.Error(res.Error)\n\t}\n\n\tfor index, err := range res.RecordErrors {\n\t\tt.Fatalf(\"record at index %d produced an error: %v\", index, err)\n\t}\n\tctx, cancel = context.WithTimeout(context.Background(), time.Second*30)\n\tdefer cancel()\n\tgroupID := makeGroupID()\n\n\tgroup, err := NewConsumerGroup(ConsumerGroupConfig{\n\t\tID: groupID,\n\t\tTopics: []string{topic},\n\t\tBrokers: []string{\"localhost:9092\"},\n\t\tHeartbeatInterval: 2 * time.Second,\n\t\tRebalanceTimeout: 2 * time.Second,\n\t\tRetentionTime: time.Hour,\n\t\tLogger: log.New(os.Stdout, \"cg-test: \", 0),\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer group.Close()\n\n\tgen, err := group.Next(ctx)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tocr, err := client.OffsetCommit(ctx, &OffsetCommitRequest{\n\t\tAddr: nil,\n\t\tGroupID: groupID,\n\t\tGenerationID: int(gen.ID),\n\t\tMemberID: gen.MemberID,\n\t\tTopics: map[string][]OffsetCommit{\n\t\t\ttopic: {\n\t\t\t\t{Partition: 0, Offset: 10},\n\t\t\t\t{Partition: 1, Offset: 10},\n\t\t\t\t{Partition: 2, Offset: 10},\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tresps := ocr.Topics[topic]\n\tif len(resps) != 3 {\n\t\tt.Fatalf(\"expected 3 offsetcommitpartition responses; got %d\", len(resps))\n\t}\n\n\tfor _, resp := range resps {\n\t\tif resp.Error != nil {\n\t\t\tt.Fatal(resp.Error)\n\t\t}\n\t}\n\n\tofr, err := client.OffsetFetch(ctx, &OffsetFetchRequest{\n\t\tGroupID: groupID,\n\t\tTopics: map[string][]int{topic: {0, 1, 2}},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif ofr.Error != nil {\n\t\tt.Error(res.Error)\n\t}\n\n\tfetresps := ofr.Topics[topic]\n\tif len(fetresps) != 3 {\n\t\tt.Fatalf(\"expected 3 offsetfetchpartition responses; got %d\", len(resps))\n\t}\n\n\tfor _, r := range fetresps {\n\t\tif r.Error != nil {\n\t\t\tt.Fatal(r.Error)\n\t\t}\n\n\t\tif r.CommittedOffset != 10 {\n\t\t\tt.Fatalf(\"expected committed offset to be 10; got: %v for partition: %v\", r.CommittedOffset, r.Partition)\n\t\t}\n\t}\n}\n" }, { "path": "offsetdelete.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/offsetdelete\"\n)\n\n// OffsetDelete deletes the offset for a consumer group on a particular topic\n// for a particular partition.\ntype OffsetDelete struct {\n\tTopic string\n\tPartition int\n}\n\n// OffsetDeleteRequest represents a request sent to a kafka broker to delete\n// the offsets for a partition on a given topic associated with a consumer group.\ntype OffsetDeleteRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// ID of the consumer group to delete the offsets for.\n\tGroupID string\n\n\t// Set of topic partitions to delete offsets for.\n\tTopics map[string][]int\n}\n\n// OffsetDeleteResponse represents a response from a kafka broker to a delete\n// offset request.\ntype OffsetDeleteResponse struct {\n\t// An error that may have occurred while attempting to delete an offset\n\tError error\n\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// Set of topic partitions that the kafka broker has additional info (error?)\n\t// for.\n\tTopics map[string][]OffsetDeletePartition\n}\n\n// OffsetDeletePartition represents the state of a status of a partition in response\n// to deleting offsets.\ntype OffsetDeletePartition struct {\n\t// ID of the partition.\n\tPartition int\n\n\t// An error that may have occurred while attempting to delete an offset for\n\t// this partition.\n\tError error\n}\n\n// OffsetDelete sends a delete offset request to a kafka broker and returns the\n// response.\nfunc (c *Client) OffsetDelete(ctx context.Context, req *OffsetDeleteRequest) (*OffsetDeleteResponse, error) {\n\ttopics := make([]offsetdelete.RequestTopic, 0, len(req.Topics))\n\n\tfor topicName, partitionIndexes := range req.Topics {\n\t\tpartitions := make([]offsetdelete.RequestPartition, len(partitionIndexes))\n\n\t\tfor i, c := range partitionIndexes {\n\t\t\tpartitions[i] = offsetdelete.RequestPartition{\n\t\t\t\tPartitionIndex: int32(c),\n\t\t\t}\n\t\t}\n\n\t\ttopics = append(topics, offsetdelete.RequestTopic{\n\t\t\tName: topicName,\n\t\t\tPartitions: partitions,\n\t\t})\n\t}\n\n\tm, err := c.roundTrip(ctx, req.Addr, &offsetdelete.Request{\n\t\tGroupID: req.GroupID,\n\t\tTopics: topics,\n\t})\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).OffsetDelete: %w\", err)\n\t}\n\tr := m.(*offsetdelete.Response)\n\n\tres := &OffsetDeleteResponse{\n\t\tError: makeError(r.ErrorCode, \"\"),\n\t\tThrottle: makeDuration(r.ThrottleTimeMs),\n\t\tTopics: make(map[string][]OffsetDeletePartition, len(r.Topics)),\n\t}\n\n\tfor _, topic := range r.Topics {\n\t\tpartitions := make([]OffsetDeletePartition, len(topic.Partitions))\n\n\t\tfor i, p := range topic.Partitions {\n\t\t\tpartitions[i] = OffsetDeletePartition{\n\t\t\t\tPartition: int(p.PartitionIndex),\n\t\t\t\tError: makeError(p.ErrorCode, \"\"),\n\t\t\t}\n\t\t}\n\n\t\tres.Topics[topic.Name] = partitions\n\t}\n\n\treturn res, nil\n}\n" }, { "path": "offsetdelete_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"log\"\n\t\"os\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc TestClientDeleteOffset(t *testing.T) {\n\tif !ktesting.KafkaIsAtLeast(\"2.4.0\") {\n\t\treturn\n\t}\n\n\ttopic := makeTopic()\n\tclient, shutdown := newLocalClientWithTopic(topic, 3)\n\tdefer shutdown()\n\tnow := time.Now()\n\n\tconst N = 10 * 3\n\trecords := make([]Record, 0, N)\n\tfor i := 0; i < N; i++ {\n\t\trecords = append(records, Record{\n\t\t\tTime: now,\n\t\t\tValue: NewBytes([]byte(\"test-message-\" + strconv.Itoa(i))),\n\t\t})\n\t}\n\tctx, cancel := context.WithTimeout(context.Background(), time.Second*30)\n\tdefer cancel()\n\tres, err := client.Produce(ctx, &ProduceRequest{\n\t\tTopic: topic,\n\t\tRequiredAcks: RequireAll,\n\t\tRecords: NewRecordReader(records...),\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif res.Error != nil {\n\t\tt.Error(res.Error)\n\t}\n\n\tfor index, err := range res.RecordErrors {\n\t\tt.Fatalf(\"record at index %d produced an error: %v\", index, err)\n\t}\n\tctx, cancel = context.WithTimeout(context.Background(), time.Second*30)\n\tdefer cancel()\n\tgroupID := makeGroupID()\n\n\tgroup, err := NewConsumerGroup(ConsumerGroupConfig{\n\t\tID: groupID,\n\t\tTopics: []string{topic},\n\t\tBrokers: []string{\"localhost:9092\"},\n\t\tHeartbeatInterval: 2 * time.Second,\n\t\tRebalanceTimeout: 2 * time.Second,\n\t\tRetentionTime: time.Hour,\n\t\tLogger: log.New(os.Stdout, \"cg-test: \", 0),\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tgen, err := group.Next(ctx)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tocr, err := client.OffsetCommit(ctx, &OffsetCommitRequest{\n\t\tAddr: nil,\n\t\tGroupID: groupID,\n\t\tGenerationID: int(gen.ID),\n\t\tMemberID: gen.MemberID,\n\t\tTopics: map[string][]OffsetCommit{\n\t\t\ttopic: {\n\t\t\t\t{Partition: 0, Offset: 10},\n\t\t\t\t{Partition: 1, Offset: 10},\n\t\t\t\t{Partition: 2, Offset: 10},\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tgroup.Close()\n\n\tresps := ocr.Topics[topic]\n\tif len(resps) != 3 {\n\t\tt.Fatalf(\"expected 3 offsetcommitpartition responses; got %d\", len(resps))\n\t}\n\n\tfor _, resp := range resps {\n\t\tif resp.Error != nil {\n\t\t\tt.Fatal(resp.Error)\n\t\t}\n\t}\n\n\tofr, err := client.OffsetFetch(ctx, &OffsetFetchRequest{\n\t\tGroupID: groupID,\n\t\tTopics: map[string][]int{topic: {0, 1, 2}},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif ofr.Error != nil {\n\t\tt.Error(res.Error)\n\t}\n\n\tfetresps := ofr.Topics[topic]\n\tif len(fetresps) != 3 {\n\t\tt.Fatalf(\"expected 3 offsetfetchpartition responses; got %d\", len(resps))\n\t}\n\n\tfor _, r := range fetresps {\n\t\tif r.Error != nil {\n\t\t\tt.Fatal(r.Error)\n\t\t}\n\n\t\tif r.CommittedOffset != 10 {\n\t\t\tt.Fatalf(\"expected committed offset to be 10; got: %v for partition: %v\", r.CommittedOffset, r.Partition)\n\t\t}\n\t}\n\n\t// Remove offsets\n\todr, err := client.OffsetDelete(ctx, &OffsetDeleteRequest{\n\t\tGroupID: groupID,\n\t\tTopics: map[string][]int{topic: {0, 1, 2}},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif odr.Error != nil {\n\t\tt.Error(odr.Error)\n\t}\n\n\t// Fetch the offsets again\n\tofr, err = client.OffsetFetch(ctx, &OffsetFetchRequest{\n\t\tGroupID: groupID,\n\t\tTopics: map[string][]int{topic: {0, 1, 2}},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif ofr.Error != nil {\n\t\tt.Error(res.Error)\n\t}\n\n\tfor _, r := range ofr.Topics[topic] {\n\t\tif r.CommittedOffset != -1 {\n\t\t\tt.Fatalf(\"expected committed offset to be -1; got: %v for partition: %v\", r.CommittedOffset, r.Partition)\n\t\t}\n\t}\n}\n" }, { "path": "offsetfetch.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/offsetfetch\"\n)\n\n// OffsetFetchRequest represents a request sent to a kafka broker to read the\n// currently committed offsets of topic partitions.\ntype OffsetFetchRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// ID of the consumer group to retrieve the offsets for.\n\tGroupID string\n\n\t// Set of topic partitions to retrieve the offsets for.\n\tTopics map[string][]int\n}\n\n// OffsetFetchResponse represents a response from a kafka broker to an offset\n// fetch request.\ntype OffsetFetchResponse struct {\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// Set of topic partitions that the kafka broker has returned offsets for.\n\tTopics map[string][]OffsetFetchPartition\n\n\t// An error that may have occurred while attempting to retrieve consumer\n\t// group offsets.\n\t//\n\t// The error contains both the kafka error code, and an error message\n\t// returned by the kafka broker. Programs may use the standard errors.Is\n\t// function to test the error against kafka error codes.\n\tError error\n}\n\n// OffsetFetchPartition represents the state of a single partition in a consumer\n// group.\ntype OffsetFetchPartition struct {\n\t// ID of the partition.\n\tPartition int\n\n\t// Last committed offsets on the partition when the request was served by\n\t// the kafka broker.\n\tCommittedOffset int64\n\n\t// Consumer group metadata for this partition.\n\tMetadata string\n\n\t// An error that may have occurred while attempting to retrieve consumer\n\t// group offsets for this partition.\n\t//\n\t// The error contains both the kafka error code, and an error message\n\t// returned by the kafka broker. Programs may use the standard errors.Is\n\t// function to test the error against kafka error codes.\n\tError error\n}\n\n// OffsetFetch sends an offset fetch request to a kafka broker and returns the\n// response.\nfunc (c *Client) OffsetFetch(ctx context.Context, req *OffsetFetchRequest) (*OffsetFetchResponse, error) {\n\n\t// Kafka version 0.10.2.x and above allow null Topics map for OffsetFetch API\n\t// which will return the result for all topics with the desired consumer group:\n\t// https://kafka.apache.org/0102/protocol.html#The_Messages_OffsetFetch\n\t// For Kafka version below 0.10.2.x this call will result in an error\n\tvar topics []offsetfetch.RequestTopic\n\n\tif len(req.Topics) > 0 {\n\t\ttopics = make([]offsetfetch.RequestTopic, 0, len(req.Topics))\n\n\t\tfor topicName, partitions := range req.Topics {\n\t\t\tindexes := make([]int32, len(partitions))\n\n\t\t\tfor i, p := range partitions {\n\t\t\t\tindexes[i] = int32(p)\n\t\t\t}\n\n\t\t\ttopics = append(topics, offsetfetch.RequestTopic{\n\t\t\t\tName: topicName,\n\t\t\t\tPartitionIndexes: indexes,\n\t\t\t})\n\t\t}\n\t}\n\n\tm, err := c.roundTrip(ctx, req.Addr, &offsetfetch.Request{\n\t\tGroupID: req.GroupID,\n\t\tTopics: topics,\n\t})\n\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).OffsetFetch: %w\", err)\n\t}\n\n\tres := m.(*offsetfetch.Response)\n\tret := &OffsetFetchResponse{\n\t\tThrottle: makeDuration(res.ThrottleTimeMs),\n\t\tTopics: make(map[string][]OffsetFetchPartition, len(res.Topics)),\n\t\tError: makeError(res.ErrorCode, \"\"),\n\t}\n\n\tfor _, t := range res.Topics {\n\t\tpartitions := make([]OffsetFetchPartition, len(t.Partitions))\n\n\t\tfor i, p := range t.Partitions {\n\t\t\tpartitions[i] = OffsetFetchPartition{\n\t\t\t\tPartition: int(p.PartitionIndex),\n\t\t\t\tCommittedOffset: p.CommittedOffset,\n\t\t\t\tMetadata: p.Metadata,\n\t\t\t\tError: makeError(p.ErrorCode, \"\"),\n\t\t\t}\n\t\t}\n\n\t\tret.Topics[t.Name] = partitions\n\t}\n\n\treturn ret, nil\n}\n\ntype offsetFetchRequestV1Topic struct {\n\t// Topic name\n\tTopic string\n\n\t// Partitions to fetch offsets\n\tPartitions []int32\n}\n\nfunc (t offsetFetchRequestV1Topic) size() int32 {\n\treturn sizeofString(t.Topic) +\n\t\tsizeofInt32Array(t.Partitions)\n}\n\nfunc (t offsetFetchRequestV1Topic) writeTo(wb *writeBuffer) {\n\twb.writeString(t.Topic)\n\twb.writeInt32Array(t.Partitions)\n}\n\ntype offsetFetchRequestV1 struct {\n\t// GroupID holds the unique group identifier\n\tGroupID string\n\n\t// Topics to fetch offsets.\n\tTopics []offsetFetchRequestV1Topic\n}\n\nfunc (t offsetFetchRequestV1) size() int32 {\n\treturn sizeofString(t.GroupID) +\n\t\tsizeofArray(len(t.Topics), func(i int) int32 { return t.Topics[i].size() })\n}\n\nfunc (t offsetFetchRequestV1) writeTo(wb *writeBuffer) {\n\twb.writeString(t.GroupID)\n\twb.writeArray(len(t.Topics), func(i int) { t.Topics[i].writeTo(wb) })\n}\n\ntype offsetFetchResponseV1PartitionResponse struct {\n\t// Partition ID\n\tPartition int32\n\n\t// Offset of last committed message\n\tOffset int64\n\n\t// Metadata client wants to keep\n\tMetadata string\n\n\t// ErrorCode holds response error code\n\tErrorCode int16\n}\n\nfunc (t offsetFetchResponseV1PartitionResponse) size() int32 {\n\treturn sizeofInt32(t.Partition) +\n\t\tsizeofInt64(t.Offset) +\n\t\tsizeofString(t.Metadata) +\n\t\tsizeofInt16(t.ErrorCode)\n}\n\nfunc (t offsetFetchResponseV1PartitionResponse) writeTo(wb *writeBuffer) {\n\twb.writeInt32(t.Partition)\n\twb.writeInt64(t.Offset)\n\twb.writeString(t.Metadata)\n\twb.writeInt16(t.ErrorCode)\n}\n\nfunc (t *offsetFetchResponseV1PartitionResponse) readFrom(r *bufio.Reader, size int) (remain int, err error) {\n\tif remain, err = readInt32(r, size, &t.Partition); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readInt64(r, remain, &t.Offset); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readString(r, remain, &t.Metadata); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readInt16(r, remain, &t.ErrorCode); err != nil {\n\t\treturn\n\t}\n\treturn\n}\n\ntype offsetFetchResponseV1Response struct {\n\t// Topic name\n\tTopic string\n\n\t// PartitionResponses holds offsets by partition\n\tPartitionResponses []offsetFetchResponseV1PartitionResponse\n}\n\nfunc (t offsetFetchResponseV1Response) size() int32 {\n\treturn sizeofString(t.Topic) +\n\t\tsizeofArray(len(t.PartitionResponses), func(i int) int32 { return t.PartitionResponses[i].size() })\n}\n\nfunc (t offsetFetchResponseV1Response) writeTo(wb *writeBuffer) {\n\twb.writeString(t.Topic)\n\twb.writeArray(len(t.PartitionResponses), func(i int) { t.PartitionResponses[i].writeTo(wb) })\n}\n\nfunc (t *offsetFetchResponseV1Response) readFrom(r *bufio.Reader, size int) (remain int, err error) {\n\tif remain, err = readString(r, size, &t.Topic); err != nil {\n\t\treturn\n\t}\n\n\tfn := func(r *bufio.Reader, size int) (fnRemain int, fnErr error) {\n\t\titem := offsetFetchResponseV1PartitionResponse{}\n\t\tif fnRemain, fnErr = (&item).readFrom(r, size); fnErr != nil {\n\t\t\treturn\n\t\t}\n\t\tt.PartitionResponses = append(t.PartitionResponses, item)\n\t\treturn\n\t}\n\tif remain, err = readArrayWith(r, remain, fn); err != nil {\n\t\treturn\n\t}\n\n\treturn\n}\n\ntype offsetFetchResponseV1 struct {\n\t// Responses holds topic partition offsets\n\tResponses []offsetFetchResponseV1Response\n}\n\nfunc (t offsetFetchResponseV1) size() int32 {\n\treturn sizeofArray(len(t.Responses), func(i int) int32 { return t.Responses[i].size() })\n}\n\nfunc (t offsetFetchResponseV1) writeTo(wb *writeBuffer) {\n\twb.writeArray(len(t.Responses), func(i int) { t.Responses[i].writeTo(wb) })\n}\n\nfunc (t *offsetFetchResponseV1) readFrom(r *bufio.Reader, size int) (remain int, err error) {\n\tfn := func(r *bufio.Reader, withSize int) (fnRemain int, fnErr error) {\n\t\titem := offsetFetchResponseV1Response{}\n\t\tif fnRemain, fnErr = (&item).readFrom(r, withSize); fnErr != nil {\n\t\t\treturn\n\t\t}\n\t\tt.Responses = append(t.Responses, item)\n\t\treturn\n\t}\n\tif remain, err = readArrayWith(r, size, fn); err != nil {\n\t\treturn\n\t}\n\n\treturn\n}\n" }, { "path": "offsetfetch_test.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"reflect\"\n\t\"testing\"\n\t\"time\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc TestOffsetFetchResponseV1(t *testing.T) {\n\titem := offsetFetchResponseV1{\n\t\tResponses: []offsetFetchResponseV1Response{\n\t\t\t{\n\t\t\t\tTopic: \"a\",\n\t\t\t\tPartitionResponses: []offsetFetchResponseV1PartitionResponse{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 2,\n\t\t\t\t\t\tOffset: 3,\n\t\t\t\t\t\tMetadata: \"b\",\n\t\t\t\t\t\tErrorCode: 4,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tb := bytes.NewBuffer(nil)\n\tw := &writeBuffer{w: b}\n\titem.writeTo(w)\n\n\tvar found offsetFetchResponseV1\n\tremain, err := (&found).readFrom(bufio.NewReader(b), b.Len())\n\tif err != nil {\n\t\tt.Error(err)\n\t\tt.FailNow()\n\t}\n\tif remain != 0 {\n\t\tt.Errorf(\"expected 0 remain, got %v\", remain)\n\t\tt.FailNow()\n\t}\n\tif !reflect.DeepEqual(item, found) {\n\t\tt.Error(\"expected item and found to be the same\")\n\t\tt.FailNow()\n\t}\n}\n\nfunc TestOffsetFetchRequestWithNoTopic(t *testing.T) {\n\tif !ktesting.KafkaIsAtLeast(\"0.10.2.0\") {\n\t\tt.Logf(\"Test %s is not applicable for kafka versions below 0.10.2.0\", t.Name())\n\t\tt.SkipNow()\n\t}\n\tctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)\n\ttopic1 := makeTopic()\n\tdefer deleteTopic(t, topic1)\n\ttopic2 := makeTopic()\n\tdefer deleteTopic(t, topic2)\n\tconsumeGroup := makeGroupID()\n\tnumMsgs := 50\n\tdefer cancel()\n\tr1 := NewReader(ReaderConfig{\n\t\tBrokers: []string{\"localhost:9092\"},\n\t\tTopic: topic1,\n\t\tGroupID: consumeGroup,\n\t\tMinBytes: 1,\n\t\tMaxBytes: 100,\n\t\tMaxWait: 100 * time.Millisecond,\n\t})\n\tdefer r1.Close()\n\tprepareReader(t, ctx, r1, makeTestSequence(numMsgs)...)\n\tr2 := NewReader(ReaderConfig{\n\t\tBrokers: []string{\"localhost:9092\"},\n\t\tTopic: topic2,\n\t\tGroupID: consumeGroup,\n\t\tMinBytes: 1,\n\t\tMaxBytes: 100,\n\t\tMaxWait: 100 * time.Millisecond,\n\t})\n\tdefer r2.Close()\n\tprepareReader(t, ctx, r2, makeTestSequence(numMsgs)...)\n\n\tfor i := 0; i < numMsgs; i++ {\n\t\tif _, err := r1.ReadMessage(ctx); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\tfor i := 0; i < numMsgs; i++ {\n\t\tif _, err := r2.ReadMessage(ctx); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\tclient := Client{Addr: TCP(\"localhost:9092\")}\n\n\ttopicOffsets, err := client.OffsetFetch(ctx, &OffsetFetchRequest{GroupID: consumeGroup})\n\n\tif err != nil {\n\t\tt.Error(err)\n\t\tt.FailNow()\n\t}\n\n\tif len(topicOffsets.Topics) != 2 {\n\t\tt.Error(err)\n\t\tt.FailNow()\n\t}\n\n}\n\nfunc TestOffsetFetchRequestWithOneTopic(t *testing.T) {\n\tctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)\n\ttopic1 := makeTopic()\n\tdefer deleteTopic(t, topic1)\n\ttopic2 := makeTopic()\n\tdefer deleteTopic(t, topic2)\n\tconsumeGroup := makeGroupID()\n\tnumMsgs := 50\n\tdefer cancel()\n\tr1 := NewReader(ReaderConfig{\n\t\tBrokers: []string{\"localhost:9092\"},\n\t\tTopic: topic1,\n\t\tGroupID: consumeGroup,\n\t\tMinBytes: 1,\n\t\tMaxBytes: 100,\n\t\tMaxWait: 100 * time.Millisecond,\n\t})\n\tdefer r1.Close()\n\tprepareReader(t, ctx, r1, makeTestSequence(numMsgs)...)\n\tr2 := NewReader(ReaderConfig{\n\t\tBrokers: []string{\"localhost:9092\"},\n\t\tTopic: topic2,\n\t\tGroupID: consumeGroup,\n\t\tMinBytes: 1,\n\t\tMaxBytes: 100,\n\t\tMaxWait: 100 * time.Millisecond,\n\t})\n\tdefer r2.Close()\n\tprepareReader(t, ctx, r2, makeTestSequence(numMsgs)...)\n\n\tfor i := 0; i < numMsgs; i++ {\n\t\tif _, err := r1.ReadMessage(ctx); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\tfor i := 0; i < numMsgs; i++ {\n\t\tif _, err := r2.ReadMessage(ctx); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n\n\tclient := Client{Addr: TCP(\"localhost:9092\")}\n\ttopicOffsets, err := client.OffsetFetch(ctx, &OffsetFetchRequest{GroupID: consumeGroup, Topics: map[string][]int{\n\t\ttopic1: {0},\n\t}})\n\n\tif err != nil {\n\t\tt.Error(err)\n\t\tt.FailNow()\n\t}\n\n\tif len(topicOffsets.Topics) != 1 {\n\t\tt.Error(err)\n\t\tt.FailNow()\n\t}\n}\n" }, { "path": "produce.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"encoding\"\n\t\"errors\"\n\t\"fmt\"\n\t\"net\"\n\t\"strconv\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n\tproduceAPI \"github.com/segmentio/kafka-go/protocol/produce\"\n)\n\ntype RequiredAcks int\n\nconst (\n\tRequireNone RequiredAcks = 0\n\tRequireOne RequiredAcks = 1\n\tRequireAll RequiredAcks = -1\n)\n\nfunc (acks RequiredAcks) String() string {\n\tswitch acks {\n\tcase RequireNone:\n\t\treturn \"none\"\n\tcase RequireOne:\n\t\treturn \"one\"\n\tcase RequireAll:\n\t\treturn \"all\"\n\tdefault:\n\t\treturn \"unknown\"\n\t}\n}\n\nfunc (acks RequiredAcks) MarshalText() ([]byte, error) {\n\treturn []byte(acks.String()), nil\n}\n\nfunc (acks *RequiredAcks) UnmarshalText(b []byte) error {\n\tswitch string(b) {\n\tcase \"none\":\n\t\t*acks = RequireNone\n\tcase \"one\":\n\t\t*acks = RequireOne\n\tcase \"all\":\n\t\t*acks = RequireAll\n\tdefault:\n\t\tx, err := strconv.ParseInt(string(b), 10, 64)\n\t\tparsed := RequiredAcks(x)\n\t\tif err != nil || (parsed != RequireNone && parsed != RequireOne && parsed != RequireAll) {\n\t\t\treturn fmt.Errorf(\"required acks must be one of none, one, or all, not %q\", b)\n\t\t}\n\t\t*acks = parsed\n\t}\n\treturn nil\n}\n\nvar (\n\t_ encoding.TextMarshaler = RequiredAcks(0)\n\t_ encoding.TextUnmarshaler = (*RequiredAcks)(nil)\n)\n\n// ProduceRequest represents a request sent to a kafka broker to produce records\n// to a topic partition.\ntype ProduceRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// The topic to produce the records to.\n\tTopic string\n\n\t// The partition to produce the records to.\n\tPartition int\n\n\t// The level of required acknowledgements to ask the kafka broker for.\n\tRequiredAcks RequiredAcks\n\n\t// The message format version used when encoding the records.\n\t//\n\t// By default, the client automatically determine which version should be\n\t// used based on the version of the Produce API supported by the server.\n\tMessageVersion int\n\n\t// An optional transaction id when producing to the kafka broker is part of\n\t// a transaction.\n\tTransactionalID string\n\n\t// The sequence of records to produce to the topic partition.\n\tRecords RecordReader\n\n\t// An optional compression algorithm to apply to the batch of records sent\n\t// to the kafka broker.\n\tCompression Compression\n}\n\n// ProduceResponse represents a response from a kafka broker to a produce\n// request.\ntype ProduceResponse struct {\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// An error that may have occurred while attempting to produce the records.\n\t//\n\t// The error contains both the kafka error code, and an error message\n\t// returned by the kafka broker. Programs may use the standard errors.Is\n\t// function to test the error against kafka error codes.\n\tError error\n\n\t// Offset of the first record that was written to the topic partition.\n\t//\n\t// This field will be zero if the kafka broker did not support Produce API\n\t// version 3 or above.\n\tBaseOffset int64\n\n\t// Time at which the broker wrote the records to the topic partition.\n\t//\n\t// This field will be zero if the kafka broker did not support Produce API\n\t// version 2 or above.\n\tLogAppendTime time.Time\n\n\t// First offset in the topic partition that the records were written to.\n\t//\n\t// This field will be zero if the kafka broker did not support Produce\n\t// API version 5 or above (or if the first offset is zero).\n\tLogStartOffset int64\n\n\t// If errors occurred writing specific records, they will be reported in\n\t// this map.\n\t//\n\t// This field will always be empty if the kafka broker did not support the\n\t// Produce API in version 8 or above.\n\tRecordErrors map[int]error\n}\n\n// Produce sends a produce request to a kafka broker and returns the response.\n//\n// If the request contained no records, an error wrapping protocol.ErrNoRecord\n// is returned.\n//\n// When the request is configured with RequiredAcks=none, both the response and\n// the error will be nil on success.\nfunc (c *Client) Produce(ctx context.Context, req *ProduceRequest) (*ProduceResponse, error) {\n\tattributes := protocol.Attributes(req.Compression) & 0x7\n\n\tm, err := c.roundTrip(ctx, req.Addr, &produceAPI.Request{\n\t\tTransactionalID: req.TransactionalID,\n\t\tAcks: int16(req.RequiredAcks),\n\t\tTimeout: c.timeoutMs(ctx, defaultProduceTimeout),\n\t\tTopics: []produceAPI.RequestTopic{{\n\t\t\tTopic: req.Topic,\n\t\t\tPartitions: []produceAPI.RequestPartition{{\n\t\t\t\tPartition: int32(req.Partition),\n\t\t\t\tRecordSet: protocol.RecordSet{\n\t\t\t\t\tAttributes: attributes,\n\t\t\t\t\tRecords: req.Records,\n\t\t\t\t},\n\t\t\t}},\n\t\t}},\n\t})\n\n\tswitch {\n\tcase err == nil:\n\tcase errors.Is(err, protocol.ErrNoRecord):\n\t\treturn new(ProduceResponse), nil\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).Produce: %w\", err)\n\t}\n\n\tif req.RequiredAcks == RequireNone {\n\t\treturn nil, nil\n\t}\n\n\tres := m.(*produceAPI.Response)\n\tif len(res.Topics) == 0 {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).Produce: %w\", protocol.ErrNoTopic)\n\t}\n\ttopic := &res.Topics[0]\n\tif len(topic.Partitions) == 0 {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).Produce: %w\", protocol.ErrNoPartition)\n\t}\n\tpartition := &topic.Partitions[0]\n\n\tret := &ProduceResponse{\n\t\tThrottle: makeDuration(res.ThrottleTimeMs),\n\t\tError: makeError(partition.ErrorCode, partition.ErrorMessage),\n\t\tBaseOffset: partition.BaseOffset,\n\t\tLogAppendTime: makeTime(partition.LogAppendTime),\n\t\tLogStartOffset: partition.LogStartOffset,\n\t}\n\n\tif len(partition.RecordErrors) != 0 {\n\t\tret.RecordErrors = make(map[int]error, len(partition.RecordErrors))\n\n\t\tfor _, recErr := range partition.RecordErrors {\n\t\t\tret.RecordErrors[int(recErr.BatchIndex)] = errors.New(recErr.BatchIndexErrorMessage)\n\t\t}\n\t}\n\n\treturn ret, nil\n}\n\ntype produceRequestV2 struct {\n\tRequiredAcks int16\n\tTimeout int32\n\tTopics []produceRequestTopicV2\n}\n\nfunc (r produceRequestV2) size() int32 {\n\treturn 2 + 4 + sizeofArray(len(r.Topics), func(i int) int32 { return r.Topics[i].size() })\n}\n\nfunc (r produceRequestV2) writeTo(wb *writeBuffer) {\n\twb.writeInt16(r.RequiredAcks)\n\twb.writeInt32(r.Timeout)\n\twb.writeArray(len(r.Topics), func(i int) { r.Topics[i].writeTo(wb) })\n}\n\ntype produceRequestTopicV2 struct {\n\tTopicName string\n\tPartitions []produceRequestPartitionV2\n}\n\nfunc (t produceRequestTopicV2) size() int32 {\n\treturn sizeofString(t.TopicName) +\n\t\tsizeofArray(len(t.Partitions), func(i int) int32 { return t.Partitions[i].size() })\n}\n\nfunc (t produceRequestTopicV2) writeTo(wb *writeBuffer) {\n\twb.writeString(t.TopicName)\n\twb.writeArray(len(t.Partitions), func(i int) { t.Partitions[i].writeTo(wb) })\n}\n\ntype produceRequestPartitionV2 struct {\n\tPartition int32\n\tMessageSetSize int32\n\tMessageSet messageSet\n}\n\nfunc (p produceRequestPartitionV2) size() int32 {\n\treturn 4 + 4 + p.MessageSet.size()\n}\n\nfunc (p produceRequestPartitionV2) writeTo(wb *writeBuffer) {\n\twb.writeInt32(p.Partition)\n\twb.writeInt32(p.MessageSetSize)\n\tp.MessageSet.writeTo(wb)\n}\n\ntype produceResponsePartitionV2 struct {\n\tPartition int32\n\tErrorCode int16\n\tOffset int64\n\tTimestamp int64\n}\n\nfunc (p produceResponsePartitionV2) size() int32 {\n\treturn 4 + 2 + 8 + 8\n}\n\nfunc (p produceResponsePartitionV2) writeTo(wb *writeBuffer) {\n\twb.writeInt32(p.Partition)\n\twb.writeInt16(p.ErrorCode)\n\twb.writeInt64(p.Offset)\n\twb.writeInt64(p.Timestamp)\n}\n\nfunc (p *produceResponsePartitionV2) readFrom(r *bufio.Reader, sz int) (remain int, err error) {\n\tif remain, err = readInt32(r, sz, &p.Partition); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readInt16(r, remain, &p.ErrorCode); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readInt64(r, remain, &p.Offset); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readInt64(r, remain, &p.Timestamp); err != nil {\n\t\treturn\n\t}\n\treturn\n}\n\ntype produceResponsePartitionV7 struct {\n\tPartition int32\n\tErrorCode int16\n\tOffset int64\n\tTimestamp int64\n\tStartOffset int64\n}\n\nfunc (p produceResponsePartitionV7) size() int32 {\n\treturn 4 + 2 + 8 + 8 + 8\n}\n\nfunc (p produceResponsePartitionV7) writeTo(wb *writeBuffer) {\n\twb.writeInt32(p.Partition)\n\twb.writeInt16(p.ErrorCode)\n\twb.writeInt64(p.Offset)\n\twb.writeInt64(p.Timestamp)\n\twb.writeInt64(p.StartOffset)\n}\n\nfunc (p *produceResponsePartitionV7) readFrom(r *bufio.Reader, sz int) (remain int, err error) {\n\tif remain, err = readInt32(r, sz, &p.Partition); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readInt16(r, remain, &p.ErrorCode); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readInt64(r, remain, &p.Offset); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readInt64(r, remain, &p.Timestamp); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readInt64(r, remain, &p.StartOffset); err != nil {\n\t\treturn\n\t}\n\treturn\n}\n" }, { "path": "produce_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/compress\"\n)\n\nfunc TestRequiredAcks(t *testing.T) {\n\tfor _, acks := range []RequiredAcks{\n\t\tRequireNone,\n\t\tRequireOne,\n\t\tRequireAll,\n\t} {\n\t\tt.Run(acks.String(), func(t *testing.T) {\n\t\t\ta := strconv.Itoa(int(acks))\n\t\t\tx := RequiredAcks(-2)\n\t\t\ty := RequiredAcks(-2)\n\t\t\tb, err := acks.MarshalText()\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\tif err := x.UnmarshalText([]byte(a)); err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif err := y.UnmarshalText(b); err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\tif x != acks {\n\t\t\t\tt.Errorf(\"required acks mismatch after marshal/unmarshal text: want=%s got=%s\", acks, x)\n\t\t\t}\n\t\t\tif y != acks {\n\t\t\t\tt.Errorf(\"required acks mismatch after marshal/unmarshal value: want=%s got=%s\", acks, y)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestClientProduce(t *testing.T) {\n\tclient, topic, shutdown := newLocalClientAndTopic()\n\tdefer shutdown()\n\n\tnow := time.Now()\n\n\tres, err := client.Produce(context.Background(), &ProduceRequest{\n\t\tTopic: topic,\n\t\tPartition: 0,\n\t\tRequiredAcks: -1,\n\t\tRecords: NewRecordReader(\n\t\t\tRecord{Time: now, Value: NewBytes([]byte(`hello-1`))},\n\t\t\tRecord{Time: now, Value: NewBytes([]byte(`hello-2`))},\n\t\t\tRecord{Time: now, Value: NewBytes([]byte(`hello-3`))},\n\t\t),\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif res.Error != nil {\n\t\tt.Error(res.Error)\n\t}\n\n\tfor index, err := range res.RecordErrors {\n\t\tt.Errorf(\"record at index %d produced an error: %v\", index, err)\n\t}\n}\n\nfunc TestClientProduceCompressed(t *testing.T) {\n\tclient, topic, shutdown := newLocalClientAndTopic()\n\tdefer shutdown()\n\n\tnow := time.Now()\n\n\tres, err := client.Produce(context.Background(), &ProduceRequest{\n\t\tTopic: topic,\n\t\tPartition: 0,\n\t\tRequiredAcks: -1,\n\t\tCompression: compress.Gzip,\n\t\tRecords: NewRecordReader(\n\t\t\tRecord{Time: now, Value: NewBytes([]byte(`hello-1`))},\n\t\t\tRecord{Time: now, Value: NewBytes([]byte(`hello-2`))},\n\t\t\tRecord{Time: now, Value: NewBytes([]byte(`hello-3`))},\n\t\t),\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif res.Error != nil {\n\t\tt.Error(res.Error)\n\t}\n\n\tfor index, err := range res.RecordErrors {\n\t\tt.Errorf(\"record at index %d produced an error: %v\", index, err)\n\t}\n}\n\nfunc TestClientProduceNilRecords(t *testing.T) {\n\tclient, topic, shutdown := newLocalClientAndTopic()\n\tdefer shutdown()\n\n\t_, err := client.Produce(context.Background(), &ProduceRequest{\n\t\tTopic: topic,\n\t\tPartition: 0,\n\t\tRequiredAcks: -1,\n\t\tRecords: nil,\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestClientProduceEmptyRecords(t *testing.T) {\n\tclient, topic, shutdown := newLocalClientAndTopic()\n\tdefer shutdown()\n\n\t_, err := client.Produce(context.Background(), &ProduceRequest{\n\t\tTopic: topic,\n\t\tPartition: 0,\n\t\tRequiredAcks: -1,\n\t\tRecords: NewRecordReader(),\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n" }, { "path": "protocol/addoffsetstotxn/addoffsetstotxn.go", "content": "package addoffsetstotxn\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v3,max=v3,tag\"`\n\n\tTransactionalID string `kafka:\"min=v0,max=v2|min=v3,max=v3,compact\"`\n\tProducerID int64 `kafka:\"min=v0,max=v3\"`\n\tProducerEpoch int16 `kafka:\"min=v0,max=v3\"`\n\tGroupID string `kafka:\"min=v0,max=v3|min=v3,max=v3,compact\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.AddOffsetsToTxn }\n\nfunc (r *Request) Transaction() string { return r.TransactionalID }\n\nvar _ protocol.TransactionalMessage = (*Request)(nil)\n\ntype Response struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v3,max=v3,tag\"`\n\n\tThrottleTimeMs int32 `kafka:\"min=v0,max=v3\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v3\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.AddOffsetsToTxn }\n" }, { "path": "protocol/addoffsetstotxn/addoffsetstotxn_test.go", "content": "package addoffsetstotxn_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/addoffsetstotxn\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nfunc TestAddOffsetsToTxnRequest(t *testing.T) {\n\tfor _, version := range []int16{0, 1, 2, 3} {\n\t\tprototest.TestRequest(t, version, &addoffsetstotxn.Request{\n\t\t\tTransactionalID: \"transactional-id-0\",\n\t\t\tProducerID: 1,\n\t\t\tProducerEpoch: 10,\n\t\t\tGroupID: \"group-id-0\",\n\t\t})\n\t}\n}\n\nfunc TestAddOffsetsToTxnResponse(t *testing.T) {\n\tfor _, version := range []int16{0, 1, 2, 3} {\n\t\tprototest.TestResponse(t, version, &addoffsetstotxn.Response{\n\t\t\tThrottleTimeMs: 10,\n\t\t\tErrorCode: 1,\n\t\t})\n\t}\n}\n" }, { "path": "protocol/addpartitionstotxn/addpartitionstotxn.go", "content": "package addpartitionstotxn\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v3,max=v3,tag\"`\n\n\tTransactionalID string `kafka:\"min=v0,max=v2|min=v3,max=v3,compact\"`\n\tProducerID int64 `kafka:\"min=v0,max=v3\"`\n\tProducerEpoch int16 `kafka:\"min=v0,max=v3\"`\n\tTopics []RequestTopic `kafka:\"min=v0,max=v3\"`\n}\n\ntype RequestTopic struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v3,max=v3,tag\"`\n\n\tName string `kafka:\"min=v0,max=v2|min=v3,max=v3,compact\"`\n\tPartitions []int32 `kafka:\"min=v0,max=v3\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.AddPartitionsToTxn }\n\nfunc (r *Request) Transaction() string { return r.TransactionalID }\n\nvar _ protocol.TransactionalMessage = (*Request)(nil)\n\ntype Response struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v3,max=v3,tag\"`\n\n\tThrottleTimeMs int32 `kafka:\"min=v0,max=v3\"`\n\tResults []ResponseResult `kafka:\"min=v0,max=v3\"`\n}\n\ntype ResponseResult struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v3,max=v3,tag\"`\n\n\tName string `kafka:\"min=v0,max=v2|min=v3,max=v3,compact\"`\n\tResults []ResponsePartition `kafka:\"min=v0,max=v3\"`\n}\n\ntype ResponsePartition struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v3,max=v3,tag\"`\n\n\tPartitionIndex int32 `kafka:\"min=v0,max=v3\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v3\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.AddPartitionsToTxn }\n" }, { "path": "protocol/addpartitionstotxn/addpartitionstotxn_test.go", "content": "package addpartitionstotxn_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/addpartitionstotxn\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nfunc TestAddPartitionsToTxnRequest(t *testing.T) {\n\tfor _, version := range []int16{0, 1, 2, 3} {\n\t\tprototest.TestRequest(t, version, &addpartitionstotxn.Request{\n\t\t\tTransactionalID: \"transaction-id-0\",\n\t\t\tProducerID: 10,\n\t\t\tProducerEpoch: 100,\n\t\t\tTopics: []addpartitionstotxn.RequestTopic{\n\t\t\t\t{\n\t\t\t\t\tName: \"topic-1\",\n\t\t\t\t\tPartitions: []int32{0, 1, 2, 3},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tName: \"topic-2\",\n\t\t\t\t\tPartitions: []int32{0, 1, 2},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n}\n\nfunc TestAddPartitionsToTxnResponse(t *testing.T) {\n\tfor _, version := range []int16{0, 1, 2, 3} {\n\t\tprototest.TestResponse(t, version, &addpartitionstotxn.Response{\n\t\t\tThrottleTimeMs: 20,\n\t\t\tResults: []addpartitionstotxn.ResponseResult{\n\t\t\t\t{\n\t\t\t\t\tName: \"topic-1\",\n\t\t\t\t\tResults: []addpartitionstotxn.ResponsePartition{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartitionIndex: 0,\n\t\t\t\t\t\t\tErrorCode: 19,\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartitionIndex: 1,\n\t\t\t\t\t\t\tErrorCode: 0,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tName: \"topic-2\",\n\t\t\t\t\tResults: []addpartitionstotxn.ResponsePartition{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartitionIndex: 0,\n\t\t\t\t\t\t\tErrorCode: 0,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n}\n" }, { "path": "protocol/alterclientquotas/alterclientquotas.go", "content": "package alterclientquotas\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\n// Detailed API definition: https://kafka.apache.org/protocol#The_Messages_AlterClientQuotas\ntype Request struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v1,max=v1,tag\"`\n\tEntries []Entry `kafka:\"min=v0,max=v1\"`\n\tValidateOnly bool `kafka:\"min=v0,max=v1\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.AlterClientQuotas }\n\nfunc (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, error) {\n\treturn cluster.Brokers[cluster.Controller], nil\n}\n\ntype Entry struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v1,max=v1,tag\"`\n\tEntities []Entity `kafka:\"min=v0,max=v1\"`\n\tOps []Ops `kafka:\"min=v0,max=v1\"`\n}\n\ntype Entity struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v1,max=v1,tag\"`\n\tEntityType string `kafka:\"min=v0,max=v0|min=v1,max=v1,compact\"`\n\tEntityName string `kafka:\"min=v0,max=v0,nullable|min=v1,max=v1,nullable,compact\"`\n}\n\ntype Ops struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v1,max=v1,tag\"`\n\tKey string `kafka:\"min=v0,max=v0|min=v1,max=v1,compact\"`\n\tValue float64 `kafka:\"min=v0,max=v1\"`\n\tRemove bool `kafka:\"min=v0,max=v1\"`\n}\n\ntype Response struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v1,max=v1,tag\"`\n\tThrottleTimeMs int32 `kafka:\"min=v0,max=v1\"`\n\tResults []ResponseQuotas `kafka:\"min=v0,max=v1\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.AlterClientQuotas }\n\ntype ResponseQuotas struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v1,max=v1,tag\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v1\"`\n\tErrorMessage string `kafka:\"min=v0,max=v1,nullable\"`\n\tEntities []Entity `kafka:\"min=v0,max=v1\"`\n}\n\nvar _ protocol.BrokerMessage = (*Request)(nil)\n" }, { "path": "protocol/alterclientquotas/alterclientquotas_test.go", "content": "package alterclientquotas_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/alterclientquotas\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nconst (\n\tv0 = 0\n\tv1 = 1\n)\n\nfunc TestAlterClientQuotasRequest(t *testing.T) {\n\tprototest.TestRequest(t, v0, &alterclientquotas.Request{\n\t\tValidateOnly: true,\n\t\tEntries: []alterclientquotas.Entry{\n\t\t\t{\n\t\t\t\tEntities: []alterclientquotas.Entity{\n\t\t\t\t\t{\n\t\t\t\t\t\tEntityType: \"client-id\",\n\t\t\t\t\t\tEntityName: \"my-client-id\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tOps: []alterclientquotas.Ops{\n\t\t\t\t\t{\n\t\t\t\t\t\tKey: \"producer_byte_rate\",\n\t\t\t\t\t\tValue: 1.0,\n\t\t\t\t\t\tRemove: false,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestRequest(t, v1, &alterclientquotas.Request{\n\t\tValidateOnly: true,\n\t\tEntries: []alterclientquotas.Entry{\n\t\t\t{\n\t\t\t\tEntities: []alterclientquotas.Entity{\n\t\t\t\t\t{\n\t\t\t\t\t\tEntityType: \"client-id\",\n\t\t\t\t\t\tEntityName: \"my-client-id\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tOps: []alterclientquotas.Ops{\n\t\t\t\t\t{\n\t\t\t\t\t\tKey: \"producer_byte_rate\",\n\t\t\t\t\t\tValue: 1.0,\n\t\t\t\t\t\tRemove: false,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n}\n\nfunc TestAlterClientQuotasResponse(t *testing.T) {\n\tprototest.TestResponse(t, v0, &alterclientquotas.Response{\n\t\tThrottleTimeMs: 500,\n\t\tResults: []alterclientquotas.ResponseQuotas{\n\t\t\t{\n\t\t\t\tErrorCode: 1,\n\t\t\t\tErrorMessage: \"foo\",\n\t\t\t\tEntities: []alterclientquotas.Entity{\n\t\t\t\t\t{\n\t\t\t\t\t\tEntityType: \"client-id\",\n\t\t\t\t\t\tEntityName: \"my-client-id\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v1, &alterclientquotas.Response{\n\t\tThrottleTimeMs: 500,\n\t\tResults: []alterclientquotas.ResponseQuotas{\n\t\t\t{\n\t\t\t\tErrorCode: 1,\n\t\t\t\tErrorMessage: \"foo\",\n\t\t\t\tEntities: []alterclientquotas.Entity{\n\t\t\t\t\t{\n\t\t\t\t\t\tEntityType: \"client-id\",\n\t\t\t\t\t\tEntityName: \"my-client-id\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n}\n" }, { "path": "protocol/alterconfigs/alterconfigs.go", "content": "package alterconfigs\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\n// Detailed API definition: https://kafka.apache.org/protocol#The_Messages_AlterConfigs\ntype Request struct {\n\tResources []RequestResources `kafka:\"min=v0,max=v1\"`\n\tValidateOnly bool `kafka:\"min=v0,max=v1\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.AlterConfigs }\n\nfunc (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, error) {\n\treturn cluster.Brokers[cluster.Controller], nil\n}\n\ntype RequestResources struct {\n\tResourceType int8 `kafka:\"min=v0,max=v1\"`\n\tResourceName string `kafka:\"min=v0,max=v1\"`\n\tConfigs []RequestConfig `kafka:\"min=v0,max=v1\"`\n}\n\ntype RequestConfig struct {\n\tName string `kafka:\"min=v0,max=v1\"`\n\tValue string `kafka:\"min=v0,max=v1,nullable\"`\n}\n\ntype Response struct {\n\tThrottleTimeMs int32 `kafka:\"min=v0,max=v1\"`\n\tResponses []ResponseResponses `kafka:\"min=v0,max=v1\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.AlterConfigs }\n\ntype ResponseResponses struct {\n\tErrorCode int16 `kafka:\"min=v0,max=v1\"`\n\tErrorMessage string `kafka:\"min=v0,max=v1,nullable\"`\n\tResourceType int8 `kafka:\"min=v0,max=v1\"`\n\tResourceName string `kafka:\"min=v0,max=v1\"`\n}\n\nvar (\n\t_ protocol.BrokerMessage = (*Request)(nil)\n)\n" }, { "path": "protocol/alterconfigs/alterconfigs_test.go", "content": "package alterconfigs_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/alterconfigs\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nconst (\n\tv0 = 0\n\tv1 = 1\n)\n\nfunc TestAlterConfigsRequest(t *testing.T) {\n\tprototest.TestRequest(t, v0, &alterconfigs.Request{\n\t\tValidateOnly: true,\n\t\tResources: []alterconfigs.RequestResources{\n\t\t\t{\n\t\t\t\tResourceType: 1,\n\t\t\t\tResourceName: \"foo\",\n\t\t\t\tConfigs: []alterconfigs.RequestConfig{\n\t\t\t\t\t{\n\t\t\t\t\t\tName: \"foo\",\n\t\t\t\t\t\tValue: \"foo\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestRequest(t, v1, &alterconfigs.Request{\n\t\tValidateOnly: true,\n\t\tResources: []alterconfigs.RequestResources{\n\t\t\t{\n\t\t\t\tResourceType: 1,\n\t\t\t\tResourceName: \"foo\",\n\t\t\t\tConfigs: []alterconfigs.RequestConfig{\n\t\t\t\t\t{\n\t\t\t\t\t\tName: \"foo\",\n\t\t\t\t\t\tValue: \"foo\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n}\n\nfunc TestAlterConfigsResponse(t *testing.T) {\n\tprototest.TestResponse(t, v0, &alterconfigs.Response{\n\t\tThrottleTimeMs: 500,\n\t\tResponses: []alterconfigs.ResponseResponses{\n\t\t\t{\n\t\t\t\tErrorCode: 1,\n\t\t\t\tErrorMessage: \"foo\",\n\t\t\t\tResourceType: 1,\n\t\t\t\tResourceName: \"foo\",\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v1, &alterconfigs.Response{\n\t\tThrottleTimeMs: 500,\n\t\tResponses: []alterconfigs.ResponseResponses{\n\t\t\t{\n\t\t\t\tErrorCode: 1,\n\t\t\t\tErrorMessage: \"foo\",\n\t\t\t\tResourceType: 1,\n\t\t\t\tResourceName: \"foo\",\n\t\t\t},\n\t\t},\n\t})\n}\n" }, { "path": "protocol/alterpartitionreassignments/alterpartitionreassignments.go", "content": "package alterpartitionreassignments\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\n// Detailed API definition: https://kafka.apache.org/protocol#The_Messages_AlterPartitionReassignments\ntype Request struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v0,max=v0,tag\"`\n\n\tTimeoutMs int32 `kafka:\"min=v0,max=v0\"`\n\tTopics []RequestTopic `kafka:\"min=v0,max=v0\"`\n}\n\ntype RequestTopic struct {\n\tName string `kafka:\"min=v0,max=v0\"`\n\tPartitions []RequestPartition `kafka:\"min=v0,max=v0\"`\n}\n\ntype RequestPartition struct {\n\tPartitionIndex int32 `kafka:\"min=v0,max=v0\"`\n\tReplicas []int32 `kafka:\"min=v0,max=v0,nullable\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey {\n\treturn protocol.AlterPartitionReassignments\n}\n\nfunc (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, error) {\n\treturn cluster.Brokers[cluster.Controller], nil\n}\n\ntype Response struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v0,max=v0,tag\"`\n\n\tThrottleTimeMs int32 `kafka:\"min=v0,max=v0\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v0\"`\n\tErrorMessage string `kafka:\"min=v0,max=v0,nullable\"`\n\tResults []ResponseResult `kafka:\"min=v0,max=v0\"`\n}\n\ntype ResponseResult struct {\n\tName string `kafka:\"min=v0,max=v0\"`\n\tPartitions []ResponsePartition `kafka:\"min=v0,max=v0\"`\n}\n\ntype ResponsePartition struct {\n\tPartitionIndex int32 `kafka:\"min=v0,max=v0\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v0\"`\n\tErrorMessage string `kafka:\"min=v0,max=v0,nullable\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey {\n\treturn protocol.AlterPartitionReassignments\n}\n" }, { "path": "protocol/alterpartitionreassignments/alterpartitionreassignments_test.go", "content": "package alterpartitionreassignments_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/alterpartitionreassignments\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nconst (\n\tv0 = 0\n)\n\nfunc TestAlterPartitionReassignmentsRequest(t *testing.T) {\n\tprototest.TestRequest(t, v0, &alterpartitionreassignments.Request{\n\t\tTimeoutMs: 1,\n\t\tTopics: []alterpartitionreassignments.RequestTopic{\n\t\t\t{\n\t\t\t\tName: \"topic-1\",\n\t\t\t\tPartitions: []alterpartitionreassignments.RequestPartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartitionIndex: 1,\n\t\t\t\t\t\tReplicas: []int32{1, 2, 3},\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tPartitionIndex: 2,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n}\n\nfunc TestAlterPartitionReassignmentsResponse(t *testing.T) {\n\tprototest.TestResponse(t, v0, &alterpartitionreassignments.Response{\n\t\tErrorCode: 1,\n\t\tErrorMessage: \"error\",\n\t\tThrottleTimeMs: 1,\n\t\tResults: []alterpartitionreassignments.ResponseResult{\n\t\t\t{\n\t\t\t\tName: \"topic-1\",\n\t\t\t\tPartitions: []alterpartitionreassignments.ResponsePartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartitionIndex: 1,\n\t\t\t\t\t\tErrorMessage: \"error\",\n\t\t\t\t\t\tErrorCode: 1,\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tPartitionIndex: 2,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n}\n" }, { "path": "protocol/alteruserscramcredentials/alteruserscramcredentials.go", "content": "package alteruserscramcredentials\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\t// We need at least one tagged field to indicate that v2+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v0,max=v0,tag\"`\n\n\tDeletions []RequestUserScramCredentialsDeletion `kafka:\"min=v0,max=v0\"`\n\tUpsertions []RequestUserScramCredentialsUpsertion `kafka:\"min=v0,max=v0\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.AlterUserScramCredentials }\n\nfunc (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, error) {\n\treturn cluster.Brokers[cluster.Controller], nil\n}\n\ntype RequestUserScramCredentialsDeletion struct {\n\t// We need at least one tagged field to indicate that v2+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v0,max=v0,tag\"`\n\n\tName string `kafka:\"min=v0,max=v0,compact\"`\n\tMechanism int8 `kafka:\"min=v0,max=v0\"`\n}\n\ntype RequestUserScramCredentialsUpsertion struct {\n\t// We need at least one tagged field to indicate that v2+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v0,max=v0,tag\"`\n\n\tName string `kafka:\"min=v0,max=v0,compact\"`\n\tMechanism int8 `kafka:\"min=v0,max=v0\"`\n\tIterations int32 `kafka:\"min=v0,max=v0\"`\n\tSalt []byte `kafka:\"min=v0,max=v0,compact\"`\n\tSaltedPassword []byte `kafka:\"min=v0,max=v0,compact\"`\n}\n\ntype Response struct {\n\t// We need at least one tagged field to indicate that v2+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v0,max=v0,tag\"`\n\n\tThrottleTimeMs int32 `kafka:\"min=v0,max=v0\"`\n\tResults []ResponseUserScramCredentials `kafka:\"min=v0,max=v0\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.AlterUserScramCredentials }\n\ntype ResponseUserScramCredentials struct {\n\t// We need at least one tagged field to indicate that v2+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v0,max=v0,tag\"`\n\n\tUser string `kafka:\"min=v0,max=v0,compact\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v0\"`\n\tErrorMessage string `kafka:\"min=v0,max=v0,nullable\"`\n}\n\nvar _ protocol.BrokerMessage = (*Request)(nil)\n" }, { "path": "protocol/alteruserscramcredentials/alteruserscramcredentials_test.go", "content": "package alteruserscramcredentials_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/alteruserscramcredentials\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nconst (\n\tv0 = 0\n)\n\nfunc TestAlterUserScramCredentialsRequest(t *testing.T) {\n\tprototest.TestRequest(t, v0, &alteruserscramcredentials.Request{\n\t\tDeletions: []alteruserscramcredentials.RequestUserScramCredentialsDeletion{\n\t\t\t{\n\t\t\t\tName: \"foo-1\",\n\t\t\t\tMechanism: 1,\n\t\t\t},\n\t\t},\n\t\tUpsertions: []alteruserscramcredentials.RequestUserScramCredentialsUpsertion{\n\t\t\t{\n\t\t\t\tName: \"foo-2\",\n\t\t\t\tMechanism: 2,\n\t\t\t\tIterations: 15000,\n\t\t\t\tSalt: []byte(\"my-salt\"),\n\t\t\t\tSaltedPassword: []byte(\"my-salted-password\"),\n\t\t\t},\n\t\t},\n\t})\n}\n\nfunc TestAlterUserScramCredentialsResponse(t *testing.T) {\n\tprototest.TestResponse(t, v0, &alteruserscramcredentials.Response{\n\t\tThrottleTimeMs: 500,\n\t\tResults: []alteruserscramcredentials.ResponseUserScramCredentials{\n\t\t\t{\n\t\t\t\tUser: \"foo\",\n\t\t\t\tErrorCode: 1,\n\t\t\t\tErrorMessage: \"foo-error\",\n\t\t\t},\n\t\t},\n\t})\n}\n" }, { "path": "protocol/apiversions/apiversions.go", "content": "package apiversions\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\t_ struct{} `kafka:\"min=v0,max=v2\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.ApiVersions }\n\ntype Response struct {\n\tErrorCode int16 `kafka:\"min=v0,max=v2\"`\n\tApiKeys []ApiKeyResponse `kafka:\"min=v0,max=v2\"`\n\tThrottleTimeMs int32 `kafka:\"min=v1,max=v2\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.ApiVersions }\n\ntype ApiKeyResponse struct {\n\tApiKey int16 `kafka:\"min=v0,max=v2\"`\n\tMinVersion int16 `kafka:\"min=v0,max=v2\"`\n\tMaxVersion int16 `kafka:\"min=v0,max=v2\"`\n}\n" }, { "path": "protocol/apiversions/apiversions_test.go", "content": "package apiversions_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/apiversions\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nconst (\n\tv0 = 0\n\tv1 = 1\n\tv2 = 2\n)\n\nfunc TestApiversionsRequest(t *testing.T) {\n\tprototest.TestRequest(t, v0, &apiversions.Request{})\n\n\tprototest.TestRequest(t, v1, &apiversions.Request{})\n\n\tprototest.TestRequest(t, v2, &apiversions.Request{})\n}\n\nfunc TestApiversionsResponse(t *testing.T) {\n\tprototest.TestResponse(t, v0, &apiversions.Response{\n\t\tErrorCode: 0,\n\t\tApiKeys: []apiversions.ApiKeyResponse{\n\t\t\t{\n\t\t\t\tApiKey: 0,\n\t\t\t\tMinVersion: 0,\n\t\t\t\tMaxVersion: 2,\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v1, &apiversions.Response{\n\t\tErrorCode: 0,\n\t\tApiKeys: []apiversions.ApiKeyResponse{\n\t\t\t{\n\t\t\t\tApiKey: 0,\n\t\t\t\tMinVersion: 0,\n\t\t\t\tMaxVersion: 2,\n\t\t\t},\n\t\t},\n\t\tThrottleTimeMs: 10,\n\t})\n\n\tprototest.TestResponse(t, v2, &apiversions.Response{\n\t\tErrorCode: 0,\n\t\tApiKeys: []apiversions.ApiKeyResponse{\n\t\t\t{\n\t\t\t\tApiKey: 0,\n\t\t\t\tMinVersion: 0,\n\t\t\t\tMaxVersion: 2,\n\t\t\t},\n\t\t},\n\t\tThrottleTimeMs: 50,\n\t})\n}\n" }, { "path": "protocol/buffer.go", "content": "package protocol\n\nimport (\n\t\"bytes\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"sync\"\n\t\"sync/atomic\"\n)\n\n// Bytes is an interface implemented by types that represent immutable\n// sequences of bytes.\n//\n// Bytes values are used to abstract the location where record keys and\n// values are read from (e.g. in-memory buffers, network sockets, files).\n//\n// The Close method should be called to release resources held by the object\n// when the program is done with it.\n//\n// Bytes values are generally not safe to use concurrently from multiple\n// goroutines.\ntype Bytes interface {\n\tio.ReadCloser\n\t// Returns the number of bytes remaining to be read from the payload.\n\tLen() int\n}\n\n// NewBytes constructs a Bytes value from b.\n//\n// The returned value references b, it does not make a copy of the backing\n// array.\n//\n// If b is nil, nil is returned to represent a null BYTES value in the kafka\n// protocol.\nfunc NewBytes(b []byte) Bytes {\n\tif b == nil {\n\t\treturn nil\n\t}\n\tr := new(bytesReader)\n\tr.Reset(b)\n\treturn r\n}\n\n// ReadAll is similar to ioutil.ReadAll, but it takes advantage of knowing the\n// length of b to minimize the memory footprint.\n//\n// The function returns a nil slice if b is nil.\nfunc ReadAll(b Bytes) ([]byte, error) {\n\tif b == nil {\n\t\treturn nil, nil\n\t}\n\ts := make([]byte, b.Len())\n\t_, err := io.ReadFull(b, s)\n\treturn s, err\n}\n\ntype bytesReader struct{ bytes.Reader }\n\nfunc (*bytesReader) Close() error { return nil }\n\ntype refCount uintptr\n\nfunc (rc *refCount) ref() { atomic.AddUintptr((*uintptr)(rc), 1) }\n\nfunc (rc *refCount) unref(onZero func()) {\n\tif atomic.AddUintptr((*uintptr)(rc), ^uintptr(0)) == 0 {\n\t\tonZero()\n\t}\n}\n\nconst (\n\t// Size of the memory buffer for a single page. We use a farily\n\t// large size here (64 KiB) because batches exchanged with kafka\n\t// tend to be multiple kilobytes in size, sometimes hundreds.\n\t// Using large pages amortizes the overhead of the page metadata\n\t// and algorithms to manage the pages.\n\tpageSize = 65536\n)\n\ntype page struct {\n\trefc refCount\n\toffset int64\n\tlength int\n\tbuffer *[pageSize]byte\n}\n\nfunc newPage(offset int64) *page {\n\tp, _ := pagePool.Get().(*page)\n\tif p != nil {\n\t\tp.offset = offset\n\t\tp.length = 0\n\t\tp.ref()\n\t} else {\n\t\tp = &page{\n\t\t\trefc: 1,\n\t\t\toffset: offset,\n\t\t\tbuffer: &[pageSize]byte{},\n\t\t}\n\t}\n\treturn p\n}\n\nfunc (p *page) ref() { p.refc.ref() }\n\nfunc (p *page) unref() { p.refc.unref(func() { pagePool.Put(p) }) }\n\nfunc (p *page) slice(begin, end int64) []byte {\n\ti, j := begin-p.offset, end-p.offset\n\n\tif i < 0 {\n\t\ti = 0\n\t} else if i > pageSize {\n\t\ti = pageSize\n\t}\n\n\tif j < 0 {\n\t\tj = 0\n\t} else if j > pageSize {\n\t\tj = pageSize\n\t}\n\n\tif i < j {\n\t\treturn p.buffer[i:j]\n\t}\n\n\treturn nil\n}\n\nfunc (p *page) Cap() int { return pageSize }\n\nfunc (p *page) Len() int { return p.length }\n\nfunc (p *page) Size() int64 { return int64(p.length) }\n\nfunc (p *page) Truncate(n int) {\n\tif n < p.length {\n\t\tp.length = n\n\t}\n}\n\nfunc (p *page) ReadAt(b []byte, off int64) (int, error) {\n\tif off -= p.offset; off < 0 || off > pageSize {\n\t\tpanic(\"offset out of range\")\n\t}\n\tif off > int64(p.length) {\n\t\treturn 0, nil\n\t}\n\treturn copy(b, p.buffer[off:p.length]), nil\n}\n\nfunc (p *page) ReadFrom(r io.Reader) (int64, error) {\n\tn, err := io.ReadFull(r, p.buffer[p.length:])\n\tif errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {\n\t\terr = nil\n\t}\n\tp.length += n\n\treturn int64(n), err\n}\n\nfunc (p *page) WriteAt(b []byte, off int64) (int, error) {\n\tif off -= p.offset; off < 0 || off > pageSize {\n\t\tpanic(\"offset out of range\")\n\t}\n\tn := copy(p.buffer[off:], b)\n\tif end := int(off) + n; end > p.length {\n\t\tp.length = end\n\t}\n\treturn n, nil\n}\n\nfunc (p *page) Write(b []byte) (int, error) {\n\treturn p.WriteAt(b, p.offset+int64(p.length))\n}\n\nvar (\n\t_ io.ReaderAt = (*page)(nil)\n\t_ io.ReaderFrom = (*page)(nil)\n\t_ io.Writer = (*page)(nil)\n\t_ io.WriterAt = (*page)(nil)\n)\n\ntype pageBuffer struct {\n\trefc refCount\n\tpages contiguousPages\n\tlength int\n\tcursor int\n}\n\nfunc newPageBuffer() *pageBuffer {\n\tb, _ := pageBufferPool.Get().(*pageBuffer)\n\tif b != nil {\n\t\tb.cursor = 0\n\t\tb.refc.ref()\n\t} else {\n\t\tb = &pageBuffer{\n\t\t\trefc: 1,\n\t\t\tpages: make(contiguousPages, 0, 16),\n\t\t}\n\t}\n\treturn b\n}\n\nfunc (pb *pageBuffer) refTo(ref *pageRef, begin, end int64) {\n\tlength := end - begin\n\n\tif length > math.MaxUint32 {\n\t\tpanic(\"reference to contiguous buffer pages exceeds the maximum size of 4 GB\")\n\t}\n\n\tref.pages = append(ref.buffer[:0], pb.pages.slice(begin, end)...)\n\tref.pages.ref()\n\tref.offset = begin\n\tref.length = uint32(length)\n}\n\nfunc (pb *pageBuffer) ref(begin, end int64) *pageRef {\n\tref := new(pageRef)\n\tpb.refTo(ref, begin, end)\n\treturn ref\n}\n\nfunc (pb *pageBuffer) unref() {\n\tpb.refc.unref(func() {\n\t\tpb.pages.unref()\n\t\tpb.pages.clear()\n\t\tpb.pages = pb.pages[:0]\n\t\tpb.length = 0\n\t\tpageBufferPool.Put(pb)\n\t})\n}\n\nfunc (pb *pageBuffer) newPage() *page {\n\treturn newPage(int64(pb.length))\n}\n\nfunc (pb *pageBuffer) Close() error {\n\treturn nil\n}\n\nfunc (pb *pageBuffer) Len() int {\n\treturn pb.length - pb.cursor\n}\n\nfunc (pb *pageBuffer) Size() int64 {\n\treturn int64(pb.length)\n}\n\nfunc (pb *pageBuffer) Discard(n int) (int, error) {\n\tremain := pb.length - pb.cursor\n\tif remain < n {\n\t\tn = remain\n\t}\n\tpb.cursor += n\n\treturn n, nil\n}\n\nfunc (pb *pageBuffer) Truncate(n int) {\n\tif n < pb.length {\n\t\tpb.length = n\n\n\t\tif n < pb.cursor {\n\t\t\tpb.cursor = n\n\t\t}\n\n\t\tfor i := range pb.pages {\n\t\t\tif p := pb.pages[i]; p.length <= n {\n\t\t\t\tn -= p.length\n\t\t\t} else {\n\t\t\t\tif n > 0 {\n\t\t\t\t\tpb.pages[i].Truncate(n)\n\t\t\t\t\ti++\n\t\t\t\t}\n\t\t\t\tpb.pages[i:].unref()\n\t\t\t\tpb.pages[i:].clear()\n\t\t\t\tpb.pages = pb.pages[:i]\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (pb *pageBuffer) Seek(offset int64, whence int) (int64, error) {\n\tc, err := seek(int64(pb.cursor), int64(pb.length), offset, whence)\n\tif err != nil {\n\t\treturn -1, err\n\t}\n\tpb.cursor = int(c)\n\treturn c, nil\n}\n\nfunc (pb *pageBuffer) ReadByte() (byte, error) {\n\tb := [1]byte{}\n\t_, err := pb.Read(b[:])\n\treturn b[0], err\n}\n\nfunc (pb *pageBuffer) Read(b []byte) (int, error) {\n\tif pb.cursor >= pb.length {\n\t\treturn 0, io.EOF\n\t}\n\tn, err := pb.ReadAt(b, int64(pb.cursor))\n\tpb.cursor += n\n\treturn n, err\n}\n\nfunc (pb *pageBuffer) ReadAt(b []byte, off int64) (int, error) {\n\treturn pb.pages.ReadAt(b, off)\n}\n\nfunc (pb *pageBuffer) ReadFrom(r io.Reader) (int64, error) {\n\tif len(pb.pages) == 0 {\n\t\tpb.pages = append(pb.pages, pb.newPage())\n\t}\n\n\trn := int64(0)\n\n\tfor {\n\t\ttail := pb.pages[len(pb.pages)-1]\n\t\tfree := tail.Cap() - tail.Len()\n\n\t\tif free == 0 {\n\t\t\ttail = pb.newPage()\n\t\t\tfree = pageSize\n\t\t\tpb.pages = append(pb.pages, tail)\n\t\t}\n\n\t\tn, err := tail.ReadFrom(r)\n\t\tpb.length += int(n)\n\t\trn += n\n\t\tif n < int64(free) {\n\t\t\treturn rn, err\n\t\t}\n\t}\n}\n\nfunc (pb *pageBuffer) WriteString(s string) (int, error) {\n\treturn pb.Write([]byte(s))\n}\n\nfunc (pb *pageBuffer) Write(b []byte) (int, error) {\n\twn := len(b)\n\tif wn == 0 {\n\t\treturn 0, nil\n\t}\n\n\tif len(pb.pages) == 0 {\n\t\tpb.pages = append(pb.pages, pb.newPage())\n\t}\n\n\tfor len(b) != 0 {\n\t\ttail := pb.pages[len(pb.pages)-1]\n\t\tfree := tail.Cap() - tail.Len()\n\n\t\tif len(b) <= free {\n\t\t\ttail.Write(b)\n\t\t\tpb.length += len(b)\n\t\t\tbreak\n\t\t}\n\n\t\ttail.Write(b[:free])\n\t\tb = b[free:]\n\n\t\tpb.length += free\n\t\tpb.pages = append(pb.pages, pb.newPage())\n\t}\n\n\treturn wn, nil\n}\n\nfunc (pb *pageBuffer) WriteAt(b []byte, off int64) (int, error) {\n\tn, err := pb.pages.WriteAt(b, off)\n\tif err != nil {\n\t\treturn n, err\n\t}\n\tif n < len(b) {\n\t\tpb.Write(b[n:])\n\t}\n\treturn len(b), nil\n}\n\nfunc (pb *pageBuffer) WriteTo(w io.Writer) (int64, error) {\n\tvar wn int\n\tvar err error\n\tpb.pages.scan(int64(pb.cursor), int64(pb.length), func(b []byte) bool {\n\t\tvar n int\n\t\tn, err = w.Write(b)\n\t\twn += n\n\t\treturn err == nil\n\t})\n\tpb.cursor += wn\n\treturn int64(wn), err\n}\n\nvar (\n\t_ io.ReaderAt = (*pageBuffer)(nil)\n\t_ io.ReaderFrom = (*pageBuffer)(nil)\n\t_ io.StringWriter = (*pageBuffer)(nil)\n\t_ io.Writer = (*pageBuffer)(nil)\n\t_ io.WriterAt = (*pageBuffer)(nil)\n\t_ io.WriterTo = (*pageBuffer)(nil)\n\n\tpagePool sync.Pool\n\tpageBufferPool sync.Pool\n)\n\ntype contiguousPages []*page\n\nfunc (pages contiguousPages) ref() {\n\tfor _, p := range pages {\n\t\tp.ref()\n\t}\n}\n\nfunc (pages contiguousPages) unref() {\n\tfor _, p := range pages {\n\t\tp.unref()\n\t}\n}\n\nfunc (pages contiguousPages) clear() {\n\tfor i := range pages {\n\t\tpages[i] = nil\n\t}\n}\n\nfunc (pages contiguousPages) ReadAt(b []byte, off int64) (int, error) {\n\trn := 0\n\n\tfor _, p := range pages.slice(off, off+int64(len(b))) {\n\t\tn, _ := p.ReadAt(b, off)\n\t\tb = b[n:]\n\t\trn += n\n\t\toff += int64(n)\n\t}\n\n\treturn rn, nil\n}\n\nfunc (pages contiguousPages) WriteAt(b []byte, off int64) (int, error) {\n\twn := 0\n\n\tfor _, p := range pages.slice(off, off+int64(len(b))) {\n\t\tn, _ := p.WriteAt(b, off)\n\t\tb = b[n:]\n\t\twn += n\n\t\toff += int64(n)\n\t}\n\n\treturn wn, nil\n}\n\nfunc (pages contiguousPages) slice(begin, end int64) contiguousPages {\n\ti := pages.indexOf(begin)\n\tj := pages.indexOf(end)\n\tif j < len(pages) {\n\t\tj++\n\t}\n\treturn pages[i:j]\n}\n\nfunc (pages contiguousPages) indexOf(offset int64) int {\n\tif len(pages) == 0 {\n\t\treturn 0\n\t}\n\treturn int((offset - pages[0].offset) / pageSize)\n}\n\nfunc (pages contiguousPages) scan(begin, end int64, f func([]byte) bool) {\n\tfor _, p := range pages.slice(begin, end) {\n\t\tif !f(p.slice(begin, end)) {\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nvar (\n\t_ io.ReaderAt = contiguousPages{}\n\t_ io.WriterAt = contiguousPages{}\n)\n\ntype pageRef struct {\n\tbuffer [2]*page\n\tpages contiguousPages\n\toffset int64\n\tcursor int64\n\tlength uint32\n\tonce uint32\n}\n\nfunc (ref *pageRef) unref() {\n\tif atomic.CompareAndSwapUint32(&ref.once, 0, 1) {\n\t\tref.pages.unref()\n\t\tref.pages.clear()\n\t\tref.pages = nil\n\t\tref.offset = 0\n\t\tref.cursor = 0\n\t\tref.length = 0\n\t}\n}\n\nfunc (ref *pageRef) Len() int { return int(ref.Size() - ref.cursor) }\n\nfunc (ref *pageRef) Size() int64 { return int64(ref.length) }\n\nfunc (ref *pageRef) Close() error { ref.unref(); return nil }\n\nfunc (ref *pageRef) String() string {\n\treturn fmt.Sprintf(\"[offset=%d cursor=%d length=%d]\", ref.offset, ref.cursor, ref.length)\n}\n\nfunc (ref *pageRef) Seek(offset int64, whence int) (int64, error) {\n\tc, err := seek(ref.cursor, int64(ref.length), offset, whence)\n\tif err != nil {\n\t\treturn -1, err\n\t}\n\tref.cursor = c\n\treturn c, nil\n}\n\nfunc (ref *pageRef) ReadByte() (byte, error) {\n\tvar c byte\n\tvar ok bool\n\tref.scan(ref.cursor, func(b []byte) bool {\n\t\tc, ok = b[0], true\n\t\treturn false\n\t})\n\tif ok {\n\t\tref.cursor++\n\t} else {\n\t\treturn 0, io.EOF\n\t}\n\treturn c, nil\n}\n\nfunc (ref *pageRef) Read(b []byte) (int, error) {\n\tif ref.cursor >= int64(ref.length) {\n\t\treturn 0, io.EOF\n\t}\n\tn, err := ref.ReadAt(b, ref.cursor)\n\tref.cursor += int64(n)\n\treturn n, err\n}\n\nfunc (ref *pageRef) ReadAt(b []byte, off int64) (int, error) {\n\tlimit := ref.offset + int64(ref.length)\n\toff += ref.offset\n\n\tif off >= limit {\n\t\treturn 0, io.EOF\n\t}\n\n\tif off+int64(len(b)) > limit {\n\t\tb = b[:limit-off]\n\t}\n\n\tif len(b) == 0 {\n\t\treturn 0, nil\n\t}\n\n\tn, err := ref.pages.ReadAt(b, off)\n\tif n == 0 && err == nil {\n\t\terr = io.EOF\n\t}\n\treturn n, err\n}\n\nfunc (ref *pageRef) WriteTo(w io.Writer) (wn int64, err error) {\n\tref.scan(ref.cursor, func(b []byte) bool {\n\t\tvar n int\n\t\tn, err = w.Write(b)\n\t\twn += int64(n)\n\t\treturn err == nil\n\t})\n\tref.cursor += wn\n\treturn\n}\n\nfunc (ref *pageRef) scan(off int64, f func([]byte) bool) {\n\tbegin := ref.offset + off\n\tend := ref.offset + int64(ref.length)\n\tref.pages.scan(begin, end, f)\n}\n\nvar (\n\t_ io.Closer = (*pageRef)(nil)\n\t_ io.Seeker = (*pageRef)(nil)\n\t_ io.Reader = (*pageRef)(nil)\n\t_ io.ReaderAt = (*pageRef)(nil)\n\t_ io.WriterTo = (*pageRef)(nil)\n)\n\ntype pageRefAllocator struct {\n\trefs []pageRef\n\thead int\n\tsize int\n}\n\nfunc (a *pageRefAllocator) newPageRef() *pageRef {\n\tif a.head == len(a.refs) {\n\t\ta.refs = make([]pageRef, a.size)\n\t\ta.head = 0\n\t}\n\tref := &a.refs[a.head]\n\ta.head++\n\treturn ref\n}\n\nfunc seek(cursor, limit, offset int64, whence int) (int64, error) {\n\tswitch whence {\n\tcase io.SeekStart:\n\t\t// absolute offset\n\tcase io.SeekCurrent:\n\t\toffset = cursor + offset\n\tcase io.SeekEnd:\n\t\toffset = limit - offset\n\tdefault:\n\t\treturn -1, fmt.Errorf(\"seek: invalid whence value: %d\", whence)\n\t}\n\tif offset < 0 {\n\t\toffset = 0\n\t}\n\tif offset > limit {\n\t\toffset = limit\n\t}\n\treturn offset, nil\n}\n\nfunc closeBytes(b Bytes) {\n\tif b != nil {\n\t\tb.Close()\n\t}\n}\n" }, { "path": "protocol/buffer_test.go", "content": "package protocol\n\nimport (\n\t\"bytes\"\n\t\"io\"\n\t\"io/ioutil\"\n\t\"testing\"\n)\n\nfunc TestPageBufferWriteReadSeek(t *testing.T) {\n\tbuffer := newPageBuffer()\n\tdefer buffer.unref()\n\n\tio.WriteString(buffer, \"Hello World!\")\n\n\tif n := buffer.Size(); n != 12 {\n\t\tt.Fatal(\"invalid size:\", n)\n\t}\n\n\tfor i := 0; i < 3; i++ {\n\t\tif n := buffer.Len(); n != 12 {\n\t\t\tt.Fatal(\"invalid length before read:\", n)\n\t\t}\n\n\t\tb, err := ioutil.ReadAll(buffer)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tif n := buffer.Len(); n != 0 {\n\t\t\tt.Fatal(\"invalid length after read:\", n)\n\t\t}\n\n\t\tif string(b) != \"Hello World!\" {\n\t\t\tt.Fatalf(\"invalid content after read #%d: %q\", i, b)\n\t\t}\n\n\t\toffset, err := buffer.Seek(0, io.SeekStart)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif offset != 0 {\n\t\t\tt.Fatalf(\"invalid offset after seek #%d: %d\", i, offset)\n\t\t}\n\t}\n}\n\nfunc TestPageRefWriteReadSeek(t *testing.T) {\n\tbuffer := newPageBuffer()\n\tdefer buffer.unref()\n\n\tio.WriteString(buffer, \"Hello World!\")\n\n\tref := buffer.ref(1, 11)\n\tdefer ref.unref()\n\n\tif n := ref.Size(); n != 10 {\n\t\tt.Fatal(\"invalid size:\", n)\n\t}\n\n\tfor i := 0; i < 3; i++ {\n\t\tif n := ref.Len(); n != 10 {\n\t\t\tt.Fatal(\"invalid length before read:\", n)\n\t\t}\n\n\t\tb, err := ioutil.ReadAll(ref)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tif n := ref.Len(); n != 0 {\n\t\t\tt.Fatal(\"invalid length after read:\", n)\n\t\t}\n\n\t\tif string(b) != \"ello World\" {\n\t\t\tt.Fatalf(\"invalid content after read #%d: %q\", i, b)\n\t\t}\n\n\t\toffset, err := ref.Seek(0, io.SeekStart)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif offset != 0 {\n\t\t\tt.Fatalf(\"invalid offset after seek #%d: %d\", i, offset)\n\t\t}\n\t}\n}\n\nfunc TestPageRefReadByte(t *testing.T) {\n\tbuffer := newPageBuffer()\n\tdefer buffer.unref()\n\n\tcontent := bytes.Repeat([]byte(\"1234567890\"), 10e3)\n\tbuffer.Write(content)\n\n\tref := buffer.ref(0, buffer.Size())\n\tdefer ref.unref()\n\n\tfor i, c := range content {\n\t\tb, err := ref.ReadByte()\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif b != c {\n\t\t\tt.Fatalf(\"byte at offset %d mismatch, expected '%c' but got '%c'\", i, c, b)\n\t\t}\n\t}\n}\n" }, { "path": "protocol/cluster.go", "content": "package protocol\n\nimport (\n\t\"fmt\"\n\t\"sort\"\n\t\"strings\"\n\t\"text/tabwriter\"\n)\n\ntype Cluster struct {\n\tClusterID string\n\tController int32\n\tBrokers map[int32]Broker\n\tTopics map[string]Topic\n}\n\nfunc (c Cluster) BrokerIDs() []int32 {\n\tbrokerIDs := make([]int32, 0, len(c.Brokers))\n\tfor id := range c.Brokers {\n\t\tbrokerIDs = append(brokerIDs, id)\n\t}\n\tsort.Slice(brokerIDs, func(i, j int) bool {\n\t\treturn brokerIDs[i] < brokerIDs[j]\n\t})\n\treturn brokerIDs\n}\n\nfunc (c Cluster) TopicNames() []string {\n\ttopicNames := make([]string, 0, len(c.Topics))\n\tfor name := range c.Topics {\n\t\ttopicNames = append(topicNames, name)\n\t}\n\tsort.Strings(topicNames)\n\treturn topicNames\n}\n\nfunc (c Cluster) IsZero() bool {\n\treturn c.ClusterID == \"\" && c.Controller == 0 && len(c.Brokers) == 0 && len(c.Topics) == 0\n}\n\nfunc (c Cluster) Format(w fmt.State, _ rune) {\n\ttw := new(tabwriter.Writer)\n\tfmt.Fprintf(w, \"CLUSTER: %q\\n\\n\", c.ClusterID)\n\n\ttw.Init(w, 0, 8, 2, ' ', 0)\n\tfmt.Fprint(tw, \" BROKER\\tHOST\\tPORT\\tRACK\\tCONTROLLER\\n\")\n\n\tfor _, id := range c.BrokerIDs() {\n\t\tbroker := c.Brokers[id]\n\t\tfmt.Fprintf(tw, \" %d\\t%s\\t%d\\t%s\\t%t\\n\", broker.ID, broker.Host, broker.Port, broker.Rack, broker.ID == c.Controller)\n\t}\n\n\ttw.Flush()\n\tfmt.Fprintln(w)\n\n\ttw.Init(w, 0, 8, 2, ' ', 0)\n\tfmt.Fprint(tw, \" TOPIC\\tPARTITIONS\\tBROKERS\\n\")\n\ttopicNames := c.TopicNames()\n\tbrokers := make(map[int32]struct{}, len(c.Brokers))\n\tbrokerIDs := make([]int32, 0, len(c.Brokers))\n\n\tfor _, name := range topicNames {\n\t\ttopic := c.Topics[name]\n\n\t\tfor _, p := range topic.Partitions {\n\t\t\tfor _, id := range p.Replicas {\n\t\t\t\tbrokers[id] = struct{}{}\n\t\t\t}\n\t\t}\n\n\t\tfor id := range brokers {\n\t\t\tbrokerIDs = append(brokerIDs, id)\n\t\t}\n\n\t\tfmt.Fprintf(tw, \" %s\\t%d\\t%s\\n\", topic.Name, len(topic.Partitions), formatBrokerIDs(brokerIDs, -1))\n\n\t\tfor id := range brokers {\n\t\t\tdelete(brokers, id)\n\t\t}\n\n\t\tbrokerIDs = brokerIDs[:0]\n\t}\n\n\ttw.Flush()\n\tfmt.Fprintln(w)\n\n\tif w.Flag('+') {\n\t\tfor _, name := range topicNames {\n\t\t\tfmt.Fprintf(w, \" TOPIC: %q\\n\\n\", name)\n\n\t\t\ttw.Init(w, 0, 8, 2, ' ', 0)\n\t\t\tfmt.Fprint(tw, \" PARTITION\\tREPLICAS\\tISR\\tOFFLINE\\n\")\n\n\t\t\tfor _, p := range c.Topics[name].Partitions {\n\t\t\t\tfmt.Fprintf(tw, \" %d\\t%s\\t%s\\t%s\\n\", p.ID,\n\t\t\t\t\tformatBrokerIDs(p.Replicas, -1),\n\t\t\t\t\tformatBrokerIDs(p.ISR, p.Leader),\n\t\t\t\t\tformatBrokerIDs(p.Offline, -1),\n\t\t\t\t)\n\t\t\t}\n\n\t\t\ttw.Flush()\n\t\t\tfmt.Fprintln(w)\n\t\t}\n\t}\n}\n\nfunc formatBrokerIDs(brokerIDs []int32, leader int32) string {\n\tif len(brokerIDs) == 0 {\n\t\treturn \"\"\n\t}\n\n\tif len(brokerIDs) == 1 {\n\t\treturn itoa(brokerIDs[0])\n\t}\n\n\tsort.Slice(brokerIDs, func(i, j int) bool {\n\t\tid1 := brokerIDs[i]\n\t\tid2 := brokerIDs[j]\n\n\t\tif id1 == leader {\n\t\t\treturn true\n\t\t}\n\n\t\tif id2 == leader {\n\t\t\treturn false\n\t\t}\n\n\t\treturn id1 < id2\n\t})\n\n\tbrokerNames := make([]string, len(brokerIDs))\n\n\tfor i, id := range brokerIDs {\n\t\tbrokerNames[i] = itoa(id)\n\t}\n\n\treturn strings.Join(brokerNames, \",\")\n}\n\nvar (\n\t_ fmt.Formatter = Cluster{}\n)\n" }, { "path": "protocol/conn.go", "content": "package protocol\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"net\"\n\t\"sync/atomic\"\n\t\"time\"\n)\n\ntype Conn struct {\n\tbuffer *bufio.Reader\n\tconn net.Conn\n\tclientID string\n\tidgen int32\n\tversions atomic.Value // map[ApiKey]int16\n}\n\nfunc NewConn(conn net.Conn, clientID string) *Conn {\n\treturn &Conn{\n\t\tbuffer: bufio.NewReader(conn),\n\t\tconn: conn,\n\t\tclientID: clientID,\n\t}\n}\n\nfunc (c *Conn) String() string {\n\treturn fmt.Sprintf(\"kafka://%s@%s->%s\", c.clientID, c.LocalAddr(), c.RemoteAddr())\n}\n\nfunc (c *Conn) Close() error {\n\treturn c.conn.Close()\n}\n\nfunc (c *Conn) Discard(n int) (int, error) {\n\treturn c.buffer.Discard(n)\n}\n\nfunc (c *Conn) Peek(n int) ([]byte, error) {\n\treturn c.buffer.Peek(n)\n}\n\nfunc (c *Conn) Read(b []byte) (int, error) {\n\treturn c.buffer.Read(b)\n}\n\nfunc (c *Conn) Write(b []byte) (int, error) {\n\treturn c.conn.Write(b)\n}\n\nfunc (c *Conn) LocalAddr() net.Addr {\n\treturn c.conn.LocalAddr()\n}\n\nfunc (c *Conn) RemoteAddr() net.Addr {\n\treturn c.conn.RemoteAddr()\n}\n\nfunc (c *Conn) SetDeadline(t time.Time) error {\n\treturn c.conn.SetDeadline(t)\n}\n\nfunc (c *Conn) SetReadDeadline(t time.Time) error {\n\treturn c.conn.SetReadDeadline(t)\n}\n\nfunc (c *Conn) SetWriteDeadline(t time.Time) error {\n\treturn c.conn.SetWriteDeadline(t)\n}\n\nfunc (c *Conn) SetVersions(versions map[ApiKey]int16) {\n\tconnVersions := make(map[ApiKey]int16, len(versions))\n\n\tfor k, v := range versions {\n\t\tconnVersions[k] = v\n\t}\n\n\tc.versions.Store(connVersions)\n}\n\nfunc (c *Conn) RoundTrip(msg Message) (Message, error) {\n\tcorrelationID := atomic.AddInt32(&c.idgen, +1)\n\tversions, _ := c.versions.Load().(map[ApiKey]int16)\n\tapiVersion := versions[msg.ApiKey()]\n\n\tif p, _ := msg.(PreparedMessage); p != nil {\n\t\tp.Prepare(apiVersion)\n\t}\n\n\tif raw, ok := msg.(RawExchanger); ok && raw.Required(versions) {\n\t\treturn raw.RawExchange(c)\n\t}\n\n\treturn RoundTrip(c, apiVersion, correlationID, c.clientID, msg)\n}\n\nvar (\n\t_ net.Conn = (*Conn)(nil)\n\t_ bufferedReader = (*Conn)(nil)\n)\n" }, { "path": "protocol/consumer/consumer.go", "content": "package consumer\n\nconst MaxVersionSupported = 1\n\ntype Subscription struct {\n\tVersion int16 `kafka:\"min=v0,max=v1\"`\n\tTopics []string `kafka:\"min=v0,max=v1\"`\n\tUserData []byte `kafka:\"min=v0,max=v1,nullable\"`\n\tOwnedPartitions []TopicPartition `kafka:\"min=v1,max=v1\"`\n}\n\ntype Assignment struct {\n\tVersion int16 `kafka:\"min=v0,max=v1\"`\n\tAssignedPartitions []TopicPartition `kafka:\"min=v0,max=v1\"`\n\tUserData []byte `kafka:\"min=v0,max=v1,nullable\"`\n}\n\ntype TopicPartition struct {\n\tTopic string `kafka:\"min=v0,max=v1\"`\n\tPartitions []int32 `kafka:\"min=v0,max=v1\"`\n}\n" }, { "path": "protocol/consumer/consumer_test.go", "content": "package consumer_test\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n\t\"github.com/segmentio/kafka-go/protocol/consumer\"\n)\n\nfunc TestSubscription(t *testing.T) {\n\tsubscription := consumer.Subscription{\n\t\tTopics: []string{\"topic-1\", \"topic-2\"},\n\t\tUserData: []byte(\"user-data\"),\n\t\tOwnedPartitions: []consumer.TopicPartition{\n\t\t\t{\n\t\t\t\tTopic: \"topic-1\",\n\t\t\t\tPartitions: []int32{1, 2, 3},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor _, version := range []int16{1, 0} {\n\t\tif version == 0 {\n\t\t\tsubscription.OwnedPartitions = nil\n\t\t}\n\t\tdata, err := protocol.Marshal(version, subscription)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tvar gotSubscription consumer.Subscription\n\t\terr = protocol.Unmarshal(data, version, &gotSubscription)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif !reflect.DeepEqual(subscription, gotSubscription) {\n\t\t\tt.Fatalf(\"unexpected result after marshal/unmarshal \\nexpected\\n %#v\\ngot\\n %#v\", subscription, gotSubscription)\n\t\t}\n\t}\n}\n" }, { "path": "protocol/createacls/createacls.go", "content": "package createacls\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\t// We need at least one tagged field to indicate that v2+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v2,max=v3,tag\"`\n\n\tCreations []RequestACLs `kafka:\"min=v0,max=v3\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.CreateAcls }\n\nfunc (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, error) {\n\treturn cluster.Brokers[cluster.Controller], nil\n}\n\ntype RequestACLs struct {\n\t// We need at least one tagged field to indicate that v2+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v2,max=v3,tag\"`\n\n\tResourceType int8 `kafka:\"min=v0,max=v3\"`\n\tResourceName string `kafka:\"min=v0,max=v1|min=v2,max=v3,compact\"`\n\tResourcePatternType int8 `kafka:\"min=v1,max=v3\"`\n\tPrincipal string `kafka:\"min=v0,max=v1|min=v2,max=v3,compact\"`\n\tHost string `kafka:\"min=v0,max=v1|min=v2,max=v3,compact\"`\n\tOperation int8 `kafka:\"min=v0,max=v3\"`\n\tPermissionType int8 `kafka:\"min=v0,max=v3\"`\n}\n\ntype Response struct {\n\t// We need at least one tagged field to indicate that v2+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v2,max=v3,tag\"`\n\n\tThrottleTimeMs int32 `kafka:\"min=v0,max=v3\"`\n\tResults []ResponseACLs `kafka:\"min=v0,max=v3\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.CreateAcls }\n\ntype ResponseACLs struct {\n\t// We need at least one tagged field to indicate that v2+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v2,max=v3,tag\"`\n\n\tErrorCode int16 `kafka:\"min=v0,max=v3\"`\n\tErrorMessage string `kafka:\"min=v0,max=v1,nullable|min=v2,max=v3,nullable,compact\"`\n}\n\nvar _ protocol.BrokerMessage = (*Request)(nil)\n" }, { "path": "protocol/createacls/createacls_test.go", "content": "package createacls_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/createacls\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nconst (\n\tv0 = 0\n\tv1 = 1\n\tv2 = 2\n\tv3 = 3\n)\n\nfunc TestCreateACLsRequest(t *testing.T) {\n\tprototest.TestRequest(t, v0, &createacls.Request{\n\t\tCreations: []createacls.RequestACLs{\n\t\t\t{\n\t\t\t\tPrincipal: \"User:alice\",\n\t\t\t\tPermissionType: 3,\n\t\t\t\tOperation: 3,\n\t\t\t\tResourceType: 2,\n\t\t\t\tResourceName: \"fake-topic-for-alice\",\n\t\t\t\tHost: \"*\",\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestRequest(t, v1, &createacls.Request{\n\t\tCreations: []createacls.RequestACLs{\n\t\t\t{\n\t\t\t\tPrincipal: \"User:alice\",\n\t\t\t\tPermissionType: 3,\n\t\t\t\tOperation: 3,\n\t\t\t\tResourceType: 2,\n\t\t\t\tResourcePatternType: 3,\n\t\t\t\tResourceName: \"fake-topic-for-alice\",\n\t\t\t\tHost: \"*\",\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestRequest(t, v2, &createacls.Request{\n\t\tCreations: []createacls.RequestACLs{\n\t\t\t{\n\t\t\t\tPrincipal: \"User:alice\",\n\t\t\t\tPermissionType: 3,\n\t\t\t\tOperation: 3,\n\t\t\t\tResourceType: 2,\n\t\t\t\tResourcePatternType: 3,\n\t\t\t\tResourceName: \"fake-topic-for-alice\",\n\t\t\t\tHost: \"*\",\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestRequest(t, v3, &createacls.Request{\n\t\tCreations: []createacls.RequestACLs{\n\t\t\t{\n\t\t\t\tPrincipal: \"User:alice\",\n\t\t\t\tPermissionType: 3,\n\t\t\t\tOperation: 3,\n\t\t\t\tResourceType: 2,\n\t\t\t\tResourcePatternType: 3,\n\t\t\t\tResourceName: \"fake-topic-for-alice\",\n\t\t\t\tHost: \"*\",\n\t\t\t},\n\t\t},\n\t})\n}\n\nfunc TestCreateACLsResponse(t *testing.T) {\n\tprototest.TestResponse(t, v0, &createacls.Response{\n\t\tThrottleTimeMs: 1,\n\t\tResults: []createacls.ResponseACLs{\n\t\t\t{\n\t\t\t\tErrorCode: 1,\n\t\t\t\tErrorMessage: \"foo\",\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v1, &createacls.Response{\n\t\tThrottleTimeMs: 1,\n\t\tResults: []createacls.ResponseACLs{\n\t\t\t{\n\t\t\t\tErrorCode: 1,\n\t\t\t\tErrorMessage: \"foo\",\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v2, &createacls.Response{\n\t\tThrottleTimeMs: 1,\n\t\tResults: []createacls.ResponseACLs{\n\t\t\t{\n\t\t\t\tErrorCode: 1,\n\t\t\t\tErrorMessage: \"foo\",\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v3, &createacls.Response{\n\t\tThrottleTimeMs: 1,\n\t\tResults: []createacls.ResponseACLs{\n\t\t\t{\n\t\t\t\tErrorCode: 1,\n\t\t\t\tErrorMessage: \"foo\",\n\t\t\t},\n\t\t},\n\t})\n\n}\n" }, { "path": "protocol/createpartitions/createpartitions.go", "content": "package createpartitions\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\n// Detailed API definition: https://kafka.apache.org/protocol#The_Messages_CreatePartitions.\n// TODO: Support version 2.\ntype Request struct {\n\tTopics []RequestTopic `kafka:\"min=v0,max=v1\"`\n\tTimeoutMs int32 `kafka:\"min=v0,max=v1\"`\n\tValidateOnly bool `kafka:\"min=v0,max=v1\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.CreatePartitions }\n\nfunc (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, error) {\n\treturn cluster.Brokers[cluster.Controller], nil\n}\n\ntype RequestTopic struct {\n\tName string `kafka:\"min=v0,max=v1\"`\n\tCount int32 `kafka:\"min=v0,max=v1\"`\n\tAssignments []RequestAssignment `kafka:\"min=v0,max=v1,nullable\"`\n}\n\ntype RequestAssignment struct {\n\tBrokerIDs []int32 `kafka:\"min=v0,max=v1\"`\n}\n\ntype Response struct {\n\tThrottleTimeMs int32 `kafka:\"min=v0,max=v1\"`\n\tResults []ResponseResult `kafka:\"min=v0,max=v1\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.CreatePartitions }\n\ntype ResponseResult struct {\n\tName string `kafka:\"min=v0,max=v1\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v1\"`\n\tErrorMessage string `kafka:\"min=v0,max=v1,nullable\"`\n}\n\nvar _ protocol.BrokerMessage = (*Request)(nil)\n" }, { "path": "protocol/createpartitions/createpartitions_test.go", "content": "package createpartitions_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/createpartitions\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nconst (\n\tv0 = 0\n\tv1 = 1\n)\n\nfunc TestCreatePartitionsRequest(t *testing.T) {\n\tprototest.TestRequest(t, v0, &createpartitions.Request{\n\t\tTopics: []createpartitions.RequestTopic{\n\t\t\t{\n\t\t\t\tName: \"foo\",\n\t\t\t\tCount: 1,\n\t\t\t\tAssignments: []createpartitions.RequestAssignment{\n\t\t\t\t\t{\n\t\t\t\t\t\tBrokerIDs: []int32{1, 2, 3},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\tTimeoutMs: 500,\n\t\tValidateOnly: false,\n\t})\n\n\tprototest.TestRequest(t, v1, &createpartitions.Request{\n\t\tTopics: []createpartitions.RequestTopic{\n\t\t\t{\n\t\t\t\tName: \"foo\",\n\t\t\t\tCount: 1,\n\t\t\t\tAssignments: []createpartitions.RequestAssignment{\n\t\t\t\t\t{\n\t\t\t\t\t\tBrokerIDs: []int32{1, 2, 3},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\tTimeoutMs: 500,\n\t\tValidateOnly: false,\n\t})\n}\n\nfunc TestCreatePartitionsResponse(t *testing.T) {\n\tprototest.TestResponse(t, v0, &createpartitions.Response{\n\t\tThrottleTimeMs: 500,\n\t\tResults: []createpartitions.ResponseResult{\n\t\t\t{\n\t\t\t\tName: \"foo\",\n\t\t\t\tErrorCode: 1,\n\t\t\t\tErrorMessage: \"foo\",\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v1, &createpartitions.Response{\n\t\tThrottleTimeMs: 500,\n\t\tResults: []createpartitions.ResponseResult{\n\t\t\t{\n\t\t\t\tName: \"foo\",\n\t\t\t\tErrorCode: 1,\n\t\t\t\tErrorMessage: \"foo\",\n\t\t\t},\n\t\t},\n\t})\n}\n" }, { "path": "protocol/createtopics/createtopics.go", "content": "package createtopics\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\t// We need at least one tagged field to indicate that v5+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v5,max=v5,tag\"`\n\n\tTopics []RequestTopic `kafka:\"min=v0,max=v5\"`\n\tTimeoutMs int32 `kafka:\"min=v0,max=v5\"`\n\tValidateOnly bool `kafka:\"min=v1,max=v5\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.CreateTopics }\n\nfunc (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, error) {\n\treturn cluster.Brokers[cluster.Controller], nil\n}\n\ntype RequestTopic struct {\n\tName string `kafka:\"min=v0,max=v5\"`\n\tNumPartitions int32 `kafka:\"min=v0,max=v5\"`\n\tReplicationFactor int16 `kafka:\"min=v0,max=v5\"`\n\tAssignments []RequestAssignment `kafka:\"min=v0,max=v5\"`\n\tConfigs []RequestConfig `kafka:\"min=v0,max=v5\"`\n}\n\ntype RequestAssignment struct {\n\tPartitionIndex int32 `kafka:\"min=v0,max=v5\"`\n\tBrokerIDs []int32 `kafka:\"min=v0,max=v5\"`\n}\n\ntype RequestConfig struct {\n\tName string `kafka:\"min=v0,max=v5\"`\n\tValue string `kafka:\"min=v0,max=v5,nullable\"`\n}\n\ntype Response struct {\n\t// We need at least one tagged field to indicate that v5+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v5,max=v5,tag\"`\n\n\tThrottleTimeMs int32 `kafka:\"min=v2,max=v5\"`\n\tTopics []ResponseTopic `kafka:\"min=v0,max=v5\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.CreateTopics }\n\ntype ResponseTopic struct {\n\tName string `kafka:\"min=v0,max=v5\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v5\"`\n\tErrorMessage string `kafka:\"min=v1,max=v5,nullable\"`\n\tNumPartitions int32 `kafka:\"min=v5,max=v5\"`\n\tReplicationFactor int16 `kafka:\"min=v5,max=v5\"`\n\n\tConfigs []ResponseTopicConfig `kafka:\"min=v5,max=v5\"`\n}\n\ntype ResponseTopicConfig struct {\n\tName string `kafka:\"min=v5,max=v5\"`\n\tValue string `kafka:\"min=v5,max=v5,nullable\"`\n\tReadOnly bool `kafka:\"min=v5,max=v5\"`\n\tConfigSource int8 `kafka:\"min=v5,max=v5\"`\n\tIsSensitive bool `kafka:\"min=v5,max=v5\"`\n}\n\nvar (\n\t_ protocol.BrokerMessage = (*Request)(nil)\n)\n" }, { "path": "protocol/decode.go", "content": "package protocol\n\nimport (\n\t\"bytes\"\n\t\"encoding/binary\"\n\t\"fmt\"\n\t\"hash/crc32\"\n\t\"io\"\n\t\"io/ioutil\"\n\t\"math\"\n\t\"reflect\"\n\t\"sync\"\n\t\"sync/atomic\"\n)\n\ntype discarder interface {\n\tDiscard(int) (int, error)\n}\n\ntype decoder struct {\n\treader io.Reader\n\tremain int\n\tbuffer [8]byte\n\terr error\n\ttable *crc32.Table\n\tcrc32 uint32\n}\n\nfunc (d *decoder) Reset(r io.Reader, n int) {\n\td.reader = r\n\td.remain = n\n\td.buffer = [8]byte{}\n\td.err = nil\n\td.table = nil\n\td.crc32 = 0\n}\n\nfunc (d *decoder) Read(b []byte) (int, error) {\n\tif d.err != nil {\n\t\treturn 0, d.err\n\t}\n\tif d.remain == 0 {\n\t\treturn 0, io.EOF\n\t}\n\tif len(b) > d.remain {\n\t\tb = b[:d.remain]\n\t}\n\tn, err := d.reader.Read(b)\n\tif n > 0 && d.table != nil {\n\t\td.crc32 = crc32.Update(d.crc32, d.table, b[:n])\n\t}\n\td.remain -= n\n\treturn n, err\n}\n\nfunc (d *decoder) ReadByte() (byte, error) {\n\tc := d.readByte()\n\treturn c, d.err\n}\n\nfunc (d *decoder) done() bool {\n\treturn d.remain == 0 || d.err != nil\n}\n\nfunc (d *decoder) setCRC(table *crc32.Table) {\n\td.table, d.crc32 = table, 0\n}\n\nfunc (d *decoder) decodeBool(v value) {\n\tv.setBool(d.readBool())\n}\n\nfunc (d *decoder) decodeInt8(v value) {\n\tv.setInt8(d.readInt8())\n}\n\nfunc (d *decoder) decodeInt16(v value) {\n\tv.setInt16(d.readInt16())\n}\n\nfunc (d *decoder) decodeInt32(v value) {\n\tv.setInt32(d.readInt32())\n}\n\nfunc (d *decoder) decodeInt64(v value) {\n\tv.setInt64(d.readInt64())\n}\n\nfunc (d *decoder) decodeFloat64(v value) {\n\tv.setFloat64(d.readFloat64())\n}\n\nfunc (d *decoder) decodeString(v value) {\n\tv.setString(d.readString())\n}\n\nfunc (d *decoder) decodeCompactString(v value) {\n\tv.setString(d.readCompactString())\n}\n\nfunc (d *decoder) decodeBytes(v value) {\n\tv.setBytes(d.readBytes())\n}\n\nfunc (d *decoder) decodeCompactBytes(v value) {\n\tv.setBytes(d.readCompactBytes())\n}\n\nfunc (d *decoder) decodeArray(v value, elemType reflect.Type, decodeElem decodeFunc) {\n\tif n := d.readInt32(); n < 0 {\n\t\tv.setArray(array{})\n\t} else {\n\t\ta := makeArray(elemType, int(n))\n\t\tfor i := 0; i < int(n) && d.remain > 0; i++ {\n\t\t\tdecodeElem(d, a.index(i))\n\t\t}\n\t\tv.setArray(a)\n\t}\n}\n\nfunc (d *decoder) decodeCompactArray(v value, elemType reflect.Type, decodeElem decodeFunc) {\n\tif n := d.readUnsignedVarInt(); n < 1 {\n\t\tv.setArray(array{})\n\t} else {\n\t\ta := makeArray(elemType, int(n-1))\n\t\tfor i := 0; i < int(n-1) && d.remain > 0; i++ {\n\t\t\tdecodeElem(d, a.index(i))\n\t\t}\n\t\tv.setArray(a)\n\t}\n}\n\nfunc (d *decoder) discardAll() {\n\td.discard(d.remain)\n}\n\nfunc (d *decoder) discard(n int) {\n\tif n > d.remain {\n\t\tn = d.remain\n\t}\n\tvar err error\n\tif r, _ := d.reader.(discarder); r != nil {\n\t\tn, err = r.Discard(n)\n\t\td.remain -= n\n\t} else {\n\t\t_, err = io.Copy(ioutil.Discard, d)\n\t}\n\td.setError(err)\n}\n\nfunc (d *decoder) read(n int) []byte {\n\tb := make([]byte, n)\n\tn, err := io.ReadFull(d, b)\n\tb = b[:n]\n\td.setError(err)\n\treturn b\n}\n\nfunc (d *decoder) writeTo(w io.Writer, n int) {\n\tlimit := d.remain\n\tif n < limit {\n\t\td.remain = n\n\t}\n\tc, err := io.Copy(w, d)\n\tif int(c) < n && err == nil {\n\t\terr = io.ErrUnexpectedEOF\n\t}\n\td.remain = limit - int(c)\n\td.setError(err)\n}\n\nfunc (d *decoder) setError(err error) {\n\tif d.err == nil && err != nil {\n\t\td.err = err\n\t\td.discardAll()\n\t}\n}\n\nfunc (d *decoder) readFull(b []byte) bool {\n\tn, err := io.ReadFull(d, b)\n\td.setError(err)\n\treturn n == len(b)\n}\n\nfunc (d *decoder) readByte() byte {\n\tif d.readFull(d.buffer[:1]) {\n\t\treturn d.buffer[0]\n\t}\n\treturn 0\n}\n\nfunc (d *decoder) readBool() bool {\n\treturn d.readByte() != 0\n}\n\nfunc (d *decoder) readInt8() int8 {\n\tif d.readFull(d.buffer[:1]) {\n\t\treturn readInt8(d.buffer[:1])\n\t}\n\treturn 0\n}\n\nfunc (d *decoder) readInt16() int16 {\n\tif d.readFull(d.buffer[:2]) {\n\t\treturn readInt16(d.buffer[:2])\n\t}\n\treturn 0\n}\n\nfunc (d *decoder) readInt32() int32 {\n\tif d.readFull(d.buffer[:4]) {\n\t\treturn readInt32(d.buffer[:4])\n\t}\n\treturn 0\n}\n\nfunc (d *decoder) readInt64() int64 {\n\tif d.readFull(d.buffer[:8]) {\n\t\treturn readInt64(d.buffer[:8])\n\t}\n\treturn 0\n}\n\nfunc (d *decoder) readFloat64() float64 {\n\tif d.readFull(d.buffer[:8]) {\n\t\treturn readFloat64(d.buffer[:8])\n\t}\n\treturn 0\n}\n\nfunc (d *decoder) readString() string {\n\tif n := d.readInt16(); n < 0 {\n\t\treturn \"\"\n\t} else {\n\t\treturn bytesToString(d.read(int(n)))\n\t}\n}\n\nfunc (d *decoder) readVarString() string {\n\tif n := d.readVarInt(); n < 0 {\n\t\treturn \"\"\n\t} else {\n\t\treturn bytesToString(d.read(int(n)))\n\t}\n}\n\nfunc (d *decoder) readCompactString() string {\n\tif n := d.readUnsignedVarInt(); n < 1 {\n\t\treturn \"\"\n\t} else {\n\t\treturn bytesToString(d.read(int(n - 1)))\n\t}\n}\n\nfunc (d *decoder) readBytes() []byte {\n\tif n := d.readInt32(); n < 0 {\n\t\treturn nil\n\t} else {\n\t\treturn d.read(int(n))\n\t}\n}\n\nfunc (d *decoder) readVarBytes() []byte {\n\tif n := d.readVarInt(); n < 0 {\n\t\treturn nil\n\t} else {\n\t\treturn d.read(int(n))\n\t}\n}\n\nfunc (d *decoder) readCompactBytes() []byte {\n\tif n := d.readUnsignedVarInt(); n < 1 {\n\t\treturn nil\n\t} else {\n\t\treturn d.read(int(n - 1))\n\t}\n}\n\nfunc (d *decoder) readVarInt() int64 {\n\tn := 11 // varints are at most 11 bytes\n\n\tif n > d.remain {\n\t\tn = d.remain\n\t}\n\n\tx := uint64(0)\n\ts := uint(0)\n\n\tfor n > 0 {\n\t\tb := d.readByte()\n\n\t\tif (b & 0x80) == 0 {\n\t\t\tx |= uint64(b) << s\n\t\t\treturn int64(x>>1) ^ -(int64(x) & 1)\n\t\t}\n\n\t\tx |= uint64(b&0x7f) << s\n\t\ts += 7\n\t\tn--\n\t}\n\n\td.setError(fmt.Errorf(\"cannot decode varint from input stream\"))\n\treturn 0\n}\n\nfunc (d *decoder) readUnsignedVarInt() uint64 {\n\tn := 11 // varints are at most 11 bytes\n\n\tif n > d.remain {\n\t\tn = d.remain\n\t}\n\n\tx := uint64(0)\n\ts := uint(0)\n\n\tfor n > 0 {\n\t\tb := d.readByte()\n\n\t\tif (b & 0x80) == 0 {\n\t\t\tx |= uint64(b) << s\n\t\t\treturn x\n\t\t}\n\n\t\tx |= uint64(b&0x7f) << s\n\t\ts += 7\n\t\tn--\n\t}\n\n\td.setError(fmt.Errorf(\"cannot decode unsigned varint from input stream\"))\n\treturn 0\n}\n\ntype decodeFunc func(*decoder, value)\n\nvar (\n\t_ io.Reader = (*decoder)(nil)\n\t_ io.ByteReader = (*decoder)(nil)\n\n\treaderFrom = reflect.TypeOf((*io.ReaderFrom)(nil)).Elem()\n)\n\nfunc decodeFuncOf(typ reflect.Type, version int16, flexible bool, tag structTag) decodeFunc {\n\tif reflect.PtrTo(typ).Implements(readerFrom) {\n\t\treturn readerDecodeFuncOf(typ)\n\t}\n\tswitch typ.Kind() {\n\tcase reflect.Bool:\n\t\treturn (*decoder).decodeBool\n\tcase reflect.Int8:\n\t\treturn (*decoder).decodeInt8\n\tcase reflect.Int16:\n\t\treturn (*decoder).decodeInt16\n\tcase reflect.Int32:\n\t\treturn (*decoder).decodeInt32\n\tcase reflect.Int64:\n\t\treturn (*decoder).decodeInt64\n\tcase reflect.Float64:\n\t\treturn (*decoder).decodeFloat64\n\tcase reflect.String:\n\t\treturn stringDecodeFuncOf(flexible, tag)\n\tcase reflect.Struct:\n\t\treturn structDecodeFuncOf(typ, version, flexible)\n\tcase reflect.Slice:\n\t\tif typ.Elem().Kind() == reflect.Uint8 { // []byte\n\t\t\treturn bytesDecodeFuncOf(flexible, tag)\n\t\t}\n\t\treturn arrayDecodeFuncOf(typ, version, flexible, tag)\n\tdefault:\n\t\tpanic(\"unsupported type: \" + typ.String())\n\t}\n}\n\nfunc stringDecodeFuncOf(flexible bool, tag structTag) decodeFunc {\n\tif flexible {\n\t\t// In flexible messages, all strings are compact\n\t\treturn (*decoder).decodeCompactString\n\t}\n\treturn (*decoder).decodeString\n}\n\nfunc bytesDecodeFuncOf(flexible bool, tag structTag) decodeFunc {\n\tif flexible {\n\t\t// In flexible messages, all arrays are compact\n\t\treturn (*decoder).decodeCompactBytes\n\t}\n\treturn (*decoder).decodeBytes\n}\n\nfunc structDecodeFuncOf(typ reflect.Type, version int16, flexible bool) decodeFunc {\n\ttype field struct {\n\t\tdecode decodeFunc\n\t\tindex index\n\t\ttagID int\n\t}\n\n\tvar fields []field\n\ttaggedFields := map[int]*field{}\n\n\tforEachStructField(typ, func(typ reflect.Type, index index, tag string) {\n\t\tforEachStructTag(tag, func(tag structTag) bool {\n\t\t\tif tag.MinVersion <= version && version <= tag.MaxVersion {\n\t\t\t\tf := field{\n\t\t\t\t\tdecode: decodeFuncOf(typ, version, flexible, tag),\n\t\t\t\t\tindex: index,\n\t\t\t\t\ttagID: tag.TagID,\n\t\t\t\t}\n\n\t\t\t\tif tag.TagID < -1 {\n\t\t\t\t\t// Normal required field\n\t\t\t\t\tfields = append(fields, f)\n\t\t\t\t} else {\n\t\t\t\t\t// Optional tagged field (flexible messages only)\n\t\t\t\t\ttaggedFields[tag.TagID] = &f\n\t\t\t\t}\n\t\t\t\treturn false\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\t})\n\n\treturn func(d *decoder, v value) {\n\t\tfor i := range fields {\n\t\t\tf := &fields[i]\n\t\t\tf.decode(d, v.fieldByIndex(f.index))\n\t\t}\n\n\t\tif flexible {\n\t\t\t// See https://cwiki.apache.org/confluence/display/KAFKA/KIP-482%3A+The+Kafka+Protocol+should+Support+Optional+Tagged+Fields\n\t\t\t// for details of tag buffers in \"flexible\" messages.\n\t\t\tn := int(d.readUnsignedVarInt())\n\n\t\t\tfor i := 0; i < n; i++ {\n\t\t\t\ttagID := int(d.readUnsignedVarInt())\n\t\t\t\tsize := int(d.readUnsignedVarInt())\n\n\t\t\t\tf, ok := taggedFields[tagID]\n\t\t\t\tif ok {\n\t\t\t\t\tf.decode(d, v.fieldByIndex(f.index))\n\t\t\t\t} else {\n\t\t\t\t\td.read(size)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc arrayDecodeFuncOf(typ reflect.Type, version int16, flexible bool, tag structTag) decodeFunc {\n\telemType := typ.Elem()\n\telemFunc := decodeFuncOf(elemType, version, flexible, tag)\n\tif flexible {\n\t\t// In flexible messages, all arrays are compact\n\t\treturn func(d *decoder, v value) { d.decodeCompactArray(v, elemType, elemFunc) }\n\t}\n\n\treturn func(d *decoder, v value) { d.decodeArray(v, elemType, elemFunc) }\n}\n\nfunc readerDecodeFuncOf(typ reflect.Type) decodeFunc {\n\ttyp = reflect.PtrTo(typ)\n\treturn func(d *decoder, v value) {\n\t\tif d.err == nil {\n\t\t\t_, err := v.iface(typ).(io.ReaderFrom).ReadFrom(d)\n\t\t\tif err != nil {\n\t\t\t\td.setError(err)\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc readInt8(b []byte) int8 {\n\treturn int8(b[0])\n}\n\nfunc readInt16(b []byte) int16 {\n\treturn int16(binary.BigEndian.Uint16(b))\n}\n\nfunc readInt32(b []byte) int32 {\n\treturn int32(binary.BigEndian.Uint32(b))\n}\n\nfunc readInt64(b []byte) int64 {\n\treturn int64(binary.BigEndian.Uint64(b))\n}\n\nfunc readFloat64(b []byte) float64 {\n\treturn math.Float64frombits(binary.BigEndian.Uint64(b))\n}\n\nfunc Unmarshal(data []byte, version int16, value interface{}) error {\n\ttyp := elemTypeOf(value)\n\tcache, _ := unmarshalers.Load().(map[versionedType]decodeFunc)\n\tkey := versionedType{typ: typ, version: version}\n\tdecode := cache[key]\n\n\tif decode == nil {\n\t\tdecode = decodeFuncOf(reflect.TypeOf(value).Elem(), version, false, structTag{\n\t\t\tMinVersion: -1,\n\t\t\tMaxVersion: -1,\n\t\t\tTagID: -2,\n\t\t\tCompact: true,\n\t\t\tNullable: true,\n\t\t})\n\n\t\tnewCache := make(map[versionedType]decodeFunc, len(cache)+1)\n\t\tnewCache[key] = decode\n\n\t\tfor typ, fun := range cache {\n\t\t\tnewCache[typ] = fun\n\t\t}\n\n\t\tunmarshalers.Store(newCache)\n\t}\n\n\td, _ := decoders.Get().(*decoder)\n\tif d == nil {\n\t\td = &decoder{reader: bytes.NewReader(nil)}\n\t}\n\n\td.remain = len(data)\n\tr, _ := d.reader.(*bytes.Reader)\n\tr.Reset(data)\n\n\tdefer func() {\n\t\tr.Reset(nil)\n\t\td.Reset(r, 0)\n\t\tdecoders.Put(d)\n\t}()\n\n\tdecode(d, valueOf(value))\n\treturn dontExpectEOF(d.err)\n}\n\nvar (\n\tdecoders sync.Pool // *decoder\n\tunmarshalers atomic.Value // map[versionedType]decodeFunc\n)\n" }, { "path": "protocol/deleteacls/deleteacls.go", "content": "package deleteacls\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\t// We need at least one tagged field to indicate that v2+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v2,max=v3,tag\"`\n\n\tFilters []RequestFilter `kafka:\"min=v0,max=v3\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.DeleteAcls }\n\nfunc (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, error) {\n\treturn cluster.Brokers[cluster.Controller], nil\n}\n\ntype RequestFilter struct {\n\t// We need at least one tagged field to indicate that v2+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v2,max=v3,tag\"`\n\n\tResourceTypeFilter int8 `kafka:\"min=v0,max=v3\"`\n\tResourceNameFilter string `kafka:\"min=v0,max=v1,nullable|min=v2,max=v3,nullable,compact\"`\n\tResourcePatternTypeFilter int8 `kafka:\"min=v1,max=v3\"`\n\tPrincipalFilter string `kafka:\"min=v0,max=v1,nullable|min=v2,max=v3,nullable,compact\"`\n\tHostFilter string `kafka:\"min=v0,max=v1,nullable|min=v2,max=v3,nullable,compact\"`\n\tOperation int8 `kafka:\"min=v0,max=v3\"`\n\tPermissionType int8 `kafka:\"min=v0,max=v3\"`\n}\n\ntype Response struct {\n\t// We need at least one tagged field to indicate that v2+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v2,max=v3,tag\"`\n\n\tThrottleTimeMs int32 `kafka:\"min=v0,max=v3\"`\n\tFilterResults []FilterResult `kafka:\"min=v0,max=v3\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.DeleteAcls }\n\ntype FilterResult struct {\n\t// We need at least one tagged field to indicate that v2+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v2,max=v3,tag\"`\n\n\tErrorCode int16 `kafka:\"min=v0,max=v3\"`\n\tErrorMessage string `kafka:\"min=v0,max=v1,nullable|min=v2,max=v3,nullable,compact\"`\n\tMatchingACLs []MatchingACL `kafka:\"min=v0,max=v3\"`\n}\n\ntype MatchingACL struct {\n\t// We need at least one tagged field to indicate that v2+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v2,max=v3,tag\"`\n\n\tErrorCode int16 `kafka:\"min=v0,max=v3\"`\n\tErrorMessage string `kafka:\"min=v0,max=v1,nullable|min=v2,max=v3,nullable,compact\"`\n\tResourceType int8 `kafka:\"min=v0,max=v3\"`\n\tResourceName string `kafka:\"min=v0,max=v1|min=v2,max=v3,compact\"`\n\tResourcePatternType int8 `kafka:\"min=v1,max=v3\"`\n\tPrincipal string `kafka:\"min=v0,max=v1|min=v2,max=v3,compact\"`\n\tHost string `kafka:\"min=v0,max=v1|min=v2,max=v3,compact\"`\n\tOperation int8 `kafka:\"min=v0,max=v3\"`\n\tPermissionType int8 `kafka:\"min=v0,max=v3\"`\n}\n\nvar _ protocol.BrokerMessage = (*Request)(nil)\n" }, { "path": "protocol/deleteacls/deleteacls_test.go", "content": "package deleteacls_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/deleteacls\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nconst (\n\tv0 = 0\n\tv1 = 1\n\tv2 = 2\n\tv3 = 3\n)\n\nfunc TestDeleteACLsRequest(t *testing.T) {\n\tprototest.TestRequest(t, v0, &deleteacls.Request{\n\t\tFilters: []deleteacls.RequestFilter{\n\t\t\t{\n\t\t\t\tResourceTypeFilter: 2,\n\t\t\t\tResourceNameFilter: \"fake-topic-for-alice\",\n\t\t\t\tPrincipalFilter: \"User:alice\",\n\t\t\t\tHostFilter: \"*\",\n\t\t\t\tOperation: 3,\n\t\t\t\tPermissionType: 3,\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestRequest(t, v1, &deleteacls.Request{\n\t\tFilters: []deleteacls.RequestFilter{\n\t\t\t{\n\t\t\t\tResourceTypeFilter: 2,\n\t\t\t\tResourceNameFilter: \"fake-topic-for-alice\",\n\t\t\t\tResourcePatternTypeFilter: 0,\n\t\t\t\tPrincipalFilter: \"User:alice\",\n\t\t\t\tHostFilter: \"*\",\n\t\t\t\tOperation: 3,\n\t\t\t\tPermissionType: 3,\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestRequest(t, v2, &deleteacls.Request{\n\t\tFilters: []deleteacls.RequestFilter{\n\t\t\t{\n\t\t\t\tResourceTypeFilter: 2,\n\t\t\t\tResourceNameFilter: \"fake-topic-for-alice\",\n\t\t\t\tResourcePatternTypeFilter: 0,\n\t\t\t\tPrincipalFilter: \"User:alice\",\n\t\t\t\tHostFilter: \"*\",\n\t\t\t\tOperation: 3,\n\t\t\t\tPermissionType: 3,\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestRequest(t, v3, &deleteacls.Request{\n\t\tFilters: []deleteacls.RequestFilter{\n\t\t\t{\n\t\t\t\tResourceTypeFilter: 2,\n\t\t\t\tResourceNameFilter: \"fake-topic-for-alice\",\n\t\t\t\tResourcePatternTypeFilter: 0,\n\t\t\t\tPrincipalFilter: \"User:alice\",\n\t\t\t\tHostFilter: \"*\",\n\t\t\t\tOperation: 3,\n\t\t\t\tPermissionType: 3,\n\t\t\t},\n\t\t},\n\t})\n}\n\nfunc TestDeleteACLsResponse(t *testing.T) {\n\tprototest.TestResponse(t, v0, &deleteacls.Response{\n\t\tThrottleTimeMs: 1,\n\t\tFilterResults: []deleteacls.FilterResult{\n\t\t\t{\n\t\t\t\tErrorCode: 1,\n\t\t\t\tErrorMessage: \"foo\",\n\t\t\t\tMatchingACLs: []deleteacls.MatchingACL{\n\t\t\t\t\t{\n\t\t\t\t\t\tErrorCode: 1,\n\t\t\t\t\t\tErrorMessage: \"bar\",\n\t\t\t\t\t\tResourceType: 2,\n\t\t\t\t\t\tResourceName: \"fake-topic-for-alice\",\n\t\t\t\t\t\tPrincipal: \"User:alice\",\n\t\t\t\t\t\tHost: \"*\",\n\t\t\t\t\t\tOperation: 3,\n\t\t\t\t\t\tPermissionType: 3,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v1, &deleteacls.Response{\n\t\tThrottleTimeMs: 1,\n\t\tFilterResults: []deleteacls.FilterResult{\n\t\t\t{\n\t\t\t\tErrorCode: 1,\n\t\t\t\tErrorMessage: \"foo\",\n\t\t\t\tMatchingACLs: []deleteacls.MatchingACL{\n\t\t\t\t\t{\n\t\t\t\t\t\tErrorCode: 1,\n\t\t\t\t\t\tErrorMessage: \"bar\",\n\t\t\t\t\t\tResourceType: 2,\n\t\t\t\t\t\tResourceName: \"fake-topic-for-alice\",\n\t\t\t\t\t\tResourcePatternType: 0,\n\t\t\t\t\t\tPrincipal: \"User:alice\",\n\t\t\t\t\t\tHost: \"*\",\n\t\t\t\t\t\tOperation: 3,\n\t\t\t\t\t\tPermissionType: 3,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v2, &deleteacls.Response{\n\t\tThrottleTimeMs: 1,\n\t\tFilterResults: []deleteacls.FilterResult{\n\t\t\t{\n\t\t\t\tErrorCode: 1,\n\t\t\t\tErrorMessage: \"foo\",\n\t\t\t\tMatchingACLs: []deleteacls.MatchingACL{\n\t\t\t\t\t{\n\t\t\t\t\t\tErrorCode: 1,\n\t\t\t\t\t\tErrorMessage: \"bar\",\n\t\t\t\t\t\tResourceType: 2,\n\t\t\t\t\t\tResourceName: \"fake-topic-for-alice\",\n\t\t\t\t\t\tResourcePatternType: 0,\n\t\t\t\t\t\tPrincipal: \"User:alice\",\n\t\t\t\t\t\tHost: \"*\",\n\t\t\t\t\t\tOperation: 3,\n\t\t\t\t\t\tPermissionType: 3,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v3, &deleteacls.Response{\n\t\tThrottleTimeMs: 1,\n\t\tFilterResults: []deleteacls.FilterResult{\n\t\t\t{\n\t\t\t\tErrorCode: 1,\n\t\t\t\tErrorMessage: \"foo\",\n\t\t\t\tMatchingACLs: []deleteacls.MatchingACL{\n\t\t\t\t\t{\n\t\t\t\t\t\tErrorCode: 1,\n\t\t\t\t\t\tErrorMessage: \"bar\",\n\t\t\t\t\t\tResourceType: 2,\n\t\t\t\t\t\tResourceName: \"fake-topic-for-alice\",\n\t\t\t\t\t\tResourcePatternType: 0,\n\t\t\t\t\t\tPrincipal: \"User:alice\",\n\t\t\t\t\t\tHost: \"*\",\n\t\t\t\t\t\tOperation: 3,\n\t\t\t\t\t\tPermissionType: 3,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n}\n" }, { "path": "protocol/deletegroups/deletegroups.go", "content": "package deletegroups\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v2,max=v2,tag\"`\n\n\tGroupIDs []string `kafka:\"min=v0,max=v2\"`\n}\n\nfunc (r *Request) Group() string {\n\t// use first group to determine group coordinator\n\tif len(r.GroupIDs) > 0 {\n\t\treturn r.GroupIDs[0]\n\t}\n\treturn \"\"\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.DeleteGroups }\n\nvar (\n\t_ protocol.GroupMessage = (*Request)(nil)\n)\n\ntype Response struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v2,max=v2,tag\"`\n\n\tThrottleTimeMs int32 `kafka:\"min=v0,max=v2\"`\n\tResponses []ResponseGroup `kafka:\"min=v0,max=v2\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.DeleteGroups }\n\ntype ResponseGroup struct {\n\tGroupID string `kafka:\"min=v0,max=v2\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v2\"`\n}\n" }, { "path": "protocol/deletegroups/deletegroups_test.go", "content": "package deletegroups_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/deletegroups\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nfunc TestDeleteGroupsRequest(t *testing.T) {\n\tfor _, version := range []int16{0, 1, 2} {\n\t\tprototest.TestRequest(t, version, &deletegroups.Request{\n\t\t\tGroupIDs: []string{\"group1\", \"group2\"},\n\t\t})\n\t}\n}\n\nfunc TestDeleteGroupsResponse(t *testing.T) {\n\tfor _, version := range []int16{0, 1, 2} {\n\t\tprototest.TestResponse(t, version, &deletegroups.Response{\n\t\t\tResponses: []deletegroups.ResponseGroup{\n\t\t\t\t{\n\t\t\t\t\tGroupID: \"group1\",\n\t\t\t\t\tErrorCode: 0,\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tGroupID: \"group2\",\n\t\t\t\t\tErrorCode: 1,\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n}\n" }, { "path": "protocol/deletetopics/deletetopics.go", "content": "package deletetopics\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\tTopicNames []string `kafka:\"min=v0,max=v3\"`\n\tTimeoutMs int32 `kafka:\"min=v0,max=v3\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.DeleteTopics }\n\nfunc (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, error) {\n\treturn cluster.Brokers[cluster.Controller], nil\n}\n\ntype Response struct {\n\tThrottleTimeMs int32 `kafka:\"min=v1,max=v3\"`\n\tResponses []ResponseTopic `kafka:\"min=v0,max=v3\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.DeleteTopics }\n\ntype ResponseTopic struct {\n\tName string `kafka:\"min=v0,max=v3\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v3\"`\n}\n\nvar (\n\t_ protocol.BrokerMessage = (*Request)(nil)\n)\n" }, { "path": "protocol/deletetopics/deletetopics_test.go", "content": "package deletetopics_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/deletetopics\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nconst (\n\tv0 = 0\n\tv1 = 1\n\tv3 = 3\n)\n\nfunc TestDeleteTopicsRequest(t *testing.T) {\n\tprototest.TestRequest(t, v0, &deletetopics.Request{\n\t\tTopicNames: []string{\"foo\", \"bar\"},\n\t\tTimeoutMs: 500,\n\t})\n\n\tprototest.TestRequest(t, v1, &deletetopics.Request{\n\t\tTopicNames: []string{\"foo\", \"bar\"},\n\t\tTimeoutMs: 500,\n\t})\n\n\tprototest.TestRequest(t, v3, &deletetopics.Request{\n\t\tTopicNames: []string{\"foo\", \"bar\"},\n\t\tTimeoutMs: 500,\n\t})\n}\n\nfunc TestDeleteTopicsResponse(t *testing.T) {\n\tprototest.TestResponse(t, v0, &deletetopics.Response{\n\t\tResponses: []deletetopics.ResponseTopic{\n\t\t\t{\n\t\t\t\tName: \"foo\",\n\t\t\t\tErrorCode: 1,\n\t\t\t},\n\t\t\t{\n\t\t\t\tName: \"bar\",\n\t\t\t\tErrorCode: 1,\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v1, &deletetopics.Response{\n\t\tThrottleTimeMs: 500,\n\t\tResponses: []deletetopics.ResponseTopic{\n\t\t\t{\n\t\t\t\tName: \"foo\",\n\t\t\t\tErrorCode: 1,\n\t\t\t},\n\t\t\t{\n\t\t\t\tName: \"bar\",\n\t\t\t\tErrorCode: 1,\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v3, &deletetopics.Response{\n\t\tThrottleTimeMs: 500,\n\t\tResponses: []deletetopics.ResponseTopic{\n\t\t\t{\n\t\t\t\tName: \"foo\",\n\t\t\t\tErrorCode: 1,\n\t\t\t},\n\t\t\t{\n\t\t\t\tName: \"bar\",\n\t\t\t\tErrorCode: 1,\n\t\t\t},\n\t\t},\n\t})\n}\n" }, { "path": "protocol/describeacls/describeacls.go", "content": "package describeacls\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\t// We need at least one tagged field to indicate that v2+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v2,max=v3,tag\"`\n\n\tFilter ACLFilter `kafka:\"min=v0,max=v3\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.DescribeAcls }\n\nfunc (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, error) {\n\treturn cluster.Brokers[cluster.Controller], nil\n}\n\ntype ACLFilter struct {\n\t// We need at least one tagged field to indicate that v2+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v2,max=v3,tag\"`\n\n\tResourceTypeFilter int8 `kafka:\"min=v0,max=v3\"`\n\tResourceNameFilter string `kafka:\"min=v0,max=v1,nullable|min=v2,max=v3,nullable,compact\"`\n\tResourcePatternTypeFilter int8 `kafka:\"min=v1,max=v3\"`\n\tPrincipalFilter string `kafka:\"min=v0,max=v1,nullable|min=v2,max=v3,nullable,compact\"`\n\tHostFilter string `kafka:\"min=v0,max=v1,nullable|min=v2,max=v3,nullable,compact\"`\n\tOperation int8 `kafka:\"min=v0,max=v3\"`\n\tPermissionType int8 `kafka:\"min=v0,max=v3\"`\n}\n\ntype Response struct {\n\t// We need at least one tagged field to indicate that v2+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v2,max=v3,tag\"`\n\n\tThrottleTimeMs int32 `kafka:\"min=v0,max=v3\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v3\"`\n\tErrorMessage string `kafka:\"min=v0,max=v1,nullable|min=v2,max=v3,nullable,compact\"`\n\tResources []Resource `kafka:\"min=v0,max=v3\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.DescribeAcls }\n\ntype Resource struct {\n\t// We need at least one tagged field to indicate that v2+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v2,max=v3,tag\"`\n\n\tResourceType int8 `kafka:\"min=v0,max=v3\"`\n\tResourceName string `kafka:\"min=v0,max=v1|min=v2,max=v3,compact\"`\n\tPatternType int8 `kafka:\"min=v1,max=v3\"`\n\tACLs []ResponseACL `kafka:\"min=v0,max=v3\"`\n}\n\ntype ResponseACL struct {\n\t// We need at least one tagged field to indicate that v2+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v2,max=v3,tag\"`\n\n\tPrincipal string `kafka:\"min=v0,max=v1|min=v2,max=v3,compact\"`\n\tHost string `kafka:\"min=v0,max=v1|min=v2,max=v3,compact\"`\n\tOperation int8 `kafka:\"min=v0,max=v3\"`\n\tPermissionType int8 `kafka:\"min=v0,max=v3\"`\n}\n\nvar _ protocol.BrokerMessage = (*Request)(nil)\n" }, { "path": "protocol/describeacls/describeacls_test.go", "content": "package describeacls_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/describeacls\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nconst (\n\tv0 = 0\n\tv1 = 1\n\tv2 = 2\n\tv3 = 3\n)\n\nfunc TestDescribeACLsRequest(t *testing.T) {\n\tprototest.TestRequest(t, v0, &describeacls.Request{\n\t\tFilter: describeacls.ACLFilter{\n\t\t\tResourceTypeFilter: 2,\n\t\t\tResourceNameFilter: \"fake-topic-for-alice\",\n\t\t\tPrincipalFilter: \"User:alice\",\n\t\t\tHostFilter: \"*\",\n\t\t\tOperation: 3,\n\t\t\tPermissionType: 3,\n\t\t},\n\t})\n\n\tprototest.TestRequest(t, v1, &describeacls.Request{\n\t\tFilter: describeacls.ACLFilter{\n\t\t\tResourceTypeFilter: 2,\n\t\t\tResourceNameFilter: \"fake-topic-for-alice\",\n\t\t\tResourcePatternTypeFilter: 0,\n\t\t\tPrincipalFilter: \"User:alice\",\n\t\t\tHostFilter: \"*\",\n\t\t\tOperation: 3,\n\t\t\tPermissionType: 3,\n\t\t},\n\t})\n\n\tprototest.TestRequest(t, v2, &describeacls.Request{\n\t\tFilter: describeacls.ACLFilter{\n\t\t\tResourceTypeFilter: 2,\n\t\t\tResourceNameFilter: \"fake-topic-for-alice\",\n\t\t\tResourcePatternTypeFilter: 0,\n\t\t\tPrincipalFilter: \"User:alice\",\n\t\t\tHostFilter: \"*\",\n\t\t\tOperation: 3,\n\t\t\tPermissionType: 3,\n\t\t},\n\t})\n\n\tprototest.TestRequest(t, v3, &describeacls.Request{\n\t\tFilter: describeacls.ACLFilter{\n\t\t\tResourceTypeFilter: 2,\n\t\t\tResourceNameFilter: \"fake-topic-for-alice\",\n\t\t\tResourcePatternTypeFilter: 0,\n\t\t\tPrincipalFilter: \"User:alice\",\n\t\t\tHostFilter: \"*\",\n\t\t\tOperation: 3,\n\t\t\tPermissionType: 3,\n\t\t},\n\t})\n}\n\nfunc TestDescribeACLsResponse(t *testing.T) {\n\tprototest.TestResponse(t, v0, &describeacls.Response{\n\t\tThrottleTimeMs: 1,\n\t\tErrorCode: 1,\n\t\tErrorMessage: \"foo\",\n\t\tResources: []describeacls.Resource{\n\t\t\t{\n\t\t\t\tResourceType: 2,\n\t\t\t\tResourceName: \"fake-topic-for-alice\",\n\t\t\t\tACLs: []describeacls.ResponseACL{\n\t\t\t\t\t{\n\t\t\t\t\t\tPrincipal: \"User:alice\",\n\t\t\t\t\t\tHost: \"*\",\n\t\t\t\t\t\tOperation: 3,\n\t\t\t\t\t\tPermissionType: 3,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v1, &describeacls.Response{\n\t\tThrottleTimeMs: 1,\n\t\tErrorCode: 1,\n\t\tErrorMessage: \"foo\",\n\t\tResources: []describeacls.Resource{\n\t\t\t{\n\t\t\t\tResourceType: 2,\n\t\t\t\tResourceName: \"fake-topic-for-alice\",\n\t\t\t\tPatternType: 3,\n\t\t\t\tACLs: []describeacls.ResponseACL{\n\t\t\t\t\t{\n\t\t\t\t\t\tPrincipal: \"User:alice\",\n\t\t\t\t\t\tHost: \"*\",\n\t\t\t\t\t\tOperation: 3,\n\t\t\t\t\t\tPermissionType: 3,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v2, &describeacls.Response{\n\t\tThrottleTimeMs: 1,\n\t\tErrorCode: 1,\n\t\tErrorMessage: \"foo\",\n\t\tResources: []describeacls.Resource{\n\t\t\t{\n\t\t\t\tResourceType: 2,\n\t\t\t\tResourceName: \"fake-topic-for-alice\",\n\t\t\t\tPatternType: 3,\n\t\t\t\tACLs: []describeacls.ResponseACL{\n\t\t\t\t\t{\n\t\t\t\t\t\tPrincipal: \"User:alice\",\n\t\t\t\t\t\tHost: \"*\",\n\t\t\t\t\t\tOperation: 3,\n\t\t\t\t\t\tPermissionType: 3,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v3, &describeacls.Response{\n\t\tThrottleTimeMs: 1,\n\t\tErrorCode: 1,\n\t\tErrorMessage: \"foo\",\n\t\tResources: []describeacls.Resource{\n\t\t\t{\n\t\t\t\tResourceType: 2,\n\t\t\t\tResourceName: \"fake-topic-for-alice\",\n\t\t\t\tPatternType: 3,\n\t\t\t\tACLs: []describeacls.ResponseACL{\n\t\t\t\t\t{\n\t\t\t\t\t\tPrincipal: \"User:alice\",\n\t\t\t\t\t\tHost: \"*\",\n\t\t\t\t\t\tOperation: 3,\n\t\t\t\t\t\tPermissionType: 3,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n}\n" }, { "path": "protocol/describeclientquotas/describeclientquotas.go", "content": "package describeclientquotas\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v1,max=v1,tag\"`\n\tComponents []Component `kafka:\"min=v0,max=v1\"`\n\tStrict bool `kafka:\"min=v0,max=v1\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.DescribeClientQuotas }\n\nfunc (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, error) {\n\treturn cluster.Brokers[cluster.Controller], nil\n}\n\ntype Component struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v1,max=v1,tag\"`\n\tEntityType string `kafka:\"min=v0,max=v1\"`\n\tMatchType int8 `kafka:\"min=v0,max=v1\"`\n\tMatch string `kafka:\"min=v0,max=v1,nullable\"`\n}\n\ntype Response struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v1,max=v1,tag\"`\n\tThrottleTimeMs int32 `kafka:\"min=v0,max=v1\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v1\"`\n\tErrorMessage string `kafka:\"min=v0,max=v0,nullable|min=v1,max=v1,nullable,compact\"`\n\tEntries []ResponseQuotas `kafka:\"min=v0,max=v1\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.DescribeClientQuotas }\n\ntype Entity struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v1,max=v1,tag\"`\n\tEntityType string `kafka:\"min=v0,max=v0|min=v1,max=v1,compact\"`\n\tEntityName string `kafka:\"min=v0,max=v0,nullable|min=v1,max=v1,nullable,compact\"`\n}\n\ntype Value struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v1,max=v1,tag\"`\n\tKey string `kafka:\"min=v0,max=v0|min=v1,max=v1,compact\"`\n\tValue float64 `kafka:\"min=v0,max=v1\"`\n}\n\ntype ResponseQuotas struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v1,max=v1,tag\"`\n\tEntities []Entity `kafka:\"min=v0,max=v1\"`\n\tValues []Value `kafka:\"min=v0,max=v1\"`\n}\n\nvar _ protocol.BrokerMessage = (*Request)(nil)\n" }, { "path": "protocol/describeclientquotas/describeclientquotas_test.go", "content": "package describeclientquotas_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/describeclientquotas\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nconst (\n\tv0 = 0\n\tv1 = 1\n)\n\nfunc TestDescribeClientQuotasRequest(t *testing.T) {\n\tprototest.TestRequest(t, v0, &describeclientquotas.Request{\n\t\tStrict: true,\n\t\tComponents: []describeclientquotas.Component{\n\t\t\t{\n\t\t\t\tEntityType: \"client-id\",\n\t\t\t\tMatchType: 0,\n\t\t\t\tMatch: \"my-client-id\",\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestRequest(t, v1, &describeclientquotas.Request{\n\t\tStrict: true,\n\t\tComponents: []describeclientquotas.Component{\n\t\t\t{\n\t\t\t\tEntityType: \"client-id\",\n\t\t\t\tMatchType: 0,\n\t\t\t\tMatch: \"my-client-id\",\n\t\t\t},\n\t\t},\n\t})\n}\n\nfunc TestDescribeClientQuotasResponse(t *testing.T) {\n\tprototest.TestResponse(t, v0, &describeclientquotas.Response{\n\t\tThrottleTimeMs: 1,\n\t\tErrorCode: 1,\n\t\tErrorMessage: \"foo\",\n\t\tEntries: []describeclientquotas.ResponseQuotas{\n\t\t\t{\n\t\t\t\tEntities: []describeclientquotas.Entity{\n\t\t\t\t\t{\n\t\t\t\t\t\tEntityType: \"client-id\",\n\t\t\t\t\t\tEntityName: \"my-client-id\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tValues: []describeclientquotas.Value{\n\t\t\t\t\t{\n\t\t\t\t\t\tKey: \"foo\",\n\t\t\t\t\t\tValue: 1.0,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v1, &describeclientquotas.Response{\n\t\tThrottleTimeMs: 1,\n\t\tErrorCode: 1,\n\t\tErrorMessage: \"foo\",\n\t\tEntries: []describeclientquotas.ResponseQuotas{\n\t\t\t{\n\t\t\t\tEntities: []describeclientquotas.Entity{\n\t\t\t\t\t{\n\t\t\t\t\t\tEntityType: \"client-id\",\n\t\t\t\t\t\tEntityName: \"my-client-id\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tValues: []describeclientquotas.Value{\n\t\t\t\t\t{\n\t\t\t\t\t\tKey: \"foo\",\n\t\t\t\t\t\tValue: 1.0,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n}\n" }, { "path": "protocol/describeconfigs/describeconfigs.go", "content": "package describeconfigs\n\nimport (\n\t\"strconv\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n)\n\nconst (\n\tresourceTypeBroker int8 = 4\n)\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\n// Detailed API definition: https://kafka.apache.org/protocol#The_Messages_DescribeConfigs\ntype Request struct {\n\tResources []RequestResource `kafka:\"min=v0,max=v3\"`\n\tIncludeSynonyms bool `kafka:\"min=v1,max=v3\"`\n\tIncludeDocumentation bool `kafka:\"min=v3,max=v3\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.DescribeConfigs }\n\nfunc (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, error) {\n\t// Broker metadata requests must be sent to the associated broker\n\tfor _, resource := range r.Resources {\n\t\tif resource.ResourceType == resourceTypeBroker {\n\t\t\tbrokerID, err := strconv.Atoi(resource.ResourceName)\n\t\t\tif err != nil {\n\t\t\t\treturn protocol.Broker{}, err\n\t\t\t}\n\n\t\t\treturn cluster.Brokers[int32(brokerID)], nil\n\t\t}\n\t}\n\n\treturn cluster.Brokers[cluster.Controller], nil\n}\n\nfunc (r *Request) Split(cluster protocol.Cluster) (\n\t[]protocol.Message,\n\tprotocol.Merger,\n\terror,\n) {\n\tmessages := []protocol.Message{}\n\ttopicsMessage := Request{}\n\n\tfor _, resource := range r.Resources {\n\t\t// Split out broker requests to separate brokers\n\t\tif resource.ResourceType == resourceTypeBroker {\n\t\t\tmessages = append(messages, &Request{\n\t\t\t\tResources: []RequestResource{resource},\n\t\t\t})\n\t\t} else {\n\t\t\ttopicsMessage.Resources = append(\n\t\t\t\ttopicsMessage.Resources, resource,\n\t\t\t)\n\t\t}\n\t}\n\n\tif len(topicsMessage.Resources) > 0 {\n\t\tmessages = append(messages, &topicsMessage)\n\t}\n\n\treturn messages, new(Response), nil\n}\n\ntype RequestResource struct {\n\tResourceType int8 `kafka:\"min=v0,max=v3\"`\n\tResourceName string `kafka:\"min=v0,max=v3\"`\n\tConfigNames []string `kafka:\"min=v0,max=v3,nullable\"`\n}\n\ntype Response struct {\n\tThrottleTimeMs int32 `kafka:\"min=v0,max=v3\"`\n\tResources []ResponseResource `kafka:\"min=v0,max=v3\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.DescribeConfigs }\n\nfunc (r *Response) Merge(requests []protocol.Message, results []interface{}) (\n\tprotocol.Message,\n\terror,\n) {\n\tresponse := &Response{}\n\n\tfor _, result := range results {\n\t\tm, err := protocol.Result(result)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tresponse.Resources = append(\n\t\t\tresponse.Resources,\n\t\t\tm.(*Response).Resources...,\n\t\t)\n\t}\n\n\treturn response, nil\n}\n\ntype ResponseResource struct {\n\tErrorCode int16 `kafka:\"min=v0,max=v3\"`\n\tErrorMessage string `kafka:\"min=v0,max=v3,nullable\"`\n\tResourceType int8 `kafka:\"min=v0,max=v3\"`\n\tResourceName string `kafka:\"min=v0,max=v3\"`\n\tConfigEntries []ResponseConfigEntry `kafka:\"min=v0,max=v3\"`\n}\n\ntype ResponseConfigEntry struct {\n\tConfigName string `kafka:\"min=v0,max=v3\"`\n\tConfigValue string `kafka:\"min=v0,max=v3,nullable\"`\n\tReadOnly bool `kafka:\"min=v0,max=v3\"`\n\tIsDefault bool `kafka:\"min=v0,max=v0\"`\n\tConfigSource int8 `kafka:\"min=v1,max=v3\"`\n\tIsSensitive bool `kafka:\"min=v0,max=v3\"`\n\tConfigSynonyms []ResponseConfigSynonym `kafka:\"min=v1,max=v3\"`\n\tConfigType int8 `kafka:\"min=v3,max=v3\"`\n\tConfigDocumentation string `kafka:\"min=v3,max=v3,nullable\"`\n}\n\ntype ResponseConfigSynonym struct {\n\tConfigName string `kafka:\"min=v1,max=v3\"`\n\tConfigValue string `kafka:\"min=v1,max=v3,nullable\"`\n\tConfigSource int8 `kafka:\"min=v1,max=v3\"`\n}\n\nvar _ protocol.BrokerMessage = (*Request)(nil)\n" }, { "path": "protocol/describeconfigs/describeconfigs_test.go", "content": "package describeconfigs\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestResponse_Merge(t *testing.T) {\n\tt.Run(\"happy path\", func(t *testing.T) {\n\t\tr := &Response{}\n\n\t\tr1 := &Response{\n\t\t\tResources: []ResponseResource{\n\t\t\t\t{ResourceName: \"r1\"},\n\t\t\t},\n\t\t}\n\t\tr2 := &Response{\n\t\t\tResources: []ResponseResource{\n\t\t\t\t{ResourceName: \"r2\"},\n\t\t\t},\n\t\t}\n\n\t\tgot, err := r.Merge([]protocol.Message{&Request{}}, []interface{}{r1, r2})\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\twant := &Response{\n\t\t\tResources: []ResponseResource{\n\t\t\t\t{ResourceName: \"r1\"},\n\t\t\t\t{ResourceName: \"r2\"},\n\t\t\t},\n\t\t}\n\n\t\tif !reflect.DeepEqual(want, got) {\n\t\t\tt.Fatalf(\"wanted response: \\n%+v, got \\n%+v\", want, got)\n\t\t}\n\t})\n\n\tt.Run(\"with errors\", func(t *testing.T) {\n\t\tr := &Response{}\n\n\t\tr1 := &Response{\n\t\t\tResources: []ResponseResource{\n\t\t\t\t{ResourceName: \"r1\"},\n\t\t\t},\n\t\t}\n\n\t\t_, err := r.Merge([]protocol.Message{&Request{}}, []interface{}{r1, io.EOF})\n\t\tif !errors.Is(err, io.EOF) {\n\t\t\tt.Fatalf(\"wanted err io.EOF, got %v\", err)\n\t\t}\n\t})\n\n\tt.Run(\"panic with unexpected type\", func(t *testing.T) {\n\t\tdefer func() {\n\t\t\tmsg := recover()\n\t\t\trequire.Equal(t, \"BUG: result must be a message or an error but not string\", fmt.Sprintf(\"%s\", msg))\n\t\t}()\n\t\tr := &Response{}\n\n\t\tr1 := &Response{\n\t\t\tResources: []ResponseResource{\n\t\t\t\t{ResourceName: \"r1\"},\n\t\t\t},\n\t\t}\n\n\t\t_, _ = r.Merge([]protocol.Message{&Request{}}, []interface{}{r1, \"how did a string got here\"})\n\t\tt.Fatal(\"did not panic\")\n\t})\n}\n" }, { "path": "protocol/describegroups/describegroups.go", "content": "package describegroups\n\nimport (\n\t\"github.com/segmentio/kafka-go/protocol\"\n)\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\n// Detailed API definition: https://kafka.apache.org/protocol#The_Messages_DescribeGroups\ntype Request struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v5,max=v5,tag\"`\n\tGroups []string `kafka:\"min=v0,max=v4|min=v5,max=v5,compact\"`\n\tIncludeAuthorizedOperations bool `kafka:\"min=v3,max=v5\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.DescribeGroups }\n\nfunc (r *Request) Group() string {\n\treturn r.Groups[0]\n}\n\nfunc (r *Request) Split(cluster protocol.Cluster) (\n\t[]protocol.Message,\n\tprotocol.Merger,\n\terror,\n) {\n\tmessages := []protocol.Message{}\n\n\t// Split requests by group since they'll need to go to different coordinators.\n\tfor _, group := range r.Groups {\n\t\tmessages = append(\n\t\t\tmessages,\n\t\t\t&Request{\n\t\t\t\tGroups: []string{group},\n\t\t\t\tIncludeAuthorizedOperations: r.IncludeAuthorizedOperations,\n\t\t\t},\n\t\t)\n\t}\n\n\treturn messages, new(Response), nil\n}\n\ntype Response struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v5,max=v5,tag\"`\n\tThrottleTimeMs int32 `kafka:\"min=v1,max=v5\"`\n\tGroups []ResponseGroup `kafka:\"min=v0,max=v5\"`\n}\n\ntype ResponseGroup struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v5,max=v5,tag\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v5\"`\n\tGroupID string `kafka:\"min=v0,max=v4|min=v5,max=v5,compact\"`\n\tGroupState string `kafka:\"min=v0,max=v4|min=v5,max=v5,compact\"`\n\tProtocolType string `kafka:\"min=v0,max=v4|min=v5,max=v5,compact\"`\n\tProtocolData string `kafka:\"min=v0,max=v4|min=v5,max=v5,compact\"`\n\tMembers []ResponseGroupMember `kafka:\"min=v0,max=v4|min=v5,max=v5,compact\"`\n\tAuthorizedOperations int32 `kafka:\"min=v3,max=v5\"`\n}\n\ntype ResponseGroupMember struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v5,max=v5,tag\"`\n\tMemberID string `kafka:\"min=v0,max=v4|min=v5,max=v5,compact\"`\n\tGroupInstanceID string `kafka:\"min=v4,max=v4,nullable|min=v5,max=v5,compact,nullable\"`\n\tClientID string `kafka:\"min=v0,max=v4|min=v5,max=v5,compact\"`\n\tClientHost string `kafka:\"min=v0,max=v4|min=v5,max=v5,compact\"`\n\tMemberMetadata []byte `kafka:\"min=v0,max=v4|min=v5,max=v5,compact\"`\n\tMemberAssignment []byte `kafka:\"min=v0,max=v4|min=v5,max=v5,compact\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.DescribeGroups }\n\nfunc (r *Response) Merge(requests []protocol.Message, results []interface{}) (\n\tprotocol.Message,\n\terror,\n) {\n\tresponse := &Response{}\n\n\tfor _, result := range results {\n\t\tm, err := protocol.Result(result)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tresponse.Groups = append(response.Groups, m.(*Response).Groups...)\n\t}\n\n\treturn response, nil\n}\n" }, { "path": "protocol/describegroups/describegroups_test.go", "content": "package describegroups_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/describegroups\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nconst (\n\tv0 = 0\n\tv1 = 1\n\tv2 = 2\n\tv3 = 3\n\tv4 = 4\n\tv5 = 5\n)\n\nfunc TestDescribeGroupsRequest(t *testing.T) {\n\tprototest.TestRequest(t, v0, &describegroups.Request{\n\t\tGroups: []string{\"test-group\"},\n\t})\n\n\tprototest.TestRequest(t, v1, &describegroups.Request{\n\t\tGroups: []string{\"test-group\"},\n\t})\n\n\tprototest.TestRequest(t, v2, &describegroups.Request{\n\t\tGroups: []string{\"test-group\"},\n\t})\n\n\tprototest.TestRequest(t, v3, &describegroups.Request{\n\t\tGroups: []string{\"test-group\"},\n\t\tIncludeAuthorizedOperations: true,\n\t})\n\n\tprototest.TestRequest(t, v4, &describegroups.Request{\n\t\tGroups: []string{\"test-group\"},\n\t\tIncludeAuthorizedOperations: true,\n\t})\n\n\tprototest.TestRequest(t, v5, &describegroups.Request{\n\t\tGroups: []string{\"test-group\"},\n\t\tIncludeAuthorizedOperations: true,\n\t})\n}\n\nfunc TestDescribeGroupsResponse(t *testing.T) {\n\tprototest.TestResponse(t, v0, &describegroups.Response{\n\t\tGroups: []describegroups.ResponseGroup{\n\t\t\t{\n\t\t\t\tErrorCode: 0,\n\t\t\t\tGroupID: \"test-group\",\n\t\t\t\tGroupState: \"Stable\",\n\t\t\t\tProtocolType: \"consumer\",\n\t\t\t\tProtocolData: \"range\",\n\t\t\t\tMembers: []describegroups.ResponseGroupMember{\n\t\t\t\t\t{\n\t\t\t\t\t\tMemberID: \"consumer-1\",\n\t\t\t\t\t\tClientID: \"client-1\",\n\t\t\t\t\t\tClientHost: \"/127.0.0.1\",\n\t\t\t\t\t\tMemberMetadata: []byte{0x00, 0x01},\n\t\t\t\t\t\tMemberAssignment: []byte{0x00, 0x02},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v1, &describegroups.Response{\n\t\tThrottleTimeMs: 100,\n\t\tGroups: []describegroups.ResponseGroup{\n\t\t\t{\n\t\t\t\tErrorCode: 0,\n\t\t\t\tGroupID: \"test-group\",\n\t\t\t\tGroupState: \"Stable\",\n\t\t\t\tProtocolType: \"consumer\",\n\t\t\t\tProtocolData: \"range\",\n\t\t\t\tMembers: []describegroups.ResponseGroupMember{\n\t\t\t\t\t{\n\t\t\t\t\t\tMemberID: \"consumer-1\",\n\t\t\t\t\t\tClientID: \"client-1\",\n\t\t\t\t\t\tClientHost: \"/127.0.0.1\",\n\t\t\t\t\t\tMemberMetadata: []byte{0x00, 0x01},\n\t\t\t\t\t\tMemberAssignment: []byte{0x00, 0x02},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v2, &describegroups.Response{\n\t\tThrottleTimeMs: 100,\n\t\tGroups: []describegroups.ResponseGroup{\n\t\t\t{\n\t\t\t\tErrorCode: 0,\n\t\t\t\tGroupID: \"test-group\",\n\t\t\t\tGroupState: \"Stable\",\n\t\t\t\tProtocolType: \"consumer\",\n\t\t\t\tProtocolData: \"range\",\n\t\t\t\tMembers: []describegroups.ResponseGroupMember{\n\t\t\t\t\t{\n\t\t\t\t\t\tMemberID: \"consumer-1\",\n\t\t\t\t\t\tClientID: \"client-1\",\n\t\t\t\t\t\tClientHost: \"/127.0.0.1\",\n\t\t\t\t\t\tMemberMetadata: []byte{0x00, 0x01},\n\t\t\t\t\t\tMemberAssignment: []byte{0x00, 0x02},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v3, &describegroups.Response{\n\t\tThrottleTimeMs: 100,\n\t\tGroups: []describegroups.ResponseGroup{\n\t\t\t{\n\t\t\t\tErrorCode: 0,\n\t\t\t\tGroupID: \"test-group\",\n\t\t\t\tGroupState: \"Stable\",\n\t\t\t\tProtocolType: \"consumer\",\n\t\t\t\tProtocolData: \"range\",\n\t\t\t\tAuthorizedOperations: 2147483647,\n\t\t\t\tMembers: []describegroups.ResponseGroupMember{\n\t\t\t\t\t{\n\t\t\t\t\t\tMemberID: \"consumer-1\",\n\t\t\t\t\t\tClientID: \"client-1\",\n\t\t\t\t\t\tClientHost: \"/127.0.0.1\",\n\t\t\t\t\t\tMemberMetadata: []byte{0x00, 0x01},\n\t\t\t\t\t\tMemberAssignment: []byte{0x00, 0x02},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v4, &describegroups.Response{\n\t\tThrottleTimeMs: 100,\n\t\tGroups: []describegroups.ResponseGroup{\n\t\t\t{\n\t\t\t\tErrorCode: 0,\n\t\t\t\tGroupID: \"test-group\",\n\t\t\t\tGroupState: \"Stable\",\n\t\t\t\tProtocolType: \"consumer\",\n\t\t\t\tProtocolData: \"range\",\n\t\t\t\tAuthorizedOperations: 2147483647,\n\t\t\t\tMembers: []describegroups.ResponseGroupMember{\n\t\t\t\t\t{\n\t\t\t\t\t\tMemberID: \"consumer-1\",\n\t\t\t\t\t\tGroupInstanceID: \"instance-1\",\n\t\t\t\t\t\tClientID: \"client-1\",\n\t\t\t\t\t\tClientHost: \"/127.0.0.1\",\n\t\t\t\t\t\tMemberMetadata: []byte{0x00, 0x01},\n\t\t\t\t\t\tMemberAssignment: []byte{0x00, 0x02},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v5, &describegroups.Response{\n\t\tThrottleTimeMs: 100,\n\t\tGroups: []describegroups.ResponseGroup{\n\t\t\t{\n\t\t\t\tErrorCode: 0,\n\t\t\t\tGroupID: \"test-group\",\n\t\t\t\tGroupState: \"Stable\",\n\t\t\t\tProtocolType: \"consumer\",\n\t\t\t\tProtocolData: \"range\",\n\t\t\t\tAuthorizedOperations: 2147483647,\n\t\t\t\tMembers: []describegroups.ResponseGroupMember{\n\t\t\t\t\t{\n\t\t\t\t\t\tMemberID: \"consumer-1\",\n\t\t\t\t\t\tGroupInstanceID: \"instance-1\",\n\t\t\t\t\t\tClientID: \"client-1\",\n\t\t\t\t\t\tClientHost: \"/127.0.0.1\",\n\t\t\t\t\t\tMemberMetadata: []byte{0x00, 0x01},\n\t\t\t\t\t\tMemberAssignment: []byte{0x00, 0x02},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n}\n" }, { "path": "protocol/describeuserscramcredentials/describeuserscramcredentials.go", "content": "package describeuserscramcredentials\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\t// We need at least one tagged field to indicate that v2+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v0,max=v0,tag\"`\n\n\tUsers []RequestUser `kafka:\"min=v0,max=v0\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.DescribeUserScramCredentials }\n\nfunc (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, error) {\n\treturn cluster.Brokers[cluster.Controller], nil\n}\n\ntype RequestUser struct {\n\t// We need at least one tagged field to indicate that v2+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v0,max=v0,tag\"`\n\n\tName string `kafka:\"min=v0,max=v0,compact\"`\n}\n\ntype Response struct {\n\t// We need at least one tagged field to indicate that v2+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v0,max=v0,tag\"`\n\n\tThrottleTimeMs int32 `kafka:\"min=v0,max=v0\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v0\"`\n\tErrorMessage string `kafka:\"min=v0,max=v0,nullable\"`\n\tResults []ResponseResult `kafka:\"min=v0,max=v0\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.DescribeUserScramCredentials }\n\ntype ResponseResult struct {\n\t// We need at least one tagged field to indicate that v2+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v0,max=v0,tag\"`\n\n\tUser string `kafka:\"min=v0,max=v0,compact\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v0\"`\n\tErrorMessage string `kafka:\"min=v0,max=v0,nullable\"`\n\tCredentialInfos []CredentialInfo `kafka:\"min=v0,max=v0\"`\n}\n\ntype CredentialInfo struct {\n\t// We need at least one tagged field to indicate that v2+ uses \"flexible\"\n\t// messages.\n\t_ struct{} `kafka:\"min=v0,max=v0,tag\"`\n\n\tMechanism int8 `kafka:\"min=v0,max=v0\"`\n\tIterations int32 `kafka:\"min=v0,max=v0\"`\n}\n\nvar _ protocol.BrokerMessage = (*Request)(nil)\n" }, { "path": "protocol/describeuserscramcredentials/describeuserscramcredentials_test.go", "content": "package describeuserscramcredentials_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/describeuserscramcredentials\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nconst (\n\tv0 = 0\n)\n\nfunc TestDescribeUserScramCredentialsRequest(t *testing.T) {\n\tprototest.TestRequest(t, v0, &describeuserscramcredentials.Request{\n\t\tUsers: []describeuserscramcredentials.RequestUser{\n\t\t\t{\n\t\t\t\tName: \"foo-1\",\n\t\t\t},\n\t\t},\n\t})\n}\n\nfunc TestDescribeUserScramCredentialsResponse(t *testing.T) {\n\tprototest.TestResponse(t, v0, &describeuserscramcredentials.Response{\n\t\tThrottleTimeMs: 500,\n\t\tResults: []describeuserscramcredentials.ResponseResult{\n\t\t\t{\n\t\t\t\tUser: \"foo\",\n\t\t\t\tErrorCode: 1,\n\t\t\t\tErrorMessage: \"foo-error\",\n\t\t\t\tCredentialInfos: []describeuserscramcredentials.CredentialInfo{\n\t\t\t\t\t{\n\t\t\t\t\t\tMechanism: 2,\n\t\t\t\t\t\tIterations: 15000,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n}\n" }, { "path": "protocol/electleaders/electleaders.go", "content": "package electleaders\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\n// Detailed API definition: https://kafka.apache.org/protocol#The_Messages_ElectLeaders\ntype Request struct {\n\tElectionType int8 `kafka:\"min=v1,max=v1\"`\n\tTopicPartitions []RequestTopicPartitions `kafka:\"min=v0,max=v1\"`\n\tTimeoutMs int32 `kafka:\"min=v0,max=v1\"`\n}\n\ntype RequestTopicPartitions struct {\n\tTopic string `kafka:\"min=v0,max=v1\"`\n\tPartitionIDs []int32 `kafka:\"min=v0,max=v1\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.ElectLeaders }\n\nfunc (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, error) {\n\treturn cluster.Brokers[cluster.Controller], nil\n}\n\ntype Response struct {\n\tThrottleTime int32 `kafka:\"min=v0,max=v1\"`\n\tErrorCode int16 `kafka:\"min=v1,max=v1\"`\n\tReplicaElectionResults []ResponseReplicaElectionResult `kafka:\"min=v0,max=v1\"`\n}\n\ntype ResponseReplicaElectionResult struct {\n\tTopic string `kafka:\"min=v0,max=v1\"`\n\tPartitionResults []ResponsePartitionResult `kafka:\"min=v0,max=v1\"`\n}\n\ntype ResponsePartitionResult struct {\n\tPartitionID int32 `kafka:\"min=v0,max=v1\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v1\"`\n\tErrorMessage string `kafka:\"min=v0,max=v1,nullable\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.ElectLeaders }\n" }, { "path": "protocol/electleaders/electleaders_test.go", "content": "package electleaders_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/electleaders\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nconst (\n\tv0 = 0\n\tv1 = 1\n)\n\nfunc TestElectLeadersRequest(t *testing.T) {\n\tprototest.TestRequest(t, v0, &electleaders.Request{\n\t\tTimeoutMs: 500,\n\t\tTopicPartitions: []electleaders.RequestTopicPartitions{\n\t\t\t{\n\t\t\t\tTopic: \"foo\",\n\t\t\t\tPartitionIDs: []int32{100, 101, 102},\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestRequest(t, v1, &electleaders.Request{\n\t\tElectionType: 1,\n\t\tTimeoutMs: 500,\n\t\tTopicPartitions: []electleaders.RequestTopicPartitions{\n\t\t\t{\n\t\t\t\tTopic: \"foo\",\n\t\t\t\tPartitionIDs: []int32{100, 101, 102},\n\t\t\t},\n\t\t},\n\t})\n}\n\nfunc TestElectLeadersResponse(t *testing.T) {\n\tprototest.TestResponse(t, v0, &electleaders.Response{\n\t\tThrottleTime: 500,\n\t\tReplicaElectionResults: []electleaders.ResponseReplicaElectionResult{\n\t\t\t{\n\t\t\t\tTopic: \"foo\",\n\t\t\t\tPartitionResults: []electleaders.ResponsePartitionResult{\n\t\t\t\t\t{PartitionID: 100, ErrorCode: 0, ErrorMessage: \"\"},\n\t\t\t\t\t{PartitionID: 101, ErrorCode: 0, ErrorMessage: \"\"},\n\t\t\t\t\t{PartitionID: 102, ErrorCode: 0, ErrorMessage: \"\"},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v1, &electleaders.Response{\n\t\tThrottleTime: 500,\n\t\tErrorCode: 1,\n\t\tReplicaElectionResults: []electleaders.ResponseReplicaElectionResult{\n\t\t\t{\n\t\t\t\tTopic: \"foo\",\n\t\t\t\tPartitionResults: []electleaders.ResponsePartitionResult{\n\t\t\t\t\t{PartitionID: 100, ErrorCode: 0, ErrorMessage: \"\"},\n\t\t\t\t\t{PartitionID: 101, ErrorCode: 0, ErrorMessage: \"\"},\n\t\t\t\t\t{PartitionID: 102, ErrorCode: 0, ErrorMessage: \"\"},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n}\n" }, { "path": "protocol/encode.go", "content": "package protocol\n\nimport (\n\t\"bytes\"\n\t\"encoding/binary\"\n\t\"fmt\"\n\t\"hash/crc32\"\n\t\"io\"\n\t\"math\"\n\t\"reflect\"\n\t\"sync\"\n\t\"sync/atomic\"\n)\n\ntype encoder struct {\n\twriter io.Writer\n\terr error\n\ttable *crc32.Table\n\tcrc32 uint32\n\tbuffer [32]byte\n}\n\ntype encoderChecksum struct {\n\treader io.Reader\n\tencoder *encoder\n}\n\nfunc (e *encoderChecksum) Read(b []byte) (int, error) {\n\tn, err := e.reader.Read(b)\n\tif n > 0 {\n\t\te.encoder.update(b[:n])\n\t}\n\treturn n, err\n}\n\nfunc (e *encoder) Reset(w io.Writer) {\n\te.writer = w\n\te.err = nil\n\te.table = nil\n\te.crc32 = 0\n\te.buffer = [32]byte{}\n}\n\nfunc (e *encoder) ReadFrom(r io.Reader) (int64, error) {\n\tif e.table != nil {\n\t\tr = &encoderChecksum{\n\t\t\treader: r,\n\t\t\tencoder: e,\n\t\t}\n\t}\n\treturn io.Copy(e.writer, r)\n}\n\nfunc (e *encoder) Write(b []byte) (int, error) {\n\tif e.err != nil {\n\t\treturn 0, e.err\n\t}\n\tn, err := e.writer.Write(b)\n\tif n > 0 {\n\t\te.update(b[:n])\n\t}\n\tif err != nil {\n\t\te.err = err\n\t}\n\treturn n, err\n}\n\nfunc (e *encoder) WriteByte(b byte) error {\n\te.buffer[0] = b\n\t_, err := e.Write(e.buffer[:1])\n\treturn err\n}\n\nfunc (e *encoder) WriteString(s string) (int, error) {\n\t// This implementation is an optimization to avoid the heap allocation that\n\t// would occur when converting the string to a []byte to call crc32.Update.\n\t//\n\t// Strings are rarely long in the kafka protocol, so the use of a 32 byte\n\t// buffer is a good comprise between keeping the encoder value small and\n\t// limiting the number of calls to Write.\n\t//\n\t// We introduced this optimization because memory profiles on the benchmarks\n\t// showed that most heap allocations were caused by this code path.\n\tn := 0\n\n\tfor len(s) != 0 {\n\t\tc := copy(e.buffer[:], s)\n\t\tw, err := e.Write(e.buffer[:c])\n\t\tn += w\n\t\tif err != nil {\n\t\t\treturn n, err\n\t\t}\n\t\ts = s[c:]\n\t}\n\n\treturn n, nil\n}\n\nfunc (e *encoder) setCRC(table *crc32.Table) {\n\te.table, e.crc32 = table, 0\n}\n\nfunc (e *encoder) update(b []byte) {\n\tif e.table != nil {\n\t\te.crc32 = crc32.Update(e.crc32, e.table, b)\n\t}\n}\n\nfunc (e *encoder) encodeBool(v value) {\n\tb := int8(0)\n\tif v.bool() {\n\t\tb = 1\n\t}\n\te.writeInt8(b)\n}\n\nfunc (e *encoder) encodeInt8(v value) {\n\te.writeInt8(v.int8())\n}\n\nfunc (e *encoder) encodeInt16(v value) {\n\te.writeInt16(v.int16())\n}\n\nfunc (e *encoder) encodeInt32(v value) {\n\te.writeInt32(v.int32())\n}\n\nfunc (e *encoder) encodeInt64(v value) {\n\te.writeInt64(v.int64())\n}\n\nfunc (e *encoder) encodeFloat64(v value) {\n\te.writeFloat64(v.float64())\n}\n\nfunc (e *encoder) encodeString(v value) {\n\te.writeString(v.string())\n}\n\nfunc (e *encoder) encodeCompactString(v value) {\n\te.writeCompactString(v.string())\n}\n\nfunc (e *encoder) encodeNullString(v value) {\n\te.writeNullString(v.string())\n}\n\nfunc (e *encoder) encodeCompactNullString(v value) {\n\te.writeCompactNullString(v.string())\n}\n\nfunc (e *encoder) encodeBytes(v value) {\n\te.writeBytes(v.bytes())\n}\n\nfunc (e *encoder) encodeCompactBytes(v value) {\n\te.writeCompactBytes(v.bytes())\n}\n\nfunc (e *encoder) encodeNullBytes(v value) {\n\te.writeNullBytes(v.bytes())\n}\n\nfunc (e *encoder) encodeCompactNullBytes(v value) {\n\te.writeCompactNullBytes(v.bytes())\n}\n\nfunc (e *encoder) encodeArray(v value, elemType reflect.Type, encodeElem encodeFunc) {\n\ta := v.array(elemType)\n\tn := a.length()\n\te.writeInt32(int32(n))\n\n\tfor i := 0; i < n; i++ {\n\t\tencodeElem(e, a.index(i))\n\t}\n}\n\nfunc (e *encoder) encodeCompactArray(v value, elemType reflect.Type, encodeElem encodeFunc) {\n\ta := v.array(elemType)\n\tn := a.length()\n\te.writeUnsignedVarInt(uint64(n + 1))\n\n\tfor i := 0; i < n; i++ {\n\t\tencodeElem(e, a.index(i))\n\t}\n}\n\nfunc (e *encoder) encodeNullArray(v value, elemType reflect.Type, encodeElem encodeFunc) {\n\ta := v.array(elemType)\n\tif a.isNil() {\n\t\te.writeInt32(-1)\n\t\treturn\n\t}\n\n\tn := a.length()\n\te.writeInt32(int32(n))\n\n\tfor i := 0; i < n; i++ {\n\t\tencodeElem(e, a.index(i))\n\t}\n}\n\nfunc (e *encoder) encodeCompactNullArray(v value, elemType reflect.Type, encodeElem encodeFunc) {\n\ta := v.array(elemType)\n\tif a.isNil() {\n\t\te.writeUnsignedVarInt(0)\n\t\treturn\n\t}\n\n\tn := a.length()\n\te.writeUnsignedVarInt(uint64(n + 1))\n\tfor i := 0; i < n; i++ {\n\t\tencodeElem(e, a.index(i))\n\t}\n}\n\nfunc (e *encoder) writeInt8(i int8) {\n\twriteInt8(e.buffer[:1], i)\n\te.Write(e.buffer[:1])\n}\n\nfunc (e *encoder) writeInt16(i int16) {\n\twriteInt16(e.buffer[:2], i)\n\te.Write(e.buffer[:2])\n}\n\nfunc (e *encoder) writeInt32(i int32) {\n\twriteInt32(e.buffer[:4], i)\n\te.Write(e.buffer[:4])\n}\n\nfunc (e *encoder) writeInt64(i int64) {\n\twriteInt64(e.buffer[:8], i)\n\te.Write(e.buffer[:8])\n}\n\nfunc (e *encoder) writeFloat64(f float64) {\n\twriteFloat64(e.buffer[:8], f)\n\te.Write(e.buffer[:8])\n}\n\nfunc (e *encoder) writeString(s string) {\n\te.writeInt16(int16(len(s)))\n\te.WriteString(s)\n}\n\nfunc (e *encoder) writeVarString(s string) {\n\te.writeVarInt(int64(len(s)))\n\te.WriteString(s)\n}\n\nfunc (e *encoder) writeCompactString(s string) {\n\te.writeUnsignedVarInt(uint64(len(s)) + 1)\n\te.WriteString(s)\n}\n\nfunc (e *encoder) writeNullString(s string) {\n\tif s == \"\" {\n\t\te.writeInt16(-1)\n\t} else {\n\t\te.writeInt16(int16(len(s)))\n\t\te.WriteString(s)\n\t}\n}\n\nfunc (e *encoder) writeCompactNullString(s string) {\n\tif s == \"\" {\n\t\te.writeUnsignedVarInt(0)\n\t} else {\n\t\te.writeUnsignedVarInt(uint64(len(s)) + 1)\n\t\te.WriteString(s)\n\t}\n}\n\nfunc (e *encoder) writeBytes(b []byte) {\n\te.writeInt32(int32(len(b)))\n\te.Write(b)\n}\n\nfunc (e *encoder) writeCompactBytes(b []byte) {\n\te.writeUnsignedVarInt(uint64(len(b)) + 1)\n\te.Write(b)\n}\n\nfunc (e *encoder) writeNullBytes(b []byte) {\n\tif b == nil {\n\t\te.writeInt32(-1)\n\t} else {\n\t\te.writeInt32(int32(len(b)))\n\t\te.Write(b)\n\t}\n}\n\nfunc (e *encoder) writeVarNullBytes(b []byte) {\n\tif b == nil {\n\t\te.writeVarInt(-1)\n\t} else {\n\t\te.writeVarInt(int64(len(b)))\n\t\te.Write(b)\n\t}\n}\n\nfunc (e *encoder) writeCompactNullBytes(b []byte) {\n\tif b == nil {\n\t\te.writeUnsignedVarInt(0)\n\t} else {\n\t\te.writeUnsignedVarInt(uint64(len(b)) + 1)\n\t\te.Write(b)\n\t}\n}\n\nfunc (e *encoder) writeNullBytesFrom(b Bytes) error {\n\tif b == nil {\n\t\te.writeInt32(-1)\n\t\treturn nil\n\t} else {\n\t\tsize := int64(b.Len())\n\t\te.writeInt32(int32(size))\n\t\tn, err := io.Copy(e, b)\n\t\tif err == nil && n != size {\n\t\t\terr = fmt.Errorf(\"size of nullable bytes does not match the number of bytes that were written (size=%d, written=%d): %w\", size, n, io.ErrUnexpectedEOF)\n\t\t}\n\t\treturn err\n\t}\n}\n\nfunc (e *encoder) writeVarNullBytesFrom(b Bytes) error {\n\tif b == nil {\n\t\te.writeVarInt(-1)\n\t\treturn nil\n\t} else {\n\t\tsize := int64(b.Len())\n\t\te.writeVarInt(size)\n\t\tn, err := io.Copy(e, b)\n\t\tif err == nil && n != size {\n\t\t\terr = fmt.Errorf(\"size of nullable bytes does not match the number of bytes that were written (size=%d, written=%d): %w\", size, n, io.ErrUnexpectedEOF)\n\t\t}\n\t\treturn err\n\t}\n}\n\nfunc (e *encoder) writeVarInt(i int64) {\n\te.writeUnsignedVarInt(uint64((i << 1) ^ (i >> 63)))\n}\n\nfunc (e *encoder) writeUnsignedVarInt(i uint64) {\n\tb := e.buffer[:]\n\tn := 0\n\n\tfor i >= 0x80 && n < len(b) {\n\t\tb[n] = byte(i) | 0x80\n\t\ti >>= 7\n\t\tn++\n\t}\n\n\tif n < len(b) {\n\t\tb[n] = byte(i)\n\t\tn++\n\t}\n\n\te.Write(b[:n])\n}\n\ntype encodeFunc func(*encoder, value)\n\nvar (\n\t_ io.ReaderFrom = (*encoder)(nil)\n\t_ io.Writer = (*encoder)(nil)\n\t_ io.ByteWriter = (*encoder)(nil)\n\t_ io.StringWriter = (*encoder)(nil)\n\n\twriterTo = reflect.TypeOf((*io.WriterTo)(nil)).Elem()\n)\n\nfunc encodeFuncOf(typ reflect.Type, version int16, flexible bool, tag structTag) encodeFunc {\n\tif reflect.PtrTo(typ).Implements(writerTo) {\n\t\treturn writerEncodeFuncOf(typ)\n\t}\n\tswitch typ.Kind() {\n\tcase reflect.Bool:\n\t\treturn (*encoder).encodeBool\n\tcase reflect.Int8:\n\t\treturn (*encoder).encodeInt8\n\tcase reflect.Int16:\n\t\treturn (*encoder).encodeInt16\n\tcase reflect.Int32:\n\t\treturn (*encoder).encodeInt32\n\tcase reflect.Int64:\n\t\treturn (*encoder).encodeInt64\n\tcase reflect.Float64:\n\t\treturn (*encoder).encodeFloat64\n\tcase reflect.String:\n\t\treturn stringEncodeFuncOf(flexible, tag)\n\tcase reflect.Struct:\n\t\treturn structEncodeFuncOf(typ, version, flexible)\n\tcase reflect.Slice:\n\t\tif typ.Elem().Kind() == reflect.Uint8 { // []byte\n\t\t\treturn bytesEncodeFuncOf(flexible, tag)\n\t\t}\n\t\treturn arrayEncodeFuncOf(typ, version, flexible, tag)\n\tdefault:\n\t\tpanic(\"unsupported type: \" + typ.String())\n\t}\n}\n\nfunc stringEncodeFuncOf(flexible bool, tag structTag) encodeFunc {\n\tswitch {\n\tcase flexible && tag.Nullable:\n\t\t// In flexible messages, all strings are compact\n\t\treturn (*encoder).encodeCompactNullString\n\tcase flexible:\n\t\t// In flexible messages, all strings are compact\n\t\treturn (*encoder).encodeCompactString\n\tcase tag.Nullable:\n\t\treturn (*encoder).encodeNullString\n\tdefault:\n\t\treturn (*encoder).encodeString\n\t}\n}\n\nfunc bytesEncodeFuncOf(flexible bool, tag structTag) encodeFunc {\n\tswitch {\n\tcase flexible && tag.Nullable:\n\t\t// In flexible messages, all arrays are compact\n\t\treturn (*encoder).encodeCompactNullBytes\n\tcase flexible:\n\t\t// In flexible messages, all arrays are compact\n\t\treturn (*encoder).encodeCompactBytes\n\tcase tag.Nullable:\n\t\treturn (*encoder).encodeNullBytes\n\tdefault:\n\t\treturn (*encoder).encodeBytes\n\t}\n}\n\nfunc structEncodeFuncOf(typ reflect.Type, version int16, flexible bool) encodeFunc {\n\ttype field struct {\n\t\tencode encodeFunc\n\t\tindex index\n\t\ttagID int\n\t}\n\n\tvar fields []field\n\tvar taggedFields []field\n\n\tforEachStructField(typ, func(typ reflect.Type, index index, tag string) {\n\t\tif typ.Size() != 0 { // skip struct{}\n\t\t\tforEachStructTag(tag, func(tag structTag) bool {\n\t\t\t\tif tag.MinVersion <= version && version <= tag.MaxVersion {\n\t\t\t\t\tf := field{\n\t\t\t\t\t\tencode: encodeFuncOf(typ, version, flexible, tag),\n\t\t\t\t\t\tindex: index,\n\t\t\t\t\t\ttagID: tag.TagID,\n\t\t\t\t\t}\n\n\t\t\t\t\tif tag.TagID < -1 {\n\t\t\t\t\t\t// Normal required field\n\t\t\t\t\t\tfields = append(fields, f)\n\t\t\t\t\t} else {\n\t\t\t\t\t\t// Optional tagged field (flexible messages only)\n\t\t\t\t\t\ttaggedFields = append(taggedFields, f)\n\t\t\t\t\t}\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t\treturn true\n\t\t\t})\n\t\t}\n\t})\n\n\treturn func(e *encoder, v value) {\n\t\tfor i := range fields {\n\t\t\tf := &fields[i]\n\t\t\tf.encode(e, v.fieldByIndex(f.index))\n\t\t}\n\n\t\tif flexible {\n\t\t\t// See https://cwiki.apache.org/confluence/display/KAFKA/KIP-482%3A+The+Kafka+Protocol+should+Support+Optional+Tagged+Fields\n\t\t\t// for details of tag buffers in \"flexible\" messages.\n\t\t\te.writeUnsignedVarInt(uint64(len(taggedFields)))\n\n\t\t\tfor i := range taggedFields {\n\t\t\t\tf := &taggedFields[i]\n\t\t\t\te.writeUnsignedVarInt(uint64(f.tagID))\n\n\t\t\t\tbuf := &bytes.Buffer{}\n\t\t\t\tse := &encoder{writer: buf}\n\t\t\t\tf.encode(se, v.fieldByIndex(f.index))\n\t\t\t\te.writeUnsignedVarInt(uint64(buf.Len()))\n\t\t\t\te.Write(buf.Bytes())\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc arrayEncodeFuncOf(typ reflect.Type, version int16, flexible bool, tag structTag) encodeFunc {\n\telemType := typ.Elem()\n\telemFunc := encodeFuncOf(elemType, version, flexible, tag)\n\tswitch {\n\tcase flexible && tag.Nullable:\n\t\t// In flexible messages, all arrays are compact\n\t\treturn func(e *encoder, v value) { e.encodeCompactNullArray(v, elemType, elemFunc) }\n\tcase flexible:\n\t\t// In flexible messages, all arrays are compact\n\t\treturn func(e *encoder, v value) { e.encodeCompactArray(v, elemType, elemFunc) }\n\tcase tag.Nullable:\n\t\treturn func(e *encoder, v value) { e.encodeNullArray(v, elemType, elemFunc) }\n\tdefault:\n\t\treturn func(e *encoder, v value) { e.encodeArray(v, elemType, elemFunc) }\n\t}\n}\n\nfunc writerEncodeFuncOf(typ reflect.Type) encodeFunc {\n\ttyp = reflect.PtrTo(typ)\n\treturn func(e *encoder, v value) {\n\t\t// Optimization to write directly into the buffer when the encoder\n\t\t// does no need to compute a crc32 checksum.\n\t\tw := io.Writer(e)\n\t\tif e.table == nil {\n\t\t\tw = e.writer\n\t\t}\n\t\t_, err := v.iface(typ).(io.WriterTo).WriteTo(w)\n\t\tif err != nil {\n\t\t\te.err = err\n\t\t}\n\t}\n}\n\nfunc writeInt8(b []byte, i int8) {\n\tb[0] = byte(i)\n}\n\nfunc writeInt16(b []byte, i int16) {\n\tbinary.BigEndian.PutUint16(b, uint16(i))\n}\n\nfunc writeInt32(b []byte, i int32) {\n\tbinary.BigEndian.PutUint32(b, uint32(i))\n}\n\nfunc writeInt64(b []byte, i int64) {\n\tbinary.BigEndian.PutUint64(b, uint64(i))\n}\n\nfunc writeFloat64(b []byte, f float64) {\n\tbinary.BigEndian.PutUint64(b, math.Float64bits(f))\n}\n\nfunc Marshal(version int16, value interface{}) ([]byte, error) {\n\ttyp := typeOf(value)\n\tcache, _ := marshalers.Load().(map[versionedType]encodeFunc)\n\tkey := versionedType{typ: typ, version: version}\n\tencode := cache[key]\n\n\tif encode == nil {\n\t\tencode = encodeFuncOf(reflect.TypeOf(value), version, false, structTag{\n\t\t\tMinVersion: -1,\n\t\t\tMaxVersion: -1,\n\t\t\tTagID: -2,\n\t\t\tCompact: true,\n\t\t\tNullable: true,\n\t\t})\n\n\t\tnewCache := make(map[versionedType]encodeFunc, len(cache)+1)\n\t\tnewCache[key] = encode\n\n\t\tfor typ, fun := range cache {\n\t\t\tnewCache[typ] = fun\n\t\t}\n\n\t\tmarshalers.Store(newCache)\n\t}\n\n\te, _ := encoders.Get().(*encoder)\n\tif e == nil {\n\t\te = &encoder{writer: new(bytes.Buffer)}\n\t}\n\n\tb, _ := e.writer.(*bytes.Buffer)\n\tdefer func() {\n\t\tb.Reset()\n\t\te.Reset(b)\n\t\tencoders.Put(e)\n\t}()\n\n\tencode(e, nonAddressableValueOf(value))\n\n\tif e.err != nil {\n\t\treturn nil, e.err\n\t}\n\n\tbuf := b.Bytes()\n\tout := make([]byte, len(buf))\n\tcopy(out, buf)\n\treturn out, nil\n}\n\ntype versionedType struct {\n\ttyp _type\n\tversion int16\n}\n\nvar (\n\tencoders sync.Pool // *encoder\n\tmarshalers atomic.Value // map[versionedType]encodeFunc\n)\n" }, { "path": "protocol/endtxn/endtxn.go", "content": "package endtxn\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v3,max=v3,tag\"`\n\n\tTransactionalID string `kafka:\"min=v0,max=v2|min=v3,max=v3,compact\"`\n\tProducerID int64 `kafka:\"min=v0,max=v3\"`\n\tProducerEpoch int16 `kafka:\"min=v0,max=v3\"`\n\tCommitted bool `kafka:\"min=v0,max=v3\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.EndTxn }\n\nfunc (r *Request) Transaction() string { return r.TransactionalID }\n\nvar _ protocol.TransactionalMessage = (*Request)(nil)\n\ntype Response struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v3,max=v3,tag\"`\n\n\tThrottleTimeMs int32 `kafka:\"min=v0,max=v3\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v3\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.EndTxn }\n" }, { "path": "protocol/endtxn/endtxn_test.go", "content": "package endtxn_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/endtxn\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nfunc TestEndTxnRequest(t *testing.T) {\n\tfor _, version := range []int16{0, 1, 2, 3} {\n\t\tprototest.TestRequest(t, version, &endtxn.Request{\n\t\t\tTransactionalID: \"transactional-id-1\",\n\t\t\tProducerID: 1,\n\t\t\tProducerEpoch: 100,\n\t\t\tCommitted: false,\n\t\t})\n\t}\n}\n\nfunc TestEndTxnResponse(t *testing.T) {\n\tfor _, version := range []int16{0, 1, 2, 3} {\n\t\tprototest.TestResponse(t, version, &endtxn.Response{\n\t\t\tThrottleTimeMs: 1000,\n\t\t\tErrorCode: 4,\n\t\t})\n\t}\n}\n" }, { "path": "protocol/error.go", "content": "package protocol\n\nimport (\n\t\"fmt\"\n)\n\n// Error represents client-side protocol errors.\ntype Error string\n\nfunc (e Error) Error() string { return string(e) }\n\nfunc Errorf(msg string, args ...interface{}) Error {\n\treturn Error(fmt.Sprintf(msg, args...))\n}\n\nconst (\n\t// ErrNoTopic is returned when a request needs to be sent to a specific.\n\tErrNoTopic Error = \"topic not found\"\n\n\t// ErrNoPartition is returned when a request needs to be sent to a specific\n\t// partition, but the client did not find it in the cluster metadata.\n\tErrNoPartition Error = \"topic partition not found\"\n\n\t// ErrNoLeader is returned when a request needs to be sent to a partition\n\t// leader, but the client could not determine what the leader was at this\n\t// time.\n\tErrNoLeader Error = \"topic partition has no leader\"\n\n\t// ErrNoRecord is returned when attempting to write a message containing an\n\t// empty record set (which kafka forbids).\n\t//\n\t// We handle this case client-side because kafka will close the connection\n\t// that it received an empty produce request on, causing all concurrent\n\t// requests to be aborted.\n\tErrNoRecord Error = \"record set contains no records\"\n\n\t// ErrNoReset is returned by ResetRecordReader when the record reader does\n\t// not support being reset.\n\tErrNoReset Error = \"record sequence does not support reset\"\n)\n\ntype TopicError struct {\n\tTopic string\n\tErr error\n}\n\nfunc NewTopicError(topic string, err error) *TopicError {\n\treturn &TopicError{Topic: topic, Err: err}\n}\n\nfunc NewErrNoTopic(topic string) *TopicError {\n\treturn NewTopicError(topic, ErrNoTopic)\n}\n\nfunc (e *TopicError) Error() string {\n\treturn fmt.Sprintf(\"%v (topic=%q)\", e.Err, e.Topic)\n}\n\nfunc (e *TopicError) Unwrap() error {\n\treturn e.Err\n}\n\ntype TopicPartitionError struct {\n\tTopic string\n\tPartition int32\n\tErr error\n}\n\nfunc NewTopicPartitionError(topic string, partition int32, err error) *TopicPartitionError {\n\treturn &TopicPartitionError{\n\t\tTopic: topic,\n\t\tPartition: partition,\n\t\tErr: err,\n\t}\n}\n\nfunc NewErrNoPartition(topic string, partition int32) *TopicPartitionError {\n\treturn NewTopicPartitionError(topic, partition, ErrNoPartition)\n}\n\nfunc NewErrNoLeader(topic string, partition int32) *TopicPartitionError {\n\treturn NewTopicPartitionError(topic, partition, ErrNoLeader)\n}\n\nfunc (e *TopicPartitionError) Error() string {\n\treturn fmt.Sprintf(\"%v (topic=%q partition=%d)\", e.Err, e.Topic, e.Partition)\n}\n\nfunc (e *TopicPartitionError) Unwrap() error {\n\treturn e.Err\n}\n" }, { "path": "protocol/fetch/fetch.go", "content": "package fetch\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n)\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\tReplicaID int32 `kafka:\"min=v0,max=v11\"`\n\tMaxWaitTime int32 `kafka:\"min=v0,max=v11\"`\n\tMinBytes int32 `kafka:\"min=v0,max=v11\"`\n\tMaxBytes int32 `kafka:\"min=v3,max=v11\"`\n\tIsolationLevel int8 `kafka:\"min=v4,max=v11\"`\n\tSessionID int32 `kafka:\"min=v7,max=v11\"`\n\tSessionEpoch int32 `kafka:\"min=v7,max=v11\"`\n\tTopics []RequestTopic `kafka:\"min=v0,max=v11\"`\n\tForgottenTopics []RequestForgottenTopic `kafka:\"min=v7,max=v11\"`\n\tRackID string `kafka:\"min=v11,max=v11\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.Fetch }\n\nfunc (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, error) {\n\tbroker := protocol.Broker{ID: -1}\n\n\tfor i := range r.Topics {\n\t\tt := &r.Topics[i]\n\n\t\ttopic, ok := cluster.Topics[t.Topic]\n\t\tif !ok {\n\t\t\treturn broker, NewError(protocol.NewErrNoTopic(t.Topic))\n\t\t}\n\n\t\tfor j := range t.Partitions {\n\t\t\tp := &t.Partitions[j]\n\n\t\t\tpartition, ok := topic.Partitions[p.Partition]\n\t\t\tif !ok {\n\t\t\t\treturn broker, NewError(protocol.NewErrNoPartition(t.Topic, p.Partition))\n\t\t\t}\n\n\t\t\tif b, ok := cluster.Brokers[partition.Leader]; !ok {\n\t\t\t\treturn broker, NewError(protocol.NewErrNoLeader(t.Topic, p.Partition))\n\t\t\t} else if broker.ID < 0 {\n\t\t\t\tbroker = b\n\t\t\t} else if b.ID != broker.ID {\n\t\t\t\treturn broker, NewError(fmt.Errorf(\"mismatching leaders (%d!=%d)\", b.ID, broker.ID))\n\t\t\t}\n\t\t}\n\t}\n\n\treturn broker, nil\n}\n\ntype RequestTopic struct {\n\tTopic string `kafka:\"min=v0,max=v11\"`\n\tPartitions []RequestPartition `kafka:\"min=v0,max=v11\"`\n}\n\ntype RequestPartition struct {\n\tPartition int32 `kafka:\"min=v0,max=v11\"`\n\tCurrentLeaderEpoch int32 `kafka:\"min=v9,max=v11\"`\n\tFetchOffset int64 `kafka:\"min=v0,max=v11\"`\n\tLogStartOffset int64 `kafka:\"min=v5,max=v11\"`\n\tPartitionMaxBytes int32 `kafka:\"min=v0,max=v11\"`\n}\n\ntype RequestForgottenTopic struct {\n\tTopic string `kafka:\"min=v7,max=v11\"`\n\tPartitions []int32 `kafka:\"min=v7,max=v11\"`\n}\n\ntype Response struct {\n\tThrottleTimeMs int32 `kafka:\"min=v1,max=v11\"`\n\tErrorCode int16 `kafka:\"min=v7,max=v11\"`\n\tSessionID int32 `kafka:\"min=v7,max=v11\"`\n\tTopics []ResponseTopic `kafka:\"min=v0,max=v11\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.Fetch }\n\ntype ResponseTopic struct {\n\tTopic string `kafka:\"min=v0,max=v11\"`\n\tPartitions []ResponsePartition `kafka:\"min=v0,max=v11\"`\n}\n\ntype ResponsePartition struct {\n\tPartition int32 `kafka:\"min=v0,max=v11\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v11\"`\n\tHighWatermark int64 `kafka:\"min=v0,max=v11\"`\n\tLastStableOffset int64 `kafka:\"min=v4,max=v11\"`\n\tLogStartOffset int64 `kafka:\"min=v5,max=v11\"`\n\tAbortedTransactions []ResponseTransaction `kafka:\"min=v4,max=v11\"`\n\tPreferredReadReplica int32 `kafka:\"min=v11,max=v11\"`\n\tRecordSet protocol.RecordSet `kafka:\"min=v0,max=v11\"`\n}\n\ntype ResponseTransaction struct {\n\tProducerID int64 `kafka:\"min=v4,max=v11\"`\n\tFirstOffset int64 `kafka:\"min=v4,max=v11\"`\n}\n\nvar (\n\t_ protocol.BrokerMessage = (*Request)(nil)\n)\n\ntype Error struct {\n\tErr error\n}\n\nfunc NewError(err error) *Error {\n\treturn &Error{Err: err}\n}\n\nfunc (e *Error) Error() string {\n\treturn fmt.Sprintf(\"fetch request error: %v\", e.Err)\n}\n\nfunc (e *Error) Unwrap() error {\n\treturn e.Err\n}\n" }, { "path": "protocol/fetch/fetch_test.go", "content": "package fetch_test\n\nimport (\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n\t\"github.com/segmentio/kafka-go/protocol/fetch\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nconst (\n\tv0 = 0\n\tv11 = 11\n)\n\nfunc TestFetchRequest(t *testing.T) {\n\tprototest.TestRequest(t, v0, &fetch.Request{\n\t\tReplicaID: -1,\n\t\tMaxWaitTime: 500,\n\t\tMinBytes: 1024,\n\t\tTopics: []fetch.RequestTopic{\n\t\t\t{\n\t\t\t\tTopic: \"topic-1\",\n\t\t\t\tPartitions: []fetch.RequestPartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\tFetchOffset: 2,\n\t\t\t\t\t\tPartitionMaxBytes: 1024,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n}\n\nfunc TestFetchResponse(t *testing.T) {\n\tt0 := time.Now().Truncate(time.Millisecond)\n\tt1 := t0.Add(1 * time.Millisecond)\n\tt2 := t0.Add(2 * time.Millisecond)\n\n\tprototest.TestResponse(t, v0, &fetch.Response{\n\t\tTopics: []fetch.ResponseTopic{\n\t\t\t{\n\t\t\t\tTopic: \"topic-1\",\n\t\t\t\tPartitions: []fetch.ResponsePartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\tHighWatermark: 1000,\n\t\t\t\t\t\tRecordSet: protocol.RecordSet{\n\t\t\t\t\t\t\tVersion: 1,\n\t\t\t\t\t\t\tRecords: protocol.NewRecordReader(\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 0, Time: t0, Key: nil, Value: prototest.String(\"msg-0\")},\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 1, Time: t1, Key: nil, Value: prototest.String(\"msg-1\")},\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 2, Time: t2, Key: prototest.Bytes([]byte{1}), Value: prototest.String(\"msg-2\")},\n\t\t\t\t\t\t\t),\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\theaders := []protocol.Header{\n\t\t{Key: \"key-1\", Value: []byte(\"value-1\")},\n\t\t{Key: \"key-2\", Value: []byte(\"value-2\")},\n\t\t{Key: \"key-3\", Value: []byte(\"value-3\")},\n\t}\n\n\tprototest.TestResponse(t, v11, &fetch.Response{\n\t\tTopics: []fetch.ResponseTopic{\n\t\t\t{\n\t\t\t\tTopic: \"topic-1\",\n\t\t\t\tPartitions: []fetch.ResponsePartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\tHighWatermark: 1000,\n\t\t\t\t\t\tRecordSet: protocol.RecordSet{\n\t\t\t\t\t\t\tVersion: 2,\n\t\t\t\t\t\t\tRecords: protocol.NewRecordReader(\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 0, Time: t0, Key: nil, Value: prototest.String(\"msg-0\"), Headers: headers},\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 1, Time: t1, Key: nil, Value: prototest.String(\"msg-1\")},\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 2, Time: t2, Key: prototest.Bytes([]byte{1}), Value: prototest.String(\"msg-2\")},\n\t\t\t\t\t\t\t),\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n}\n\nfunc BenchmarkFetchResponse(b *testing.B) {\n\tt0 := time.Now().Truncate(time.Millisecond)\n\tt1 := t0.Add(1 * time.Millisecond)\n\tt2 := t0.Add(2 * time.Millisecond)\n\n\tprototest.BenchmarkResponse(b, v0, &fetch.Response{\n\t\tTopics: []fetch.ResponseTopic{\n\t\t\t{\n\t\t\t\tTopic: \"topic-1\",\n\t\t\t\tPartitions: []fetch.ResponsePartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\tHighWatermark: 1000,\n\t\t\t\t\t\tRecordSet: protocol.RecordSet{\n\t\t\t\t\t\t\tVersion: 1,\n\t\t\t\t\t\t\tRecords: protocol.NewRecordReader(\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 0, Time: t0, Key: nil, Value: prototest.String(\"msg-0\")},\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 1, Time: t1, Key: nil, Value: prototest.String(\"msg-1\")},\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 2, Time: t2, Key: prototest.Bytes([]byte{1}), Value: prototest.String(\"msg-2\")},\n\t\t\t\t\t\t\t),\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\theaders := []protocol.Header{\n\t\t{Key: \"key-1\", Value: []byte(\"value-1\")},\n\t\t{Key: \"key-2\", Value: []byte(\"value-2\")},\n\t\t{Key: \"key-3\", Value: []byte(\"value-3\")},\n\t}\n\n\tprototest.BenchmarkResponse(b, v11, &fetch.Response{\n\t\tTopics: []fetch.ResponseTopic{\n\t\t\t{\n\t\t\t\tTopic: \"topic-1\",\n\t\t\t\tPartitions: []fetch.ResponsePartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\tHighWatermark: 1000,\n\t\t\t\t\t\tRecordSet: protocol.RecordSet{\n\t\t\t\t\t\t\tVersion: 2,\n\t\t\t\t\t\t\tRecords: protocol.NewRecordReader(\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 0, Time: t0, Key: nil, Value: prototest.String(\"msg-0\"), Headers: headers},\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 1, Time: t1, Key: nil, Value: prototest.String(\"msg-1\")},\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 2, Time: t2, Key: prototest.Bytes([]byte{1}), Value: prototest.String(\"msg-2\")},\n\t\t\t\t\t\t\t),\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n}\n" }, { "path": "protocol/findcoordinator/findcoordinator.go", "content": "package findcoordinator\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\tKey string `kafka:\"min=v0,max=v2\"`\n\tKeyType int8 `kafka:\"min=v1,max=v2\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.FindCoordinator }\n\ntype Response struct {\n\tThrottleTimeMs int32 `kafka:\"min=v1,max=v2\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v2\"`\n\tErrorMessage string `kafka:\"min=v1,max=v2,nullable\"`\n\tNodeID int32 `kafka:\"min=v0,max=v2\"`\n\tHost string `kafka:\"min=v0,max=v2\"`\n\tPort int32 `kafka:\"min=v0,max=v2\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.FindCoordinator }\n" }, { "path": "protocol/heartbeat/heartbeat.go", "content": "package heartbeat\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\n// Detailed API definition: https://kafka.apache.org/protocol#The_Messages_Heartbeat\ntype Request struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v4,max=v4,tag\"`\n\n\tGroupID string `kafka:\"min=v0,max=v4\"`\n\tGenerationID int32 `kafka:\"min=v0,max=v4\"`\n\tMemberID string `kafka:\"min=v0,max=v4\"`\n\tGroupInstanceID string `kafka:\"min=v3,max=v4,nullable\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey {\n\treturn protocol.Heartbeat\n}\n\ntype Response struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v4,max=v4,tag\"`\n\n\tErrorCode int16 `kafka:\"min=v0,max=v4\"`\n\tThrottleTimeMs int32 `kafka:\"min=v1,max=v4\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey {\n\treturn protocol.Heartbeat\n}\n" }, { "path": "protocol/heartbeat/heartbeat_test.go", "content": "package heartbeat_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/heartbeat\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nfunc TestHeartbeatRequest(t *testing.T) {\n\t// Versions 0-3 have all the same fields.\n\tfor _, version := range []int16{0, 1, 2, 3} {\n\t\tprototest.TestRequest(t, version, &heartbeat.Request{\n\t\t\tGroupID: \"group-1\",\n\t\t\tGenerationID: 1,\n\t\t\tMemberID: \"member-1\",\n\t\t})\n\t}\n\n\tfor _, version := range []int16{4} {\n\t\tprototest.TestRequest(t, version, &heartbeat.Request{\n\t\t\tGroupID: \"group-2\",\n\t\t\tGenerationID: 10,\n\t\t\tMemberID: \"member-2\",\n\t\t\tGroupInstanceID: \"instace-1\",\n\t\t})\n\t}\n}\n\nfunc TestHeartbeatResponse(t *testing.T) {\n\tfor _, version := range []int16{0} {\n\t\tprototest.TestResponse(t, version, &heartbeat.Response{\n\t\t\tErrorCode: 4,\n\t\t})\n\t}\n\n\t// Versions 1-4 have all the same fields.\n\tfor _, version := range []int16{1, 2, 3, 4} {\n\t\tprototest.TestResponse(t, version, &heartbeat.Response{\n\t\t\tErrorCode: 4,\n\t\t\tThrottleTimeMs: 10,\n\t\t})\n\t}\n}\n" }, { "path": "protocol/incrementalalterconfigs/incrementalalterconfigs.go", "content": "package incrementalalterconfigs\n\nimport (\n\t\"errors\"\n\t\"strconv\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n)\n\nconst (\n\tresourceTypeBroker int8 = 4\n)\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\n// Detailed API definition: https://kafka.apache.org/protocol#The_Messages_IncrementalAlterConfigs\ntype Request struct {\n\tResources []RequestResource `kafka:\"min=v0,max=v0\"`\n\tValidateOnly bool `kafka:\"min=v0,max=v0\"`\n}\n\ntype RequestResource struct {\n\tResourceType int8 `kafka:\"min=v0,max=v0\"`\n\tResourceName string `kafka:\"min=v0,max=v0\"`\n\tConfigs []RequestConfig `kafka:\"min=v0,max=v0\"`\n}\n\ntype RequestConfig struct {\n\tName string `kafka:\"min=v0,max=v0\"`\n\tConfigOperation int8 `kafka:\"min=v0,max=v0\"`\n\tValue string `kafka:\"min=v0,max=v0,nullable\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.IncrementalAlterConfigs }\n\nfunc (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, error) {\n\t// Check that at most only one broker is being updated.\n\t//\n\t// TODO: Support updating multiple brokers in a single request.\n\tbrokers := map[string]struct{}{}\n\tfor _, resource := range r.Resources {\n\t\tif resource.ResourceType == resourceTypeBroker {\n\t\t\tbrokers[resource.ResourceName] = struct{}{}\n\t\t}\n\t}\n\tif len(brokers) > 1 {\n\t\treturn protocol.Broker{},\n\t\t\terrors.New(\"Updating more than one broker in a single request is not supported yet\")\n\t}\n\n\tfor _, resource := range r.Resources {\n\t\tif resource.ResourceType == resourceTypeBroker {\n\t\t\tbrokerID, err := strconv.Atoi(resource.ResourceName)\n\t\t\tif err != nil {\n\t\t\t\treturn protocol.Broker{}, err\n\t\t\t}\n\n\t\t\treturn cluster.Brokers[int32(brokerID)], nil\n\t\t}\n\t}\n\n\treturn cluster.Brokers[cluster.Controller], nil\n}\n\ntype Response struct {\n\tThrottleTimeMs int32 `kafka:\"min=v0,max=v0\"`\n\tResponses []ResponseAlterResponse `kafka:\"min=v0,max=v0\"`\n}\n\ntype ResponseAlterResponse struct {\n\tErrorCode int16 `kafka:\"min=v0,max=v0\"`\n\tErrorMessage string `kafka:\"min=v0,max=v0,nullable\"`\n\tResourceType int8 `kafka:\"min=v0,max=v0\"`\n\tResourceName string `kafka:\"min=v0,max=v0\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.IncrementalAlterConfigs }\n" }, { "path": "protocol/incrementalalterconfigs/incrementalalterconfigs_test.go", "content": "package incrementalalterconfigs_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n\t\"github.com/segmentio/kafka-go/protocol/incrementalalterconfigs\"\n)\n\nconst (\n\tresourceTypeTopic int8 = 2\n\tresourceTypeBroker int8 = 4\n)\n\nfunc TestMetadataRequestBroker(t *testing.T) {\n\treq := &incrementalalterconfigs.Request{\n\t\tResources: []incrementalalterconfigs.RequestResource{\n\t\t\t{\n\t\t\t\tResourceType: resourceTypeBroker,\n\t\t\t\tResourceName: \"1\",\n\t\t\t\tConfigs: []incrementalalterconfigs.RequestConfig{\n\t\t\t\t\t{\n\t\t\t\t\t\tName: \"test-name1\",\n\t\t\t\t\t\tValue: \"test-value1\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tResourceType: resourceTypeBroker,\n\t\t\t\tResourceName: \"1\",\n\t\t\t\tConfigs: []incrementalalterconfigs.RequestConfig{\n\t\t\t\t\t{\n\t\t\t\t\t\tName: \"test-name2\",\n\t\t\t\t\t\tValue: \"test-value2\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tResourceType: resourceTypeTopic,\n\t\t\t\tResourceName: \"test-topic1\",\n\t\t\t\tConfigs: []incrementalalterconfigs.RequestConfig{\n\t\t\t\t\t{\n\t\t\t\t\t\tName: \"test-name3\",\n\t\t\t\t\t\tValue: \"test-value3\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tResourceType: resourceTypeTopic,\n\t\t\t\tResourceName: \"test-topic2\",\n\t\t\t\tConfigs: []incrementalalterconfigs.RequestConfig{\n\t\t\t\t\t{\n\t\t\t\t\t\tName: \"test-name4\",\n\t\t\t\t\t\tValue: \"test-value4\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\tb, err := req.Broker(protocol.Cluster{\n\t\tBrokers: map[int32]protocol.Broker{\n\t\t\t0: {\n\t\t\t\tID: 0,\n\t\t\t},\n\t\t\t1: {\n\t\t\t\tID: 1,\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(\n\t\t\t\"Unexpected error getting request broker\",\n\t\t\t\"expected\", nil,\n\t\t\t\"got\", err,\n\t\t)\n\t}\n\tif b.ID != 1 {\n\t\tt.Error(\n\t\t\t\"Unexpected id returned for request broker\",\n\t\t\t\"expected\", 1,\n\t\t\t\"got\", b.ID,\n\t\t)\n\t}\n\n\treq = &incrementalalterconfigs.Request{\n\t\tResources: []incrementalalterconfigs.RequestResource{\n\t\t\t{\n\t\t\t\tResourceType: resourceTypeBroker,\n\t\t\t\tResourceName: \"1\",\n\t\t\t\tConfigs: []incrementalalterconfigs.RequestConfig{\n\t\t\t\t\t{\n\t\t\t\t\t\tName: \"test-name1\",\n\t\t\t\t\t\tValue: \"test-value1\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tResourceType: resourceTypeBroker,\n\t\t\t\tResourceName: \"2\",\n\t\t\t\tConfigs: []incrementalalterconfigs.RequestConfig{\n\t\t\t\t\t{\n\t\t\t\t\t\tName: \"test-name2\",\n\t\t\t\t\t\tValue: \"test-value2\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\t_, err = req.Broker(protocol.Cluster{\n\t\tBrokers: map[int32]protocol.Broker{\n\t\t\t0: {\n\t\t\t\tID: 0,\n\t\t\t},\n\t\t\t1: {\n\t\t\t\tID: 1,\n\t\t\t},\n\t\t\t2: {\n\t\t\t\tID: 1,\n\t\t\t},\n\t\t},\n\t})\n\tif err == nil {\n\t\tt.Fatal(\n\t\t\t\"Unexpected error getting request broker\",\n\t\t\t\"expected\", \"non-nil\",\n\t\t\t\"got\", err,\n\t\t)\n\t}\n}\n" }, { "path": "protocol/initproducerid/initproducerid.go", "content": "package initproducerid\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v2,max=v4,tag\"`\n\n\tTransactionalID string `kafka:\"min=v0,max=v4,nullable\"`\n\tTransactionTimeoutMs int32 `kafka:\"min=v0,max=v4\"`\n\tProducerID int64 `kafka:\"min=v3,max=v4\"`\n\tProducerEpoch int16 `kafka:\"min=v3,max=v4\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.InitProducerId }\n\nfunc (r *Request) Transaction() string { return r.TransactionalID }\n\nvar _ protocol.TransactionalMessage = (*Request)(nil)\n\ntype Response struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v2,max=v4,tag\"`\n\n\tThrottleTimeMs int32 `kafka:\"min=v0,max=v4\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v4\"`\n\tProducerID int64 `kafka:\"min=v0,max=v4\"`\n\tProducerEpoch int16 `kafka:\"min=v0,max=v4\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.InitProducerId }\n" }, { "path": "protocol/initproducerid/initproducerid_test.go", "content": "package initproducerid_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/initproducerid\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nfunc TestInitProducerIDRequest(t *testing.T) {\n\tfor _, version := range []int16{0, 1, 2} {\n\t\tprototest.TestRequest(t, version, &initproducerid.Request{\n\t\t\tTransactionalID: \"transactional-id-0\",\n\t\t\tTransactionTimeoutMs: 1000,\n\t\t})\n\t}\n\n\t// Version 2 added:\n\t// ProducerID\n\t// ProducerEpoch\n\tfor _, version := range []int16{3, 4} {\n\t\tprototest.TestRequest(t, version, &initproducerid.Request{\n\t\t\tTransactionalID: \"transactional-id-0\",\n\t\t\tTransactionTimeoutMs: 1000,\n\t\t\tProducerID: 10,\n\t\t\tProducerEpoch: 5,\n\t\t})\n\t}\n}\n\nfunc TestInitProducerIDResponse(t *testing.T) {\n\tfor _, version := range []int16{0, 1, 2, 3, 4} {\n\t\tprototest.TestResponse(t, version, &initproducerid.Response{\n\t\t\tThrottleTimeMs: 1000,\n\t\t\tErrorCode: 9,\n\t\t\tProducerID: 10,\n\t\t\tProducerEpoch: 1000,\n\t\t})\n\t}\n}\n" }, { "path": "protocol/joingroup/joingroup.go", "content": "package joingroup\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v6,max=v7,tag\"`\n\n\tGroupID string `kafka:\"min=v0,max=v5|min=v6,max=v7,compact\"`\n\tSessionTimeoutMS int32 `kafka:\"min=v0,max=v7\"`\n\tRebalanceTimeoutMS int32 `kafka:\"min=v1,max=v7\"`\n\tMemberID string `kafka:\"min=v0,max=v5|min=v6,max=v7,compact\"`\n\tGroupInstanceID string `kafka:\"min=v5,max=v5,nullable|min=v6,max=v7,compact,nullable\"`\n\tProtocolType string `kafka:\"min=v0,max=v5|min=v6,max=v7,compact\"`\n\tProtocols []RequestProtocol `kafka:\"min=v0,max=v7\"`\n}\n\ntype RequestProtocol struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v6,max=v7,tag\"`\n\n\tName string `kafka:\"min=v0,max=v5|min=v6,max=v7,compact\"`\n\tMetadata []byte `kafka:\"min=v0,max=v5|min=v6,max=v7,compact\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey {\n\treturn protocol.JoinGroup\n}\n\nfunc (r *Request) Group() string { return r.GroupID }\n\nvar _ protocol.GroupMessage = (*Request)(nil)\n\ntype Response struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v6,max=v7,tag\"`\n\n\tThrottleTimeMS int32 `kafka:\"min=v2,max=v7\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v7\"`\n\tGenerationID int32 `kafka:\"min=v0,max=v7\"`\n\tProtocolType string `kafka:\"min=v7,max=v7,compact,nullable\"`\n\tProtocolName string `kafka:\"min=v0,max=v5|min=v6,max=v6,compact|min=v7,max=v7,compact,nullable\"`\n\tLeaderID string `kafka:\"min=v0,max=v5|min=v6,max=v7,compact\"`\n\tMemberID string `kafka:\"min=v0,max=v5|min=v6,max=v7,compact\"`\n\tMembers []ResponseMember `kafka:\"min=v0,max=v7\"`\n}\n\ntype ResponseMember struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v6,max=v7,tag\"`\n\n\tMemberID string `kafka:\"min=v0,max=v5|min=v6,max=v7,compact\"`\n\tGroupInstanceID string `kafka:\"min=v5,max=v5,nullable|min=v6,max=v7,nullable,compact\"`\n\tMetadata []byte `kafka:\"min=v0,max=v5|min=v6,max=v7,compact\"`\n}\n\ntype ResponseMemberMetadata struct{}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.JoinGroup }\n" }, { "path": "protocol/joingroup/joingroup_test.go", "content": "package joingroup_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/joingroup\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nfunc TestJoinGroupReq(t *testing.T) {\n\tfor _, version := range []int16{0} {\n\t\tprototest.TestRequest(t, version, &joingroup.Request{\n\t\t\tGroupID: \"group-id\",\n\t\t\tSessionTimeoutMS: 10,\n\t\t\tMemberID: \"member-id\",\n\t\t\tProtocolType: \"protocol-type\",\n\t\t\tProtocols: []joingroup.RequestProtocol{\n\t\t\t\t{\n\t\t\t\t\tName: \"protocol-1\",\n\t\t\t\t\tMetadata: []byte{0, 1, 2, 3, 4},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n\n\t// Version 1 added\n\t// RebalanceTimeoutMS\n\tfor _, version := range []int16{1, 2, 3, 4} {\n\t\tprototest.TestRequest(t, version, &joingroup.Request{\n\t\t\tGroupID: \"group-id\",\n\t\t\tSessionTimeoutMS: 10,\n\t\t\tRebalanceTimeoutMS: 10,\n\t\t\tMemberID: \"member-id\",\n\t\t\tProtocolType: \"protocol-type\",\n\t\t\tProtocols: []joingroup.RequestProtocol{\n\t\t\t\t{\n\t\t\t\t\tName: \"protocol-1\",\n\t\t\t\t\tMetadata: []byte{0, 1, 2, 3, 4},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n\n\t// Version 5 added\n\t// GroupInstanceID\n\tfor _, version := range []int16{5, 6, 7} {\n\t\tprototest.TestRequest(t, version, &joingroup.Request{\n\t\t\tGroupID: \"group-id\",\n\t\t\tSessionTimeoutMS: 10,\n\t\t\tRebalanceTimeoutMS: 10,\n\t\t\tMemberID: \"member-id\",\n\t\t\tProtocolType: \"protocol-type\",\n\t\t\tGroupInstanceID: \"group-instance-id\",\n\t\t\tProtocols: []joingroup.RequestProtocol{\n\t\t\t\t{\n\t\t\t\t\tName: \"protocol-1\",\n\t\t\t\t\tMetadata: []byte{0, 1, 2, 3, 4},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n}\n\nfunc TestJoinGroupResp(t *testing.T) {\n\tfor _, version := range []int16{0, 1} {\n\t\tprototest.TestResponse(t, version, &joingroup.Response{\n\t\t\tErrorCode: 10,\n\t\t\tGenerationID: 10,\n\t\t\tProtocolName: \"protocol-name\",\n\t\t\tLeaderID: \"leader\",\n\t\t\tMemberID: \"member-id-1\",\n\t\t\tMembers: []joingroup.ResponseMember{\n\t\t\t\t{\n\t\t\t\t\tMemberID: \"member-id-2\",\n\t\t\t\t\tMetadata: []byte{0, 1, 2, 3, 4},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n\n\t// Version 2 added\n\t// ThrottleTimeMS\n\tfor _, version := range []int16{2, 3, 4} {\n\t\tprototest.TestResponse(t, version, &joingroup.Response{\n\t\t\tErrorCode: 10,\n\t\t\tGenerationID: 10,\n\t\t\tThrottleTimeMS: 100,\n\t\t\tProtocolName: \"protocol-name\",\n\t\t\tLeaderID: \"leader\",\n\t\t\tMemberID: \"member-id-1\",\n\t\t\tMembers: []joingroup.ResponseMember{\n\t\t\t\t{\n\t\t\t\t\tMemberID: \"member-id-2\",\n\t\t\t\t\tMetadata: []byte{0, 1, 2, 3, 4},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n\n\t// Version 5 added\n\t// ResponseMember.GroupInstanceID\n\tfor _, version := range []int16{5, 6} {\n\t\tprototest.TestResponse(t, version, &joingroup.Response{\n\t\t\tErrorCode: 10,\n\t\t\tGenerationID: 10,\n\t\t\tThrottleTimeMS: 100,\n\t\t\tProtocolName: \"protocol-name\",\n\t\t\tLeaderID: \"leader\",\n\t\t\tMemberID: \"member-id-1\",\n\t\t\tMembers: []joingroup.ResponseMember{\n\t\t\t\t{\n\t\t\t\t\tMemberID: \"member-id-2\",\n\t\t\t\t\tMetadata: []byte{0, 1, 2, 3, 4},\n\t\t\t\t\tGroupInstanceID: \"group-instance-id\",\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n\n\t// Version 7 added\n\t// ProtocolType\n\tfor _, version := range []int16{7} {\n\t\tprototest.TestResponse(t, version, &joingroup.Response{\n\t\t\tErrorCode: 10,\n\t\t\tGenerationID: 10,\n\t\t\tThrottleTimeMS: 100,\n\t\t\tProtocolName: \"protocol-name\",\n\t\t\tProtocolType: \"protocol-type\",\n\t\t\tLeaderID: \"leader\",\n\t\t\tMemberID: \"member-id-1\",\n\t\t\tMembers: []joingroup.ResponseMember{\n\t\t\t\t{\n\t\t\t\t\tMemberID: \"member-id-2\",\n\t\t\t\t\tMetadata: []byte{0, 1, 2, 3, 4},\n\t\t\t\t\tGroupInstanceID: \"group-instance-id\",\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n}\n" }, { "path": "protocol/leavegroup/leavegroup.go", "content": "package leavegroup\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v4,max=v4,tag\"`\n\n\tGroupID string `kafka:\"min=v0,max=v2|min=v3,max=v4,compact\"`\n\tMemberID string `kafka:\"min=v0,max=v2\"`\n\tMembers []RequestMember `kafka:\"min=v3,max=v4\"`\n}\n\nfunc (r *Request) Prepare(apiVersion int16) {\n\tif apiVersion < 3 {\n\t\tif len(r.Members) > 0 {\n\t\t\tr.MemberID = r.Members[0].MemberID\n\t\t}\n\t}\n}\n\ntype RequestMember struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v4,max=v4,tag\"`\n\n\tMemberID string `kafka:\"min=v3,max=v3|min=v4,max=v4,compact\"`\n\tGroupInstanceID string `kafka:\"min=v3,max=v3,nullable|min=v4,max=v4,nullable,compact\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.LeaveGroup }\n\nfunc (r *Request) Group() string { return r.GroupID }\n\nvar (\n\t_ protocol.GroupMessage = (*Request)(nil)\n\t_ protocol.PreparedMessage = (*Request)(nil)\n)\n\ntype Response struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v4,max=v4,tag\"`\n\n\tErrorCode int16 `kafka:\"min=v0,max=v4\"`\n\tThrottleTimeMS int32 `kafka:\"min=v1,max=v4\"`\n\tMembers []ResponseMember `kafka:\"min=v3,max=v4\"`\n}\n\ntype ResponseMember struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v4,max=v4,tag\"`\n\n\tMemberID string `kafka:\"min=v3,max=v3|min=v4,max=v4,compact\"`\n\tGroupInstanceID string `kafka:\"min=v3,max=v3,nullable|min=v4,max=v4,nullable,compact\"`\n\tErrorCode int16 `kafka:\"min=v3,max=v4\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.LeaveGroup }\n" }, { "path": "protocol/leavegroup/leavegroup_test.go", "content": "package leavegroup_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/leavegroup\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nfunc TestLeaveGroupReq(t *testing.T) {\n\tfor _, version := range []int16{0, 1, 2} {\n\t\tprototest.TestRequest(t, version, &leavegroup.Request{\n\t\t\tGroupID: \"group-id\",\n\t\t\tMemberID: \"member-id\",\n\t\t})\n\t}\n\n\t// Version 3 added\n\t// Members\n\t// removed\n\t// MemberID\n\tfor _, version := range []int16{3, 4} {\n\t\tprototest.TestRequest(t, version, &leavegroup.Request{\n\t\t\tGroupID: \"group-id\",\n\t\t\tMembers: []leavegroup.RequestMember{\n\t\t\t\t{\n\t\t\t\t\tMemberID: \"member-id-1\",\n\t\t\t\t\tGroupInstanceID: \"group-instance-id\",\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n}\n\nfunc TestLeaveGroupResp(t *testing.T) {\n\tfor _, version := range []int16{0} {\n\t\tprototest.TestResponse(t, version, &leavegroup.Response{\n\t\t\tErrorCode: 10,\n\t\t})\n\t}\n\n\t// Version 1 added\n\t// ThrottleTimeMS\n\tfor _, version := range []int16{1, 2} {\n\t\tprototest.TestResponse(t, version, &leavegroup.Response{\n\t\t\tErrorCode: 10,\n\t\t\tThrottleTimeMS: 100,\n\t\t})\n\t}\n\n\t// Version 3 added\n\t// Members\n\tfor _, version := range []int16{3, 4} {\n\t\tprototest.TestResponse(t, version, &leavegroup.Response{\n\t\t\tErrorCode: 10,\n\t\t\tThrottleTimeMS: 100,\n\t\t\tMembers: []leavegroup.ResponseMember{\n\t\t\t\t{\n\t\t\t\t\tMemberID: \"member-id-1\",\n\t\t\t\t\tGroupInstanceID: \"group-instance-id\",\n\t\t\t\t\tErrorCode: 10,\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n}\n" }, { "path": "protocol/listgroups/listgroups.go", "content": "package listgroups\n\nimport (\n\t\"github.com/segmentio/kafka-go/protocol\"\n)\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\n// Detailed API definition: https://kafka.apache.org/protocol#The_Messages_ListGroups\ntype Request struct {\n\t_ struct{} `kafka:\"min=v0,max=v2\"`\n\tbrokerID int32\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.ListGroups }\n\nfunc (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, error) {\n\treturn cluster.Brokers[r.brokerID], nil\n}\n\nfunc (r *Request) Split(cluster protocol.Cluster) (\n\t[]protocol.Message,\n\tprotocol.Merger,\n\terror,\n) {\n\tmessages := []protocol.Message{}\n\n\tfor _, broker := range cluster.Brokers {\n\t\tmessages = append(messages, &Request{brokerID: broker.ID})\n\t}\n\n\treturn messages, new(Response), nil\n}\n\ntype Response struct {\n\tThrottleTimeMs int32 `kafka:\"min=v1,max=v2\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v2\"`\n\tGroups []ResponseGroup `kafka:\"min=v0,max=v2\"`\n}\n\ntype ResponseGroup struct {\n\tGroupID string `kafka:\"min=v0,max=v2\"`\n\tProtocolType string `kafka:\"min=v0,max=v2\"`\n\n\t// Use this to store which broker returned the response\n\tBrokerID int32 `kafka:\"-\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.ListGroups }\n\nfunc (r *Response) Merge(requests []protocol.Message, results []interface{}) (\n\tprotocol.Message,\n\terror,\n) {\n\tresponse := &Response{}\n\n\tfor r, result := range results {\n\t\tm, err := protocol.Result(result)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tbrokerResp := m.(*Response)\n\t\trespGroups := []ResponseGroup{}\n\n\t\tfor _, brokerResp := range brokerResp.Groups {\n\t\t\trespGroups = append(\n\t\t\t\trespGroups,\n\t\t\t\tResponseGroup{\n\t\t\t\t\tGroupID: brokerResp.GroupID,\n\t\t\t\t\tProtocolType: brokerResp.ProtocolType,\n\t\t\t\t\tBrokerID: requests[r].(*Request).brokerID,\n\t\t\t\t},\n\t\t\t)\n\t\t}\n\n\t\tresponse.Groups = append(response.Groups, respGroups...)\n\t}\n\n\treturn response, nil\n}\n" }, { "path": "protocol/listoffsets/listoffsets.go", "content": "package listoffsets\n\nimport (\n\t\"sort\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n)\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\tReplicaID int32 `kafka:\"min=v1,max=v5\"`\n\tIsolationLevel int8 `kafka:\"min=v2,max=v5\"`\n\tTopics []RequestTopic `kafka:\"min=v1,max=v5\"`\n}\n\ntype RequestTopic struct {\n\tTopic string `kafka:\"min=v1,max=v5\"`\n\tPartitions []RequestPartition `kafka:\"min=v1,max=v5\"`\n}\n\ntype RequestPartition struct {\n\tPartition int32 `kafka:\"min=v1,max=v5\"`\n\tCurrentLeaderEpoch int32 `kafka:\"min=v4,max=v5\"`\n\tTimestamp int64 `kafka:\"min=v1,max=v5\"`\n\t// v0 of the API predates kafka 0.10, and doesn't make much sense to\n\t// use so we chose not to support it. It had this extra field to limit\n\t// the number of offsets returned, which has been removed in v1.\n\t//\n\t// MaxNumOffsets int32 `kafka:\"min=v0,max=v0\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.ListOffsets }\n\nfunc (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, error) {\n\t// Expects r to be a request that was returned by Map, will likely panic\n\t// or produce the wrong result if that's not the case.\n\tpartition := r.Topics[0].Partitions[0].Partition\n\ttopic := r.Topics[0].Topic\n\n\tfor _, p := range cluster.Topics[topic].Partitions {\n\t\tif p.ID == partition {\n\t\t\treturn cluster.Brokers[p.Leader], nil\n\t\t}\n\t}\n\n\treturn protocol.Broker{ID: -1}, nil\n}\n\nfunc (r *Request) Split(cluster protocol.Cluster) ([]protocol.Message, protocol.Merger, error) {\n\t// Because kafka refuses to answer ListOffsets requests containing multiple\n\t// entries of unique topic/partition pairs, we submit multiple requests on\n\t// the wire and merge their results back.\n\t//\n\t// ListOffsets requests also need to be sent to partition leaders, to keep\n\t// the logic simple we simply split each offset request into a single\n\t// message. This may cause a bit more requests to be sent on the wire but\n\t// it keeps the code sane, we can still optimize the aggregation mechanism\n\t// later if it becomes a problem.\n\t//\n\t// Really the idea here is to shield applications from having to deal with\n\t// the limitation of the kafka server, so they can request any combinations\n\t// of topic/partition/offsets.\n\trequests := make([]Request, 0, 2*len(r.Topics))\n\n\tfor _, t := range r.Topics {\n\t\tfor _, p := range t.Partitions {\n\t\t\trequests = append(requests, Request{\n\t\t\t\tReplicaID: r.ReplicaID,\n\t\t\t\tIsolationLevel: r.IsolationLevel,\n\t\t\t\tTopics: []RequestTopic{{\n\t\t\t\t\tTopic: t.Topic,\n\t\t\t\t\tPartitions: []RequestPartition{{\n\t\t\t\t\t\tPartition: p.Partition,\n\t\t\t\t\t\tCurrentLeaderEpoch: p.CurrentLeaderEpoch,\n\t\t\t\t\t\tTimestamp: p.Timestamp,\n\t\t\t\t\t}},\n\t\t\t\t}},\n\t\t\t})\n\t\t}\n\t}\n\n\tmessages := make([]protocol.Message, len(requests))\n\n\tfor i := range requests {\n\t\tmessages[i] = &requests[i]\n\t}\n\n\treturn messages, new(Response), nil\n}\n\ntype Response struct {\n\tThrottleTimeMs int32 `kafka:\"min=v2,max=v5\"`\n\tTopics []ResponseTopic `kafka:\"min=v1,max=v5\"`\n}\n\ntype ResponseTopic struct {\n\tTopic string `kafka:\"min=v1,max=v5\"`\n\tPartitions []ResponsePartition `kafka:\"min=v1,max=v5\"`\n}\n\ntype ResponsePartition struct {\n\tPartition int32 `kafka:\"min=v1,max=v5\"`\n\tErrorCode int16 `kafka:\"min=v1,max=v5\"`\n\tTimestamp int64 `kafka:\"min=v1,max=v5\"`\n\tOffset int64 `kafka:\"min=v1,max=v5\"`\n\tLeaderEpoch int32 `kafka:\"min=v4,max=v5\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.ListOffsets }\n\nfunc (r *Response) Merge(requests []protocol.Message, results []interface{}) (protocol.Message, error) {\n\ttype topicPartition struct {\n\t\ttopic string\n\t\tpartition int32\n\t}\n\n\t// Kafka doesn't always return the timestamp in the response, for example\n\t// when the request sends -2 (for the first offset) it always returns -1,\n\t// probably to indicate that the timestamp is unknown. This means that we\n\t// can't correlate the requests and responses based on their timestamps,\n\t// the primary key is the topic/partition pair.\n\t//\n\t// To make the API a bit friendly, we reconstructing an index of topic\n\t// partitions to the timestamps that were requested, and override the\n\t// timestamp value in the response.\n\ttimestamps := make([]map[topicPartition]int64, len(requests))\n\n\tfor i, m := range requests {\n\t\treq := m.(*Request)\n\t\tts := make(map[topicPartition]int64, len(req.Topics))\n\n\t\tfor _, t := range req.Topics {\n\t\t\tfor _, p := range t.Partitions {\n\t\t\t\tts[topicPartition{\n\t\t\t\t\ttopic: t.Topic,\n\t\t\t\t\tpartition: p.Partition,\n\t\t\t\t}] = p.Timestamp\n\t\t\t}\n\t\t}\n\n\t\ttimestamps[i] = ts\n\t}\n\n\ttopics := make(map[string][]ResponsePartition)\n\terrors := 0\n\n\tfor i, res := range results {\n\t\tm, err := protocol.Result(res)\n\t\tif err != nil {\n\t\t\tfor _, t := range requests[i].(*Request).Topics {\n\t\t\t\tpartitions := topics[t.Topic]\n\n\t\t\t\tfor _, p := range t.Partitions {\n\t\t\t\t\tpartitions = append(partitions, ResponsePartition{\n\t\t\t\t\t\tPartition: p.Partition,\n\t\t\t\t\t\tErrorCode: -1, // UNKNOWN, can we do better?\n\t\t\t\t\t\tTimestamp: -1,\n\t\t\t\t\t\tOffset: -1,\n\t\t\t\t\t\tLeaderEpoch: -1,\n\t\t\t\t\t})\n\t\t\t\t}\n\n\t\t\t\ttopics[t.Topic] = partitions\n\t\t\t}\n\t\t\terrors++\n\t\t\tcontinue\n\t\t}\n\n\t\tresponse := m.(*Response)\n\n\t\tif r.ThrottleTimeMs < response.ThrottleTimeMs {\n\t\t\tr.ThrottleTimeMs = response.ThrottleTimeMs\n\t\t}\n\n\t\tfor _, t := range response.Topics {\n\t\t\tfor _, p := range t.Partitions {\n\t\t\t\tif timestamp, ok := timestamps[i][topicPartition{\n\t\t\t\t\ttopic: t.Topic,\n\t\t\t\t\tpartition: p.Partition,\n\t\t\t\t}]; ok {\n\t\t\t\t\tp.Timestamp = timestamp\n\t\t\t\t}\n\t\t\t\ttopics[t.Topic] = append(topics[t.Topic], p)\n\t\t\t}\n\t\t}\n\n\t}\n\n\tif errors > 0 && errors == len(results) {\n\t\t_, err := protocol.Result(results[0])\n\t\treturn nil, err\n\t}\n\n\tr.Topics = make([]ResponseTopic, 0, len(topics))\n\n\tfor topicName, partitions := range topics {\n\t\tr.Topics = append(r.Topics, ResponseTopic{\n\t\t\tTopic: topicName,\n\t\t\tPartitions: partitions,\n\t\t})\n\t}\n\n\tsort.Slice(r.Topics, func(i, j int) bool {\n\t\treturn r.Topics[i].Topic < r.Topics[j].Topic\n\t})\n\n\tfor _, t := range r.Topics {\n\t\tsort.Slice(t.Partitions, func(i, j int) bool {\n\t\t\tp1 := &t.Partitions[i]\n\t\t\tp2 := &t.Partitions[j]\n\n\t\t\tif p1.Partition != p2.Partition {\n\t\t\t\treturn p1.Partition < p2.Partition\n\t\t\t}\n\n\t\t\treturn p1.Offset < p2.Offset\n\t\t})\n\t}\n\n\treturn r, nil\n}\n\nvar (\n\t_ protocol.BrokerMessage = (*Request)(nil)\n\t_ protocol.Splitter = (*Request)(nil)\n\t_ protocol.Merger = (*Response)(nil)\n)\n" }, { "path": "protocol/listoffsets/listoffsets_test.go", "content": "package listoffsets_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/listoffsets\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nconst (\n\tv1 = 1\n\tv4 = 4\n)\n\nfunc TestListOffsetsRequest(t *testing.T) {\n\tprototest.TestRequest(t, v1, &listoffsets.Request{\n\t\tReplicaID: 1,\n\t\tTopics: []listoffsets.RequestTopic{\n\t\t\t{\n\t\t\t\tTopic: \"topic-1\",\n\t\t\t\tPartitions: []listoffsets.RequestPartition{\n\t\t\t\t\t{Partition: 0, Timestamp: 1e9},\n\t\t\t\t\t{Partition: 1, Timestamp: 1e9},\n\t\t\t\t\t{Partition: 2, Timestamp: 1e9},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestRequest(t, v4, &listoffsets.Request{\n\t\tReplicaID: 1,\n\t\tIsolationLevel: 2,\n\t\tTopics: []listoffsets.RequestTopic{\n\t\t\t{\n\t\t\t\tTopic: \"topic-1\",\n\t\t\t\tPartitions: []listoffsets.RequestPartition{\n\t\t\t\t\t{Partition: 0, Timestamp: 1e9},\n\t\t\t\t\t{Partition: 1, Timestamp: 1e9},\n\t\t\t\t\t{Partition: 2, Timestamp: 1e9},\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tTopic: \"topic-2\",\n\t\t\t\tPartitions: []listoffsets.RequestPartition{\n\t\t\t\t\t{Partition: 0, CurrentLeaderEpoch: 10, Timestamp: 1e9},\n\t\t\t\t\t{Partition: 1, CurrentLeaderEpoch: 11, Timestamp: 1e9},\n\t\t\t\t\t{Partition: 2, CurrentLeaderEpoch: 12, Timestamp: 1e9},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n}\n\nfunc TestListOffsetsResponse(t *testing.T) {\n\tprototest.TestResponse(t, v1, &listoffsets.Response{\n\t\tTopics: []listoffsets.ResponseTopic{\n\t\t\t{\n\t\t\t\tTopic: \"topic-1\",\n\t\t\t\tPartitions: []listoffsets.ResponsePartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\tErrorCode: 0,\n\t\t\t\t\t\tTimestamp: 1e9,\n\t\t\t\t\t\tOffset: 1234567890,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v4, &listoffsets.Response{\n\t\tThrottleTimeMs: 1234,\n\t\tTopics: []listoffsets.ResponseTopic{\n\t\t\t{\n\t\t\t\tTopic: \"topic-1\",\n\t\t\t\tPartitions: []listoffsets.ResponsePartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\tErrorCode: 0,\n\t\t\t\t\t\tTimestamp: 1e9,\n\t\t\t\t\t\tOffset: 1234567890,\n\t\t\t\t\t\tLeaderEpoch: 10,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t\t{\n\t\t\t\tTopic: \"topic-2\",\n\t\t\t\tPartitions: []listoffsets.ResponsePartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\tErrorCode: 0,\n\t\t\t\t\t\tTimestamp: 1e9,\n\t\t\t\t\t\tOffset: 1234567890,\n\t\t\t\t\t\tLeaderEpoch: 10,\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\tErrorCode: 2,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n}\n" }, { "path": "protocol/listpartitionreassignments/listpartitionreassignments.go", "content": "package listpartitionreassignments\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\n// Detailed API definition: https://kafka.apache.org/protocol#The_Messages_ListPartitionReassignments.\n\ntype Request struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v0,max=v0,tag\"`\n\n\tTimeoutMs int32 `kafka:\"min=v0,max=v0\"`\n\tTopics []RequestTopic `kafka:\"min=v0,max=v0,nullable\"`\n}\n\ntype RequestTopic struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v0,max=v0,tag\"`\n\n\tName string `kafka:\"min=v0,max=v0\"`\n\tPartitionIndexes []int32 `kafka:\"min=v0,max=v0\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey {\n\treturn protocol.ListPartitionReassignments\n}\n\nfunc (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, error) {\n\treturn cluster.Brokers[cluster.Controller], nil\n}\n\ntype Response struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v0,max=v0,tag\"`\n\n\tThrottleTimeMs int32 `kafka:\"min=v0,max=v0\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v0\"`\n\tErrorMessage string `kafka:\"min=v0,max=v0,nullable\"`\n\tTopics []ResponseTopic `kafka:\"min=v0,max=v0\"`\n}\n\ntype ResponseTopic struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v0,max=v0,tag\"`\n\n\tName string `kafka:\"min=v0,max=v0\"`\n\tPartitions []ResponsePartition `kafka:\"min=v0,max=v0\"`\n}\n\ntype ResponsePartition struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v0,max=v0,tag\"`\n\n\tPartitionIndex int32 `kafka:\"min=v0,max=v0\"`\n\tReplicas []int32 `kafka:\"min=v0,max=v0\"`\n\tAddingReplicas []int32 `kafka:\"min=v0,max=v0\"`\n\tRemovingReplicas []int32 `kafka:\"min=v0,max=v0\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey {\n\treturn protocol.ListPartitionReassignments\n}\n" }, { "path": "protocol/listpartitionreassignments/listpartitionreassignments_test.go", "content": "package listpartitionreassignments_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/listpartitionreassignments\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nconst (\n\tv0 = 0\n)\n\nfunc TestListPartitionReassignmentsRequest(t *testing.T) {\n\tprototest.TestRequest(t, v0, &listpartitionreassignments.Request{\n\t\tTopics: []listpartitionreassignments.RequestTopic{\n\t\t\t{\n\t\t\t\tName: \"topic-1\",\n\t\t\t\tPartitionIndexes: []int32{1, 2, 3},\n\t\t\t},\n\t\t},\n\t})\n}\n\nfunc TestListPartitionReassignmentsResponse(t *testing.T) {\n\tprototest.TestResponse(t, v0, &listpartitionreassignments.Response{\n\t\tTopics: []listpartitionreassignments.ResponseTopic{\n\t\t\t{\n\t\t\t\tName: \"topic-1\",\n\t\t\t\tPartitions: []listpartitionreassignments.ResponsePartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartitionIndex: 1,\n\t\t\t\t\t\tReplicas: []int32{1, 2, 3},\n\t\t\t\t\t\tAddingReplicas: []int32{4, 5, 6},\n\t\t\t\t\t\tRemovingReplicas: []int32{7, 8, 9},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n}\n" }, { "path": "protocol/metadata/metadata.go", "content": "package metadata\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\tTopicNames []string `kafka:\"min=v0,max=v8,nullable\"`\n\tAllowAutoTopicCreation bool `kafka:\"min=v4,max=v8\"`\n\tIncludeClusterAuthorizedOperations bool `kafka:\"min=v8,max=v8\"`\n\tIncludeTopicAuthorizedOperations bool `kafka:\"min=v8,max=v8\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.Metadata }\n\ntype Response struct {\n\tThrottleTimeMs int32 `kafka:\"min=v3,max=v8\"`\n\tBrokers []ResponseBroker `kafka:\"min=v0,max=v8\"`\n\tClusterID string `kafka:\"min=v2,max=v8,nullable\"`\n\tControllerID int32 `kafka:\"min=v1,max=v8\"`\n\tTopics []ResponseTopic `kafka:\"min=v0,max=v8\"`\n\tClusterAuthorizedOperations int32 `kafka:\"min=v8,max=v8\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.Metadata }\n\ntype ResponseBroker struct {\n\tNodeID int32 `kafka:\"min=v0,max=v8\"`\n\tHost string `kafka:\"min=v0,max=v8\"`\n\tPort int32 `kafka:\"min=v0,max=v8\"`\n\tRack string `kafka:\"min=v1,max=v8,nullable\"`\n}\n\ntype ResponseTopic struct {\n\tErrorCode int16 `kafka:\"min=v0,max=v8\"`\n\tName string `kafka:\"min=v0,max=v8\"`\n\tIsInternal bool `kafka:\"min=v1,max=v8\"`\n\tPartitions []ResponsePartition `kafka:\"min=v0,max=v8\"`\n\tTopicAuthorizedOperations int32 `kafka:\"min=v8,max=v8\"`\n}\n\ntype ResponsePartition struct {\n\tErrorCode int16 `kafka:\"min=v0,max=v8\"`\n\tPartitionIndex int32 `kafka:\"min=v0,max=v8\"`\n\tLeaderID int32 `kafka:\"min=v0,max=v8\"`\n\tLeaderEpoch int32 `kafka:\"min=v7,max=v8\"`\n\tReplicaNodes []int32 `kafka:\"min=v0,max=v8\"`\n\tIsrNodes []int32 `kafka:\"min=v0,max=v8\"`\n\tOfflineReplicas []int32 `kafka:\"min=v5,max=v8\"`\n}\n" }, { "path": "protocol/metadata/metadata_test.go", "content": "package metadata_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/metadata\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nconst (\n\tv0 = 0\n\tv1 = 1\n\tv4 = 4\n\tv8 = 8\n)\n\nfunc TestMetadataRequest(t *testing.T) {\n\tprototest.TestRequest(t, v0, &metadata.Request{\n\t\tTopicNames: nil,\n\t})\n\n\tprototest.TestRequest(t, v4, &metadata.Request{\n\t\tTopicNames: []string{\"hello\", \"world\"},\n\t\tAllowAutoTopicCreation: true,\n\t})\n\n\tprototest.TestRequest(t, v8, &metadata.Request{\n\t\tTopicNames: []string{\"hello\", \"world\"},\n\t\tAllowAutoTopicCreation: true,\n\t\tIncludeClusterAuthorizedOperations: true,\n\t\tIncludeTopicAuthorizedOperations: true,\n\t})\n}\n\nfunc TestMetadataResponse(t *testing.T) {\n\tprototest.TestResponse(t, v0, &metadata.Response{\n\t\tBrokers: []metadata.ResponseBroker{\n\t\t\t{\n\t\t\t\tNodeID: 0,\n\t\t\t\tHost: \"127.0.0.1\",\n\t\t\t\tPort: 9092,\n\t\t\t},\n\t\t\t{\n\t\t\t\tNodeID: 1,\n\t\t\t\tHost: \"127.0.0.1\",\n\t\t\t\tPort: 9093,\n\t\t\t},\n\t\t},\n\t\tTopics: []metadata.ResponseTopic{\n\t\t\t{\n\t\t\t\tName: \"topic-1\",\n\t\t\t\tPartitions: []metadata.ResponsePartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartitionIndex: 0,\n\t\t\t\t\t\tLeaderID: 1,\n\t\t\t\t\t\tReplicaNodes: []int32{0},\n\t\t\t\t\t\tIsrNodes: []int32{1, 0},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v1, &metadata.Response{\n\t\tControllerID: 1,\n\t\tBrokers: []metadata.ResponseBroker{\n\t\t\t{\n\t\t\t\tNodeID: 0,\n\t\t\t\tHost: \"127.0.0.1\",\n\t\t\t\tPort: 9092,\n\t\t\t\tRack: \"rack-1\",\n\t\t\t},\n\t\t\t{\n\t\t\t\tNodeID: 1,\n\t\t\t\tHost: \"127.0.0.1\",\n\t\t\t\tPort: 9093,\n\t\t\t\tRack: \"rack-2\",\n\t\t\t},\n\t\t},\n\t\tTopics: []metadata.ResponseTopic{\n\t\t\t{\n\t\t\t\tName: \"topic-1\",\n\t\t\t\tIsInternal: true,\n\t\t\t\tPartitions: []metadata.ResponsePartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartitionIndex: 0,\n\t\t\t\t\t\tLeaderID: 1,\n\t\t\t\t\t\tReplicaNodes: []int32{0},\n\t\t\t\t\t\tIsrNodes: []int32{1, 0},\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tPartitionIndex: 1,\n\t\t\t\t\t\tLeaderID: 0,\n\t\t\t\t\t\tReplicaNodes: []int32{1},\n\t\t\t\t\t\tIsrNodes: []int32{0, 1},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v8, &metadata.Response{\n\t\tThrottleTimeMs: 123,\n\t\tClusterID: \"test\",\n\t\tControllerID: 1,\n\t\tClusterAuthorizedOperations: 0x01,\n\t\tBrokers: []metadata.ResponseBroker{\n\t\t\t{\n\t\t\t\tNodeID: 0,\n\t\t\t\tHost: \"127.0.0.1\",\n\t\t\t\tPort: 9092,\n\t\t\t\tRack: \"rack-1\",\n\t\t\t},\n\t\t\t{\n\t\t\t\tNodeID: 1,\n\t\t\t\tHost: \"127.0.0.1\",\n\t\t\t\tPort: 9093,\n\t\t\t\tRack: \"rack-2\",\n\t\t\t},\n\t\t},\n\t\tTopics: []metadata.ResponseTopic{\n\t\t\t{\n\t\t\t\tName: \"topic-1\",\n\t\t\t\tPartitions: []metadata.ResponsePartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartitionIndex: 0,\n\t\t\t\t\t\tLeaderID: 1,\n\t\t\t\t\t\tLeaderEpoch: 1234567890,\n\t\t\t\t\t\tReplicaNodes: []int32{0},\n\t\t\t\t\t\tIsrNodes: []int32{0},\n\t\t\t\t\t\tOfflineReplicas: []int32{1},\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tErrorCode: 1,\n\t\t\t\t\t\tReplicaNodes: []int32{},\n\t\t\t\t\t\tIsrNodes: []int32{},\n\t\t\t\t\t\tOfflineReplicas: []int32{},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tTopicAuthorizedOperations: 0x01,\n\t\t\t},\n\t\t},\n\t})\n}\n\nfunc BenchmarkMetadataRequest(b *testing.B) {\n\tprototest.BenchmarkRequest(b, v8, &metadata.Request{\n\t\tTopicNames: []string{\"hello\", \"world\"},\n\t\tAllowAutoTopicCreation: true,\n\t\tIncludeClusterAuthorizedOperations: true,\n\t\tIncludeTopicAuthorizedOperations: true,\n\t})\n}\n\nfunc BenchmarkMetadataResponse(b *testing.B) {\n\tprototest.BenchmarkResponse(b, v8, &metadata.Response{\n\t\tThrottleTimeMs: 123,\n\t\tClusterID: \"test\",\n\t\tControllerID: 1,\n\t\tClusterAuthorizedOperations: 0x01,\n\t\tBrokers: []metadata.ResponseBroker{\n\t\t\t{\n\t\t\t\tNodeID: 0,\n\t\t\t\tHost: \"127.0.0.1\",\n\t\t\t\tPort: 9092,\n\t\t\t\tRack: \"rack-1\",\n\t\t\t},\n\t\t\t{\n\t\t\t\tNodeID: 1,\n\t\t\t\tHost: \"127.0.0.1\",\n\t\t\t\tPort: 9093,\n\t\t\t\tRack: \"rack-2\",\n\t\t\t},\n\t\t},\n\t\tTopics: []metadata.ResponseTopic{\n\t\t\t{\n\t\t\t\tName: \"topic-1\",\n\t\t\t\tPartitions: []metadata.ResponsePartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartitionIndex: 0,\n\t\t\t\t\t\tLeaderID: 1,\n\t\t\t\t\t\tLeaderEpoch: 1234567890,\n\t\t\t\t\t\tReplicaNodes: []int32{0},\n\t\t\t\t\t\tIsrNodes: []int32{0},\n\t\t\t\t\t\tOfflineReplicas: []int32{1},\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tErrorCode: 1,\n\t\t\t\t\t\tReplicaNodes: []int32{},\n\t\t\t\t\t\tIsrNodes: []int32{},\n\t\t\t\t\t\tOfflineReplicas: []int32{},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tTopicAuthorizedOperations: 0x01,\n\t\t\t},\n\t\t},\n\t})\n\n}\n" }, { "path": "protocol/offsetcommit/offsetcommit.go", "content": "package offsetcommit\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\tGroupID string `kafka:\"min=v0,max=v7\"`\n\tGenerationID int32 `kafka:\"min=v1,max=v7\"`\n\tMemberID string `kafka:\"min=v1,max=v7\"`\n\tRetentionTimeMs int64 `kafka:\"min=v2,max=v4\"`\n\tGroupInstanceID string `kafka:\"min=v7,max=v7,nullable\"`\n\tTopics []RequestTopic `kafka:\"min=v0,max=v7\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.OffsetCommit }\n\nfunc (r *Request) Group() string { return r.GroupID }\n\ntype RequestTopic struct {\n\tName string `kafka:\"min=v0,max=v7\"`\n\tPartitions []RequestPartition `kafka:\"min=v0,max=v7\"`\n}\n\ntype RequestPartition struct {\n\tPartitionIndex int32 `kafka:\"min=v0,max=v7\"`\n\tCommittedOffset int64 `kafka:\"min=v0,max=v7\"`\n\tCommitTimestamp int64 `kafka:\"min=v1,max=v1\"`\n\tCommittedLeaderEpoch int32 `kafka:\"min=v6,max=v7\"`\n\tCommittedMetadata string `kafka:\"min=v0,max=v7,nullable\"`\n}\n\nvar (\n\t_ protocol.GroupMessage = (*Request)(nil)\n)\n\ntype Response struct {\n\tThrottleTimeMs int32 `kafka:\"min=v3,max=v7\"`\n\tTopics []ResponseTopic `kafka:\"min=v0,max=v7\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.OffsetCommit }\n\ntype ResponseTopic struct {\n\tName string `kafka:\"min=v0,max=v7\"`\n\tPartitions []ResponsePartition `kafka:\"min=v0,max=v7\"`\n}\n\ntype ResponsePartition struct {\n\tPartitionIndex int32 `kafka:\"min=v0,max=v7\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v7\"`\n}\n" }, { "path": "protocol/offsetcommit/offsetcommit_test.go", "content": "package offsetcommit_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/offsetcommit\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nfunc TestOffsetCommitRequest(t *testing.T) {\n\tfor _, version := range []int16{0} {\n\t\tprototest.TestRequest(t, version, &offsetcommit.Request{\n\t\t\tGroupID: \"group-0\",\n\t\t\tTopics: []offsetcommit.RequestTopic{\n\t\t\t\t{\n\t\t\t\t\tName: \"topic-0\",\n\t\t\t\t\tPartitions: []offsetcommit.RequestPartition{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartitionIndex: 0,\n\t\t\t\t\t\t\tCommittedOffset: 1,\n\t\t\t\t\t\t\tCommittedMetadata: \"meta-0-0\",\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartitionIndex: 1,\n\t\t\t\t\t\t\tCommittedOffset: 2,\n\t\t\t\t\t\t\tCommittedMetadata: \"meta-0-1\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n\n\t// Version 1 added:\n\t// GenerationID\n\t// MemberID\n\t// RequestTopic.RequestPartition.CommitTimestamp\n\tfor _, version := range []int16{1} {\n\t\tprototest.TestRequest(t, version, &offsetcommit.Request{\n\t\t\tGroupID: \"group-1\",\n\t\t\tGenerationID: 1,\n\t\t\tMemberID: \"member-1\",\n\t\t\tTopics: []offsetcommit.RequestTopic{\n\t\t\t\t{\n\t\t\t\t\tName: \"topic-1\",\n\t\t\t\t\tPartitions: []offsetcommit.RequestPartition{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartitionIndex: 0,\n\t\t\t\t\t\t\tCommittedOffset: 1,\n\t\t\t\t\t\t\tCommittedMetadata: \"meta-1-0\",\n\t\t\t\t\t\t\tCommitTimestamp: 10,\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartitionIndex: 1,\n\t\t\t\t\t\t\tCommittedOffset: 2,\n\t\t\t\t\t\t\tCommittedMetadata: \"meta-1-1\",\n\t\t\t\t\t\t\tCommitTimestamp: 11,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n\n\t// Version 2 added:\n\t// RetentionTimeMs\n\t// Version 2 removed:\n\t// RequestTopic.RequestPartition.CommitTimestamp\n\t// Fields are the same through version 4.\n\tfor _, version := range []int16{2, 3, 4} {\n\t\tprototest.TestRequest(t, version, &offsetcommit.Request{\n\t\t\tGroupID: \"group-2\",\n\t\t\tGenerationID: 1,\n\t\t\tMemberID: \"member-2\",\n\t\t\tRetentionTimeMs: 1999,\n\t\t\tTopics: []offsetcommit.RequestTopic{\n\t\t\t\t{\n\t\t\t\t\tName: \"topic-2\",\n\t\t\t\t\tPartitions: []offsetcommit.RequestPartition{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartitionIndex: 0,\n\t\t\t\t\t\t\tCommittedOffset: 1,\n\t\t\t\t\t\t\tCommittedMetadata: \"meta-2-0\",\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartitionIndex: 1,\n\t\t\t\t\t\t\tCommittedOffset: 2,\n\t\t\t\t\t\t\tCommittedMetadata: \"meta-2-1\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n\n\t// Version 5 removed:\n\t// RetentionTimeMs\n\tfor _, version := range []int16{5} {\n\t\tprototest.TestRequest(t, version, &offsetcommit.Request{\n\t\t\tGroupID: \"group-3\",\n\t\t\tGenerationID: 1,\n\t\t\tMemberID: \"member-3\",\n\t\t\tTopics: []offsetcommit.RequestTopic{\n\t\t\t\t{\n\t\t\t\t\tName: \"topic-3\",\n\t\t\t\t\tPartitions: []offsetcommit.RequestPartition{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartitionIndex: 0,\n\t\t\t\t\t\t\tCommittedOffset: 1,\n\t\t\t\t\t\t\tCommittedMetadata: \"meta-3-0\",\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartitionIndex: 1,\n\t\t\t\t\t\t\tCommittedOffset: 2,\n\t\t\t\t\t\t\tCommittedMetadata: \"meta-3-1\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n\n\t// Version 6 added:\n\t// RequestTopic.RequestPartition.CommittedLeaderEpoch\n\tfor _, version := range []int16{6} {\n\t\tprototest.TestRequest(t, version, &offsetcommit.Request{\n\t\t\tGroupID: \"group-4\",\n\t\t\tGenerationID: 1,\n\t\t\tMemberID: \"member-4\",\n\t\t\tTopics: []offsetcommit.RequestTopic{\n\t\t\t\t{\n\t\t\t\t\tName: \"topic-4\",\n\t\t\t\t\tPartitions: []offsetcommit.RequestPartition{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartitionIndex: 0,\n\t\t\t\t\t\t\tCommittedOffset: 1,\n\t\t\t\t\t\t\tCommittedMetadata: \"meta-4-0\",\n\t\t\t\t\t\t\tCommittedLeaderEpoch: 10,\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartitionIndex: 1,\n\t\t\t\t\t\t\tCommittedOffset: 2,\n\t\t\t\t\t\t\tCommittedMetadata: \"meta-4-1\",\n\t\t\t\t\t\t\tCommittedLeaderEpoch: 11,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n\n\t// Version 7 added:\n\t// GroupInstanceID\n\tfor _, version := range []int16{7} {\n\t\tprototest.TestRequest(t, version, &offsetcommit.Request{\n\t\t\tGroupID: \"group-5\",\n\t\t\tGenerationID: 1,\n\t\t\tMemberID: \"member-5\",\n\t\t\tGroupInstanceID: \"instance-5\",\n\t\t\tTopics: []offsetcommit.RequestTopic{\n\t\t\t\t{\n\t\t\t\t\tName: \"topic-4\",\n\t\t\t\t\tPartitions: []offsetcommit.RequestPartition{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartitionIndex: 0,\n\t\t\t\t\t\t\tCommittedOffset: 1,\n\t\t\t\t\t\t\tCommittedMetadata: \"meta-5-0\",\n\t\t\t\t\t\t\tCommittedLeaderEpoch: 10,\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartitionIndex: 1,\n\t\t\t\t\t\t\tCommittedOffset: 2,\n\t\t\t\t\t\t\tCommittedMetadata: \"meta-5-1\",\n\t\t\t\t\t\t\tCommittedLeaderEpoch: 11,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n}\n\nfunc TestOffsetCommitResponse(t *testing.T) {\n\t// Fields are the same through version 2.\n\tfor _, version := range []int16{0, 1, 2} {\n\t\tprototest.TestResponse(t, version, &offsetcommit.Response{\n\t\t\tTopics: []offsetcommit.ResponseTopic{\n\t\t\t\t{\n\t\t\t\t\tName: \"topic-1\",\n\t\t\t\t\tPartitions: []offsetcommit.ResponsePartition{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartitionIndex: 4,\n\t\t\t\t\t\t\tErrorCode: 34,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n\n\t// Version 3 added:\n\t// ThrottleTimeMs\n\t// Field are the same through version 7.\n\tfor _, version := range []int16{3, 4, 5, 6, 7} {\n\t\tprototest.TestResponse(t, version, &offsetcommit.Response{\n\t\t\tThrottleTimeMs: 10000,\n\t\t\tTopics: []offsetcommit.ResponseTopic{\n\t\t\t\t{\n\t\t\t\t\tName: \"topic-2\",\n\t\t\t\t\tPartitions: []offsetcommit.ResponsePartition{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartitionIndex: 2,\n\t\t\t\t\t\t\tErrorCode: 3,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n}\n" }, { "path": "protocol/offsetdelete/offsetdelete.go", "content": "package offsetdelete\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\tGroupID string `kafka:\"min=v0,max=v0\"`\n\tTopics []RequestTopic `kafka:\"min=v0,max=v0\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.OffsetDelete }\n\nfunc (r *Request) Group() string { return r.GroupID }\n\ntype RequestTopic struct {\n\tName string `kafka:\"min=v0,max=v0\"`\n\tPartitions []RequestPartition `kafka:\"min=v0,max=v0\"`\n}\n\ntype RequestPartition struct {\n\tPartitionIndex int32 `kafka:\"min=v0,max=v0\"`\n}\n\nvar (\n\t_ protocol.GroupMessage = (*Request)(nil)\n)\n\ntype Response struct {\n\tErrorCode int16 `kafka:\"min=v0,max=v0\"`\n\tThrottleTimeMs int32 `kafka:\"min=v0,max=v0\"`\n\tTopics []ResponseTopic `kafka:\"min=v0,max=v0\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.OffsetDelete }\n\ntype ResponseTopic struct {\n\tName string `kafka:\"min=v0,max=v0\"`\n\tPartitions []ResponsePartition `kafka:\"min=v0,max=v0\"`\n}\n\ntype ResponsePartition struct {\n\tPartitionIndex int32 `kafka:\"min=v0,max=v0\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v0\"`\n}\n" }, { "path": "protocol/offsetdelete/offsetdelete_test.go", "content": "package offsetdelete_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/offsetdelete\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nfunc TestOffsetDeleteRequest(t *testing.T) {\n\tfor _, version := range []int16{0} {\n\t\tprototest.TestRequest(t, version, &offsetdelete.Request{\n\t\t\tGroupID: \"group-0\",\n\t\t\tTopics: []offsetdelete.RequestTopic{\n\t\t\t\t{\n\t\t\t\t\tName: \"topic-0\",\n\t\t\t\t\tPartitions: []offsetdelete.RequestPartition{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartitionIndex: 0,\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartitionIndex: 1,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n}\n\nfunc TestOffsetDeleteResponse(t *testing.T) {\n\tfor _, version := range []int16{0} {\n\t\tprototest.TestResponse(t, version, &offsetdelete.Response{\n\t\t\tErrorCode: 0,\n\t\t\tTopics: []offsetdelete.ResponseTopic{\n\t\t\t\t{\n\t\t\t\t\tName: \"topic-0\",\n\t\t\t\t\tPartitions: []offsetdelete.ResponsePartition{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartitionIndex: 0,\n\t\t\t\t\t\t\tErrorCode: 1,\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartitionIndex: 1,\n\t\t\t\t\t\t\tErrorCode: 1,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n}\n" }, { "path": "protocol/offsetfetch/offsetfetch.go", "content": "package offsetfetch\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\tGroupID string `kafka:\"min=v0,max=v5\"`\n\tTopics []RequestTopic `kafka:\"min=v0,max=v5,nullable\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.OffsetFetch }\n\nfunc (r *Request) Group() string { return r.GroupID }\n\ntype RequestTopic struct {\n\tName string `kafka:\"min=v0,max=v5\"`\n\tPartitionIndexes []int32 `kafka:\"min=v0,max=v5\"`\n}\n\nvar (\n\t_ protocol.GroupMessage = (*Request)(nil)\n)\n\ntype Response struct {\n\tThrottleTimeMs int32 `kafka:\"min=v3,max=v5\"`\n\tTopics []ResponseTopic `kafka:\"min=v0,max=v5\"`\n\tErrorCode int16 `kafka:\"min=v2,max=v5\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.OffsetFetch }\n\ntype ResponseTopic struct {\n\tName string `kafka:\"min=v0,max=v5\"`\n\tPartitions []ResponsePartition `kafka:\"min=v0,max=v5\"`\n}\n\ntype ResponsePartition struct {\n\tPartitionIndex int32 `kafka:\"min=v0,max=v5\"`\n\tCommittedOffset int64 `kafka:\"min=v0,max=v5\"`\n\tComittedLeaderEpoch int32 `kafka:\"min=v5,max=v5\"`\n\tMetadata string `kafka:\"min=v0,max=v5,nullable\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v5\"`\n}\n" }, { "path": "protocol/produce/produce.go", "content": "package produce\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n)\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\tTransactionalID string `kafka:\"min=v3,max=v8,nullable\"`\n\tAcks int16 `kafka:\"min=v0,max=v8\"`\n\tTimeout int32 `kafka:\"min=v0,max=v8\"`\n\tTopics []RequestTopic `kafka:\"min=v0,max=v8\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.Produce }\n\nfunc (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, error) {\n\tbroker := protocol.Broker{ID: -1}\n\n\tfor i := range r.Topics {\n\t\tt := &r.Topics[i]\n\n\t\ttopic, ok := cluster.Topics[t.Topic]\n\t\tif !ok {\n\t\t\treturn broker, NewError(protocol.NewErrNoTopic(t.Topic))\n\t\t}\n\n\t\tfor j := range t.Partitions {\n\t\t\tp := &t.Partitions[j]\n\n\t\t\tpartition, ok := topic.Partitions[p.Partition]\n\t\t\tif !ok {\n\t\t\t\treturn broker, NewError(protocol.NewErrNoPartition(t.Topic, p.Partition))\n\t\t\t}\n\n\t\t\tif b, ok := cluster.Brokers[partition.Leader]; !ok {\n\t\t\t\treturn broker, NewError(protocol.NewErrNoLeader(t.Topic, p.Partition))\n\t\t\t} else if broker.ID < 0 {\n\t\t\t\tbroker = b\n\t\t\t} else if b.ID != broker.ID {\n\t\t\t\treturn broker, NewError(fmt.Errorf(\"mismatching leaders (%d!=%d)\", b.ID, broker.ID))\n\t\t\t}\n\t\t}\n\t}\n\n\treturn broker, nil\n}\n\nfunc (r *Request) Prepare(apiVersion int16) {\n\t// Determine which version of the message should be used, based on which\n\t// version of the Produce API is supported by the server.\n\t//\n\t// In version 0.11, kafka gives this error:\n\t//\n\t// org.apache.kafka.common.record.InvalidRecordException\n\t// Produce requests with version 3 are only allowed to contain record batches with magic version.\n\t//\n\t// In version 2.x, kafka refuses the message claiming that the CRC32\n\t// checksum is invalid.\n\tvar recordVersion int8\n\n\tif apiVersion < 3 {\n\t\trecordVersion = 1\n\t} else {\n\t\trecordVersion = 2\n\t}\n\n\tfor i := range r.Topics {\n\t\tt := &r.Topics[i]\n\n\t\tfor j := range t.Partitions {\n\t\t\tp := &t.Partitions[j]\n\n\t\t\t// Allow the program to overload the version if really needed.\n\t\t\tif p.RecordSet.Version == 0 {\n\t\t\t\tp.RecordSet.Version = recordVersion\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (r *Request) HasResponse() bool {\n\treturn r.Acks != 0\n}\n\ntype RequestTopic struct {\n\tTopic string `kafka:\"min=v0,max=v8\"`\n\tPartitions []RequestPartition `kafka:\"min=v0,max=v8\"`\n}\n\ntype RequestPartition struct {\n\tPartition int32 `kafka:\"min=v0,max=v8\"`\n\tRecordSet protocol.RecordSet `kafka:\"min=v0,max=v8\"`\n}\n\ntype Response struct {\n\tTopics []ResponseTopic `kafka:\"min=v0,max=v8\"`\n\tThrottleTimeMs int32 `kafka:\"min=v1,max=v8\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.Produce }\n\ntype ResponseTopic struct {\n\tTopic string `kafka:\"min=v0,max=v8\"`\n\tPartitions []ResponsePartition `kafka:\"min=v0,max=v8\"`\n}\n\ntype ResponsePartition struct {\n\tPartition int32 `kafka:\"min=v0,max=v8\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v8\"`\n\tBaseOffset int64 `kafka:\"min=v0,max=v8\"`\n\tLogAppendTime int64 `kafka:\"min=v2,max=v8\"`\n\tLogStartOffset int64 `kafka:\"min=v5,max=v8\"`\n\tRecordErrors []ResponseError `kafka:\"min=v8,max=v8\"`\n\tErrorMessage string `kafka:\"min=v8,max=v8,nullable\"`\n}\n\ntype ResponseError struct {\n\tBatchIndex int32 `kafka:\"min=v8,max=v8\"`\n\tBatchIndexErrorMessage string `kafka:\"min=v8,max=v8,nullable\"`\n}\n\nvar (\n\t_ protocol.BrokerMessage = (*Request)(nil)\n\t_ protocol.PreparedMessage = (*Request)(nil)\n)\n\ntype Error struct {\n\tErr error\n}\n\nfunc NewError(err error) *Error {\n\treturn &Error{Err: err}\n}\n\nfunc (e *Error) Error() string {\n\treturn fmt.Sprintf(\"fetch request error: %v\", e.Err)\n}\n\nfunc (e *Error) Unwrap() error {\n\treturn e.Err\n}\n" }, { "path": "protocol/produce/produce_test.go", "content": "package produce_test\n\nimport (\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n\t\"github.com/segmentio/kafka-go/protocol/produce\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n)\n\nconst (\n\tv0 = 0\n\tv3 = 3\n\tv5 = 5\n\tv8 = 8\n)\n\nfunc TestProduceRequest(t *testing.T) {\n\tt0 := time.Now().Truncate(time.Millisecond)\n\tt1 := t0.Add(1 * time.Millisecond)\n\tt2 := t0.Add(2 * time.Millisecond)\n\n\tprototest.TestRequest(t, v0, &produce.Request{\n\t\tAcks: 1,\n\t\tTimeout: 500,\n\t\tTopics: []produce.RequestTopic{\n\t\t\t{\n\t\t\t\tTopic: \"topic-1\",\n\t\t\t\tPartitions: []produce.RequestPartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\tRecordSet: protocol.RecordSet{\n\t\t\t\t\t\t\tVersion: 1,\n\t\t\t\t\t\t\tRecords: protocol.NewRecordReader(\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 0, Time: t0, Key: nil, Value: nil},\n\t\t\t\t\t\t\t),\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\tRecordSet: protocol.RecordSet{\n\t\t\t\t\t\t\tVersion: 1,\n\t\t\t\t\t\t\tRecords: protocol.NewRecordReader(\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 0, Time: t0, Key: nil, Value: prototest.String(\"msg-0\")},\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 1, Time: t1, Key: nil, Value: prototest.String(\"msg-1\")},\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 2, Time: t2, Key: prototest.Bytes([]byte{1}), Value: prototest.String(\"msg-2\")},\n\t\t\t\t\t\t\t),\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\n\t\t\t{\n\t\t\t\tTopic: \"topic-2\",\n\t\t\t\tPartitions: []produce.RequestPartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\tRecordSet: protocol.RecordSet{\n\t\t\t\t\t\t\tVersion: 1,\n\t\t\t\t\t\t\tAttributes: protocol.Gzip,\n\t\t\t\t\t\t\tRecords: protocol.NewRecordReader(\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 0, Time: t0, Key: nil, Value: prototest.String(\"msg-0\")},\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 1, Time: t1, Key: nil, Value: prototest.String(\"msg-1\")},\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 2, Time: t2, Key: prototest.Bytes([]byte{1}), Value: prototest.String(\"msg-2\")},\n\t\t\t\t\t\t\t),\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestRequest(t, v3, &produce.Request{\n\t\tTransactionalID: \"1234\",\n\t\tAcks: 1,\n\t\tTimeout: 500,\n\t\tTopics: []produce.RequestTopic{\n\t\t\t{\n\t\t\t\tTopic: \"topic-1\",\n\t\t\t\tPartitions: []produce.RequestPartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\tRecordSet: protocol.RecordSet{\n\t\t\t\t\t\t\tVersion: 1,\n\t\t\t\t\t\t\tRecords: protocol.NewRecordReader(\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 0, Time: t0, Key: nil, Value: nil},\n\t\t\t\t\t\t\t),\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\tRecordSet: protocol.RecordSet{\n\t\t\t\t\t\t\tVersion: 1,\n\t\t\t\t\t\t\tRecords: protocol.NewRecordReader(\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 0, Time: t0, Key: nil, Value: prototest.String(\"msg-0\")},\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 1, Time: t1, Key: nil, Value: prototest.String(\"msg-1\")},\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 2, Time: t2, Key: prototest.Bytes([]byte{1}), Value: prototest.String(\"msg-2\")},\n\t\t\t\t\t\t\t),\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\theaders := []protocol.Header{\n\t\t{Key: \"key-1\", Value: []byte(\"value-1\")},\n\t\t{Key: \"key-2\", Value: []byte(\"value-2\")},\n\t\t{Key: \"key-3\", Value: []byte(\"value-3\")},\n\t}\n\n\tprototest.TestRequest(t, v5, &produce.Request{\n\t\tTransactionalID: \"1234\",\n\t\tAcks: 1,\n\t\tTimeout: 500,\n\t\tTopics: []produce.RequestTopic{\n\t\t\t{\n\t\t\t\tTopic: \"topic-1\",\n\t\t\t\tPartitions: []produce.RequestPartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\tRecordSet: protocol.RecordSet{\n\t\t\t\t\t\t\tVersion: 2,\n\t\t\t\t\t\t\tRecords: protocol.NewRecordReader(\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 0, Time: t0, Key: nil, Value: prototest.String(\"msg-0\"), Headers: headers},\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 1, Time: t1, Key: nil, Value: prototest.String(\"msg-1\")},\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 2, Time: t2, Key: prototest.Bytes([]byte{1}), Value: prototest.String(\"msg-2\")},\n\t\t\t\t\t\t\t),\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\n\t\t\t{\n\t\t\t\tTopic: \"topic-2\",\n\t\t\t\tPartitions: []produce.RequestPartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\tRecordSet: protocol.RecordSet{\n\t\t\t\t\t\t\tVersion: 2,\n\t\t\t\t\t\t\tAttributes: protocol.Snappy,\n\t\t\t\t\t\t\tRecords: protocol.NewRecordReader(\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 0, Time: t0, Key: nil, Value: prototest.String(\"msg-0\"), Headers: headers},\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 1, Time: t1, Key: nil, Value: prototest.String(\"msg-1\")},\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 2, Time: t2, Key: prototest.Bytes([]byte{1}), Value: prototest.String(\"msg-2\")},\n\t\t\t\t\t\t\t),\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n}\n\nfunc TestProduceResponse(t *testing.T) {\n\tprototest.TestResponse(t, v0, &produce.Response{\n\t\tTopics: []produce.ResponseTopic{\n\t\t\t{\n\t\t\t\tTopic: \"topic-1\",\n\t\t\t\tPartitions: []produce.ResponsePartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\tErrorCode: 0,\n\t\t\t\t\t\tBaseOffset: 0,\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\tErrorCode: 0,\n\t\t\t\t\t\tBaseOffset: 42,\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestResponse(t, v8, &produce.Response{\n\t\tTopics: []produce.ResponseTopic{\n\t\t\t{\n\t\t\t\tTopic: \"topic-1\",\n\t\t\t\tPartitions: []produce.ResponsePartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\tErrorCode: 0,\n\t\t\t\t\t\tBaseOffset: 42,\n\t\t\t\t\t\tLogAppendTime: 1e9,\n\t\t\t\t\t\tLogStartOffset: 10,\n\t\t\t\t\t\tRecordErrors: []produce.ResponseError{},\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\tErrorCode: 1,\n\t\t\t\t\t\tRecordErrors: []produce.ResponseError{\n\t\t\t\t\t\t\t{BatchIndex: 1, BatchIndexErrorMessage: \"message-1\"},\n\t\t\t\t\t\t\t{BatchIndex: 2, BatchIndexErrorMessage: \"message-2\"},\n\t\t\t\t\t\t\t{BatchIndex: 3, BatchIndexErrorMessage: \"message-3\"},\n\t\t\t\t\t\t},\n\t\t\t\t\t\tErrorMessage: \"something went wrong\",\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n}\n\nfunc BenchmarkProduceRequest(b *testing.B) {\n\tt0 := time.Now().Truncate(time.Millisecond)\n\tt1 := t0.Add(1 * time.Millisecond)\n\tt2 := t0.Add(2 * time.Millisecond)\n\n\tprototest.BenchmarkRequest(b, v3, &produce.Request{\n\t\tTransactionalID: \"1234\",\n\t\tAcks: 1,\n\t\tTimeout: 500,\n\t\tTopics: []produce.RequestTopic{\n\t\t\t{\n\t\t\t\tTopic: \"topic-1\",\n\t\t\t\tPartitions: []produce.RequestPartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\tRecordSet: protocol.RecordSet{\n\t\t\t\t\t\t\tVersion: 1,\n\t\t\t\t\t\t\tRecords: protocol.NewRecordReader(\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 0, Time: t0, Key: nil, Value: nil},\n\t\t\t\t\t\t\t),\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\tRecordSet: protocol.RecordSet{\n\t\t\t\t\t\t\tVersion: 1,\n\t\t\t\t\t\t\tRecords: protocol.NewRecordReader(\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 0, Time: t0, Key: nil, Value: prototest.String(\"msg-0\")},\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 1, Time: t1, Key: nil, Value: prototest.String(\"msg-1\")},\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 2, Time: t2, Key: prototest.Bytes([]byte{1}), Value: prototest.String(\"msg-2\")},\n\t\t\t\t\t\t\t),\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\theaders := []protocol.Header{\n\t\t{Key: \"key-1\", Value: []byte(\"value-1\")},\n\t\t{Key: \"key-2\", Value: []byte(\"value-2\")},\n\t\t{Key: \"key-3\", Value: []byte(\"value-3\")},\n\t}\n\n\tprototest.BenchmarkRequest(b, v5, &produce.Request{\n\t\tTransactionalID: \"1234\",\n\t\tAcks: 1,\n\t\tTimeout: 500,\n\t\tTopics: []produce.RequestTopic{\n\t\t\t{\n\t\t\t\tTopic: \"topic-1\",\n\t\t\t\tPartitions: []produce.RequestPartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\tRecordSet: protocol.RecordSet{\n\t\t\t\t\t\t\tVersion: 2,\n\t\t\t\t\t\t\tRecords: protocol.NewRecordReader(\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 0, Time: t0, Key: nil, Value: prototest.String(\"msg-0\"), Headers: headers},\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 1, Time: t1, Key: nil, Value: prototest.String(\"msg-1\")},\n\t\t\t\t\t\t\t\tprotocol.Record{Offset: 2, Time: t2, Key: prototest.Bytes([]byte{1}), Value: prototest.String(\"msg-2\")},\n\t\t\t\t\t\t\t),\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n}\n" }, { "path": "protocol/protocol.go", "content": "package protocol\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"strings\"\n)\n\n// Message is an interface implemented by all request and response types of the\n// kafka protocol.\n//\n// This interface is used mostly as a safe-guard to provide a compile-time check\n// for values passed to functions dealing kafka message types.\ntype Message interface {\n\tApiKey() ApiKey\n}\n\ntype ApiKey int16\n\nfunc (k ApiKey) String() string {\n\tif i := int(k); i >= 0 && i < len(apiNames) {\n\t\treturn apiNames[i]\n\t}\n\treturn strconv.Itoa(int(k))\n}\n\nfunc (k ApiKey) MinVersion() int16 { return k.apiType().minVersion() }\n\nfunc (k ApiKey) MaxVersion() int16 { return k.apiType().maxVersion() }\n\nfunc (k ApiKey) SelectVersion(minVersion, maxVersion int16) int16 {\n\tmin := k.MinVersion()\n\tmax := k.MaxVersion()\n\tswitch {\n\tcase min > maxVersion:\n\t\treturn min\n\tcase max < maxVersion:\n\t\treturn max\n\tdefault:\n\t\treturn maxVersion\n\t}\n}\n\nfunc (k ApiKey) apiType() apiType {\n\tif i := int(k); i >= 0 && i < len(apiTypes) {\n\t\treturn apiTypes[i]\n\t}\n\treturn apiType{}\n}\n\nconst (\n\tProduce ApiKey = 0\n\tFetch ApiKey = 1\n\tListOffsets ApiKey = 2\n\tMetadata ApiKey = 3\n\tLeaderAndIsr ApiKey = 4\n\tStopReplica ApiKey = 5\n\tUpdateMetadata ApiKey = 6\n\tControlledShutdown ApiKey = 7\n\tOffsetCommit ApiKey = 8\n\tOffsetFetch ApiKey = 9\n\tFindCoordinator ApiKey = 10\n\tJoinGroup ApiKey = 11\n\tHeartbeat ApiKey = 12\n\tLeaveGroup ApiKey = 13\n\tSyncGroup ApiKey = 14\n\tDescribeGroups ApiKey = 15\n\tListGroups ApiKey = 16\n\tSaslHandshake ApiKey = 17\n\tApiVersions ApiKey = 18\n\tCreateTopics ApiKey = 19\n\tDeleteTopics ApiKey = 20\n\tDeleteRecords ApiKey = 21\n\tInitProducerId ApiKey = 22\n\tOffsetForLeaderEpoch ApiKey = 23\n\tAddPartitionsToTxn ApiKey = 24\n\tAddOffsetsToTxn ApiKey = 25\n\tEndTxn ApiKey = 26\n\tWriteTxnMarkers ApiKey = 27\n\tTxnOffsetCommit ApiKey = 28\n\tDescribeAcls ApiKey = 29\n\tCreateAcls ApiKey = 30\n\tDeleteAcls ApiKey = 31\n\tDescribeConfigs ApiKey = 32\n\tAlterConfigs ApiKey = 33\n\tAlterReplicaLogDirs ApiKey = 34\n\tDescribeLogDirs ApiKey = 35\n\tSaslAuthenticate ApiKey = 36\n\tCreatePartitions ApiKey = 37\n\tCreateDelegationToken ApiKey = 38\n\tRenewDelegationToken ApiKey = 39\n\tExpireDelegationToken ApiKey = 40\n\tDescribeDelegationToken ApiKey = 41\n\tDeleteGroups ApiKey = 42\n\tElectLeaders ApiKey = 43\n\tIncrementalAlterConfigs ApiKey = 44\n\tAlterPartitionReassignments ApiKey = 45\n\tListPartitionReassignments ApiKey = 46\n\tOffsetDelete ApiKey = 47\n\tDescribeClientQuotas ApiKey = 48\n\tAlterClientQuotas ApiKey = 49\n\tDescribeUserScramCredentials ApiKey = 50\n\tAlterUserScramCredentials ApiKey = 51\n\n\tnumApis = 52\n)\n\nvar apiNames = [numApis]string{\n\tProduce: \"Produce\",\n\tFetch: \"Fetch\",\n\tListOffsets: \"ListOffsets\",\n\tMetadata: \"Metadata\",\n\tLeaderAndIsr: \"LeaderAndIsr\",\n\tStopReplica: \"StopReplica\",\n\tUpdateMetadata: \"UpdateMetadata\",\n\tControlledShutdown: \"ControlledShutdown\",\n\tOffsetCommit: \"OffsetCommit\",\n\tOffsetFetch: \"OffsetFetch\",\n\tFindCoordinator: \"FindCoordinator\",\n\tJoinGroup: \"JoinGroup\",\n\tHeartbeat: \"Heartbeat\",\n\tLeaveGroup: \"LeaveGroup\",\n\tSyncGroup: \"SyncGroup\",\n\tDescribeGroups: \"DescribeGroups\",\n\tListGroups: \"ListGroups\",\n\tSaslHandshake: \"SaslHandshake\",\n\tApiVersions: \"ApiVersions\",\n\tCreateTopics: \"CreateTopics\",\n\tDeleteTopics: \"DeleteTopics\",\n\tDeleteRecords: \"DeleteRecords\",\n\tInitProducerId: \"InitProducerId\",\n\tOffsetForLeaderEpoch: \"OffsetForLeaderEpoch\",\n\tAddPartitionsToTxn: \"AddPartitionsToTxn\",\n\tAddOffsetsToTxn: \"AddOffsetsToTxn\",\n\tEndTxn: \"EndTxn\",\n\tWriteTxnMarkers: \"WriteTxnMarkers\",\n\tTxnOffsetCommit: \"TxnOffsetCommit\",\n\tDescribeAcls: \"DescribeAcls\",\n\tCreateAcls: \"CreateAcls\",\n\tDeleteAcls: \"DeleteAcls\",\n\tDescribeConfigs: \"DescribeConfigs\",\n\tAlterConfigs: \"AlterConfigs\",\n\tAlterReplicaLogDirs: \"AlterReplicaLogDirs\",\n\tDescribeLogDirs: \"DescribeLogDirs\",\n\tSaslAuthenticate: \"SaslAuthenticate\",\n\tCreatePartitions: \"CreatePartitions\",\n\tCreateDelegationToken: \"CreateDelegationToken\",\n\tRenewDelegationToken: \"RenewDelegationToken\",\n\tExpireDelegationToken: \"ExpireDelegationToken\",\n\tDescribeDelegationToken: \"DescribeDelegationToken\",\n\tDeleteGroups: \"DeleteGroups\",\n\tElectLeaders: \"ElectLeaders\",\n\tIncrementalAlterConfigs: \"IncrementalAlterConfigs\",\n\tAlterPartitionReassignments: \"AlterPartitionReassignments\",\n\tListPartitionReassignments: \"ListPartitionReassignments\",\n\tOffsetDelete: \"OffsetDelete\",\n\tDescribeClientQuotas: \"DescribeClientQuotas\",\n\tAlterClientQuotas: \"AlterClientQuotas\",\n\tDescribeUserScramCredentials: \"DescribeUserScramCredentials\",\n\tAlterUserScramCredentials: \"AlterUserScramCredentials\",\n}\n\ntype messageType struct {\n\tversion int16\n\tflexible bool\n\tgotype reflect.Type\n\tdecode decodeFunc\n\tencode encodeFunc\n}\n\nfunc (t *messageType) new() Message {\n\treturn reflect.New(t.gotype).Interface().(Message)\n}\n\ntype apiType struct {\n\trequests []messageType\n\tresponses []messageType\n}\n\nfunc (t apiType) minVersion() int16 {\n\tif len(t.requests) == 0 {\n\t\treturn 0\n\t}\n\treturn t.requests[0].version\n}\n\nfunc (t apiType) maxVersion() int16 {\n\tif len(t.requests) == 0 {\n\t\treturn 0\n\t}\n\treturn t.requests[len(t.requests)-1].version\n}\n\nvar apiTypes [numApis]apiType\n\n// Register is automatically called by sub-packages are imported to install a\n// new pair of request/response message types.\nfunc Register(req, res Message) {\n\tk1 := req.ApiKey()\n\tk2 := res.ApiKey()\n\n\tif k1 != k2 {\n\t\tpanic(fmt.Sprintf(\"[%T/%T]: request and response API keys mismatch: %d != %d\", req, res, k1, k2))\n\t}\n\n\tapiTypes[k1] = apiType{\n\t\trequests: typesOf(req),\n\t\tresponses: typesOf(res),\n\t}\n}\n\n// OverrideTypeMessage is an interface implemented by messages that want to override the standard\n// request/response types for a given API.\ntype OverrideTypeMessage interface {\n\tTypeKey() OverrideTypeKey\n}\n\ntype OverrideTypeKey int16\n\nconst (\n\tRawProduceOverride OverrideTypeKey = 0\n)\n\nvar overrideApiTypes [numApis]map[OverrideTypeKey]apiType\n\nfunc RegisterOverride(req, res Message, key OverrideTypeKey) {\n\tk1 := req.ApiKey()\n\tk2 := res.ApiKey()\n\n\tif k1 != k2 {\n\t\tpanic(fmt.Sprintf(\"[%T/%T]: request and response API keys mismatch: %d != %d\", req, res, k1, k2))\n\t}\n\n\tif overrideApiTypes[k1] == nil {\n\t\toverrideApiTypes[k1] = make(map[OverrideTypeKey]apiType)\n\t}\n\toverrideApiTypes[k1][key] = apiType{\n\t\trequests: typesOf(req),\n\t\tresponses: typesOf(res),\n\t}\n}\n\nfunc typesOf(v interface{}) []messageType {\n\treturn makeTypes(reflect.TypeOf(v).Elem())\n}\n\nfunc makeTypes(t reflect.Type) []messageType {\n\tminVersion := int16(-1)\n\tmaxVersion := int16(-1)\n\n\t// All future versions will be flexible (according to spec), so don't need to\n\t// worry about maxes here.\n\tminFlexibleVersion := int16(-1)\n\n\tforEachStructField(t, func(_ reflect.Type, _ index, tag string) {\n\t\tforEachStructTag(tag, func(tag structTag) bool {\n\t\t\tif minVersion < 0 || tag.MinVersion < minVersion {\n\t\t\t\tminVersion = tag.MinVersion\n\t\t\t}\n\t\t\tif maxVersion < 0 || tag.MaxVersion > maxVersion {\n\t\t\t\tmaxVersion = tag.MaxVersion\n\t\t\t}\n\t\t\tif tag.TagID > -2 && (minFlexibleVersion < 0 || tag.MinVersion < minFlexibleVersion) {\n\t\t\t\tminFlexibleVersion = tag.MinVersion\n\t\t\t}\n\t\t\treturn true\n\t\t})\n\t})\n\n\ttypes := make([]messageType, 0, (maxVersion-minVersion)+1)\n\n\tfor v := minVersion; v <= maxVersion; v++ {\n\t\tflexible := minFlexibleVersion >= 0 && v >= minFlexibleVersion\n\n\t\ttypes = append(types, messageType{\n\t\t\tversion: v,\n\t\t\tgotype: t,\n\t\t\tflexible: flexible,\n\t\t\tdecode: decodeFuncOf(t, v, flexible, structTag{}),\n\t\t\tencode: encodeFuncOf(t, v, flexible, structTag{}),\n\t\t})\n\t}\n\n\treturn types\n}\n\ntype structTag struct {\n\tMinVersion int16\n\tMaxVersion int16\n\tCompact bool\n\tNullable bool\n\tTagID int\n}\n\nfunc forEachStructTag(tag string, do func(structTag) bool) {\n\tif tag == \"-\" {\n\t\treturn // special case to ignore the field\n\t}\n\n\tforEach(tag, '|', func(s string) bool {\n\t\ttag := structTag{\n\t\t\tMinVersion: -1,\n\t\t\tMaxVersion: -1,\n\n\t\t\t// Legitimate tag IDs can start at 0. We use -1 as a placeholder to indicate\n\t\t\t// that the message type is flexible, so that leaves -2 as the default for\n\t\t\t// indicating that there is no tag ID and the message is not flexible.\n\t\t\tTagID: -2,\n\t\t}\n\n\t\tvar err error\n\t\tforEach(s, ',', func(s string) bool {\n\t\t\tswitch {\n\t\t\tcase strings.HasPrefix(s, \"min=\"):\n\t\t\t\ttag.MinVersion, err = parseVersion(s[4:])\n\t\t\tcase strings.HasPrefix(s, \"max=\"):\n\t\t\t\ttag.MaxVersion, err = parseVersion(s[4:])\n\t\t\tcase s == \"tag\":\n\t\t\t\ttag.TagID = -1\n\t\t\tcase strings.HasPrefix(s, \"tag=\"):\n\t\t\t\ttag.TagID, err = strconv.Atoi(s[4:])\n\t\t\tcase s == \"compact\":\n\t\t\t\ttag.Compact = true\n\t\t\tcase s == \"nullable\":\n\t\t\t\ttag.Nullable = true\n\t\t\tdefault:\n\t\t\t\terr = fmt.Errorf(\"unrecognized option: %q\", s)\n\t\t\t}\n\t\t\treturn err == nil\n\t\t})\n\n\t\tif err != nil {\n\t\t\tpanic(fmt.Errorf(\"malformed struct tag: %w\", err))\n\t\t}\n\n\t\tif tag.MinVersion < 0 && tag.MaxVersion >= 0 {\n\t\t\tpanic(fmt.Errorf(\"missing minimum version in struct tag: %q\", s))\n\t\t}\n\n\t\tif tag.MaxVersion < 0 && tag.MinVersion >= 0 {\n\t\t\tpanic(fmt.Errorf(\"missing maximum version in struct tag: %q\", s))\n\t\t}\n\n\t\tif tag.MinVersion > tag.MaxVersion {\n\t\t\tpanic(fmt.Errorf(\"invalid version range in struct tag: %q\", s))\n\t\t}\n\n\t\treturn do(tag)\n\t})\n}\n\nfunc forEach(s string, sep byte, do func(string) bool) bool {\n\tfor len(s) != 0 {\n\t\tp := \"\"\n\t\ti := strings.IndexByte(s, sep)\n\t\tif i < 0 {\n\t\t\tp, s = s, \"\"\n\t\t} else {\n\t\t\tp, s = s[:i], s[i+1:]\n\t\t}\n\t\tif !do(p) {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\nfunc forEachStructField(t reflect.Type, do func(reflect.Type, index, string)) {\n\tfor i, n := 0, t.NumField(); i < n; i++ {\n\t\tf := t.Field(i)\n\n\t\tif f.PkgPath != \"\" && f.Name != \"_\" {\n\t\t\tcontinue\n\t\t}\n\n\t\tkafkaTag, ok := f.Tag.Lookup(\"kafka\")\n\t\tif !ok {\n\t\t\tkafkaTag = \"|\"\n\t\t}\n\n\t\tdo(f.Type, indexOf(f), kafkaTag)\n\t}\n}\n\nfunc parseVersion(s string) (int16, error) {\n\tif !strings.HasPrefix(s, \"v\") {\n\t\treturn 0, fmt.Errorf(\"invalid version number: %q\", s)\n\t}\n\ti, err := strconv.ParseInt(s[1:], 10, 16)\n\tif err != nil {\n\t\treturn 0, fmt.Errorf(\"invalid version number: %q: %w\", s, err)\n\t}\n\tif i < 0 {\n\t\treturn 0, fmt.Errorf(\"invalid negative version number: %q\", s)\n\t}\n\treturn int16(i), nil\n}\n\nfunc dontExpectEOF(err error) error {\n\tif err != nil {\n\t\tif errors.Is(err, io.EOF) {\n\t\t\treturn io.ErrUnexpectedEOF\n\t\t}\n\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\ntype Broker struct {\n\tRack string\n\tHost string\n\tPort int32\n\tID int32\n}\n\nfunc (b Broker) String() string {\n\treturn net.JoinHostPort(b.Host, itoa(b.Port))\n}\n\nfunc (b Broker) Format(w fmt.State, v rune) {\n\tswitch v {\n\tcase 'd':\n\t\tio.WriteString(w, itoa(b.ID))\n\tcase 's':\n\t\tio.WriteString(w, b.String())\n\tcase 'v':\n\t\tio.WriteString(w, itoa(b.ID))\n\t\tio.WriteString(w, \" \")\n\t\tio.WriteString(w, b.String())\n\t\tif b.Rack != \"\" {\n\t\t\tio.WriteString(w, \" \")\n\t\t\tio.WriteString(w, b.Rack)\n\t\t}\n\t}\n}\n\nfunc itoa(i int32) string {\n\treturn strconv.Itoa(int(i))\n}\n\ntype Topic struct {\n\tName string\n\tError int16\n\tPartitions map[int32]Partition\n}\n\ntype Partition struct {\n\tID int32\n\tError int16\n\tLeader int32\n\tReplicas []int32\n\tISR []int32\n\tOffline []int32\n}\n\n// RawExchanger is an extention to the Message interface to allow messages\n// to control the request response cycle for the message. This is currently\n// only used to facilitate v0 SASL Authenticate requests being written in\n// a non-standard fashion when the SASL Handshake was done at v0 but not\n// when done at v1.\ntype RawExchanger interface {\n\t// Required should return true when a RawExchange is needed.\n\t// The passed in versions are the negotiated versions for the connection\n\t// performing the request.\n\tRequired(versions map[ApiKey]int16) bool\n\t// RawExchange is given the raw connection to the broker and the Message\n\t// is responsible for writing itself to the connection as well as reading\n\t// the response.\n\tRawExchange(rw io.ReadWriter) (Message, error)\n}\n\n// BrokerMessage is an extension of the Message interface implemented by some\n// request types to customize the broker assignment logic.\ntype BrokerMessage interface {\n\t// Given a representation of the kafka cluster state as argument, returns\n\t// the broker that the message should be routed to.\n\tBroker(Cluster) (Broker, error)\n}\n\n// GroupMessage is an extension of the Message interface implemented by some\n// request types to inform the program that they should be routed to a group\n// coordinator.\ntype GroupMessage interface {\n\t// Returns the group configured on the message.\n\tGroup() string\n}\n\n// TransactionalMessage is an extension of the Message interface implemented by some\n// request types to inform the program that they should be routed to a transaction\n// coordinator.\ntype TransactionalMessage interface {\n\t// Returns the transactional id configured on the message.\n\tTransaction() string\n}\n\n// PreparedMessage is an extension of the Message interface implemented by some\n// request types which may need to run some pre-processing on their state before\n// being sent.\ntype PreparedMessage interface {\n\t// Prepares the message before being sent to a kafka broker using the API\n\t// version passed as argument.\n\tPrepare(apiVersion int16)\n}\n\n// Splitter is an interface implemented by messages that can be split into\n// multiple requests and have their results merged back by a Merger.\ntype Splitter interface {\n\t// For a given cluster layout, returns the list of messages constructed\n\t// from the receiver for each requests that should be sent to the cluster.\n\t// The second return value is a Merger which can be used to merge back the\n\t// results of each request into a single message (or an error).\n\tSplit(Cluster) ([]Message, Merger, error)\n}\n\n// Merger is an interface implemented by messages which can merge multiple\n// results into one response.\ntype Merger interface {\n\t// Given a list of message and associated results, merge them back into a\n\t// response (or an error). The results must be either Message or error\n\t// values, other types should trigger a panic.\n\tMerge(messages []Message, results []interface{}) (Message, error)\n}\n\n// Result converts r to a Message or an error, or panics if r could not be\n// converted to these types.\nfunc Result(r interface{}) (Message, error) {\n\tswitch v := r.(type) {\n\tcase Message:\n\t\treturn v, nil\n\tcase error:\n\t\treturn nil, v\n\tdefault:\n\t\tpanic(fmt.Errorf(\"BUG: result must be a message or an error but not %T\", v))\n\t}\n}\n" }, { "path": "protocol/protocol_test.go", "content": "package protocol\n\nimport (\n\t\"bytes\"\n\t\"math\"\n\t\"reflect\"\n\t\"testing\"\n)\n\ntype testType struct {\n\tField1 string `kafka:\"min=v0,max=v4,nullable\"`\n\tField2 int16 `kafka:\"min=v2,max=v4\"`\n\tField3 []byte `kafka:\"min=v2,max=v4,nullable\"`\n\tSubTypes []testSubType `kafka:\"min=v1,max=v4\"`\n\n\tTaggedField1 int8 `kafka:\"min=v3,max=v4,tag=0\"`\n\tTaggedField2 string `kafka:\"min=v4,max=v4,tag=1\"`\n}\n\ntype testSubType struct {\n\tSubField1 int8 `kafka:\"min=v1,max=v4\"`\n}\n\nfunc TestMakeFlexibleTypes(t *testing.T) {\n\ttypes := makeTypes(reflect.TypeOf(&testType{}).Elem())\n\tif len(types) != 5 {\n\t\tt.Error(\n\t\t\t\"Wrong number of types\",\n\t\t\t\"expected\", 5,\n\t\t\t\"got\", len(types),\n\t\t)\n\t}\n\n\tfv := []int16{}\n\n\tfor _, to := range types {\n\t\tif to.flexible {\n\t\t\tfv = append(fv, to.version)\n\t\t}\n\t}\n\n\tif !reflect.DeepEqual([]int16{3, 4}, fv) {\n\t\tt.Error(\n\t\t\t\"Unexpected flexible versions\",\n\t\t\t\"expected\", []int16{3, 4},\n\t\t\t\"got\", fv,\n\t\t)\n\t}\n}\n\nfunc TestEncodeDecodeFlexibleType(t *testing.T) {\n\tf := &testType{\n\t\tField1: \"value1\",\n\t\tField2: 15,\n\t\tField3: []byte(\"hello\"),\n\t\tSubTypes: []testSubType{\n\t\t\t{\n\t\t\t\tSubField1: 2,\n\t\t\t},\n\t\t\t{\n\t\t\t\tSubField1: 3,\n\t\t\t},\n\t\t},\n\n\t\tTaggedField1: 34,\n\t\tTaggedField2: \"taggedValue2\",\n\t}\n\n\tb := &bytes.Buffer{}\n\te := &encoder{writer: b}\n\n\ttypes := makeTypes(reflect.TypeOf(&testType{}).Elem())\n\tft := types[4]\n\tft.encode(e, valueOf(f))\n\tif e.err != nil {\n\t\tt.Error(\n\t\t\t\"Error during encoding\",\n\t\t\t\"expected\", nil,\n\t\t\t\"got\", e.err,\n\t\t)\n\t}\n\n\texp := []byte{\n\t\t// size of \"value1\" + 1\n\t\t7,\n\t\t// \"value1\"\n\t\t118, 97, 108, 117, 101, 49,\n\t\t// 15 as 16-bit int\n\t\t0, 15,\n\t\t// size of []byte(\"hello\") + 1\n\t\t6,\n\t\t// []byte(\"hello\")\n\t\t104, 101, 108, 108, 111,\n\t\t// size of []SubTypes + 1\n\t\t3,\n\t\t// 2 as 8-bit int\n\t\t2,\n\t\t// tag buffer for first SubType struct\n\t\t0,\n\t\t// 3 as 8-bit int\n\t\t3,\n\t\t// tag buffer for second SubType struct\n\t\t0,\n\t\t// number of tagged fields\n\t\t2,\n\t\t// id of first tagged field\n\t\t0,\n\t\t// size of first tagged field\n\t\t1,\n\t\t// 34 as 8-bit int\n\t\t34,\n\t\t// id of second tagged field\n\t\t1,\n\t\t// size of second tagged field\n\t\t13,\n\t\t// size of \"taggedValue2\" + 1\n\t\t13,\n\t\t// \"taggedValue2\"\n\t\t116, 97, 103, 103, 101, 100, 86, 97, 108, 117, 101, 50,\n\t}\n\n\tif !reflect.DeepEqual(exp, b.Bytes()) {\n\t\tt.Error(\n\t\t\t\"Wrong encoded output\",\n\t\t\t\"expected\", exp,\n\t\t\t\"got\", b.Bytes(),\n\t\t)\n\t}\n\n\tb = &bytes.Buffer{}\n\tb.Write(exp)\n\td := &decoder{reader: b, remain: len(exp)}\n\n\tf2 := &testType{}\n\tft.decode(d, valueOf(f2))\n\tif d.err != nil {\n\t\tt.Error(\n\t\t\t\"Error during decoding\",\n\t\t\t\"expected\", nil,\n\t\t\t\"got\", e.err,\n\t\t)\n\t}\n\n\tif !reflect.DeepEqual(f, f2) {\n\t\tt.Error(\n\t\t\t\"Decoded value does not equal encoded one\",\n\t\t\t\"expected\", *f,\n\t\t\t\"got\", *f2,\n\t\t)\n\t}\n}\n\nfunc TestVarInts(t *testing.T) {\n\ttype tc struct {\n\t\tinput int64\n\t\texpVarInt []byte\n\t\texpUVarInt []byte\n\t}\n\n\ttcs := []tc{\n\t\t{\n\t\t\tinput: 12,\n\t\t\texpVarInt: []byte{24},\n\t\t\texpUVarInt: []byte{12},\n\t\t},\n\t\t{\n\t\t\tinput: 63,\n\t\t\texpVarInt: []byte{126},\n\t\t\texpUVarInt: []byte{63},\n\t\t},\n\t\t{\n\t\t\tinput: -64,\n\t\t\texpVarInt: []byte{127},\n\t\t\texpUVarInt: []byte{192, 255, 255, 255, 255, 255, 255, 255, 255, 1},\n\t\t},\n\t\t{\n\t\t\tinput: 64,\n\t\t\texpVarInt: []byte{128, 1},\n\t\t\texpUVarInt: []byte{64},\n\t\t},\n\t\t{\n\t\t\tinput: 127,\n\t\t\texpVarInt: []byte{254, 1},\n\t\t\texpUVarInt: []byte{127},\n\t\t},\n\t\t{\n\t\t\tinput: 128,\n\t\t\texpVarInt: []byte{128, 2},\n\t\t\texpUVarInt: []byte{128, 1},\n\t\t},\n\t\t{\n\t\t\tinput: 129,\n\t\t\texpVarInt: []byte{130, 2},\n\t\t\texpUVarInt: []byte{129, 1},\n\t\t},\n\t\t{\n\t\t\tinput: 12345,\n\t\t\texpVarInt: []byte{242, 192, 1},\n\t\t\texpUVarInt: []byte{185, 96},\n\t\t},\n\t\t{\n\t\t\tinput: 123456789101112,\n\t\t\texpVarInt: []byte{240, 232, 249, 224, 144, 146, 56},\n\t\t\texpUVarInt: []byte{184, 244, 188, 176, 136, 137, 28},\n\t\t},\n\t}\n\n\tfor _, tc := range tcs {\n\t\tb := &bytes.Buffer{}\n\t\te := &encoder{writer: b}\n\t\te.writeVarInt(tc.input)\n\t\tif e.err != nil {\n\t\t\tt.Errorf(\n\t\t\t\t\"Unexpected error encoding %d as varInt: %+v\",\n\t\t\t\ttc.input,\n\t\t\t\te.err,\n\t\t\t)\n\t\t}\n\t\tif !reflect.DeepEqual(b.Bytes(), tc.expVarInt) {\n\t\t\tt.Error(\n\t\t\t\t\"Wrong output encoding value\", tc.input, \"as varInt\",\n\t\t\t\t\"expected\", tc.expVarInt,\n\t\t\t\t\"got\", b.Bytes(),\n\t\t\t)\n\t\t}\n\t\texpLen := sizeOfVarInt(tc.input)\n\t\tif expLen != len(b.Bytes()) {\n\t\t\tt.Error(\n\t\t\t\t\"Wrong sizeOf for\", tc.input, \"as varInt\",\n\t\t\t\t\"expected\", expLen,\n\t\t\t\t\"got\", len(b.Bytes()),\n\t\t\t)\n\t\t}\n\n\t\td := &decoder{reader: b, remain: len(b.Bytes())}\n\t\tv := d.readVarInt()\n\t\tif v != tc.input {\n\t\t\tt.Error(\n\t\t\t\t\"Decoded varInt value does not equal encoded one\",\n\t\t\t\t\"expected\", tc.input,\n\t\t\t\t\"got\", v,\n\t\t\t)\n\t\t}\n\n\t\tb = &bytes.Buffer{}\n\t\te = &encoder{writer: b}\n\t\te.writeUnsignedVarInt(uint64(tc.input))\n\t\tif e.err != nil {\n\t\t\tt.Errorf(\n\t\t\t\t\"Unexpected error encoding %d as unsignedVarInt: %+v\",\n\t\t\t\ttc.input,\n\t\t\t\te.err,\n\t\t\t)\n\t\t}\n\t\tif !reflect.DeepEqual(b.Bytes(), tc.expUVarInt) {\n\t\t\tt.Error(\n\t\t\t\t\"Wrong output encoding value\", tc.input, \"as unsignedVarInt\",\n\t\t\t\t\"expected\", tc.expUVarInt,\n\t\t\t\t\"got\", b.Bytes(),\n\t\t\t)\n\t\t}\n\t\texpLen = sizeOfUnsignedVarInt(uint64(tc.input))\n\t\tif expLen != len(b.Bytes()) {\n\t\t\tt.Error(\n\t\t\t\t\"Wrong sizeOf for\", tc.input, \"as unsignedVarInt\",\n\t\t\t\t\"expected\", expLen,\n\t\t\t\t\"got\", len(b.Bytes()),\n\t\t\t)\n\t\t}\n\n\t\td = &decoder{reader: b, remain: len(b.Bytes())}\n\t\tv = int64(d.readUnsignedVarInt())\n\t\tif v != tc.input {\n\t\t\tt.Error(\n\t\t\t\t\"Decoded unsignedVarInt value does not equal encoded one\",\n\t\t\t\t\"expected\", tc.input,\n\t\t\t\t\"got\", v,\n\t\t\t)\n\t\t}\n\n\t}\n}\n\nfunc TestFloat64(t *testing.T) {\n\ttype tc struct {\n\t\tinput float64\n\t\texpected []byte\n\t}\n\n\ttcs := []tc{\n\t\t{\n\t\t\tinput: 0.0,\n\t\t\texpected: []byte{0, 0, 0, 0, 0, 0, 0, 0},\n\t\t},\n\t\t{\n\t\t\tinput: math.MaxFloat64,\n\t\t\texpected: []byte{127, 239, 255, 255, 255, 255, 255, 255},\n\t\t},\n\t\t{\n\t\t\tinput: -math.MaxFloat64,\n\t\t\texpected: []byte{255, 239, 255, 255, 255, 255, 255, 255},\n\t\t},\n\t\t{\n\t\t\tinput: math.SmallestNonzeroFloat64,\n\t\t\texpected: []byte{0, 0, 0, 0, 0, 0, 0, 1},\n\t\t},\n\t\t{\n\t\t\tinput: -math.SmallestNonzeroFloat64,\n\t\t\texpected: []byte{128, 0, 0, 0, 0, 0, 0, 1},\n\t\t},\n\t}\n\n\tfor _, tc := range tcs {\n\t\tb := &bytes.Buffer{}\n\t\te := &encoder{writer: b}\n\t\te.writeFloat64(tc.input)\n\t\tif e.err != nil {\n\t\t\tt.Errorf(\n\t\t\t\t\"Unexpected error encoding %f as float64: %+v\",\n\t\t\t\ttc.input,\n\t\t\t\te.err,\n\t\t\t)\n\t\t}\n\t\tif !reflect.DeepEqual(b.Bytes(), tc.expected) {\n\t\t\tt.Error(\n\t\t\t\t\"Wrong output encoding value\", tc.input, \"as float64\",\n\t\t\t\t\"expected\", tc.expected,\n\t\t\t\t\"got\", b.Bytes(),\n\t\t\t)\n\t\t}\n\n\t\td := &decoder{reader: b, remain: len(b.Bytes())}\n\t\tv := d.readFloat64()\n\t\tif v != tc.input {\n\t\t\tt.Error(\n\t\t\t\t\"Decoded float64 value does not equal encoded one\",\n\t\t\t\t\"expected\", tc.input,\n\t\t\t\t\"got\", v,\n\t\t\t)\n\t\t}\n\t}\n}\n" }, { "path": "protocol/prototest/bytes.go", "content": "package prototest\n\nimport (\n\t\"github.com/segmentio/kafka-go/protocol\"\n)\n\n// Bytes constructs a Bytes which exposes the content of b.\nfunc Bytes(b []byte) protocol.Bytes {\n\treturn protocol.NewBytes(b)\n}\n\n// String constructs a Bytes which exposes the content of s.\nfunc String(s string) protocol.Bytes {\n\treturn protocol.NewBytes([]byte(s))\n}\n" }, { "path": "protocol/prototest/prototest.go", "content": "package prototest\n\nimport (\n\t\"bytes\"\n\t\"errors\"\n\t\"io\"\n\t\"reflect\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n)\n\nfunc deepEqual(x1, x2 interface{}) bool {\n\tif x1 == nil {\n\t\treturn x2 == nil\n\t}\n\tif r1, ok := x1.(protocol.RecordReader); ok {\n\t\tif r2, ok := x2.(protocol.RecordReader); ok {\n\t\t\treturn deepEqualRecords(r1, r2)\n\t\t}\n\t\treturn false\n\t}\n\tif b1, ok := x1.(protocol.Bytes); ok {\n\t\tif b2, ok := x2.(protocol.Bytes); ok {\n\t\t\treturn deepEqualBytes(b1, b2)\n\t\t}\n\t\treturn false\n\t}\n\tif t1, ok := x1.(time.Time); ok {\n\t\tif t2, ok := x2.(time.Time); ok {\n\t\t\treturn t1.Equal(t2)\n\t\t}\n\t\treturn false\n\t}\n\treturn deepEqualValue(reflect.ValueOf(x1), reflect.ValueOf(x2))\n}\n\nfunc deepEqualValue(v1, v2 reflect.Value) bool {\n\tt1 := v1.Type()\n\tt2 := v2.Type()\n\n\tif t1 != t2 {\n\t\treturn false\n\t}\n\n\tswitch v1.Kind() {\n\tcase reflect.Bool:\n\t\treturn v1.Bool() == v2.Bool()\n\tcase reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:\n\t\treturn v1.Int() == v2.Int()\n\tcase reflect.Float64:\n\t\treturn v1.Float() == v2.Float()\n\tcase reflect.String:\n\t\treturn v1.String() == v2.String()\n\tcase reflect.Struct:\n\t\treturn deepEqualStruct(v1, v2)\n\tcase reflect.Ptr:\n\t\treturn deepEqualPtr(v1, v2)\n\tcase reflect.Slice:\n\t\treturn deepEqualSlice(v1, v2)\n\tdefault:\n\t\tpanic(\"comparing values of unsupported type: \" + v1.Type().String())\n\t}\n}\n\nfunc deepEqualPtr(v1, v2 reflect.Value) bool {\n\tif v1.IsNil() {\n\t\treturn v2.IsNil()\n\t}\n\treturn deepEqual(v1.Elem().Interface(), v2.Elem().Interface())\n}\n\nfunc deepEqualStruct(v1, v2 reflect.Value) bool {\n\tt := v1.Type()\n\tn := t.NumField()\n\n\tfor i := 0; i < n; i++ {\n\t\tf := t.Field(i)\n\n\t\tif f.PkgPath != \"\" { // ignore unexported fields\n\t\t\tcontinue\n\t\t}\n\n\t\tf1 := v1.Field(i)\n\t\tf2 := v2.Field(i)\n\n\t\tif !deepEqual(f1.Interface(), f2.Interface()) {\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n\nfunc deepEqualSlice(v1, v2 reflect.Value) bool {\n\tt := v1.Type()\n\te := t.Elem()\n\n\tif e.Kind() == reflect.Uint8 { // []byte\n\t\treturn bytes.Equal(v1.Bytes(), v2.Bytes())\n\t}\n\n\tn1 := v1.Len()\n\tn2 := v2.Len()\n\n\tif n1 != n2 {\n\t\treturn false\n\t}\n\n\tfor i := 0; i < n1; i++ {\n\t\tf1 := v1.Index(i)\n\t\tf2 := v2.Index(i)\n\n\t\tif !deepEqual(f1.Interface(), f2.Interface()) {\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n}\n\nfunc deepEqualBytes(s1, s2 protocol.Bytes) bool {\n\tif s1 == nil {\n\t\treturn s2 == nil\n\t}\n\n\tif s2 == nil {\n\t\treturn false\n\t}\n\n\tn1 := s1.Len()\n\tn2 := s2.Len()\n\n\tif n1 != n2 {\n\t\treturn false\n\t}\n\n\tb1 := make([]byte, n1)\n\tb2 := make([]byte, n2)\n\n\tif _, err := s1.(io.ReaderAt).ReadAt(b1, 0); err != nil {\n\t\tpanic(err)\n\t}\n\n\tif _, err := s2.(io.ReaderAt).ReadAt(b2, 0); err != nil {\n\t\tpanic(err)\n\t}\n\n\treturn bytes.Equal(b1, b2)\n}\n\nfunc deepEqualRecords(r1, r2 protocol.RecordReader) bool {\n\tfor {\n\t\trec1, err1 := r1.ReadRecord()\n\t\trec2, err2 := r2.ReadRecord()\n\n\t\tif err1 != nil || err2 != nil {\n\t\t\treturn errors.Is(err1, err2)\n\t\t}\n\n\t\tif !deepEqualRecord(rec1, rec2) {\n\t\t\treturn false\n\t\t}\n\t}\n}\n\nfunc deepEqualRecord(r1, r2 *protocol.Record) bool {\n\tif r1.Offset != r2.Offset {\n\t\treturn false\n\t}\n\n\tif !r1.Time.Equal(r2.Time) {\n\t\treturn false\n\t}\n\n\tif !deepEqualBytes(r1.Key, r2.Key) {\n\t\treturn false\n\t}\n\n\tif !deepEqualBytes(r1.Value, r2.Value) {\n\t\treturn false\n\t}\n\n\treturn deepEqual(r1.Headers, r2.Headers)\n}\n" }, { "path": "protocol/prototest/reflect.go", "content": "package prototest\n\nimport (\n\t\"bytes\"\n\t\"errors\"\n\t\"io\"\n\t\"reflect\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n)\n\nvar (\n\trecordReader = reflect.TypeOf((*protocol.RecordReader)(nil)).Elem()\n)\n\nfunc closeMessage(m protocol.Message) {\n\tforEachField(reflect.ValueOf(m), func(v reflect.Value) {\n\t\tif v.Type().Implements(recordReader) {\n\t\t\trr := v.Interface().(protocol.RecordReader)\n\t\t\tfor {\n\t\t\t\tr, err := rr.ReadRecord()\n\t\t\t\tif err != nil {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tif r.Key != nil {\n\t\t\t\t\tr.Key.Close()\n\t\t\t\t}\n\t\t\t\tif r.Value != nil {\n\t\t\t\t\tr.Value.Close()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t})\n}\n\nfunc load(v interface{}) (reset func()) {\n\treturn loadValue(reflect.ValueOf(v))\n}\n\nfunc loadValue(v reflect.Value) (reset func()) {\n\tresets := []func(){}\n\n\tforEachField(v, func(f reflect.Value) {\n\t\tswitch x := f.Interface().(type) {\n\t\tcase protocol.RecordReader:\n\t\t\trecords := loadRecords(x)\n\t\t\tresetFunc := func() {\n\t\t\t\tf.Set(reflect.ValueOf(protocol.NewRecordReader(makeRecords(records)...)))\n\t\t\t}\n\t\t\tresetFunc()\n\t\t\tresets = append(resets, resetFunc)\n\t\tcase io.Reader:\n\t\t\tbuf, _ := io.ReadAll(x)\n\t\t\tresetFunc := func() {\n\t\t\t\tf.Set(reflect.ValueOf(bytes.NewBuffer(buf)))\n\t\t\t}\n\t\t\tresetFunc()\n\t\t\tresets = append(resets, resetFunc)\n\t\t}\n\t})\n\n\treturn func() {\n\t\tfor _, f := range resets {\n\t\t\tf()\n\t\t}\n\t}\n}\n\nfunc forEachField(v reflect.Value, do func(reflect.Value)) {\n\tfor v.Kind() == reflect.Ptr {\n\t\tif v.IsNil() {\n\t\t\treturn\n\t\t}\n\t\tv = v.Elem()\n\t}\n\n\tswitch v.Kind() {\n\tcase reflect.Slice:\n\t\tfor i, n := 0, v.Len(); i < n; i++ {\n\t\t\tforEachField(v.Index(i), do)\n\t\t}\n\n\tcase reflect.Struct:\n\t\tfor i, n := 0, v.NumField(); i < n; i++ {\n\t\t\tforEachField(v.Field(i), do)\n\t\t}\n\n\tdefault:\n\t\tdo(v)\n\t}\n}\n\ntype memoryRecord struct {\n\toffset int64\n\ttime time.Time\n\tkey []byte\n\tvalue []byte\n\theaders []protocol.Header\n}\n\nfunc (m *memoryRecord) Record() protocol.Record {\n\treturn protocol.Record{\n\t\tOffset: m.offset,\n\t\tTime: m.time,\n\t\tKey: protocol.NewBytes(m.key),\n\t\tValue: protocol.NewBytes(m.value),\n\t\tHeaders: m.headers,\n\t}\n}\n\nfunc makeRecords(memoryRecords []memoryRecord) []protocol.Record {\n\trecords := make([]protocol.Record, len(memoryRecords))\n\tfor i, m := range memoryRecords {\n\t\trecords[i] = m.Record()\n\t}\n\treturn records\n}\n\nfunc loadRecords(r protocol.RecordReader) []memoryRecord {\n\trecords := []memoryRecord{}\n\n\tfor {\n\t\trec, err := r.ReadRecord()\n\t\tif err != nil {\n\t\t\tif errors.Is(err, io.EOF) {\n\t\t\t\treturn records\n\t\t\t}\n\t\t\tpanic(err)\n\t\t}\n\t\trecords = append(records, memoryRecord{\n\t\t\toffset: rec.Offset,\n\t\t\ttime: rec.Time,\n\t\t\tkey: readAll(rec.Key),\n\t\t\tvalue: readAll(rec.Value),\n\t\t\theaders: rec.Headers,\n\t\t})\n\t}\n}\n\nfunc readAll(bytes protocol.Bytes) []byte {\n\tif bytes != nil {\n\t\tdefer bytes.Close()\n\t}\n\tb, err := protocol.ReadAll(bytes)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn b\n}\n" }, { "path": "protocol/prototest/request.go", "content": "package prototest\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"io\"\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n)\n\nfunc TestRequest(t *testing.T, version int16, msg protocol.Message) {\n\treset := load(msg)\n\n\tt.Run(fmt.Sprintf(\"v%d\", version), func(t *testing.T) {\n\t\tb := &bytes.Buffer{}\n\n\t\tif err := protocol.WriteRequest(b, version, 1234, \"me\", msg); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\treset()\n\n\t\tt.Logf(\"\\n%s\\n\", hex.Dump(b.Bytes()))\n\n\t\tapiVersion, correlationID, clientID, req, err := protocol.ReadRequest(b)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif apiVersion != version {\n\t\t\tt.Errorf(\"api version mismatch: %d != %d\", apiVersion, version)\n\t\t}\n\t\tif correlationID != 1234 {\n\t\t\tt.Errorf(\"correlation id mismatch: %d != %d\", correlationID, 1234)\n\t\t}\n\t\tif clientID != \"me\" {\n\t\t\tt.Errorf(\"client id mismatch: %q != %q\", clientID, \"me\")\n\t\t}\n\t\tif !deepEqual(msg, req) {\n\t\t\tt.Errorf(\"request message mismatch:\")\n\t\t\tt.Logf(\"expected: %+v\", msg)\n\t\t\tt.Logf(\"found: %+v\", req)\n\t\t}\n\t})\n}\n\n// TestRequestWithOverride validates requests that have an overridden type. For requests with type overrides, we\n// double-serialize the request to ensure the resulting encoding of the overridden and original type are identical.\nfunc TestRequestWithOverride(t *testing.T, version int16, msg protocol.Message) {\n\treset := load(msg)\n\n\tt.Run(fmt.Sprintf(\"v%d\", version), func(t *testing.T) {\n\t\tb1 := &bytes.Buffer{}\n\n\t\tif err := protocol.WriteRequest(b1, version, 1234, \"me\", msg); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\treset()\n\t\tt.Logf(\"\\n%s\\n\", hex.Dump(b1.Bytes()))\n\n\t\t_, _, _, req, err := protocol.ReadRequest(b1)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tb2 := &bytes.Buffer{}\n\t\tif err := protocol.WriteRequest(b2, version, 1234, \"me\", req); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tif !deepEqual(b1, b2) {\n\t\t\tt.Errorf(\"request message mismatch:\")\n\t\t\tt.Logf(\"expected: %+v\", hex.Dump(b1.Bytes()))\n\t\t\tt.Logf(\"found: %+v\", hex.Dump(b2.Bytes()))\n\t\t}\n\t})\n}\n\nfunc BenchmarkRequest(b *testing.B, version int16, msg protocol.Message) {\n\treset := load(msg)\n\n\tb.Run(fmt.Sprintf(\"v%d\", version), func(b *testing.B) {\n\t\tbuffer := &bytes.Buffer{}\n\t\tbuffer.Grow(1024)\n\n\t\tb.Run(\"read\", func(b *testing.B) {\n\t\t\tw := io.Writer(buffer)\n\n\t\t\tif err := protocol.WriteRequest(w, version, 1234, \"client\", msg); err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\n\t\t\treset()\n\n\t\t\tp := buffer.Bytes()\n\t\t\tx := bytes.NewReader(p)\n\t\t\tr := bufio.NewReader(x)\n\n\t\t\tfor i := 0; i < b.N; i++ {\n\t\t\t\t_, _, _, req, err := protocol.ReadRequest(r)\n\t\t\t\tif err != nil {\n\t\t\t\t\tb.Fatal(err)\n\t\t\t\t}\n\t\t\t\tcloseMessage(req)\n\t\t\t\tx.Reset(p)\n\t\t\t\tr.Reset(x)\n\t\t\t}\n\n\t\t\tb.SetBytes(int64(len(p)))\n\t\t\tbuffer.Reset()\n\t\t})\n\n\t\tb.Run(\"write\", func(b *testing.B) {\n\t\t\tw := io.Writer(buffer)\n\t\t\tn := int64(0)\n\n\t\t\tfor i := 0; i < b.N; i++ {\n\t\t\t\tif err := protocol.WriteRequest(w, version, 1234, \"client\", msg); err != nil {\n\t\t\t\t\tb.Fatal(err)\n\t\t\t\t}\n\t\t\t\treset()\n\t\t\t\tn = int64(buffer.Len())\n\t\t\t\tbuffer.Reset()\n\t\t\t}\n\n\t\t\tb.SetBytes(n)\n\t\t})\n\t})\n}\n" }, { "path": "protocol/prototest/response.go", "content": "package prototest\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"io\"\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n)\n\nfunc TestResponse(t *testing.T, version int16, msg protocol.Message) {\n\treset := load(msg)\n\n\tt.Run(fmt.Sprintf(\"v%d\", version), func(t *testing.T) {\n\t\tb := &bytes.Buffer{}\n\n\t\tif err := protocol.WriteResponse(b, version, 1234, msg); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\treset()\n\n\t\tt.Logf(\"\\n%s\", hex.Dump(b.Bytes()))\n\n\t\tcorrelationID, res, err := protocol.ReadResponse(b, msg.ApiKey(), version)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t\tif correlationID != 1234 {\n\t\t\tt.Errorf(\"correlation id mismatch: %d != %d\", correlationID, 1234)\n\t\t}\n\t\tif !deepEqual(msg, res) {\n\t\t\tt.Errorf(\"response message mismatch:\")\n\t\t\tt.Logf(\"expected: %+v\", msg)\n\t\t\tt.Logf(\"found: %+v\", res)\n\t\t}\n\t\tcloseMessage(res)\n\t})\n}\n\nfunc BenchmarkResponse(b *testing.B, version int16, msg protocol.Message) {\n\treset := load(msg)\n\n\tb.Run(fmt.Sprintf(\"v%d\", version), func(b *testing.B) {\n\t\tapiKey := msg.ApiKey()\n\t\tbuffer := &bytes.Buffer{}\n\t\tbuffer.Grow(1024)\n\n\t\tb.Run(\"read\", func(b *testing.B) {\n\t\t\tw := io.Writer(buffer)\n\n\t\t\tif err := protocol.WriteResponse(w, version, 1234, msg); err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\n\t\t\treset()\n\n\t\t\tp := buffer.Bytes()\n\t\t\tx := bytes.NewReader(p)\n\t\t\tr := bufio.NewReader(x)\n\n\t\t\tfor i := 0; i < b.N; i++ {\n\t\t\t\t_, res, err := protocol.ReadResponse(r, apiKey, version)\n\t\t\t\tif err != nil {\n\t\t\t\t\tb.Fatal(err)\n\t\t\t\t}\n\t\t\t\tcloseMessage(res)\n\t\t\t\tx.Reset(p)\n\t\t\t\tr.Reset(x)\n\t\t\t}\n\n\t\t\tb.SetBytes(int64(len(p)))\n\t\t\tbuffer.Reset()\n\t\t})\n\n\t\tb.Run(\"write\", func(b *testing.B) {\n\t\t\tw := io.Writer(buffer)\n\t\t\tn := int64(0)\n\n\t\t\tfor i := 0; i < b.N; i++ {\n\t\t\t\tif err := protocol.WriteResponse(w, version, 1234, msg); err != nil {\n\t\t\t\t\tb.Fatal(err)\n\t\t\t\t}\n\t\t\t\treset()\n\t\t\t\tn = int64(buffer.Len())\n\t\t\t\tbuffer.Reset()\n\t\t\t}\n\n\t\t\tb.SetBytes(n)\n\t\t})\n\t})\n}\n" }, { "path": "protocol/rawproduce/rawproduce.go", "content": "package rawproduce\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n\t\"github.com/segmentio/kafka-go/protocol/produce\"\n)\n\nfunc init() {\n\t// Register a type override so that raw produce requests will be encoded with the correct type.\n\treq := &Request{}\n\tprotocol.RegisterOverride(req, &produce.Response{}, req.TypeKey())\n}\n\ntype Request struct {\n\tTransactionalID string `kafka:\"min=v3,max=v8,nullable\"`\n\tAcks int16 `kafka:\"min=v0,max=v8\"`\n\tTimeout int32 `kafka:\"min=v0,max=v8\"`\n\tTopics []RequestTopic `kafka:\"min=v0,max=v8\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.Produce }\n\nfunc (r *Request) TypeKey() protocol.OverrideTypeKey { return protocol.RawProduceOverride }\n\nfunc (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, error) {\n\tbroker := protocol.Broker{ID: -1}\n\n\tfor i := range r.Topics {\n\t\tt := &r.Topics[i]\n\n\t\ttopic, ok := cluster.Topics[t.Topic]\n\t\tif !ok {\n\t\t\treturn broker, NewError(protocol.NewErrNoTopic(t.Topic))\n\t\t}\n\n\t\tfor j := range t.Partitions {\n\t\t\tp := &t.Partitions[j]\n\n\t\t\tpartition, ok := topic.Partitions[p.Partition]\n\t\t\tif !ok {\n\t\t\t\treturn broker, NewError(protocol.NewErrNoPartition(t.Topic, p.Partition))\n\t\t\t}\n\n\t\t\tif b, ok := cluster.Brokers[partition.Leader]; !ok {\n\t\t\t\treturn broker, NewError(protocol.NewErrNoLeader(t.Topic, p.Partition))\n\t\t\t} else if broker.ID < 0 {\n\t\t\t\tbroker = b\n\t\t\t} else if b.ID != broker.ID {\n\t\t\t\treturn broker, NewError(fmt.Errorf(\"mismatching leaders (%d!=%d)\", b.ID, broker.ID))\n\t\t\t}\n\t\t}\n\t}\n\n\treturn broker, nil\n}\n\nfunc (r *Request) HasResponse() bool {\n\treturn r.Acks != 0\n}\n\ntype RequestTopic struct {\n\tTopic string `kafka:\"min=v0,max=v8\"`\n\tPartitions []RequestPartition `kafka:\"min=v0,max=v8\"`\n}\n\ntype RequestPartition struct {\n\tPartition int32 `kafka:\"min=v0,max=v8\"`\n\tRecordSet protocol.RawRecordSet `kafka:\"min=v0,max=v8\"`\n}\n\nvar (\n\t_ protocol.BrokerMessage = (*Request)(nil)\n)\n\ntype Error struct {\n\tErr error\n}\n\nfunc NewError(err error) *Error {\n\treturn &Error{Err: err}\n}\n\nfunc (e *Error) Error() string {\n\treturn fmt.Sprintf(\"fetch request error: %v\", e.Err)\n}\n\nfunc (e *Error) Unwrap() error {\n\treturn e.Err\n}\n" }, { "path": "protocol/rawproduce/rawproduce_test.go", "content": "package rawproduce_test\n\nimport (\n\t\"bytes\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n\t\"github.com/segmentio/kafka-go/protocol/rawproduce\"\n)\n\nconst (\n\tv0 = 0\n\tv3 = 3\n\tv5 = 5\n)\n\nfunc TestRawProduceRequest(t *testing.T) {\n\tt0 := time.Now().Truncate(time.Millisecond)\n\tt1 := t0.Add(1 * time.Millisecond)\n\tt2 := t0.Add(2 * time.Millisecond)\n\n\tprototest.TestRequestWithOverride(t, v0, &rawproduce.Request{\n\t\tAcks: 1,\n\t\tTimeout: 500,\n\t\tTopics: []rawproduce.RequestTopic{\n\t\t\t{\n\t\t\t\tTopic: \"topic-1\",\n\t\t\t\tPartitions: []rawproduce.RequestPartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\tRecordSet: NewRawRecordSet(protocol.NewRecordReader(\n\t\t\t\t\t\t\tprotocol.Record{Offset: 0, Time: t0, Key: nil, Value: nil},\n\t\t\t\t\t\t), 1, 0),\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\tRecordSet: NewRawRecordSet(protocol.NewRecordReader(\n\t\t\t\t\t\t\tprotocol.Record{Offset: 0, Time: t0, Key: nil, Value: prototest.String(\"msg-0\")},\n\t\t\t\t\t\t\tprotocol.Record{Offset: 1, Time: t1, Key: nil, Value: prototest.String(\"msg-1\")},\n\t\t\t\t\t\t\tprotocol.Record{Offset: 2, Time: t2, Key: prototest.Bytes([]byte{1}), Value: prototest.String(\"msg-2\")},\n\t\t\t\t\t\t), 1, 0),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\n\t\t\t{\n\t\t\t\tTopic: \"topic-2\",\n\t\t\t\tPartitions: []rawproduce.RequestPartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\tRecordSet: NewRawRecordSet(protocol.NewRecordReader(\n\t\t\t\t\t\t\tprotocol.Record{Offset: 0, Time: t0, Key: nil, Value: prototest.String(\"msg-0\")},\n\t\t\t\t\t\t\tprotocol.Record{Offset: 1, Time: t1, Key: nil, Value: prototest.String(\"msg-1\")},\n\t\t\t\t\t\t\tprotocol.Record{Offset: 2, Time: t2, Key: prototest.Bytes([]byte{1}), Value: prototest.String(\"msg-2\")},\n\t\t\t\t\t\t), 1, protocol.Gzip),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\tprototest.TestRequestWithOverride(t, v3, &rawproduce.Request{\n\t\tTransactionalID: \"1234\",\n\t\tAcks: 1,\n\t\tTimeout: 500,\n\t\tTopics: []rawproduce.RequestTopic{\n\t\t\t{\n\t\t\t\tTopic: \"topic-1\",\n\t\t\t\tPartitions: []rawproduce.RequestPartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\tRecordSet: NewRawRecordSet(protocol.NewRecordReader(\n\t\t\t\t\t\t\tprotocol.Record{Offset: 0, Time: t0, Key: nil, Value: nil},\n\t\t\t\t\t\t), 1, 0),\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\tRecordSet: NewRawRecordSet(protocol.NewRecordReader(\n\t\t\t\t\t\t\tprotocol.Record{Offset: 0, Time: t0, Key: nil, Value: prototest.String(\"msg-0\")},\n\t\t\t\t\t\t\tprotocol.Record{Offset: 1, Time: t1, Key: nil, Value: prototest.String(\"msg-1\")},\n\t\t\t\t\t\t\tprotocol.Record{Offset: 2, Time: t2, Key: prototest.Bytes([]byte{1}), Value: prototest.String(\"msg-2\")},\n\t\t\t\t\t\t), 1, 0),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\theaders := []protocol.Header{\n\t\t{Key: \"key-1\", Value: []byte(\"value-1\")},\n\t\t{Key: \"key-2\", Value: []byte(\"value-2\")},\n\t\t{Key: \"key-3\", Value: []byte(\"value-3\")},\n\t}\n\n\tprototest.TestRequestWithOverride(t, v5, &rawproduce.Request{\n\t\tTransactionalID: \"1234\",\n\t\tAcks: 1,\n\t\tTimeout: 500,\n\t\tTopics: []rawproduce.RequestTopic{\n\t\t\t{\n\t\t\t\tTopic: \"topic-1\",\n\t\t\t\tPartitions: []rawproduce.RequestPartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\tRecordSet: NewRawRecordSet(protocol.NewRecordReader(\n\t\t\t\t\t\t\tprotocol.Record{Offset: 0, Time: t0, Key: nil, Value: prototest.String(\"msg-0\"), Headers: headers},\n\t\t\t\t\t\t\tprotocol.Record{Offset: 1, Time: t1, Key: nil, Value: prototest.String(\"msg-1\")},\n\t\t\t\t\t\t\tprotocol.Record{Offset: 2, Time: t2, Key: prototest.Bytes([]byte{1}), Value: prototest.String(\"msg-2\")},\n\t\t\t\t\t\t), 2, 0),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\n\t\t\t{\n\t\t\t\tTopic: \"topic-2\",\n\t\t\t\tPartitions: []rawproduce.RequestPartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\tRecordSet: NewRawRecordSet(protocol.NewRecordReader(\n\t\t\t\t\t\t\tprotocol.Record{Offset: 0, Time: t0, Key: nil, Value: prototest.String(\"msg-0\"), Headers: headers},\n\t\t\t\t\t\t\tprotocol.Record{Offset: 1, Time: t1, Key: nil, Value: prototest.String(\"msg-1\")},\n\t\t\t\t\t\t\tprotocol.Record{Offset: 2, Time: t2, Key: prototest.Bytes([]byte{1}), Value: prototest.String(\"msg-2\")},\n\t\t\t\t\t\t), 2, protocol.Snappy),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n}\n\nfunc NewRawRecordSet(reader protocol.RecordReader, version int8, attr protocol.Attributes) protocol.RawRecordSet {\n\trs := protocol.RecordSet{Version: version, Attributes: attr, Records: reader}\n\tbuf := &bytes.Buffer{}\n\trs.WriteTo(buf)\n\n\treturn protocol.RawRecordSet{\n\t\tReader: buf,\n\t}\n}\n\nfunc BenchmarkProduceRequest(b *testing.B) {\n\tt0 := time.Now().Truncate(time.Millisecond)\n\tt1 := t0.Add(1 * time.Millisecond)\n\tt2 := t0.Add(2 * time.Millisecond)\n\n\tprototest.BenchmarkRequest(b, v3, &rawproduce.Request{\n\t\tTransactionalID: \"1234\",\n\t\tAcks: 1,\n\t\tTimeout: 500,\n\t\tTopics: []rawproduce.RequestTopic{\n\t\t\t{\n\t\t\t\tTopic: \"topic-1\",\n\t\t\t\tPartitions: []rawproduce.RequestPartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\tRecordSet: NewRawRecordSet(protocol.NewRecordReader(\n\t\t\t\t\t\t\tprotocol.Record{Offset: 0, Time: t0, Key: nil, Value: nil},\n\t\t\t\t\t\t), 1, 0),\n\t\t\t\t\t},\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\tRecordSet: NewRawRecordSet(protocol.NewRecordReader(\n\t\t\t\t\t\t\tprotocol.Record{Offset: 0, Time: t0, Key: nil, Value: prototest.String(\"msg-0\")},\n\t\t\t\t\t\t\tprotocol.Record{Offset: 1, Time: t1, Key: nil, Value: prototest.String(\"msg-1\")},\n\t\t\t\t\t\t\tprotocol.Record{Offset: 2, Time: t2, Key: prototest.Bytes([]byte{1}), Value: prototest.String(\"msg-2\")},\n\t\t\t\t\t\t), 1, 0),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\theaders := []protocol.Header{\n\t\t{Key: \"key-1\", Value: []byte(\"value-1\")},\n\t\t{Key: \"key-2\", Value: []byte(\"value-2\")},\n\t\t{Key: \"key-3\", Value: []byte(\"value-3\")},\n\t}\n\n\tprototest.BenchmarkRequest(b, v5, &rawproduce.Request{\n\t\tTransactionalID: \"1234\",\n\t\tAcks: 1,\n\t\tTimeout: 500,\n\t\tTopics: []rawproduce.RequestTopic{\n\t\t\t{\n\t\t\t\tTopic: \"topic-1\",\n\t\t\t\tPartitions: []rawproduce.RequestPartition{\n\t\t\t\t\t{\n\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\tRecordSet: NewRawRecordSet(protocol.NewRecordReader(\n\t\t\t\t\t\t\tprotocol.Record{Offset: 0, Time: t0, Key: nil, Value: prototest.String(\"msg-0\"), Headers: headers},\n\t\t\t\t\t\t\tprotocol.Record{Offset: 1, Time: t1, Key: nil, Value: prototest.String(\"msg-1\")},\n\t\t\t\t\t\t\tprotocol.Record{Offset: 2, Time: t2, Key: prototest.Bytes([]byte{1}), Value: prototest.String(\"msg-2\")},\n\t\t\t\t\t\t), 2, 0),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n}\n" }, { "path": "protocol/record.go", "content": "package protocol\n\nimport (\n\t\"bytes\"\n\t\"encoding/binary\"\n\t\"fmt\"\n\t\"io\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/compress\"\n)\n\n// Attributes is a bitset representing special attributes set on records.\ntype Attributes int16\n\nconst (\n\tGzip Attributes = Attributes(compress.Gzip) // 1\n\tSnappy Attributes = Attributes(compress.Snappy) // 2\n\tLz4 Attributes = Attributes(compress.Lz4) // 3\n\tZstd Attributes = Attributes(compress.Zstd) // 4\n\tTransactional Attributes = 1 << 4\n\tControl Attributes = 1 << 5\n)\n\nfunc (a Attributes) Compression() compress.Compression {\n\treturn compress.Compression(a & 7)\n}\n\nfunc (a Attributes) Transactional() bool {\n\treturn (a & Transactional) != 0\n}\n\nfunc (a Attributes) Control() bool {\n\treturn (a & Control) != 0\n}\n\nfunc (a Attributes) String() string {\n\ts := a.Compression().String()\n\tif a.Transactional() {\n\t\ts += \"+transactional\"\n\t}\n\tif a.Control() {\n\t\ts += \"+control\"\n\t}\n\treturn s\n}\n\n// Header represents a single entry in a list of record headers.\ntype Header struct {\n\tKey string\n\tValue []byte\n}\n\n// Record is an interface representing a single kafka record.\n//\n// Record values are not safe to use concurrently from multiple goroutines.\ntype Record struct {\n\t// The offset at which the record exists in a topic partition. This value\n\t// is ignored in produce requests.\n\tOffset int64\n\n\t// Returns the time of the record. This value may be omitted in produce\n\t// requests to let kafka set the time when it saves the record.\n\tTime time.Time\n\n\t// Returns a byte sequence containing the key of this record. The returned\n\t// sequence may be nil to indicate that the record has no key. If the record\n\t// is part of a RecordSet, the content of the key must remain valid at least\n\t// until the record set is closed (or until the key is closed).\n\tKey Bytes\n\n\t// Returns a byte sequence containing the value of this record. The returned\n\t// sequence may be nil to indicate that the record has no value. If the\n\t// record is part of a RecordSet, the content of the value must remain valid\n\t// at least until the record set is closed (or until the value is closed).\n\tValue Bytes\n\n\t// Returns the list of headers associated with this record. The returned\n\t// slice may be reused across calls, the program should use it as an\n\t// immutable value.\n\tHeaders []Header\n}\n\n// RecordSet represents a sequence of records in Produce requests and Fetch\n// responses. All v0, v1, and v2 formats are supported.\ntype RecordSet struct {\n\t// The message version that this record set will be represented as, valid\n\t// values are 1, or 2.\n\t//\n\t// When reading, this is the value of the highest version used in the\n\t// batches that compose the record set.\n\t//\n\t// When writing, this value dictates the format that the records will be\n\t// encoded in.\n\tVersion int8\n\n\t// Attributes set on the record set.\n\t//\n\t// When reading, the attributes are the combination of all attributes in\n\t// the batches that compose the record set.\n\t//\n\t// When writing, the attributes apply to the whole sequence of records in\n\t// the set.\n\tAttributes Attributes\n\n\t// A reader exposing the sequence of records.\n\t//\n\t// When reading a RecordSet from an io.Reader, the Records field will be a\n\t// *RecordStream. If the program needs to access the details of each batch\n\t// that compose the stream, it may use type assertions to access the\n\t// underlying types of each batch.\n\tRecords RecordReader\n}\n\n// bufferedReader is an interface implemented by types like bufio.Reader, which\n// we use to optimize prefix reads by accessing the internal buffer directly\n// through calls to Peek.\ntype bufferedReader interface {\n\tDiscard(int) (int, error)\n\tPeek(int) ([]byte, error)\n}\n\n// bytesBuffer is an interface implemented by types like bytes.Buffer, which we\n// use to optimize prefix reads by accessing the internal buffer directly\n// through calls to Bytes.\ntype bytesBuffer interface {\n\tBytes() []byte\n}\n\n// magicByteOffset is the position of the magic byte in all versions of record\n// sets in the kafka protocol.\nconst magicByteOffset = 16\n\n// ReadFrom reads the representation of a record set from r into rs, returning\n// the number of bytes consumed from r, and an non-nil error if the record set\n// could not be read.\nfunc (rs *RecordSet) ReadFrom(r io.Reader) (int64, error) {\n\td, _ := r.(*decoder)\n\tif d == nil {\n\t\td = &decoder{\n\t\t\treader: r,\n\t\t\tremain: 4,\n\t\t}\n\t}\n\n\t*rs = RecordSet{}\n\tlimit := d.remain\n\tsize := d.readInt32()\n\n\tif d.err != nil {\n\t\treturn int64(limit - d.remain), d.err\n\t}\n\n\tif size <= 0 {\n\t\treturn 4, nil\n\t}\n\n\tstream := &RecordStream{\n\t\tRecords: make([]RecordReader, 0, 4),\n\t}\n\n\tvar err error\n\td.remain = int(size)\n\n\tfor d.remain > 0 && err == nil {\n\t\tvar version byte\n\n\t\tif d.remain < (magicByteOffset + 1) {\n\t\t\tif len(stream.Records) != 0 {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\treturn 4, fmt.Errorf(\"impossible record set shorter than %d bytes\", magicByteOffset+1)\n\t\t}\n\n\t\tswitch r := d.reader.(type) {\n\t\tcase bufferedReader:\n\t\t\tb, err := r.Peek(magicByteOffset + 1)\n\t\t\tif err != nil {\n\t\t\t\tn, _ := r.Discard(len(b))\n\t\t\t\treturn 4 + int64(n), dontExpectEOF(err)\n\t\t\t}\n\t\t\tversion = b[magicByteOffset]\n\t\tcase bytesBuffer:\n\t\t\tversion = r.Bytes()[magicByteOffset]\n\t\tdefault:\n\t\t\tb := make([]byte, magicByteOffset+1)\n\t\t\tif n, err := io.ReadFull(d.reader, b); err != nil {\n\t\t\t\treturn 4 + int64(n), dontExpectEOF(err)\n\t\t\t}\n\t\t\tversion = b[magicByteOffset]\n\t\t\t// Reconstruct the prefix that we had to read to determine the version\n\t\t\t// of the record set from the magic byte.\n\t\t\t//\n\t\t\t// Technically this may recursively stack readers when consuming all\n\t\t\t// items of the batch, which could hurt performance. In practice this\n\t\t\t// path should not be taken tho, since the decoder would read from a\n\t\t\t// *bufio.Reader which implements the bufferedReader interface.\n\t\t\td.reader = io.MultiReader(bytes.NewReader(b), d.reader)\n\t\t}\n\n\t\tvar tmp RecordSet\n\t\tswitch version {\n\t\tcase 0, 1:\n\t\t\terr = tmp.readFromVersion1(d)\n\t\tcase 2:\n\t\t\terr = tmp.readFromVersion2(d)\n\t\tdefault:\n\t\t\terr = fmt.Errorf(\"unsupported message version %d for message of size %d\", version, size)\n\t\t}\n\n\t\tif tmp.Version > rs.Version {\n\t\t\trs.Version = tmp.Version\n\t\t}\n\n\t\trs.Attributes |= tmp.Attributes\n\n\t\tif tmp.Records != nil {\n\t\t\tstream.Records = append(stream.Records, tmp.Records)\n\t\t}\n\t}\n\n\tif len(stream.Records) != 0 {\n\t\trs.Records = stream\n\t\t// Ignore errors if we've successfully read records, so the\n\t\t// program can keep making progress.\n\t\terr = nil\n\t}\n\n\td.discardAll()\n\trn := 4 + (int(size) - d.remain)\n\td.remain = limit - rn\n\treturn int64(rn), err\n}\n\n// WriteTo writes the representation of rs into w. The value of rs.Version\n// dictates which format that the record set will be represented as.\n//\n// The error will be ErrNoRecord if rs contained no records.\n//\n// Note: since this package is only compatible with kafka 0.10 and above, the\n// method never produces messages in version 0. If rs.Version is zero, the\n// method defaults to producing messages in version 1.\nfunc (rs *RecordSet) WriteTo(w io.Writer) (int64, error) {\n\tif rs.Records == nil {\n\t\treturn 0, ErrNoRecord\n\t}\n\n\t// This optimization avoids rendering the record set in an intermediary\n\t// buffer when the writer is already a pageBuffer, which is a common case\n\t// due to the way WriteRequest and WriteResponse are implemented.\n\tbuffer, _ := w.(*pageBuffer)\n\tbufferOffset := int64(0)\n\n\tif buffer != nil {\n\t\tbufferOffset = buffer.Size()\n\t} else {\n\t\tbuffer = newPageBuffer()\n\t\tdefer buffer.unref()\n\t}\n\n\tsize := packUint32(0)\n\tbuffer.Write(size[:]) // size placeholder\n\n\tvar err error\n\tswitch rs.Version {\n\tcase 0, 1:\n\t\terr = rs.writeToVersion1(buffer, bufferOffset+4)\n\tcase 2:\n\t\terr = rs.writeToVersion2(buffer, bufferOffset+4)\n\tdefault:\n\t\terr = fmt.Errorf(\"unsupported record set version %d\", rs.Version)\n\t}\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tn := buffer.Size() - bufferOffset\n\tif n == 0 {\n\t\tsize = packUint32(^uint32(0))\n\t} else {\n\t\tsize = packUint32(uint32(n) - 4)\n\t}\n\tbuffer.WriteAt(size[:], bufferOffset)\n\n\t// This condition indicates that the output writer received by `WriteTo` was\n\t// not a *pageBuffer, in which case we need to flush the buffered records\n\t// data into it.\n\tif buffer != w {\n\t\treturn buffer.WriteTo(w)\n\t}\n\n\treturn n, nil\n}\n\n// RawRecordSet represents a record set for a RawProduce request. The record set is\n// represented as a raw sequence of pre-encoded record set bytes.\ntype RawRecordSet struct {\n\t// Reader exposes the raw sequence of record set bytes.\n\tReader io.Reader\n}\n\n// ReadFrom reads the representation of a record set from r into rrs. It re-uses the\n// existing RecordSet.ReadFrom implementation to first read/decode data into a RecordSet,\n// then writes/encodes the RecordSet to a buffer referenced by the RawRecordSet.\n//\n// Note: re-using the RecordSet.ReadFrom implementation makes this suboptimal from a\n// performance standpoint as it requires an extra copy of the record bytes. Holding off\n// on optimizing, as this code path is only invoked in tests.\nfunc (rrs *RawRecordSet) ReadFrom(r io.Reader) (int64, error) {\n\trs := &RecordSet{}\n\tn, err := rs.ReadFrom(r)\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\n\tbuf := &bytes.Buffer{}\n\trs.WriteTo(buf)\n\t*rrs = RawRecordSet{\n\t\tReader: buf,\n\t}\n\n\treturn n, nil\n}\n\n// WriteTo writes the RawRecordSet to an io.Writer. Since this is a raw record set representation, all that is\n// done here is copying bytes from the underlying reader to the specified writer.\nfunc (rrs *RawRecordSet) WriteTo(w io.Writer) (int64, error) {\n\tif rrs.Reader == nil {\n\t\treturn 0, ErrNoRecord\n\t}\n\n\treturn io.Copy(w, rrs.Reader)\n}\n\nfunc makeTime(t int64) time.Time {\n\treturn time.Unix(t/1000, (t%1000)*int64(time.Millisecond))\n}\n\nfunc timestamp(t time.Time) int64 {\n\tif t.IsZero() {\n\t\treturn 0\n\t}\n\treturn t.UnixNano() / int64(time.Millisecond)\n}\n\nfunc packUint32(u uint32) (b [4]byte) {\n\tbinary.BigEndian.PutUint32(b[:], u)\n\treturn\n}\n\nfunc packUint64(u uint64) (b [8]byte) {\n\tbinary.BigEndian.PutUint64(b[:], u)\n\treturn\n}\n" }, { "path": "protocol/record_batch.go", "content": "package protocol\n\nimport (\n\t\"errors\"\n\t\"io\"\n\t\"time\"\n)\n\n// RecordReader is an interface representing a sequence of records. Record sets\n// are used in both produce and fetch requests to represent the sequence of\n// records that are sent to or receive from kafka brokers.\n//\n// RecordSet values are not safe to use concurrently from multiple goroutines.\ntype RecordReader interface {\n\t// Returns the next record in the set, or io.EOF if the end of the sequence\n\t// has been reached.\n\t//\n\t// The returned Record is guaranteed to be valid until the next call to\n\t// ReadRecord. If the program needs to retain the Record value it must make\n\t// a copy.\n\tReadRecord() (*Record, error)\n}\n\n// NewRecordReader constructs a reader exposing the records passed as arguments.\nfunc NewRecordReader(records ...Record) RecordReader {\n\tswitch len(records) {\n\tcase 0:\n\t\treturn emptyRecordReader{}\n\tdefault:\n\t\tr := &recordReader{records: make([]Record, len(records))}\n\t\tcopy(r.records, records)\n\t\treturn r\n\t}\n}\n\n// MultiRecordReader merges multiple record batches into one.\nfunc MultiRecordReader(batches ...RecordReader) RecordReader {\n\tswitch len(batches) {\n\tcase 0:\n\t\treturn emptyRecordReader{}\n\tcase 1:\n\t\treturn batches[0]\n\tdefault:\n\t\tm := &multiRecordReader{batches: make([]RecordReader, len(batches))}\n\t\tcopy(m.batches, batches)\n\t\treturn m\n\t}\n}\n\nfunc forEachRecord(r RecordReader, f func(int, *Record) error) error {\n\tfor i := 0; ; i++ {\n\t\trec, err := r.ReadRecord()\n\n\t\tif err != nil {\n\t\t\tif errors.Is(err, io.EOF) {\n\t\t\t\terr = nil\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\n\t\tif err := handleRecord(i, rec, f); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n}\n\nfunc handleRecord(i int, r *Record, f func(int, *Record) error) error {\n\tif r.Key != nil {\n\t\tdefer r.Key.Close()\n\t}\n\tif r.Value != nil {\n\t\tdefer r.Value.Close()\n\t}\n\treturn f(i, r)\n}\n\ntype recordReader struct {\n\trecords []Record\n\tindex int\n}\n\nfunc (r *recordReader) ReadRecord() (*Record, error) {\n\tif i := r.index; i >= 0 && i < len(r.records) {\n\t\tr.index++\n\t\treturn &r.records[i], nil\n\t}\n\treturn nil, io.EOF\n}\n\ntype multiRecordReader struct {\n\tbatches []RecordReader\n\tindex int\n}\n\nfunc (m *multiRecordReader) ReadRecord() (*Record, error) {\n\tfor {\n\t\tif m.index == len(m.batches) {\n\t\t\treturn nil, io.EOF\n\t\t}\n\t\tr, err := m.batches[m.index].ReadRecord()\n\t\tif err == nil {\n\t\t\treturn r, nil\n\t\t}\n\t\tif !errors.Is(err, io.EOF) {\n\t\t\treturn nil, err\n\t\t}\n\t\tm.index++\n\t}\n}\n\n// optimizedRecordReader is an implementation of a RecordReader which exposes a\n// sequence.\ntype optimizedRecordReader struct {\n\trecords []optimizedRecord\n\tindex int\n\tbuffer Record\n\theaders [][]Header\n}\n\nfunc (r *optimizedRecordReader) ReadRecord() (*Record, error) {\n\tif i := r.index; i >= 0 && i < len(r.records) {\n\t\trec := &r.records[i]\n\t\tr.index++\n\t\tr.buffer = Record{\n\t\t\tOffset: rec.offset,\n\t\t\tTime: rec.time(),\n\t\t\tKey: rec.key(),\n\t\t\tValue: rec.value(),\n\t\t}\n\t\tif i < len(r.headers) {\n\t\t\tr.buffer.Headers = r.headers[i]\n\t\t}\n\t\treturn &r.buffer, nil\n\t}\n\treturn nil, io.EOF\n}\n\ntype optimizedRecord struct {\n\toffset int64\n\ttimestamp int64\n\tkeyRef *pageRef\n\tvalueRef *pageRef\n}\n\nfunc (r *optimizedRecord) time() time.Time {\n\treturn makeTime(r.timestamp)\n}\n\nfunc (r *optimizedRecord) key() Bytes {\n\treturn makeBytes(r.keyRef)\n}\n\nfunc (r *optimizedRecord) value() Bytes {\n\treturn makeBytes(r.valueRef)\n}\n\nfunc makeBytes(ref *pageRef) Bytes {\n\tif ref == nil {\n\t\treturn nil\n\t}\n\treturn ref\n}\n\ntype emptyRecordReader struct{}\n\nfunc (emptyRecordReader) ReadRecord() (*Record, error) { return nil, io.EOF }\n\n// ControlRecord represents a record read from a control batch.\ntype ControlRecord struct {\n\tOffset int64\n\tTime time.Time\n\tVersion int16\n\tType int16\n\tData []byte\n\tHeaders []Header\n}\n\nfunc ReadControlRecord(r *Record) (*ControlRecord, error) {\n\tif r.Key != nil {\n\t\tdefer r.Key.Close()\n\t}\n\tif r.Value != nil {\n\t\tdefer r.Value.Close()\n\t}\n\n\tk, err := ReadAll(r.Key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif k == nil {\n\t\treturn nil, Error(\"invalid control record with nil key\")\n\t}\n\tif len(k) != 4 {\n\t\treturn nil, Errorf(\"invalid control record with key of size %d\", len(k))\n\t}\n\n\tv, err := ReadAll(r.Value)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tc := &ControlRecord{\n\t\tOffset: r.Offset,\n\t\tTime: r.Time,\n\t\tVersion: readInt16(k[:2]),\n\t\tType: readInt16(k[2:]),\n\t\tData: v,\n\t\tHeaders: r.Headers,\n\t}\n\n\treturn c, nil\n}\n\nfunc (cr *ControlRecord) Key() Bytes {\n\tk := make([]byte, 4)\n\twriteInt16(k[:2], cr.Version)\n\twriteInt16(k[2:], cr.Type)\n\treturn NewBytes(k)\n}\n\nfunc (cr *ControlRecord) Value() Bytes {\n\treturn NewBytes(cr.Data)\n}\n\nfunc (cr *ControlRecord) Record() Record {\n\treturn Record{\n\t\tOffset: cr.Offset,\n\t\tTime: cr.Time,\n\t\tKey: cr.Key(),\n\t\tValue: cr.Value(),\n\t\tHeaders: cr.Headers,\n\t}\n}\n\n// ControlBatch is an implementation of the RecordReader interface representing\n// control batches returned by kafka brokers.\ntype ControlBatch struct {\n\tAttributes Attributes\n\tPartitionLeaderEpoch int32\n\tBaseOffset int64\n\tProducerID int64\n\tProducerEpoch int16\n\tBaseSequence int32\n\tRecords RecordReader\n}\n\n// NewControlBatch constructs a control batch from the list of records passed as\n// arguments.\nfunc NewControlBatch(records ...ControlRecord) *ControlBatch {\n\trawRecords := make([]Record, len(records))\n\tfor i, cr := range records {\n\t\trawRecords[i] = cr.Record()\n\t}\n\treturn &ControlBatch{\n\t\tRecords: NewRecordReader(rawRecords...),\n\t}\n}\n\nfunc (c *ControlBatch) ReadRecord() (*Record, error) {\n\treturn c.Records.ReadRecord()\n}\n\nfunc (c *ControlBatch) ReadControlRecord() (*ControlRecord, error) {\n\tr, err := c.ReadRecord()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif r.Key != nil {\n\t\tdefer r.Key.Close()\n\t}\n\tif r.Value != nil {\n\t\tdefer r.Value.Close()\n\t}\n\treturn ReadControlRecord(r)\n}\n\nfunc (c *ControlBatch) Offset() int64 {\n\treturn c.BaseOffset\n}\n\nfunc (c *ControlBatch) Version() int {\n\treturn 2\n}\n\n// RecordBatch is an implementation of the RecordReader interface representing\n// regular record batches (v2).\ntype RecordBatch struct {\n\tAttributes Attributes\n\tPartitionLeaderEpoch int32\n\tBaseOffset int64\n\tProducerID int64\n\tProducerEpoch int16\n\tBaseSequence int32\n\tRecords RecordReader\n}\n\nfunc (r *RecordBatch) ReadRecord() (*Record, error) {\n\treturn r.Records.ReadRecord()\n}\n\nfunc (r *RecordBatch) Offset() int64 {\n\treturn r.BaseOffset\n}\n\nfunc (r *RecordBatch) Version() int {\n\treturn 2\n}\n\n// MessageSet is an implementation of the RecordReader interface representing\n// regular message sets (v1).\ntype MessageSet struct {\n\tAttributes Attributes\n\tBaseOffset int64\n\tRecords RecordReader\n}\n\nfunc (m *MessageSet) ReadRecord() (*Record, error) {\n\treturn m.Records.ReadRecord()\n}\n\nfunc (m *MessageSet) Offset() int64 {\n\treturn m.BaseOffset\n}\n\nfunc (m *MessageSet) Version() int {\n\treturn 1\n}\n\n// RecordStream is an implementation of the RecordReader interface which\n// combines multiple underlying RecordReader and only expose records that\n// are not from control batches.\ntype RecordStream struct {\n\tRecords []RecordReader\n\tindex int\n}\n\nfunc (s *RecordStream) ReadRecord() (*Record, error) {\n\tfor {\n\t\tif s.index < 0 || s.index >= len(s.Records) {\n\t\t\treturn nil, io.EOF\n\t\t}\n\n\t\tif _, isControl := s.Records[s.index].(*ControlBatch); isControl {\n\t\t\ts.index++\n\t\t\tcontinue\n\t\t}\n\n\t\tr, err := s.Records[s.index].ReadRecord()\n\t\tif err != nil {\n\t\t\tif errors.Is(err, io.EOF) {\n\t\t\t\ts.index++\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\treturn r, err\n\t}\n}\n" }, { "path": "protocol/record_batch_test.go", "content": "package protocol\n\nimport (\n\t\"errors\"\n\t\"io\"\n\t\"reflect\"\n\t\"testing\"\n\t\"time\"\n)\n\ntype memoryRecord struct {\n\toffset int64\n\ttime time.Time\n\tkey []byte\n\tvalue []byte\n\theaders []Header\n}\n\nfunc (m *memoryRecord) Record() Record {\n\treturn Record{\n\t\tOffset: m.offset,\n\t\tTime: m.time,\n\t\tKey: NewBytes(m.key),\n\t\tValue: NewBytes(m.value),\n\t\tHeaders: m.headers,\n\t}\n}\n\nfunc makeRecords(memoryRecords []memoryRecord) []Record {\n\trecords := make([]Record, len(memoryRecords))\n\tfor i, m := range memoryRecords {\n\t\trecords[i] = m.Record()\n\t}\n\treturn records\n}\n\nfunc TestRecordReader(t *testing.T) {\n\tnow := time.Now()\n\n\trecords := []memoryRecord{\n\t\t{\n\t\t\toffset: 1,\n\t\t\ttime: now,\n\t\t\tkey: []byte(\"key-1\"),\n\t\t},\n\t\t{\n\t\t\toffset: 2,\n\t\t\ttime: now.Add(time.Millisecond),\n\t\t\tvalue: []byte(\"value-1\"),\n\t\t},\n\t\t{\n\t\t\toffset: 3,\n\t\t\ttime: now.Add(time.Second),\n\t\t\tkey: []byte(\"key-3\"),\n\t\t\tvalue: []byte(\"value-3\"),\n\t\t\theaders: []Header{\n\t\t\t\t{Key: \"answer\", Value: []byte(\"42\")},\n\t\t\t},\n\t\t},\n\t}\n\n\tr1 := NewRecordReader(makeRecords(records)...)\n\tr2 := NewRecordReader(makeRecords(records)...)\n\tassertRecords(t, r1, r2)\n}\n\nfunc TestMultiRecordReader(t *testing.T) {\n\tnow := time.Now()\n\n\trecords := []memoryRecord{\n\t\t{\n\t\t\toffset: 1,\n\t\t\ttime: now,\n\t\t\tkey: []byte(\"key-1\"),\n\t\t},\n\t\t{\n\t\t\toffset: 2,\n\t\t\ttime: now.Add(time.Millisecond),\n\t\t\tvalue: []byte(\"value-1\"),\n\t\t},\n\t\t{\n\t\t\toffset: 3,\n\t\t\ttime: now.Add(time.Second),\n\t\t\tkey: []byte(\"key-3\"),\n\t\t\tvalue: []byte(\"value-3\"),\n\t\t\theaders: []Header{\n\t\t\t\t{Key: \"answer\", Value: []byte(\"42\")},\n\t\t\t},\n\t\t},\n\t}\n\n\tr1 := NewRecordReader(makeRecords(records)...)\n\tr2 := MultiRecordReader(\n\t\tNewRecordReader(makeRecords(records[:1])...),\n\t\tNewRecordReader(makeRecords(records[1:])...),\n\t)\n\tassertRecords(t, r1, r2)\n}\n\nfunc TestControlRecord(t *testing.T) {\n\tnow := time.Now()\n\n\trecords := []ControlRecord{\n\t\t{\n\t\t\tOffset: 1,\n\t\t\tTime: now,\n\t\t\tVersion: 2,\n\t\t\tType: 3,\n\t\t},\n\t\t{\n\t\t\tOffset: 2,\n\t\t\tTime: now.Add(time.Second),\n\t\t\tVersion: 4,\n\t\t\tType: 5,\n\t\t\tData: []byte(\"Hello World!\"),\n\t\t\tHeaders: []Header{\n\t\t\t\t{Key: \"answer\", Value: []byte(\"42\")},\n\t\t\t},\n\t\t},\n\t}\n\n\tbatch := NewControlBatch(records...)\n\tfound := make([]ControlRecord, 0, len(records))\n\n\tfor {\n\t\tr, err := batch.ReadControlRecord()\n\t\tif err != nil {\n\t\t\tif !errors.Is(err, io.EOF) {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t\tfound = append(found, *r)\n\t}\n\n\tif !reflect.DeepEqual(records, found) {\n\t\tt.Error(\"control records mismatch\")\n\t}\n}\n\nfunc assertRecords(t *testing.T, r1, r2 RecordReader) {\n\tt.Helper()\n\n\tfor {\n\t\trec1, err1 := r1.ReadRecord()\n\t\trec2, err2 := r2.ReadRecord()\n\n\t\tif err1 != nil || err2 != nil {\n\t\t\tif !errors.Is(err1, err2) {\n\t\t\t\tt.Error(\"errors mismatch:\")\n\t\t\t\tt.Log(\"expected:\", err2)\n\t\t\t\tt.Log(\"found: \", err1)\n\t\t\t}\n\t\t\treturn\n\t\t}\n\n\t\tif !equalRecords(rec1, rec2) {\n\t\t\tt.Error(\"records mismatch:\")\n\t\t\tt.Logf(\"expected: %+v\", rec2)\n\t\t\tt.Logf(\"found: %+v\", rec1)\n\t\t}\n\t}\n}\n\nfunc equalRecords(r1, r2 *Record) bool {\n\tif r1.Offset != r2.Offset {\n\t\treturn false\n\t}\n\n\tif !r1.Time.Equal(r2.Time) {\n\t\treturn false\n\t}\n\n\tk1 := readAll(r1.Key)\n\tk2 := readAll(r2.Key)\n\n\tif !reflect.DeepEqual(k1, k2) {\n\t\treturn false\n\t}\n\n\tv1 := readAll(r1.Value)\n\tv2 := readAll(r2.Value)\n\n\tif !reflect.DeepEqual(v1, v2) {\n\t\treturn false\n\t}\n\n\treturn reflect.DeepEqual(r1.Headers, r2.Headers)\n}\n\nfunc readAll(bytes Bytes) []byte {\n\tif bytes != nil {\n\t\tdefer bytes.Close()\n\t}\n\tb, err := ReadAll(bytes)\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn b\n}\n" }, { "path": "protocol/record_v1.go", "content": "package protocol\n\nimport (\n\t\"errors\"\n\t\"hash/crc32\"\n\t\"io\"\n\t\"math\"\n\t\"time\"\n)\n\nfunc readMessage(b *pageBuffer, d *decoder) (attributes int8, baseOffset, timestamp int64, key, value Bytes, err error) {\n\tmd := decoder{\n\t\treader: d,\n\t\tremain: 12,\n\t}\n\n\tbaseOffset = md.readInt64()\n\tmd.remain = int(md.readInt32())\n\n\tcrc := uint32(md.readInt32())\n\tmd.setCRC(crc32.IEEETable)\n\tmagicByte := md.readInt8()\n\tattributes = md.readInt8()\n\ttimestamp = int64(0)\n\n\tif magicByte != 0 {\n\t\ttimestamp = md.readInt64()\n\t}\n\n\tkeyOffset := b.Size()\n\tkeyLength := int(md.readInt32())\n\thasKey := keyLength >= 0\n\tif hasKey {\n\t\tmd.writeTo(b, keyLength)\n\t\tkey = b.ref(keyOffset, b.Size())\n\t}\n\n\tvalueOffset := b.Size()\n\tvalueLength := int(md.readInt32())\n\thasValue := valueLength >= 0\n\tif hasValue {\n\t\tmd.writeTo(b, valueLength)\n\t\tvalue = b.ref(valueOffset, b.Size())\n\t}\n\n\tif md.crc32 != crc {\n\t\terr = Errorf(\"crc32 checksum mismatch (computed=%d found=%d)\", md.crc32, crc)\n\t} else {\n\t\terr = dontExpectEOF(md.err)\n\t}\n\n\treturn\n}\n\nfunc (rs *RecordSet) readFromVersion1(d *decoder) error {\n\tvar records RecordReader\n\n\tb := newPageBuffer()\n\tdefer b.unref()\n\n\tattributes, baseOffset, timestamp, key, value, err := readMessage(b, d)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif compression := Attributes(attributes).Compression(); compression == 0 {\n\t\trecords = &message{\n\t\t\tRecord: Record{\n\t\t\t\tOffset: baseOffset,\n\t\t\t\tTime: makeTime(timestamp),\n\t\t\t\tKey: key,\n\t\t\t\tValue: value,\n\t\t\t},\n\t\t}\n\t} else {\n\t\t// Can we have a non-nil key when reading a compressed message?\n\t\tif key != nil {\n\t\t\tkey.Close()\n\t\t}\n\t\tif value == nil {\n\t\t\trecords = emptyRecordReader{}\n\t\t} else {\n\t\t\tdefer value.Close()\n\n\t\t\tcodec := compression.Codec()\n\t\t\tif codec == nil {\n\t\t\t\treturn Errorf(\"unsupported compression codec: %d\", compression)\n\t\t\t}\n\t\t\tdecompressor := codec.NewReader(value)\n\t\t\tdefer decompressor.Close()\n\n\t\t\tb := newPageBuffer()\n\t\t\tdefer b.unref()\n\n\t\t\td := &decoder{\n\t\t\t\treader: decompressor,\n\t\t\t\tremain: math.MaxInt32,\n\t\t\t}\n\n\t\t\tr := &recordReader{\n\t\t\t\trecords: make([]Record, 0, 32),\n\t\t\t}\n\n\t\t\tfor !d.done() {\n\t\t\t\t_, offset, timestamp, key, value, err := readMessage(b, d)\n\t\t\t\tif err != nil {\n\t\t\t\t\tif errors.Is(err, io.ErrUnexpectedEOF) {\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t\tfor _, rec := range r.records {\n\t\t\t\t\t\tcloseBytes(rec.Key)\n\t\t\t\t\t\tcloseBytes(rec.Value)\n\t\t\t\t\t}\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tr.records = append(r.records, Record{\n\t\t\t\t\tOffset: offset,\n\t\t\t\t\tTime: makeTime(timestamp),\n\t\t\t\t\tKey: key,\n\t\t\t\t\tValue: value,\n\t\t\t\t})\n\t\t\t}\n\n\t\t\tif baseOffset != 0 {\n\t\t\t\t// https://kafka.apache.org/documentation/#messageset\n\t\t\t\t//\n\t\t\t\t// In version 1, to avoid server side re-compression, only the\n\t\t\t\t// wrapper message will be assigned an offset. The inner messages\n\t\t\t\t// will have relative offsets. The absolute offset can be computed\n\t\t\t\t// using the offset from the outer message, which corresponds to the\n\t\t\t\t// offset assigned to the last inner message.\n\t\t\t\tlastRelativeOffset := int64(len(r.records)) - 1\n\n\t\t\t\tfor i := range r.records {\n\t\t\t\t\tr.records[i].Offset = baseOffset - (lastRelativeOffset - r.records[i].Offset)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\trecords = r\n\t\t}\n\t}\n\n\t*rs = RecordSet{\n\t\tVersion: 1,\n\t\tAttributes: Attributes(attributes),\n\t\tRecords: records,\n\t}\n\n\treturn nil\n}\n\nfunc (rs *RecordSet) writeToVersion1(buffer *pageBuffer, bufferOffset int64) error {\n\tattributes := rs.Attributes\n\trecords := rs.Records\n\n\tif compression := attributes.Compression(); compression != 0 {\n\t\tif codec := compression.Codec(); codec != nil {\n\t\t\t// In the message format version 1, compression is achieved by\n\t\t\t// compressing the value of a message which recursively contains\n\t\t\t// the representation of the compressed message set.\n\t\t\tsubset := *rs\n\t\t\tsubset.Attributes &= ^7 // erase compression\n\n\t\t\tif err := subset.writeToVersion1(buffer, bufferOffset); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\tcompressed := newPageBuffer()\n\t\t\tdefer compressed.unref()\n\n\t\t\tcompressor := codec.NewWriter(compressed)\n\t\t\tdefer compressor.Close()\n\n\t\t\tvar err error\n\t\t\tbuffer.pages.scan(bufferOffset, buffer.Size(), func(b []byte) bool {\n\t\t\t\t_, err = compressor.Write(b)\n\t\t\t\treturn err == nil\n\t\t\t})\n\t\t\tif err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tif err := compressor.Close(); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\n\t\t\tbuffer.Truncate(int(bufferOffset))\n\n\t\t\trecords = &message{\n\t\t\t\tRecord: Record{\n\t\t\t\t\tValue: compressed,\n\t\t\t\t},\n\t\t\t}\n\t\t}\n\t}\n\n\te := encoder{writer: buffer}\n\tcurrentTimestamp := timestamp(time.Now())\n\n\treturn forEachRecord(records, func(i int, r *Record) error {\n\t\tt := timestamp(r.Time)\n\t\tif t == 0 {\n\t\t\tt = currentTimestamp\n\t\t}\n\n\t\tmessageOffset := buffer.Size()\n\t\te.writeInt64(int64(i))\n\t\te.writeInt32(0) // message size placeholder\n\t\te.writeInt32(0) // crc32 placeholder\n\t\te.setCRC(crc32.IEEETable)\n\t\te.writeInt8(1) // magic byte: version 1\n\t\te.writeInt8(int8(attributes))\n\t\te.writeInt64(t)\n\n\t\tif err := e.writeNullBytesFrom(r.Key); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tif err := e.writeNullBytesFrom(r.Value); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tb0 := packUint32(uint32(buffer.Size() - (messageOffset + 12)))\n\t\tb1 := packUint32(e.crc32)\n\n\t\tbuffer.WriteAt(b0[:], messageOffset+8)\n\t\tbuffer.WriteAt(b1[:], messageOffset+12)\n\t\te.setCRC(nil)\n\t\treturn nil\n\t})\n}\n\ntype message struct {\n\tRecord Record\n\tread bool\n}\n\nfunc (m *message) ReadRecord() (*Record, error) {\n\tif m.read {\n\t\treturn nil, io.EOF\n\t}\n\tm.read = true\n\treturn &m.Record, nil\n}\n" }, { "path": "protocol/record_v2.go", "content": "package protocol\n\nimport (\n\t\"fmt\"\n\t\"hash/crc32\"\n\t\"io\"\n\t\"time\"\n)\n\nfunc (rs *RecordSet) readFromVersion2(d *decoder) error {\n\tbaseOffset := d.readInt64()\n\tbatchLength := d.readInt32()\n\n\tif int(batchLength) > d.remain || d.err != nil {\n\t\td.discardAll()\n\t\treturn nil\n\t}\n\n\tdec := &decoder{\n\t\treader: d,\n\t\tremain: int(batchLength),\n\t}\n\n\tpartitionLeaderEpoch := dec.readInt32()\n\tmagicByte := dec.readInt8()\n\tcrc := dec.readInt32()\n\n\tdec.setCRC(crc32.MakeTable(crc32.Castagnoli))\n\n\tattributes := dec.readInt16()\n\tlastOffsetDelta := dec.readInt32()\n\tfirstTimestamp := dec.readInt64()\n\tmaxTimestamp := dec.readInt64()\n\tproducerID := dec.readInt64()\n\tproducerEpoch := dec.readInt16()\n\tbaseSequence := dec.readInt32()\n\tnumRecords := dec.readInt32()\n\treader := io.Reader(dec)\n\n\t// unused\n\t_ = lastOffsetDelta\n\t_ = maxTimestamp\n\n\tif compression := Attributes(attributes).Compression(); compression != 0 {\n\t\tcodec := compression.Codec()\n\t\tif codec == nil {\n\t\t\treturn fmt.Errorf(\"unsupported compression codec (%d)\", compression)\n\t\t}\n\t\tdecompressor := codec.NewReader(reader)\n\t\tdefer decompressor.Close()\n\t\treader = decompressor\n\t}\n\n\tbuffer := newPageBuffer()\n\tdefer buffer.unref()\n\n\t_, err := buffer.ReadFrom(reader)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif dec.crc32 != uint32(crc) {\n\t\treturn fmt.Errorf(\"crc32 checksum mismatch (computed=%d found=%d)\", dec.crc32, uint32(crc))\n\t}\n\n\trecordsLength := buffer.Len()\n\tdec.reader = buffer\n\tdec.remain = recordsLength\n\n\trecords := make([]optimizedRecord, numRecords)\n\t// These are two lazy allocators that will be used to optimize allocation of\n\t// page references for keys and values.\n\t//\n\t// By default, no memory is allocated and on first use, numRecords page refs\n\t// are allocated in a contiguous memory space, and the allocators return\n\t// pointers into those arrays for each page ref that get requested.\n\t//\n\t// The reasoning is that kafka partitions typically have records of a single\n\t// form, which either have no keys, no values, or both keys and values.\n\t// Using lazy allocators adapts nicely to these patterns to only allocate\n\t// the memory that is needed by the program, while still reducing the number\n\t// of malloc calls made by the program.\n\t//\n\t// Using a single allocator for both keys and values keeps related values\n\t// close by in memory, making access to the records more friendly to CPU\n\t// caches.\n\talloc := pageRefAllocator{size: int(numRecords)}\n\t// Following the same reasoning that kafka partitions will typically have\n\t// records with repeating formats, we expect to either find records with\n\t// no headers, or records which always contain headers.\n\t//\n\t// To reduce the memory footprint when records have no headers, the Header\n\t// slices are lazily allocated in a separate array.\n\theaders := ([][]Header)(nil)\n\n\tfor i := range records {\n\t\tr := &records[i]\n\t\t_ = dec.readVarInt() // record length (unused)\n\t\t_ = dec.readInt8() // record attributes (unused)\n\t\ttimestampDelta := dec.readVarInt()\n\t\toffsetDelta := dec.readVarInt()\n\n\t\tr.offset = baseOffset + offsetDelta\n\t\tr.timestamp = firstTimestamp + timestampDelta\n\n\t\tkeyLength := dec.readVarInt()\n\t\tkeyOffset := int64(recordsLength - dec.remain)\n\t\tif keyLength > 0 {\n\t\t\tdec.discard(int(keyLength))\n\t\t}\n\n\t\tvalueLength := dec.readVarInt()\n\t\tvalueOffset := int64(recordsLength - dec.remain)\n\t\tif valueLength > 0 {\n\t\t\tdec.discard(int(valueLength))\n\t\t}\n\n\t\tif numHeaders := dec.readVarInt(); numHeaders > 0 {\n\t\t\tif headers == nil {\n\t\t\t\theaders = make([][]Header, numRecords)\n\t\t\t}\n\n\t\t\th := make([]Header, numHeaders)\n\n\t\t\tfor i := range h {\n\t\t\t\th[i] = Header{\n\t\t\t\t\tKey: dec.readVarString(),\n\t\t\t\t\tValue: dec.readVarBytes(),\n\t\t\t\t}\n\t\t\t}\n\n\t\t\theaders[i] = h\n\t\t}\n\n\t\tif dec.err != nil {\n\t\t\trecords = records[:i]\n\t\t\tbreak\n\t\t}\n\n\t\tif keyLength >= 0 {\n\t\t\tr.keyRef = alloc.newPageRef()\n\t\t\tbuffer.refTo(r.keyRef, keyOffset, keyOffset+keyLength)\n\t\t}\n\n\t\tif valueLength >= 0 {\n\t\t\tr.valueRef = alloc.newPageRef()\n\t\t\tbuffer.refTo(r.valueRef, valueOffset, valueOffset+valueLength)\n\t\t}\n\t}\n\n\t// Note: it's unclear whether kafka 0.11+ still truncates the responses,\n\t// all attempts I made at constructing a test to trigger a truncation have\n\t// failed. I kept this code here as a safeguard but it may never execute.\n\tif dec.err != nil && len(records) == 0 {\n\t\treturn dec.err\n\t}\n\n\t*rs = RecordSet{\n\t\tVersion: magicByte,\n\t\tAttributes: Attributes(attributes),\n\t\tRecords: &optimizedRecordReader{\n\t\t\trecords: records,\n\t\t\theaders: headers,\n\t\t},\n\t}\n\n\tif rs.Attributes.Control() {\n\t\trs.Records = &ControlBatch{\n\t\t\tAttributes: rs.Attributes,\n\t\t\tPartitionLeaderEpoch: partitionLeaderEpoch,\n\t\t\tBaseOffset: baseOffset,\n\t\t\tProducerID: producerID,\n\t\t\tProducerEpoch: producerEpoch,\n\t\t\tBaseSequence: baseSequence,\n\t\t\tRecords: rs.Records,\n\t\t}\n\t} else {\n\t\trs.Records = &RecordBatch{\n\t\t\tAttributes: rs.Attributes,\n\t\t\tPartitionLeaderEpoch: partitionLeaderEpoch,\n\t\t\tBaseOffset: baseOffset,\n\t\t\tProducerID: producerID,\n\t\t\tProducerEpoch: producerEpoch,\n\t\t\tBaseSequence: baseSequence,\n\t\t\tRecords: rs.Records,\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (rs *RecordSet) writeToVersion2(buffer *pageBuffer, bufferOffset int64) error {\n\trecords := rs.Records\n\tnumRecords := int32(0)\n\n\te := &encoder{writer: buffer}\n\te.writeInt64(0) // base offset | 0 +8\n\te.writeInt32(0) // placeholder for record batch length | 8 +4\n\te.writeInt32(-1) // partition leader epoch | 12 +3\n\te.writeInt8(2) // magic byte | 16 +1\n\te.writeInt32(0) // placeholder for crc32 checksum | 17 +4\n\te.writeInt16(int16(rs.Attributes)) // attributes | 21 +2\n\te.writeInt32(0) // placeholder for lastOffsetDelta | 23 +4\n\te.writeInt64(0) // placeholder for firstTimestamp | 27 +8\n\te.writeInt64(0) // placeholder for maxTimestamp | 35 +8\n\te.writeInt64(-1) // producer id | 43 +8\n\te.writeInt16(-1) // producer epoch | 51 +2\n\te.writeInt32(-1) // base sequence | 53 +4\n\te.writeInt32(0) // placeholder for numRecords | 57 +4\n\n\tvar compressor io.WriteCloser\n\tif compression := rs.Attributes.Compression(); compression != 0 {\n\t\tif codec := compression.Codec(); codec != nil {\n\t\t\tcompressor = codec.NewWriter(buffer)\n\t\t\te.writer = compressor\n\t\t}\n\t}\n\n\tcurrentTimestamp := timestamp(time.Now())\n\tlastOffsetDelta := int32(0)\n\tfirstTimestamp := int64(0)\n\tmaxTimestamp := int64(0)\n\n\terr := forEachRecord(records, func(i int, r *Record) error {\n\t\tt := timestamp(r.Time)\n\t\tif t == 0 {\n\t\t\tt = currentTimestamp\n\t\t}\n\t\tif i == 0 {\n\t\t\tfirstTimestamp = t\n\t\t}\n\t\tif t > maxTimestamp {\n\t\t\tmaxTimestamp = t\n\t\t}\n\n\t\ttimestampDelta := t - firstTimestamp\n\t\toffsetDelta := int64(i)\n\t\tlastOffsetDelta = int32(offsetDelta)\n\n\t\tlength := 1 + // attributes\n\t\t\tsizeOfVarInt(timestampDelta) +\n\t\t\tsizeOfVarInt(offsetDelta) +\n\t\t\tsizeOfVarNullBytesIface(r.Key) +\n\t\t\tsizeOfVarNullBytesIface(r.Value) +\n\t\t\tsizeOfVarInt(int64(len(r.Headers)))\n\n\t\tfor _, h := range r.Headers {\n\t\t\tlength += sizeOfVarString(h.Key) + sizeOfVarNullBytes(h.Value)\n\t\t}\n\n\t\te.writeVarInt(int64(length))\n\t\te.writeInt8(0) // record attributes (unused)\n\t\te.writeVarInt(timestampDelta)\n\t\te.writeVarInt(offsetDelta)\n\n\t\tif err := e.writeVarNullBytesFrom(r.Key); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tif err := e.writeVarNullBytesFrom(r.Value); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\te.writeVarInt(int64(len(r.Headers)))\n\n\t\tfor _, h := range r.Headers {\n\t\t\te.writeVarString(h.Key)\n\t\t\te.writeVarNullBytes(h.Value)\n\t\t}\n\n\t\tnumRecords++\n\t\treturn nil\n\t})\n\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tif compressor != nil {\n\t\tif err := compressor.Close(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif numRecords == 0 {\n\t\treturn ErrNoRecord\n\t}\n\n\tb2 := packUint32(uint32(lastOffsetDelta))\n\tb3 := packUint64(uint64(firstTimestamp))\n\tb4 := packUint64(uint64(maxTimestamp))\n\tb5 := packUint32(uint32(numRecords))\n\n\tbuffer.WriteAt(b2[:], bufferOffset+23)\n\tbuffer.WriteAt(b3[:], bufferOffset+27)\n\tbuffer.WriteAt(b4[:], bufferOffset+35)\n\tbuffer.WriteAt(b5[:], bufferOffset+57)\n\n\ttotalLength := buffer.Size() - bufferOffset\n\tbatchLength := totalLength - 12\n\n\tchecksum := uint32(0)\n\tcrcTable := crc32.MakeTable(crc32.Castagnoli)\n\n\tbuffer.pages.scan(bufferOffset+21, bufferOffset+totalLength, func(chunk []byte) bool {\n\t\tchecksum = crc32.Update(checksum, crcTable, chunk)\n\t\treturn true\n\t})\n\n\tb0 := packUint32(uint32(batchLength))\n\tb1 := packUint32(checksum)\n\n\tbuffer.WriteAt(b0[:], bufferOffset+8)\n\tbuffer.WriteAt(b1[:], bufferOffset+17)\n\treturn nil\n}\n" }, { "path": "protocol/reflect.go", "content": "//go:build !unsafe\n// +build !unsafe\n\npackage protocol\n\nimport (\n\t\"reflect\"\n)\n\ntype index []int\n\ntype _type struct{ typ reflect.Type }\n\nfunc typeOf(x interface{}) _type {\n\treturn makeType(reflect.TypeOf(x))\n}\n\nfunc elemTypeOf(x interface{}) _type {\n\treturn makeType(reflect.TypeOf(x).Elem())\n}\n\nfunc makeType(t reflect.Type) _type {\n\treturn _type{typ: t}\n}\n\ntype value struct {\n\tval reflect.Value\n}\n\nfunc nonAddressableValueOf(x interface{}) value {\n\treturn value{val: reflect.ValueOf(x)}\n}\n\nfunc valueOf(x interface{}) value {\n\treturn value{val: reflect.ValueOf(x).Elem()}\n}\n\nfunc (v value) bool() bool { return v.val.Bool() }\n\nfunc (v value) int8() int8 { return int8(v.int64()) }\n\nfunc (v value) int16() int16 { return int16(v.int64()) }\n\nfunc (v value) int32() int32 { return int32(v.int64()) }\n\nfunc (v value) int64() int64 { return v.val.Int() }\n\nfunc (v value) float64() float64 { return v.val.Float() }\n\nfunc (v value) string() string { return v.val.String() }\n\nfunc (v value) bytes() []byte { return v.val.Bytes() }\n\nfunc (v value) iface(t reflect.Type) interface{} { return v.val.Addr().Interface() }\n\nfunc (v value) array(t reflect.Type) array { return array(v) }\n\nfunc (v value) setBool(b bool) { v.val.SetBool(b) }\n\nfunc (v value) setInt8(i int8) { v.setInt64(int64(i)) }\n\nfunc (v value) setInt16(i int16) { v.setInt64(int64(i)) }\n\nfunc (v value) setInt32(i int32) { v.setInt64(int64(i)) }\n\nfunc (v value) setInt64(i int64) { v.val.SetInt(i) }\n\nfunc (v value) setFloat64(f float64) { v.val.SetFloat(f) }\n\nfunc (v value) setString(s string) { v.val.SetString(s) }\n\nfunc (v value) setBytes(b []byte) { v.val.SetBytes(b) }\n\nfunc (v value) setArray(a array) {\n\tif a.val.IsValid() {\n\t\tv.val.Set(a.val)\n\t} else {\n\t\tv.val.Set(reflect.Zero(v.val.Type()))\n\t}\n}\n\nfunc (v value) fieldByIndex(i index) value {\n\treturn value{val: v.val.FieldByIndex(i)}\n}\n\ntype array struct {\n\tval reflect.Value\n}\n\nfunc makeArray(t reflect.Type, n int) array {\n\treturn array{val: reflect.MakeSlice(reflect.SliceOf(t), n, n)}\n}\n\nfunc (a array) index(i int) value { return value{val: a.val.Index(i)} }\n\nfunc (a array) length() int { return a.val.Len() }\n\nfunc (a array) isNil() bool { return a.val.IsNil() }\n\nfunc indexOf(s reflect.StructField) index { return index(s.Index) }\n\nfunc bytesToString(b []byte) string { return string(b) }\n" }, { "path": "protocol/reflect_unsafe.go", "content": "//go:build unsafe\n// +build unsafe\n\npackage protocol\n\nimport (\n\t\"reflect\"\n\t\"unsafe\"\n)\n\ntype iface struct {\n\ttyp unsafe.Pointer\n\tptr unsafe.Pointer\n}\n\ntype slice struct {\n\tptr unsafe.Pointer\n\tlen int\n\tcap int\n}\n\ntype index uintptr\n\ntype _type struct {\n\tptr unsafe.Pointer\n}\n\nfunc typeOf(x interface{}) _type {\n\treturn _type{ptr: ((*iface)(unsafe.Pointer(&x))).typ}\n}\n\nfunc elemTypeOf(x interface{}) _type {\n\treturn makeType(reflect.TypeOf(x).Elem())\n}\n\nfunc makeType(t reflect.Type) _type {\n\treturn _type{ptr: ((*iface)(unsafe.Pointer(&t))).ptr}\n}\n\ntype value struct {\n\tptr unsafe.Pointer\n}\n\nfunc nonAddressableValueOf(x interface{}) value {\n\treturn valueOf(x)\n}\n\nfunc valueOf(x interface{}) value {\n\treturn value{ptr: ((*iface)(unsafe.Pointer(&x))).ptr}\n}\n\nfunc makeValue(t reflect.Type) value {\n\treturn value{ptr: unsafe.Pointer(reflect.New(t).Pointer())}\n}\n\nfunc (v value) bool() bool { return *(*bool)(v.ptr) }\n\nfunc (v value) int8() int8 { return *(*int8)(v.ptr) }\n\nfunc (v value) int16() int16 { return *(*int16)(v.ptr) }\n\nfunc (v value) int32() int32 { return *(*int32)(v.ptr) }\n\nfunc (v value) int64() int64 { return *(*int64)(v.ptr) }\n\nfunc (v value) float64() float64 { return *(*float64)(v.ptr) }\n\nfunc (v value) string() string { return *(*string)(v.ptr) }\n\nfunc (v value) bytes() []byte { return *(*[]byte)(v.ptr) }\n\nfunc (v value) iface(t reflect.Type) interface{} {\n\treturn *(*interface{})(unsafe.Pointer(&iface{\n\t\ttyp: ((*iface)(unsafe.Pointer(&t))).ptr,\n\t\tptr: v.ptr,\n\t}))\n}\n\nfunc (v value) array(t reflect.Type) array {\n\treturn array{\n\t\tsize: uintptr(t.Size()),\n\t\telem: ((*slice)(v.ptr)).ptr,\n\t\tlen: ((*slice)(v.ptr)).len,\n\t}\n}\n\nfunc (v value) setBool(b bool) { *(*bool)(v.ptr) = b }\n\nfunc (v value) setInt8(i int8) { *(*int8)(v.ptr) = i }\n\nfunc (v value) setInt16(i int16) { *(*int16)(v.ptr) = i }\n\nfunc (v value) setInt32(i int32) { *(*int32)(v.ptr) = i }\n\nfunc (v value) setInt64(i int64) { *(*int64)(v.ptr) = i }\n\nfunc (v value) setFloat64(f float64) { *(*float64)(v.ptr) = f }\n\nfunc (v value) setString(s string) { *(*string)(v.ptr) = s }\n\nfunc (v value) setBytes(b []byte) { *(*[]byte)(v.ptr) = b }\n\nfunc (v value) setArray(a array) { *(*slice)(v.ptr) = slice{ptr: a.elem, len: a.len, cap: a.len} }\n\nfunc (v value) fieldByIndex(i index) value {\n\treturn value{ptr: unsafe.Pointer(uintptr(v.ptr) + uintptr(i))}\n}\n\ntype array struct {\n\telem unsafe.Pointer\n\tsize uintptr\n\tlen int\n}\n\nvar (\n\temptyArray struct{}\n)\n\nfunc makeArray(t reflect.Type, n int) array {\n\tvar elem unsafe.Pointer\n\tvar size = uintptr(t.Size())\n\tif n == 0 {\n\t\telem = unsafe.Pointer(&emptyArray)\n\t} else {\n\t\telem = unsafe_NewArray(((*iface)(unsafe.Pointer(&t))).ptr, n)\n\t}\n\treturn array{elem: elem, size: size, len: n}\n}\n\nfunc (a array) index(i int) value {\n\treturn value{ptr: unsafe.Pointer(uintptr(a.elem) + (uintptr(i) * a.size))}\n}\n\nfunc (a array) length() int { return a.len }\n\nfunc (a array) isNil() bool { return a.elem == nil }\n\nfunc indexOf(s reflect.StructField) index { return index(s.Offset) }\n\nfunc bytesToString(b []byte) string { return *(*string)(unsafe.Pointer(&b)) }\n\n//go:linkname unsafe_NewArray reflect.unsafe_NewArray\nfunc unsafe_NewArray(rtype unsafe.Pointer, length int) unsafe.Pointer\n" }, { "path": "protocol/request.go", "content": "package protocol\n\nimport (\n\t\"fmt\"\n\t\"io\"\n)\n\nfunc ReadRequest(r io.Reader) (apiVersion int16, correlationID int32, clientID string, msg Message, err error) {\n\td := &decoder{reader: r, remain: 4}\n\tsize := d.readInt32()\n\n\tif err = d.err; err != nil {\n\t\terr = dontExpectEOF(err)\n\t\treturn\n\t}\n\n\td.remain = int(size)\n\tapiKey := ApiKey(d.readInt16())\n\tapiVersion = d.readInt16()\n\tcorrelationID = d.readInt32()\n\tclientID = d.readString()\n\n\tif i := int(apiKey); i < 0 || i >= len(apiTypes) {\n\t\terr = fmt.Errorf(\"unsupported api key: %d\", i)\n\t\treturn\n\t}\n\n\tif err = d.err; err != nil {\n\t\terr = dontExpectEOF(err)\n\t\treturn\n\t}\n\n\tt := &apiTypes[apiKey]\n\tif t == nil {\n\t\terr = fmt.Errorf(\"unsupported api: %s\", apiNames[apiKey])\n\t\treturn\n\t}\n\n\tminVersion := t.minVersion()\n\tmaxVersion := t.maxVersion()\n\n\tif apiVersion < minVersion || apiVersion > maxVersion {\n\t\terr = fmt.Errorf(\"unsupported %s version: v%d not in range v%d-v%d\", apiKey, apiVersion, minVersion, maxVersion)\n\t\treturn\n\t}\n\n\treq := &t.requests[apiVersion-minVersion]\n\n\tif req.flexible {\n\t\t// In the flexible case, there's a tag buffer at the end of the request header\n\t\ttaggedCount := int(d.readUnsignedVarInt())\n\t\tfor i := 0; i < taggedCount; i++ {\n\t\t\td.readUnsignedVarInt() // tagID\n\t\t\tsize := d.readUnsignedVarInt()\n\n\t\t\t// Just throw away the values for now\n\t\t\td.read(int(size))\n\t\t}\n\t}\n\n\tmsg = req.new()\n\treq.decode(d, valueOf(msg))\n\td.discardAll()\n\n\tif err = d.err; err != nil {\n\t\terr = dontExpectEOF(err)\n\t}\n\n\treturn\n}\n\nfunc WriteRequest(w io.Writer, apiVersion int16, correlationID int32, clientID string, msg Message) error {\n\tapiKey := msg.ApiKey()\n\n\tif i := int(apiKey); i < 0 || i >= len(apiTypes) {\n\t\treturn fmt.Errorf(\"unsupported api key: %d\", i)\n\t}\n\n\tt := &apiTypes[apiKey]\n\tif t == nil {\n\t\treturn fmt.Errorf(\"unsupported api: %s\", apiNames[apiKey])\n\t}\n\n\tif typedMessage, ok := msg.(OverrideTypeMessage); ok {\n\t\ttypeKey := typedMessage.TypeKey()\n\t\toverrideType := overrideApiTypes[apiKey][typeKey]\n\t\tt = &overrideType\n\t}\n\n\tminVersion := t.minVersion()\n\tmaxVersion := t.maxVersion()\n\n\tif apiVersion < minVersion || apiVersion > maxVersion {\n\t\treturn fmt.Errorf(\"unsupported %s version: v%d not in range v%d-v%d\", apiKey, apiVersion, minVersion, maxVersion)\n\t}\n\n\tr := &t.requests[apiVersion-minVersion]\n\tv := valueOf(msg)\n\tb := newPageBuffer()\n\tdefer b.unref()\n\n\te := &encoder{writer: b}\n\te.writeInt32(0) // placeholder for the request size\n\te.writeInt16(int16(apiKey))\n\te.writeInt16(apiVersion)\n\te.writeInt32(correlationID)\n\n\tif r.flexible {\n\t\t// Flexible messages use a nullable string for the client ID, then extra space for a\n\t\t// tag buffer, which begins with a size value. Since we're not writing any fields into the\n\t\t// latter, we can just write zero for now.\n\t\t//\n\t\t// See\n\t\t// https://cwiki.apache.org/confluence/display/KAFKA/KIP-482%3A+The+Kafka+Protocol+should+Support+Optional+Tagged+Fields\n\t\t// for details.\n\t\te.writeNullString(clientID)\n\t\te.writeUnsignedVarInt(0)\n\t} else {\n\t\t// Technically, recent versions of kafka interpret this field as a nullable\n\t\t// string, however kafka 0.10 expected a non-nullable string and fails with\n\t\t// a NullPointerException when it receives a null client id.\n\t\te.writeString(clientID)\n\t}\n\tr.encode(e, v)\n\terr := e.err\n\n\tif err == nil {\n\t\tsize := packUint32(uint32(b.Size()) - 4)\n\t\tb.WriteAt(size[:], 0)\n\t\t_, err = b.WriteTo(w)\n\t}\n\n\treturn err\n}\n" }, { "path": "protocol/response.go", "content": "package protocol\n\nimport (\n\t\"crypto/tls\"\n\t\"encoding/binary\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n)\n\nfunc ReadResponse(r io.Reader, apiKey ApiKey, apiVersion int16) (correlationID int32, msg Message, err error) {\n\tif i := int(apiKey); i < 0 || i >= len(apiTypes) {\n\t\terr = fmt.Errorf(\"unsupported api key: %d\", i)\n\t\treturn\n\t}\n\n\tt := &apiTypes[apiKey]\n\tif t == nil {\n\t\terr = fmt.Errorf(\"unsupported api: %s\", apiNames[apiKey])\n\t\treturn\n\t}\n\n\tminVersion := t.minVersion()\n\tmaxVersion := t.maxVersion()\n\n\tif apiVersion < minVersion || apiVersion > maxVersion {\n\t\terr = fmt.Errorf(\"unsupported %s version: v%d not in range v%d-v%d\", apiKey, apiVersion, minVersion, maxVersion)\n\t\treturn\n\t}\n\n\td := &decoder{reader: r, remain: 4}\n\tsize := d.readInt32()\n\n\tif err = d.err; err != nil {\n\t\terr = dontExpectEOF(err)\n\t\treturn\n\t}\n\n\td.remain = int(size)\n\tcorrelationID = d.readInt32()\n\tif err = d.err; err != nil {\n\t\tif errors.Is(err, io.ErrUnexpectedEOF) {\n\t\t\t// If a Writer/Reader is configured without TLS and connects\n\t\t\t// to a broker expecting TLS the only message we return to the\n\t\t\t// caller is io.ErrUnexpetedEOF which is opaque. This section\n\t\t\t// tries to determine if that's what has happened.\n\t\t\t// We first deconstruct the initial 4 bytes of the message\n\t\t\t// from the size which was read earlier.\n\t\t\t// Next, we examine those bytes to see if they looks like a TLS\n\t\t\t// error message. If they do we wrap the io.ErrUnexpectedEOF\n\t\t\t// with some context.\n\t\t\tif looksLikeUnexpectedTLS(size) {\n\t\t\t\terr = fmt.Errorf(\"%w: broker appears to be expecting TLS\", io.ErrUnexpectedEOF)\n\t\t\t}\n\t\t\treturn\n\t\t}\n\t\terr = dontExpectEOF(err)\n\t\treturn\n\t}\n\n\tres := &t.responses[apiVersion-minVersion]\n\n\tif res.flexible {\n\t\t// In the flexible case, there's a tag buffer at the end of the response header\n\t\ttaggedCount := int(d.readUnsignedVarInt())\n\t\tfor i := 0; i < taggedCount; i++ {\n\t\t\td.readUnsignedVarInt() // tagID\n\t\t\tsize := d.readUnsignedVarInt()\n\n\t\t\t// Just throw away the values for now\n\t\t\td.read(int(size))\n\t\t}\n\t}\n\n\tmsg = res.new()\n\tres.decode(d, valueOf(msg))\n\td.discardAll()\n\n\tif err = d.err; err != nil {\n\t\terr = dontExpectEOF(err)\n\t}\n\n\treturn\n}\n\nfunc WriteResponse(w io.Writer, apiVersion int16, correlationID int32, msg Message) error {\n\tapiKey := msg.ApiKey()\n\n\tif i := int(apiKey); i < 0 || i >= len(apiTypes) {\n\t\treturn fmt.Errorf(\"unsupported api key: %d\", i)\n\t}\n\n\tt := &apiTypes[apiKey]\n\tif t == nil {\n\t\treturn fmt.Errorf(\"unsupported api: %s\", apiNames[apiKey])\n\t}\n\n\tif typedMessage, ok := msg.(OverrideTypeMessage); ok {\n\t\ttypeKey := typedMessage.TypeKey()\n\t\toverrideType := overrideApiTypes[apiKey][typeKey]\n\t\tt = &overrideType\n\t}\n\n\tminVersion := t.minVersion()\n\tmaxVersion := t.maxVersion()\n\n\tif apiVersion < minVersion || apiVersion > maxVersion {\n\t\treturn fmt.Errorf(\"unsupported %s version: v%d not in range v%d-v%d\", apiKey, apiVersion, minVersion, maxVersion)\n\t}\n\n\tr := &t.responses[apiVersion-minVersion]\n\tv := valueOf(msg)\n\tb := newPageBuffer()\n\tdefer b.unref()\n\n\te := &encoder{writer: b}\n\te.writeInt32(0) // placeholder for the response size\n\te.writeInt32(correlationID)\n\tif r.flexible {\n\t\t// Flexible messages use extra space for a tag buffer,\n\t\t// which begins with a size value. Since we're not writing any fields into the\n\t\t// latter, we can just write zero for now.\n\t\t//\n\t\t// See\n\t\t// https://cwiki.apache.org/confluence/display/KAFKA/KIP-482%3A+The+Kafka+Protocol+should+Support+Optional+Tagged+Fields\n\t\t// for details.\n\t\te.writeUnsignedVarInt(0)\n\t}\n\tr.encode(e, v)\n\terr := e.err\n\n\tif err == nil {\n\t\tsize := packUint32(uint32(b.Size()) - 4)\n\t\tb.WriteAt(size[:], 0)\n\t\t_, err = b.WriteTo(w)\n\t}\n\n\treturn err\n}\n\nconst (\n\ttlsAlertByte byte = 0x15\n)\n\n// looksLikeUnexpectedTLS returns true if the size passed in resemble\n// the TLS alert message that is returned to a client which sends\n// an invalid ClientHello message.\nfunc looksLikeUnexpectedTLS(size int32) bool {\n\tvar sizeBytes [4]byte\n\tbinary.BigEndian.PutUint32(sizeBytes[:], uint32(size))\n\n\tif sizeBytes[0] != tlsAlertByte {\n\t\treturn false\n\t}\n\tversion := int(sizeBytes[1])<<8 | int(sizeBytes[2])\n\treturn version <= tls.VersionTLS13 && version >= tls.VersionTLS10\n}\n" }, { "path": "protocol/response_test.go", "content": "package protocol\n\nimport (\n\t\"bytes\"\n\t\"errors\"\n\t\"io\"\n\t\"strings\"\n\t\"testing\"\n)\n\nfunc TestReadResponseUnexpectedTLSDetection(t *testing.T) {\n\tvar buf bytes.Buffer\n\n\tbuf.Write([]byte{tlsAlertByte, 0x03, 0x03, 10, 0, 0, 0})\n\n\tcorrelationID, _, err := ReadResponse(&buf, ApiVersions, 0)\n\tif !errors.Is(err, io.ErrUnexpectedEOF) {\n\t\tt.Fatalf(\"expected an io.ErrUnexpectedEOF from ReadResponse got %v\", err)\n\t}\n\n\tif !strings.Contains(err.Error(), \"broker appears to be expecting TLS\") {\n\t\tt.Fatalf(\"expected error messae to contain %s got %s\", \"broker appears to be expecting TLS\", err.Error())\n\t}\n\n\tif correlationID != 0 {\n\t\tt.Fatalf(\"expected correlationID of 0 got %d\", correlationID)\n\t}\n}\n" }, { "path": "protocol/roundtrip.go", "content": "package protocol\n\nimport (\n\t\"io\"\n)\n\n// RoundTrip sends a request to a kafka broker and returns the response.\nfunc RoundTrip(rw io.ReadWriter, apiVersion int16, correlationID int32, clientID string, req Message) (Message, error) {\n\tif err := WriteRequest(rw, apiVersion, correlationID, clientID, req); err != nil {\n\t\treturn nil, err\n\t}\n\tif !hasResponse(req) {\n\t\treturn nil, nil\n\t}\n\tid, res, err := ReadResponse(rw, req.ApiKey(), apiVersion)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif id != correlationID {\n\t\treturn nil, Errorf(\"correlation id mismatch (expected=%d, found=%d)\", correlationID, id)\n\t}\n\treturn res, nil\n}\n\nfunc hasResponse(msg Message) bool {\n\tx, _ := msg.(interface{ HasResponse() bool })\n\treturn x == nil || x.HasResponse()\n}\n" }, { "path": "protocol/saslauthenticate/saslauthenticate.go", "content": "package saslauthenticate\n\nimport (\n\t\"encoding/binary\"\n\t\"io\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n)\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\tAuthBytes []byte `kafka:\"min=v0,max=v1\"`\n}\n\nfunc (r *Request) RawExchange(rw io.ReadWriter) (protocol.Message, error) {\n\tif err := r.writeTo(rw); err != nil {\n\t\treturn nil, err\n\t}\n\treturn r.readResp(rw)\n}\n\nfunc (*Request) Required(versions map[protocol.ApiKey]int16) bool {\n\tconst v0 = 0\n\treturn versions[protocol.SaslHandshake] == v0\n}\n\nfunc (r *Request) writeTo(w io.Writer) error {\n\tsize := len(r.AuthBytes) + 4\n\tbuf := make([]byte, size)\n\tbinary.BigEndian.PutUint32(buf[:4], uint32(len(r.AuthBytes)))\n\tcopy(buf[4:], r.AuthBytes)\n\t_, err := w.Write(buf)\n\treturn err\n}\n\nfunc (r *Request) readResp(read io.Reader) (protocol.Message, error) {\n\tvar lenBuf [4]byte\n\tif _, err := io.ReadFull(read, lenBuf[:]); err != nil {\n\t\treturn nil, err\n\t}\n\trespLen := int32(binary.BigEndian.Uint32(lenBuf[:]))\n\tdata := make([]byte, respLen)\n\n\tif _, err := io.ReadFull(read, data[:]); err != nil {\n\t\treturn nil, err\n\t}\n\treturn &Response{\n\t\tAuthBytes: data,\n\t}, nil\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.SaslAuthenticate }\n\ntype Response struct {\n\tErrorCode int16 `kafka:\"min=v0,max=v1\"`\n\tErrorMessage string `kafka:\"min=v0,max=v1,nullable\"`\n\tAuthBytes []byte `kafka:\"min=v0,max=v1\"`\n\tSessionLifetimeMs int64 `kafka:\"min=v1,max=v1\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.SaslAuthenticate }\n\nvar _ protocol.RawExchanger = (*Request)(nil)\n" }, { "path": "protocol/saslhandshake/saslhandshake.go", "content": "package saslhandshake\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\tMechanism string `kafka:\"min=v0,max=v1\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.SaslHandshake }\n\ntype Response struct {\n\tErrorCode int16 `kafka:\"min=v0,max=v1\"`\n\tMechanisms []string `kafka:\"min=v0,max=v1\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.SaslHandshake }\n" }, { "path": "protocol/size.go", "content": "package protocol\n\nimport (\n\t\"math/bits\"\n)\n\nfunc sizeOfVarString(s string) int {\n\treturn sizeOfVarInt(int64(len(s))) + len(s)\n}\n\nfunc sizeOfVarNullBytes(b []byte) int {\n\tif b == nil {\n\t\treturn sizeOfVarInt(-1)\n\t}\n\tn := len(b)\n\treturn sizeOfVarInt(int64(n)) + n\n}\n\nfunc sizeOfVarNullBytesIface(b Bytes) int {\n\tif b == nil {\n\t\treturn sizeOfVarInt(-1)\n\t}\n\tn := b.Len()\n\treturn sizeOfVarInt(int64(n)) + n\n}\n\nfunc sizeOfVarInt(i int64) int {\n\treturn sizeOfUnsignedVarInt(uint64((i << 1) ^ (i >> 63))) // zig-zag encoding\n}\n\nfunc sizeOfUnsignedVarInt(i uint64) int {\n\treturn (bits.Len64(i|1) + 6) / 7\n}\n" }, { "path": "protocol/syncgroup/syncgroup.go", "content": "package syncgroup\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v4,max=v5,tag\"`\n\n\tGroupID string `kafka:\"min=v0,max=v3|min=v4,max=v5,compact\"`\n\tGenerationID int32 `kafka:\"min=v0,max=v5|min=v4,max=v5,compact\"`\n\tMemberID string `kafka:\"min=v0,max=v3|min=v4,max=v5,compact\"`\n\tGroupInstanceID string `kafka:\"min=v3,max=v3,nullable|min=v4,max=v5,nullable,compact\"`\n\tProtocolType string `kafka:\"min=v5,max=v5\"`\n\tProtocolName string `kafka:\"min=v5,max=v5\"`\n\tAssignments []RequestAssignment `kafka:\"min=v0,max=v5\"`\n}\n\ntype RequestAssignment struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v4,max=v5,tag\"`\n\n\tMemberID string `kafka:\"min=v0,max=v3|min=v4,max=v5,compact\"`\n\tAssignment []byte `kafka:\"min=v0,max=v3|min=v4,max=v5,compact\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.SyncGroup }\n\nfunc (r *Request) Group() string { return r.GroupID }\n\nvar _ protocol.GroupMessage = (*Request)(nil)\n\ntype Response struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v4,max=v5,tag\"`\n\n\tThrottleTimeMS int32 `kafka:\"min=v1,max=v5\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v5\"`\n\tProtocolType string `kafka:\"min=v5,max=v5\"`\n\tProtocolName string `kafka:\"min=v5,max=v5\"`\n\tAssignments []byte `kafka:\"min=v0,max=v3|min=v4,max=v5,compact\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.SyncGroup }\n" }, { "path": "protocol/syncgroup/syncgroup_test.go", "content": "package syncgroup_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n\t\"github.com/segmentio/kafka-go/protocol/syncgroup\"\n)\n\nfunc TestSyncGroupReq(t *testing.T) {\n\tfor _, version := range []int16{0, 1, 2} {\n\t\tprototest.TestRequest(t, version, &syncgroup.Request{\n\t\t\tGroupID: \"group-id-1\",\n\t\t\tGenerationID: 10,\n\t\t\tMemberID: \"member-id-1\",\n\t\t\tAssignments: []syncgroup.RequestAssignment{\n\t\t\t\t{\n\t\t\t\t\tMemberID: \"member-id-2\",\n\t\t\t\t\tAssignment: []byte{0, 1, 2, 3, 4},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n\n\t// Version 3 added:\n\t// GroupInstanceID\n\tfor _, version := range []int16{3, 4} {\n\t\tprototest.TestRequest(t, version, &syncgroup.Request{\n\t\t\tGroupID: \"group-id-1\",\n\t\t\tGenerationID: 10,\n\t\t\tMemberID: \"member-id-1\",\n\t\t\tGroupInstanceID: \"group-instance-id\",\n\t\t\tAssignments: []syncgroup.RequestAssignment{\n\t\t\t\t{\n\t\t\t\t\tMemberID: \"member-id-2\",\n\t\t\t\t\tAssignment: []byte{0, 1, 2, 3, 4},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n\n\t// Version 5 added\n\t// ProtocolType\n\t// ProtocolName\n\tfor _, version := range []int16{5} {\n\t\tprototest.TestRequest(t, version, &syncgroup.Request{\n\t\t\tGroupID: \"group-id-1\",\n\t\t\tGenerationID: 10,\n\t\t\tMemberID: \"member-id-1\",\n\t\t\tGroupInstanceID: \"group-instance-id\",\n\t\t\tProtocolType: \"protocol-type\",\n\t\t\tProtocolName: \"protocol-name\",\n\t\t\tAssignments: []syncgroup.RequestAssignment{\n\t\t\t\t{\n\t\t\t\t\tMemberID: \"member-id-2\",\n\t\t\t\t\tAssignment: []byte{0, 1, 2, 3, 4},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n}\n\nfunc TestSyncGroupResp(t *testing.T) {\n\tfor _, version := range []int16{0} {\n\t\tprototest.TestResponse(t, version, &syncgroup.Response{\n\t\t\tErrorCode: 10,\n\t\t\tAssignments: []byte{0, 1, 2, 3, 4},\n\t\t})\n\t}\n\n\t// Version 1 added\n\t// ThrottleTimeMS\n\tfor _, version := range []int16{1, 2, 3, 4} {\n\t\tprototest.TestResponse(t, version, &syncgroup.Response{\n\t\t\tErrorCode: 10,\n\t\t\tThrottleTimeMS: 1,\n\t\t\tAssignments: []byte{0, 1, 2, 3, 4},\n\t\t})\n\t}\n\n\t// Version 5 added\n\t// ProtocolType\n\t// ProtocolName\n\tfor _, version := range []int16{5} {\n\t\tprototest.TestResponse(t, version, &syncgroup.Response{\n\t\t\tErrorCode: 10,\n\t\t\tThrottleTimeMS: 1,\n\t\t\tProtocolType: \"protocol-type\",\n\t\t\tProtocolName: \"protocol-name\",\n\t\t\tAssignments: []byte{0, 1, 2, 3, 4},\n\t\t})\n\t}\n}\n" }, { "path": "protocol/txnoffsetcommit/txnoffsetcommit.go", "content": "package txnoffsetcommit\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Response{})\n}\n\ntype Request struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v3,max=v3,tag\"`\n\n\tTransactionalID string `kafka:\"min=v0,max=v2|min=v3,max=v3,compact\"`\n\tGroupID string `kafka:\"min=v0,max=v2|min=v3,max=v3,compact\"`\n\tProducerID int64 `kafka:\"min=v0,max=v3\"`\n\tProducerEpoch int16 `kafka:\"min=v0,max=v3\"`\n\tGenerationID int32 `kafka:\"min=v3,max=v3\"`\n\tMemberID string `kafka:\"min=v3,max=v3,compact\"`\n\tGroupInstanceID string `kafka:\"min=v3,max=v3,compact,nullable\"`\n\tTopics []RequestTopic `kafka:\"min=v0,max=v3\"`\n}\n\ntype RequestTopic struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v3,max=v3,tag\"`\n\n\tName string `kafka:\"min=v0,max=v2|min=v3,max=v3,compact\"`\n\tPartitions []RequestPartition `kafka:\"min=v0,max=v3\"`\n}\n\ntype RequestPartition struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v3,max=v3,tag\"`\n\n\tPartition int32 `kafka:\"min=v0,max=v3\"`\n\tCommittedOffset int64 `kafka:\"min=v0,max=v3\"`\n\tCommittedLeaderEpoch int32 `kafka:\"min=v2,max=v3\"`\n\tCommittedMetadata string `kafka:\"min=v0,max=v2|min=v3,max=v3,nullable,compact\"`\n}\n\nfunc (r *Request) ApiKey() protocol.ApiKey { return protocol.TxnOffsetCommit }\n\nfunc (r *Request) Group() string { return r.GroupID }\n\nvar _ protocol.GroupMessage = (*Request)(nil)\n\ntype Response struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v3,max=v3,tag\"`\n\n\tThrottleTimeMs int32 `kafka:\"min=v0,max=v3\"`\n\tTopics []ResponseTopic `kafka:\"min=v0,max=v3\"`\n}\n\ntype ResponseTopic struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v3,max=v3,tag\"`\n\n\tName string `kafka:\"min=v0,max=v2|min=v3,max=v3,compact\"`\n\tPartitions []ResponsePartition `kafka:\"min=v0,max=v3\"`\n}\n\ntype ResponsePartition struct {\n\t// We need at least one tagged field to indicate that this is a \"flexible\" message\n\t// type.\n\t_ struct{} `kafka:\"min=v3,max=v3,tag\"`\n\n\tPartition int32 `kafka:\"min=v0,max=v3\"`\n\tErrorCode int16 `kafka:\"min=v0,max=v3\"`\n}\n\nfunc (r *Response) ApiKey() protocol.ApiKey { return protocol.TxnOffsetCommit }\n" }, { "path": "protocol/txnoffsetcommit/txnoffsetcommit_test.go", "content": "package txnoffsetcommit_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/prototest\"\n\t\"github.com/segmentio/kafka-go/protocol/txnoffsetcommit\"\n)\n\nfunc TestTxnOffsetCommitRequest(t *testing.T) {\n\tfor _, version := range []int16{0, 1} {\n\t\tprototest.TestRequest(t, version, &txnoffsetcommit.Request{\n\t\t\tTransactionalID: \"transactional-id-0\",\n\t\t\tGroupID: \"group-0\",\n\t\t\tProducerID: 10,\n\t\t\tProducerEpoch: 100,\n\t\t\tTopics: []txnoffsetcommit.RequestTopic{\n\t\t\t\t{\n\t\t\t\t\tName: \"topic-0\",\n\t\t\t\t\tPartitions: []txnoffsetcommit.RequestPartition{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\t\tCommittedOffset: 10,\n\t\t\t\t\t\t\tCommittedMetadata: \"meta-0-0\",\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\t\tCommittedOffset: 10,\n\t\t\t\t\t\t\tCommittedMetadata: \"meta-0-1\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tName: \"topic-1\",\n\t\t\t\t\tPartitions: []txnoffsetcommit.RequestPartition{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\t\tCommittedOffset: 10,\n\t\t\t\t\t\t\tCommittedMetadata: \"meta-1-0\",\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\t\tCommittedOffset: 10,\n\t\t\t\t\t\t\tCommittedMetadata: \"meta-1-1\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n\n\t// Version 2 added:\n\t// Topics.RequestTopic.Partitions.CommittedLeaderEpoch\n\tfor _, version := range []int16{2} {\n\t\tprototest.TestRequest(t, version, &txnoffsetcommit.Request{\n\t\t\tTransactionalID: \"transactional-id-0\",\n\t\t\tGroupID: \"group-0\",\n\t\t\tProducerID: 10,\n\t\t\tProducerEpoch: 100,\n\t\t\tTopics: []txnoffsetcommit.RequestTopic{\n\t\t\t\t{\n\t\t\t\t\tName: \"topic-0\",\n\t\t\t\t\tPartitions: []txnoffsetcommit.RequestPartition{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\t\tCommittedOffset: 10,\n\t\t\t\t\t\t\tCommittedLeaderEpoch: 100,\n\t\t\t\t\t\t\tCommittedMetadata: \"meta-0-0\",\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\t\tCommittedOffset: 10,\n\t\t\t\t\t\t\tCommittedLeaderEpoch: 100,\n\t\t\t\t\t\t\tCommittedMetadata: \"meta-0-1\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tName: \"topic-1\",\n\t\t\t\t\tPartitions: []txnoffsetcommit.RequestPartition{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\t\tCommittedOffset: 10,\n\t\t\t\t\t\t\tCommittedLeaderEpoch: 100,\n\t\t\t\t\t\t\tCommittedMetadata: \"meta-1-0\",\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\t\tCommittedOffset: 10,\n\t\t\t\t\t\t\tCommittedLeaderEpoch: 100,\n\t\t\t\t\t\t\tCommittedMetadata: \"meta-1-1\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n\n\t// Version 3 added:\n\t// GenerationID\n\t// MemberID\n\t// GroupInstanceID\n\tfor _, version := range []int16{3} {\n\t\tprototest.TestRequest(t, version, &txnoffsetcommit.Request{\n\t\t\tTransactionalID: \"transactional-id-0\",\n\t\t\tGroupID: \"group-0\",\n\t\t\tProducerID: 10,\n\t\t\tProducerEpoch: 100,\n\t\t\tGenerationID: 2,\n\t\t\tMemberID: \"member-0\",\n\t\t\tGroupInstanceID: \"group-instance-id-0\",\n\t\t\tTopics: []txnoffsetcommit.RequestTopic{\n\t\t\t\t{\n\t\t\t\t\tName: \"topic-0\",\n\t\t\t\t\tPartitions: []txnoffsetcommit.RequestPartition{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\t\tCommittedOffset: 10,\n\t\t\t\t\t\t\tCommittedLeaderEpoch: 100,\n\t\t\t\t\t\t\tCommittedMetadata: \"meta-0-0\",\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\t\tCommittedOffset: 10,\n\t\t\t\t\t\t\tCommittedLeaderEpoch: 100,\n\t\t\t\t\t\t\tCommittedMetadata: \"meta-0-1\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tName: \"topic-1\",\n\t\t\t\t\tPartitions: []txnoffsetcommit.RequestPartition{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\t\tCommittedOffset: 10,\n\t\t\t\t\t\t\tCommittedLeaderEpoch: 100,\n\t\t\t\t\t\t\tCommittedMetadata: \"meta-1-0\",\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\t\tCommittedOffset: 10,\n\t\t\t\t\t\t\tCommittedLeaderEpoch: 100,\n\t\t\t\t\t\t\tCommittedMetadata: \"meta-1-1\",\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n}\n\nfunc TestTxnOffsetCommitResponse(t *testing.T) {\n\tfor _, version := range []int16{0, 1, 2, 3} {\n\t\tprototest.TestResponse(t, version, &txnoffsetcommit.Response{\n\t\t\tThrottleTimeMs: 10,\n\t\t\tTopics: []txnoffsetcommit.ResponseTopic{\n\t\t\t\t{\n\t\t\t\t\tName: \"topic-0\",\n\t\t\t\t\tPartitions: []txnoffsetcommit.ResponsePartition{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\t\tErrorCode: 0,\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\t\tErrorCode: 10,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tName: \"topic-1\",\n\t\t\t\t\tPartitions: []txnoffsetcommit.ResponsePartition{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\t\tErrorCode: 0,\n\t\t\t\t\t\t},\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartition: 1,\n\t\t\t\t\t\t\tErrorCode: 10,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}\n}\n" }, { "path": "protocol.go", "content": "package kafka\n\nimport (\n\t\"encoding/binary\"\n\t\"fmt\"\n\t\"strconv\"\n)\n\ntype ApiVersion struct {\n\tApiKey int16\n\tMinVersion int16\n\tMaxVersion int16\n}\n\nfunc (v ApiVersion) Format(w fmt.State, r rune) {\n\tswitch r {\n\tcase 's':\n\t\tfmt.Fprint(w, apiKey(v.ApiKey))\n\tcase 'd':\n\t\tswitch {\n\t\tcase w.Flag('-'):\n\t\t\tfmt.Fprint(w, v.MinVersion)\n\t\tcase w.Flag('+'):\n\t\t\tfmt.Fprint(w, v.MaxVersion)\n\t\tdefault:\n\t\t\tfmt.Fprint(w, v.ApiKey)\n\t\t}\n\tcase 'v':\n\t\tswitch {\n\t\tcase w.Flag('-'):\n\t\t\tfmt.Fprintf(w, \"v%d\", v.MinVersion)\n\t\tcase w.Flag('+'):\n\t\t\tfmt.Fprintf(w, \"v%d\", v.MaxVersion)\n\t\tcase w.Flag('#'):\n\t\t\tfmt.Fprintf(w, \"kafka.ApiVersion{ApiKey:%d MinVersion:%d MaxVersion:%d}\", v.ApiKey, v.MinVersion, v.MaxVersion)\n\t\tdefault:\n\t\t\tfmt.Fprintf(w, \"%s[v%d:v%d]\", apiKey(v.ApiKey), v.MinVersion, v.MaxVersion)\n\t\t}\n\t}\n}\n\ntype apiKey int16\n\nconst (\n\tproduce apiKey = 0\n\tfetch apiKey = 1\n\tlistOffsets apiKey = 2\n\tmetadata apiKey = 3\n\tleaderAndIsr apiKey = 4\n\tstopReplica apiKey = 5\n\tupdateMetadata apiKey = 6\n\tcontrolledShutdown apiKey = 7\n\toffsetCommit apiKey = 8\n\toffsetFetch apiKey = 9\n\tfindCoordinator apiKey = 10\n\tjoinGroup apiKey = 11\n\theartbeat apiKey = 12\n\tleaveGroup apiKey = 13\n\tsyncGroup apiKey = 14\n\tdescribeGroups apiKey = 15\n\tlistGroups apiKey = 16\n\tsaslHandshake apiKey = 17\n\tapiVersions apiKey = 18\n\tcreateTopics apiKey = 19\n\tdeleteTopics apiKey = 20\n\tdeleteRecords apiKey = 21\n\tinitProducerId apiKey = 22\n\toffsetForLeaderEpoch apiKey = 23\n\taddPartitionsToTxn apiKey = 24\n\taddOffsetsToTxn apiKey = 25\n\tendTxn apiKey = 26\n\twriteTxnMarkers apiKey = 27\n\ttxnOffsetCommit apiKey = 28\n\tdescribeAcls apiKey = 29\n\tcreateAcls apiKey = 30\n\tdeleteAcls apiKey = 31\n\tdescribeConfigs apiKey = 32\n\talterConfigs apiKey = 33\n\talterReplicaLogDirs apiKey = 34\n\tdescribeLogDirs apiKey = 35\n\tsaslAuthenticate apiKey = 36\n\tcreatePartitions apiKey = 37\n\tcreateDelegationToken apiKey = 38\n\trenewDelegationToken apiKey = 39\n\texpireDelegationToken apiKey = 40\n\tdescribeDelegationToken apiKey = 41\n\tdeleteGroups apiKey = 42\n\telectLeaders apiKey = 43\n\tincrementalAlterConfigs apiKey = 44\n\talterPartitionReassignments apiKey = 45\n\tlistPartitionReassignments apiKey = 46\n\toffsetDelete apiKey = 47\n)\n\nfunc (k apiKey) String() string {\n\tif i := int(k); i >= 0 && i < len(apiKeyStrings) {\n\t\treturn apiKeyStrings[i]\n\t}\n\treturn strconv.Itoa(int(k))\n}\n\ntype apiVersion int16\n\nconst (\n\tv0 = 0\n\tv1 = 1\n\tv2 = 2\n\tv3 = 3\n\tv5 = 5\n\tv6 = 6\n\tv7 = 7\n\tv10 = 10\n\n\t// Unused protocol versions: v4, v8, v9.\n)\n\nvar apiKeyStrings = [...]string{\n\tproduce: \"Produce\",\n\tfetch: \"Fetch\",\n\tlistOffsets: \"ListOffsets\",\n\tmetadata: \"Metadata\",\n\tleaderAndIsr: \"LeaderAndIsr\",\n\tstopReplica: \"StopReplica\",\n\tupdateMetadata: \"UpdateMetadata\",\n\tcontrolledShutdown: \"ControlledShutdown\",\n\toffsetCommit: \"OffsetCommit\",\n\toffsetFetch: \"OffsetFetch\",\n\tfindCoordinator: \"FindCoordinator\",\n\tjoinGroup: \"JoinGroup\",\n\theartbeat: \"Heartbeat\",\n\tleaveGroup: \"LeaveGroup\",\n\tsyncGroup: \"SyncGroup\",\n\tdescribeGroups: \"DescribeGroups\",\n\tlistGroups: \"ListGroups\",\n\tsaslHandshake: \"SaslHandshake\",\n\tapiVersions: \"ApiVersions\",\n\tcreateTopics: \"CreateTopics\",\n\tdeleteTopics: \"DeleteTopics\",\n\tdeleteRecords: \"DeleteRecords\",\n\tinitProducerId: \"InitProducerId\",\n\toffsetForLeaderEpoch: \"OffsetForLeaderEpoch\",\n\taddPartitionsToTxn: \"AddPartitionsToTxn\",\n\taddOffsetsToTxn: \"AddOffsetsToTxn\",\n\tendTxn: \"EndTxn\",\n\twriteTxnMarkers: \"WriteTxnMarkers\",\n\ttxnOffsetCommit: \"TxnOffsetCommit\",\n\tdescribeAcls: \"DescribeAcls\",\n\tcreateAcls: \"CreateAcls\",\n\tdeleteAcls: \"DeleteAcls\",\n\tdescribeConfigs: \"DescribeConfigs\",\n\talterConfigs: \"AlterConfigs\",\n\talterReplicaLogDirs: \"AlterReplicaLogDirs\",\n\tdescribeLogDirs: \"DescribeLogDirs\",\n\tsaslAuthenticate: \"SaslAuthenticate\",\n\tcreatePartitions: \"CreatePartitions\",\n\tcreateDelegationToken: \"CreateDelegationToken\",\n\trenewDelegationToken: \"RenewDelegationToken\",\n\texpireDelegationToken: \"ExpireDelegationToken\",\n\tdescribeDelegationToken: \"DescribeDelegationToken\",\n\tdeleteGroups: \"DeleteGroups\",\n\telectLeaders: \"ElectLeaders\",\n\tincrementalAlterConfigs: \"IncrementalAlfterConfigs\",\n\talterPartitionReassignments: \"AlterPartitionReassignments\",\n\tlistPartitionReassignments: \"ListPartitionReassignments\",\n\toffsetDelete: \"OffsetDelete\",\n}\n\ntype requestHeader struct {\n\tSize int32\n\tApiKey int16\n\tApiVersion int16\n\tCorrelationID int32\n\tClientID string\n}\n\nfunc (h requestHeader) size() int32 {\n\treturn 4 + 2 + 2 + 4 + sizeofString(h.ClientID)\n}\n\nfunc (h requestHeader) writeTo(wb *writeBuffer) {\n\twb.writeInt32(h.Size)\n\twb.writeInt16(h.ApiKey)\n\twb.writeInt16(h.ApiVersion)\n\twb.writeInt32(h.CorrelationID)\n\twb.writeString(h.ClientID)\n}\n\ntype request interface {\n\tsize() int32\n\twritable\n}\n\nfunc makeInt8(b []byte) int8 {\n\treturn int8(b[0])\n}\n\nfunc makeInt16(b []byte) int16 {\n\treturn int16(binary.BigEndian.Uint16(b))\n}\n\nfunc makeInt32(b []byte) int32 {\n\treturn int32(binary.BigEndian.Uint32(b))\n}\n\nfunc makeInt64(b []byte) int64 {\n\treturn int64(binary.BigEndian.Uint64(b))\n}\n\nfunc expectZeroSize(sz int, err error) error {\n\tif err == nil && sz != 0 {\n\t\terr = fmt.Errorf(\"reading a response left %d unread bytes\", sz)\n\t}\n\treturn err\n}\n" }, { "path": "protocol_test.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"fmt\"\n\t\"reflect\"\n\t\"testing\"\n)\n\nfunc TestApiVersionsFormat(t *testing.T) {\n\tfor _, test := range []struct {\n\t\tversion ApiVersion\n\t\tformat string\n\t\toutput string\n\t}{\n\t\t{version: ApiVersion{1, 2, 5}, format: \"%s\", output: \"Fetch\"},\n\t\t{version: ApiVersion{1, 2, 5}, format: \"%d\", output: \"1\"},\n\t\t{version: ApiVersion{1, 2, 5}, format: \"%-d\", output: \"2\"},\n\t\t{version: ApiVersion{1, 2, 5}, format: \"%+d\", output: \"5\"},\n\t\t{version: ApiVersion{1, 2, 5}, format: \"%v\", output: \"Fetch[v2:v5]\"},\n\t\t{version: ApiVersion{1, 2, 5}, format: \"%-v\", output: \"v2\"},\n\t\t{version: ApiVersion{1, 2, 5}, format: \"%+v\", output: \"v5\"},\n\t\t{version: ApiVersion{1, 2, 5}, format: \"%#v\", output: \"kafka.ApiVersion{ApiKey:1 MinVersion:2 MaxVersion:5}\"},\n\t} {\n\t\tt.Run(test.output, func(t *testing.T) {\n\t\t\tif s := fmt.Sprintf(test.format, test.version); s != test.output {\n\t\t\t\tt.Error(\"output mismatch:\", s, \"!=\", test.output)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestProtocol(t *testing.T) {\n\ttests := []interface{}{\n\t\tint8(42),\n\t\tint16(42),\n\t\tint32(42),\n\t\tint64(42),\n\t\t\"\",\n\t\t\"Hello World!\",\n\t\t[]byte(nil),\n\t\t[]byte(\"Hello World!\"),\n\n\t\trequestHeader{\n\t\t\tSize: 26,\n\t\t\tApiKey: int16(offsetCommit),\n\t\t\tApiVersion: int16(v2),\n\t\t\tCorrelationID: 42,\n\t\t\tClientID: \"Hello World!\",\n\t\t},\n\n\t\tmessage{\n\t\t\tMagicByte: 1,\n\t\t\tTimestamp: 42,\n\t\t\tKey: nil,\n\t\t\tValue: []byte(\"Hello World!\"),\n\t\t},\n\n\t\ttopicMetadataRequestV1{\"A\", \"B\", \"C\"},\n\n\t\tmetadataResponseV1{\n\t\t\tBrokers: []brokerMetadataV1{\n\t\t\t\t{NodeID: 1, Host: \"localhost\", Port: 9001},\n\t\t\t\t{NodeID: 2, Host: \"localhost\", Port: 9002, Rack: \"rack2\"},\n\t\t\t},\n\t\t\tControllerID: 2,\n\t\t\tTopics: []topicMetadataV1{\n\t\t\t\t{TopicErrorCode: 0, Internal: true, Partitions: []partitionMetadataV1{{\n\t\t\t\t\tPartitionErrorCode: 0,\n\t\t\t\t\tPartitionID: 1,\n\t\t\t\t\tLeader: 2,\n\t\t\t\t\tReplicas: []int32{1},\n\t\t\t\t\tIsr: []int32{1},\n\t\t\t\t}}},\n\t\t\t},\n\t\t},\n\n\t\ttopicMetadataRequestV6{\n\t\t\tTopics: []string{\"A\", \"B\", \"C\"},\n\t\t\tAllowAutoTopicCreation: true,\n\t\t},\n\n\t\tmetadataResponseV6{\n\t\t\tBrokers: []brokerMetadataV1{\n\t\t\t\t{NodeID: 1, Host: \"localhost\", Port: 9001},\n\t\t\t\t{NodeID: 2, Host: \"localhost\", Port: 9002, Rack: \"rack2\"},\n\t\t\t},\n\t\t\tClusterId: \"cluster\",\n\t\t\tControllerID: 2,\n\t\t\tTopics: []topicMetadataV6{\n\t\t\t\t{TopicErrorCode: 0, Internal: true, Partitions: []partitionMetadataV6{{\n\t\t\t\t\tPartitionErrorCode: 0,\n\t\t\t\t\tPartitionID: 1,\n\t\t\t\t\tLeader: 2,\n\t\t\t\t\tReplicas: []int32{1},\n\t\t\t\t\tIsr: []int32{1},\n\t\t\t\t\tOfflineReplicas: []int32{1},\n\t\t\t\t}}},\n\t\t\t},\n\t\t},\n\n\t\tlistOffsetRequestV1{\n\t\t\tReplicaID: 1,\n\t\t\tTopics: []listOffsetRequestTopicV1{\n\t\t\t\t{TopicName: \"A\", Partitions: []listOffsetRequestPartitionV1{\n\t\t\t\t\t{Partition: 0, Time: -1},\n\t\t\t\t\t{Partition: 1, Time: -1},\n\t\t\t\t\t{Partition: 2, Time: -1},\n\t\t\t\t}},\n\t\t\t\t{TopicName: \"B\", Partitions: []listOffsetRequestPartitionV1{\n\t\t\t\t\t{Partition: 0, Time: -2},\n\t\t\t\t}},\n\t\t\t\t{TopicName: \"C\", Partitions: []listOffsetRequestPartitionV1{\n\t\t\t\t\t{Partition: 0, Time: 42},\n\t\t\t\t}},\n\t\t\t},\n\t\t},\n\n\t\tlistOffsetResponseV1{\n\t\t\t{TopicName: \"A\", PartitionOffsets: []partitionOffsetV1{\n\t\t\t\t{Partition: 0, Timestamp: 42, Offset: 1},\n\t\t\t}},\n\t\t\t{TopicName: \"B\", PartitionOffsets: []partitionOffsetV1{\n\t\t\t\t{Partition: 0, Timestamp: 43, Offset: 10},\n\t\t\t\t{Partition: 1, Timestamp: 44, Offset: 100},\n\t\t\t}},\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(fmt.Sprintf(\"%T\", test), func(t *testing.T) {\n\t\t\tb := &bytes.Buffer{}\n\t\t\tr := bufio.NewReader(b)\n\t\t\tw := &writeBuffer{w: b}\n\t\t\tw.write(test)\n\n\t\t\tif size := int(sizeof(test)); size != b.Len() {\n\t\t\t\tt.Error(\"invalid size:\", size, \"!=\", b.Len())\n\t\t\t}\n\n\t\t\tv := reflect.New(reflect.TypeOf(test))\n\t\t\tn := b.Len()\n\n\t\t\tn, err := read(r, n, v.Interface())\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\t\t\tif n != 0 {\n\t\t\t\tt.Errorf(\"%d unread bytes\", n)\n\t\t\t}\n\n\t\t\tif !reflect.DeepEqual(test, v.Elem().Interface()) {\n\t\t\t\tt.Error(\"values don't match:\")\n\t\t\t\tt.Logf(\"expected: %#v\", test)\n\t\t\t\tt.Logf(\"found: %#v\", v.Elem().Interface())\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "rawproduce.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"net\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n\tproduceAPI \"github.com/segmentio/kafka-go/protocol/produce\"\n\t\"github.com/segmentio/kafka-go/protocol/rawproduce\"\n)\n\n// RawProduceRequest represents a request sent to a kafka broker to produce records\n// to a topic partition. The request contains a pre-encoded/raw record set.\ntype RawProduceRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// The topic to produce the records to.\n\tTopic string\n\n\t// The partition to produce the records to.\n\tPartition int\n\n\t// The level of required acknowledgements to ask the kafka broker for.\n\tRequiredAcks RequiredAcks\n\n\t// The message format version used when encoding the records.\n\t//\n\t// By default, the client automatically determine which version should be\n\t// used based on the version of the Produce API supported by the server.\n\tMessageVersion int\n\n\t// An optional transaction id when producing to the kafka broker is part of\n\t// a transaction.\n\tTransactionalID string\n\n\t// The sequence of records to produce to the topic partition.\n\tRawRecords protocol.RawRecordSet\n}\n\n// RawProduce sends a raw produce request to a kafka broker and returns the response.\n//\n// If the request contained no records, an error wrapping protocol.ErrNoRecord\n// is returned.\n//\n// When the request is configured with RequiredAcks=none, both the response and\n// the error will be nil on success.\nfunc (c *Client) RawProduce(ctx context.Context, req *RawProduceRequest) (*ProduceResponse, error) {\n\tm, err := c.roundTrip(ctx, req.Addr, &rawproduce.Request{\n\t\tTransactionalID: req.TransactionalID,\n\t\tAcks: int16(req.RequiredAcks),\n\t\tTimeout: c.timeoutMs(ctx, defaultProduceTimeout),\n\t\tTopics: []rawproduce.RequestTopic{{\n\t\t\tTopic: req.Topic,\n\t\t\tPartitions: []rawproduce.RequestPartition{{\n\t\t\t\tPartition: int32(req.Partition),\n\t\t\t\tRecordSet: req.RawRecords,\n\t\t\t}},\n\t\t}},\n\t})\n\n\tswitch {\n\tcase err == nil:\n\tcase errors.Is(err, protocol.ErrNoRecord):\n\t\treturn new(ProduceResponse), nil\n\tdefault:\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).RawProduce: %w\", err)\n\t}\n\n\tif req.RequiredAcks == RequireNone {\n\t\treturn nil, nil\n\t}\n\n\tres := m.(*produceAPI.Response)\n\tif len(res.Topics) == 0 {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).RawProduce: %w\", protocol.ErrNoTopic)\n\t}\n\ttopic := &res.Topics[0]\n\tif len(topic.Partitions) == 0 {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).RawProduce: %w\", protocol.ErrNoPartition)\n\t}\n\tpartition := &topic.Partitions[0]\n\n\tret := &ProduceResponse{\n\t\tThrottle: makeDuration(res.ThrottleTimeMs),\n\t\tError: makeError(partition.ErrorCode, partition.ErrorMessage),\n\t\tBaseOffset: partition.BaseOffset,\n\t\tLogAppendTime: makeTime(partition.LogAppendTime),\n\t\tLogStartOffset: partition.LogStartOffset,\n\t}\n\n\tif len(partition.RecordErrors) != 0 {\n\t\tret.RecordErrors = make(map[int]error, len(partition.RecordErrors))\n\n\t\tfor _, recErr := range partition.RecordErrors {\n\t\t\tret.RecordErrors[int(recErr.BatchIndex)] = errors.New(recErr.BatchIndexErrorMessage)\n\t\t}\n\t}\n\n\treturn ret, nil\n}\n" }, { "path": "rawproduce_test.go", "content": "package kafka\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc TestClientRawProduce(t *testing.T) {\n\t// The RawProduce request records are encoded in the format introduced in Kafka 0.11.0.\n\tif !ktesting.KafkaIsAtLeast(\"0.11.0\") {\n\t\tt.Skip(\"Skipping because the RawProduce request is not supported by Kafka versions below 0.11.0\")\n\t}\n\n\tclient, topic, shutdown := newLocalClientAndTopic()\n\tdefer shutdown()\n\n\tnow := time.Now()\n\n\tres, err := client.RawProduce(context.Background(), &RawProduceRequest{\n\t\tTopic: topic,\n\t\tPartition: 0,\n\t\tRequiredAcks: -1,\n\t\tRawRecords: NewRawRecordSet(NewRecordReader(\n\t\t\tRecord{Time: now, Value: NewBytes([]byte(`hello-1`))},\n\t\t\tRecord{Time: now, Value: NewBytes([]byte(`hello-2`))},\n\t\t\tRecord{Time: now, Value: NewBytes([]byte(`hello-3`))},\n\t\t), 0),\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif res.Error != nil {\n\t\tt.Error(res.Error)\n\t}\n\n\tfor index, err := range res.RecordErrors {\n\t\tt.Errorf(\"record at index %d produced an error: %v\", index, err)\n\t}\n}\n\nfunc TestClientRawProduceCompressed(t *testing.T) {\n\t// The RawProduce request records are encoded in the format introduced in Kafka 0.11.0.\n\tif !ktesting.KafkaIsAtLeast(\"0.11.0\") {\n\t\tt.Skip(\"Skipping because the RawProduce request is not supported by Kafka versions below 0.11.0\")\n\t}\n\n\tclient, topic, shutdown := newLocalClientAndTopic()\n\tdefer shutdown()\n\n\tnow := time.Now()\n\n\tres, err := client.RawProduce(context.Background(), &RawProduceRequest{\n\t\tTopic: topic,\n\t\tPartition: 0,\n\t\tRequiredAcks: -1,\n\t\tRawRecords: NewRawRecordSet(NewRecordReader(\n\t\t\tRecord{Time: now, Value: NewBytes([]byte(`hello-1`))},\n\t\t\tRecord{Time: now, Value: NewBytes([]byte(`hello-2`))},\n\t\t\tRecord{Time: now, Value: NewBytes([]byte(`hello-3`))},\n\t\t), protocol.Gzip),\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif res.Error != nil {\n\t\tt.Error(res.Error)\n\t}\n\n\tfor index, err := range res.RecordErrors {\n\t\tt.Errorf(\"record at index %d produced an error: %v\", index, err)\n\t}\n}\n\nfunc TestClientRawProduceNilRecords(t *testing.T) {\n\tclient, topic, shutdown := newLocalClientAndTopic()\n\tdefer shutdown()\n\n\t_, err := client.RawProduce(context.Background(), &RawProduceRequest{\n\t\tTopic: topic,\n\t\tPartition: 0,\n\t\tRequiredAcks: -1,\n\t\tRawRecords: protocol.RawRecordSet{Reader: nil},\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestClientRawProduceEmptyRecords(t *testing.T) {\n\tclient, topic, shutdown := newLocalClientAndTopic()\n\tdefer shutdown()\n\n\t_, err := client.Produce(context.Background(), &ProduceRequest{\n\t\tTopic: topic,\n\t\tPartition: 0,\n\t\tRequiredAcks: -1,\n\t\tRecords: NewRecordReader(),\n\t})\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc NewRawRecordSet(reader protocol.RecordReader, attr protocol.Attributes) protocol.RawRecordSet {\n\trs := protocol.RecordSet{Version: 2, Attributes: attr, Records: reader}\n\tbuf := &bytes.Buffer{}\n\trs.WriteTo(buf)\n\n\treturn protocol.RawRecordSet{\n\t\tReader: buf,\n\t}\n}\n" }, { "path": "read.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"reflect\"\n)\n\nvar errShortRead = errors.New(\"not enough bytes available to load the response\")\n\nfunc peekRead(r *bufio.Reader, sz int, n int, f func([]byte)) (int, error) {\n\tif n > sz {\n\t\treturn sz, errShortRead\n\t}\n\tb, err := r.Peek(n)\n\tif err != nil {\n\t\treturn sz, err\n\t}\n\tf(b)\n\treturn discardN(r, sz, n)\n}\n\nfunc readInt8(r *bufio.Reader, sz int, v *int8) (int, error) {\n\treturn peekRead(r, sz, 1, func(b []byte) { *v = makeInt8(b) })\n}\n\nfunc readInt16(r *bufio.Reader, sz int, v *int16) (int, error) {\n\treturn peekRead(r, sz, 2, func(b []byte) { *v = makeInt16(b) })\n}\n\nfunc readInt32(r *bufio.Reader, sz int, v *int32) (int, error) {\n\treturn peekRead(r, sz, 4, func(b []byte) { *v = makeInt32(b) })\n}\n\nfunc readInt64(r *bufio.Reader, sz int, v *int64) (int, error) {\n\treturn peekRead(r, sz, 8, func(b []byte) { *v = makeInt64(b) })\n}\n\nfunc readVarInt(r *bufio.Reader, sz int, v *int64) (remain int, err error) {\n\t// Optimistically assume that most of the time, there will be data buffered\n\t// in the reader. If this is not the case, the buffer will be refilled after\n\t// consuming zero bytes from the input.\n\tinput, _ := r.Peek(r.Buffered())\n\tx := uint64(0)\n\ts := uint(0)\n\n\tfor {\n\t\tif len(input) > sz {\n\t\t\tinput = input[:sz]\n\t\t}\n\n\t\tfor i, b := range input {\n\t\t\tif b < 0x80 {\n\t\t\t\tx |= uint64(b) << s\n\t\t\t\t*v = int64(x>>1) ^ -(int64(x) & 1)\n\t\t\t\tn, err := r.Discard(i + 1)\n\t\t\t\treturn sz - n, err\n\t\t\t}\n\n\t\t\tx |= uint64(b&0x7f) << s\n\t\t\ts += 7\n\t\t}\n\n\t\t// Make room in the input buffer to load more data from the underlying\n\t\t// stream. The x and s variables are left untouched, ensuring that the\n\t\t// varint decoding can continue on the next loop iteration.\n\t\tn, _ := r.Discard(len(input))\n\t\tsz -= n\n\t\tif sz == 0 {\n\t\t\treturn 0, errShortRead\n\t\t}\n\n\t\t// Fill the buffer: ask for one more byte, but in practice the reader\n\t\t// will load way more from the underlying stream.\n\t\tif _, err := r.Peek(1); err != nil {\n\t\t\tif errors.Is(err, io.EOF) {\n\t\t\t\terr = errShortRead\n\t\t\t}\n\t\t\treturn sz, err\n\t\t}\n\n\t\t// Grab as many bytes as possible from the buffer, then go on to the\n\t\t// next loop iteration which is going to consume it.\n\t\tinput, _ = r.Peek(r.Buffered())\n\t}\n}\n\nfunc readBool(r *bufio.Reader, sz int, v *bool) (int, error) {\n\treturn peekRead(r, sz, 1, func(b []byte) { *v = b[0] != 0 })\n}\n\nfunc readString(r *bufio.Reader, sz int, v *string) (int, error) {\n\treturn readStringWith(r, sz, func(r *bufio.Reader, sz int, n int) (remain int, err error) {\n\t\t*v, remain, err = readNewString(r, sz, n)\n\t\treturn\n\t})\n}\n\nfunc readStringWith(r *bufio.Reader, sz int, cb func(*bufio.Reader, int, int) (int, error)) (int, error) {\n\tvar err error\n\tvar len int16\n\n\tif sz, err = readInt16(r, sz, &len); err != nil {\n\t\treturn sz, err\n\t}\n\n\tn := int(len)\n\tif n > sz {\n\t\treturn sz, errShortRead\n\t}\n\n\treturn cb(r, sz, n)\n}\n\nfunc readNewString(r *bufio.Reader, sz int, n int) (string, int, error) {\n\tb, sz, err := readNewBytes(r, sz, n)\n\treturn string(b), sz, err\n}\n\nfunc readBytes(r *bufio.Reader, sz int, v *[]byte) (int, error) {\n\treturn readBytesWith(r, sz, func(r *bufio.Reader, sz int, n int) (remain int, err error) {\n\t\t*v, remain, err = readNewBytes(r, sz, n)\n\t\treturn\n\t})\n}\n\nfunc readBytesWith(r *bufio.Reader, sz int, cb func(*bufio.Reader, int, int) (int, error)) (int, error) {\n\tvar err error\n\tvar n int\n\n\tif sz, err = readArrayLen(r, sz, &n); err != nil {\n\t\treturn sz, err\n\t}\n\n\tif n > sz {\n\t\treturn sz, errShortRead\n\t}\n\n\treturn cb(r, sz, n)\n}\n\nfunc readNewBytes(r *bufio.Reader, sz int, n int) ([]byte, int, error) {\n\tvar err error\n\tvar b []byte\n\tvar shortRead bool\n\n\tif n > 0 {\n\t\tif sz < n {\n\t\t\tn = sz\n\t\t\tshortRead = true\n\t\t}\n\n\t\tb = make([]byte, n)\n\t\tn, err = io.ReadFull(r, b)\n\t\tb = b[:n]\n\t\tsz -= n\n\n\t\tif err == nil && shortRead {\n\t\t\terr = errShortRead\n\t\t}\n\t}\n\n\treturn b, sz, err\n}\n\nfunc readArrayLen(r *bufio.Reader, sz int, n *int) (int, error) {\n\tvar err error\n\tvar len int32\n\tif sz, err = readInt32(r, sz, &len); err != nil {\n\t\treturn sz, err\n\t}\n\t*n = int(len)\n\treturn sz, nil\n}\n\nfunc readArrayWith(r *bufio.Reader, sz int, cb func(*bufio.Reader, int) (int, error)) (int, error) {\n\tvar err error\n\tvar len int32\n\n\tif sz, err = readInt32(r, sz, &len); err != nil {\n\t\treturn sz, err\n\t}\n\n\tfor n := int(len); n > 0; n-- {\n\t\tif sz, err = cb(r, sz); err != nil {\n\t\t\tbreak\n\t\t}\n\t}\n\n\treturn sz, err\n}\n\nfunc readStringArray(r *bufio.Reader, sz int, v *[]string) (remain int, err error) {\n\tvar content []string\n\tfn := func(r *bufio.Reader, size int) (fnRemain int, fnErr error) {\n\t\tvar value string\n\t\tif fnRemain, fnErr = readString(r, size, &value); fnErr != nil {\n\t\t\treturn\n\t\t}\n\t\tcontent = append(content, value)\n\t\treturn\n\t}\n\tif remain, err = readArrayWith(r, sz, fn); err != nil {\n\t\treturn\n\t}\n\n\t*v = content\n\treturn\n}\n\nfunc readMapStringInt32(r *bufio.Reader, sz int, v *map[string][]int32) (remain int, err error) {\n\tvar len int32\n\tif remain, err = readInt32(r, sz, &len); err != nil {\n\t\treturn\n\t}\n\n\tcontent := make(map[string][]int32, len)\n\tfor i := 0; i < int(len); i++ {\n\t\tvar key string\n\t\tvar values []int32\n\n\t\tif remain, err = readString(r, remain, &key); err != nil {\n\t\t\treturn\n\t\t}\n\n\t\tfn := func(r *bufio.Reader, size int) (fnRemain int, fnErr error) {\n\t\t\tvar value int32\n\t\t\tif fnRemain, fnErr = readInt32(r, size, &value); fnErr != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tvalues = append(values, value)\n\t\t\treturn\n\t\t}\n\t\tif remain, err = readArrayWith(r, remain, fn); err != nil {\n\t\t\treturn\n\t\t}\n\n\t\tcontent[key] = values\n\t}\n\t*v = content\n\n\treturn\n}\n\nfunc read(r *bufio.Reader, sz int, a interface{}) (int, error) {\n\tswitch v := a.(type) {\n\tcase *int8:\n\t\treturn readInt8(r, sz, v)\n\tcase *int16:\n\t\treturn readInt16(r, sz, v)\n\tcase *int32:\n\t\treturn readInt32(r, sz, v)\n\tcase *int64:\n\t\treturn readInt64(r, sz, v)\n\tcase *bool:\n\t\treturn readBool(r, sz, v)\n\tcase *string:\n\t\treturn readString(r, sz, v)\n\tcase *[]byte:\n\t\treturn readBytes(r, sz, v)\n\t}\n\tswitch v := reflect.ValueOf(a).Elem(); v.Kind() {\n\tcase reflect.Struct:\n\t\treturn readStruct(r, sz, v)\n\tcase reflect.Slice:\n\t\treturn readSlice(r, sz, v)\n\tdefault:\n\t\tpanic(fmt.Sprintf(\"unsupported type: %T\", a))\n\t}\n}\n\nfunc readStruct(r *bufio.Reader, sz int, v reflect.Value) (int, error) {\n\tvar err error\n\tfor i, n := 0, v.NumField(); i != n; i++ {\n\t\tif sz, err = read(r, sz, v.Field(i).Addr().Interface()); err != nil {\n\t\t\treturn sz, err\n\t\t}\n\t}\n\treturn sz, nil\n}\n\nfunc readSlice(r *bufio.Reader, sz int, v reflect.Value) (int, error) {\n\tvar err error\n\tvar len int32\n\n\tif sz, err = readInt32(r, sz, &len); err != nil {\n\t\treturn sz, err\n\t}\n\n\tif n := int(len); n < 0 {\n\t\tv.Set(reflect.Zero(v.Type()))\n\t} else {\n\t\tv.Set(reflect.MakeSlice(v.Type(), n, n))\n\n\t\tfor i := 0; i != n; i++ {\n\t\t\tif sz, err = read(r, sz, v.Index(i).Addr().Interface()); err != nil {\n\t\t\t\treturn sz, err\n\t\t\t}\n\t\t}\n\t}\n\n\treturn sz, nil\n}\n\nfunc readFetchResponseHeaderV2(r *bufio.Reader, size int) (throttle int32, watermark int64, remain int, err error) {\n\tvar n int32\n\tvar p struct {\n\t\tPartition int32\n\t\tErrorCode int16\n\t\tHighwaterMarkOffset int64\n\t\tMessageSetSize int32\n\t}\n\n\tif remain, err = readInt32(r, size, &throttle); err != nil {\n\t\treturn\n\t}\n\n\tif remain, err = readInt32(r, remain, &n); err != nil {\n\t\treturn\n\t}\n\n\t// This error should never trigger, unless there's a bug in the kafka client\n\t// or server.\n\tif n != 1 {\n\t\terr = fmt.Errorf(\"1 kafka topic was expected in the fetch response but the client received %d\", n)\n\t\treturn\n\t}\n\n\t// We ignore the topic name because we've requests messages for a single\n\t// topic, unless there's a bug in the kafka server we will have received\n\t// the name of the topic that we requested.\n\tif remain, err = discardString(r, remain); err != nil {\n\t\treturn\n\t}\n\n\tif remain, err = readInt32(r, remain, &n); err != nil {\n\t\treturn\n\t}\n\n\t// This error should never trigger, unless there's a bug in the kafka client\n\t// or server.\n\tif n != 1 {\n\t\terr = fmt.Errorf(\"1 kafka partition was expected in the fetch response but the client received %d\", n)\n\t\treturn\n\t}\n\n\tif remain, err = read(r, remain, &p); err != nil {\n\t\treturn\n\t}\n\n\tif p.ErrorCode != 0 {\n\t\terr = Error(p.ErrorCode)\n\t\treturn\n\t}\n\n\t// This error should never trigger, unless there's a bug in the kafka client\n\t// or server.\n\tif remain != int(p.MessageSetSize) {\n\t\terr = fmt.Errorf(\"the size of the message set in a fetch response doesn't match the number of remaining bytes (message set size = %d, remaining bytes = %d)\", p.MessageSetSize, remain)\n\t\treturn\n\t}\n\n\twatermark = p.HighwaterMarkOffset\n\treturn\n}\n\nfunc readFetchResponseHeaderV5(r *bufio.Reader, size int) (throttle int32, watermark int64, remain int, err error) {\n\tvar n int32\n\ttype AbortedTransaction struct {\n\t\tProducerId int64\n\t\tFirstOffset int64\n\t}\n\tvar p struct {\n\t\tPartition int32\n\t\tErrorCode int16\n\t\tHighwaterMarkOffset int64\n\t\tLastStableOffset int64\n\t\tLogStartOffset int64\n\t}\n\tvar messageSetSize int32\n\tvar abortedTransactions []AbortedTransaction\n\n\tif remain, err = readInt32(r, size, &throttle); err != nil {\n\t\treturn\n\t}\n\n\tif remain, err = readInt32(r, remain, &n); err != nil {\n\t\treturn\n\t}\n\n\t// This error should never trigger, unless there's a bug in the kafka client\n\t// or server.\n\tif n != 1 {\n\t\terr = fmt.Errorf(\"1 kafka topic was expected in the fetch response but the client received %d\", n)\n\t\treturn\n\t}\n\n\t// We ignore the topic name because we've requests messages for a single\n\t// topic, unless there's a bug in the kafka server we will have received\n\t// the name of the topic that we requested.\n\tif remain, err = discardString(r, remain); err != nil {\n\t\treturn\n\t}\n\n\tif remain, err = readInt32(r, remain, &n); err != nil {\n\t\treturn\n\t}\n\n\t// This error should never trigger, unless there's a bug in the kafka client\n\t// or server.\n\tif n != 1 {\n\t\terr = fmt.Errorf(\"1 kafka partition was expected in the fetch response but the client received %d\", n)\n\t\treturn\n\t}\n\n\tif remain, err = read(r, remain, &p); err != nil {\n\t\treturn\n\t}\n\n\tvar abortedTransactionLen int\n\tif remain, err = readArrayLen(r, remain, &abortedTransactionLen); err != nil {\n\t\treturn\n\t}\n\n\tif abortedTransactionLen == -1 {\n\t\tabortedTransactions = nil\n\t} else {\n\t\tabortedTransactions = make([]AbortedTransaction, abortedTransactionLen)\n\t\tfor i := 0; i < abortedTransactionLen; i++ {\n\t\t\tif remain, err = read(r, remain, &abortedTransactions[i]); err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\n\tif p.ErrorCode != 0 {\n\t\terr = Error(p.ErrorCode)\n\t\treturn\n\t}\n\n\tremain, err = readInt32(r, remain, &messageSetSize)\n\tif err != nil {\n\t\treturn\n\t}\n\n\t// This error should never trigger, unless there's a bug in the kafka client\n\t// or server.\n\tif remain != int(messageSetSize) {\n\t\terr = fmt.Errorf(\"the size of the message set in a fetch response doesn't match the number of remaining bytes (message set size = %d, remaining bytes = %d)\", messageSetSize, remain)\n\t\treturn\n\t}\n\n\twatermark = p.HighwaterMarkOffset\n\treturn\n\n}\n\nfunc readFetchResponseHeaderV10(r *bufio.Reader, size int) (throttle int32, watermark int64, remain int, err error) {\n\tvar n int32\n\tvar errorCode int16\n\ttype AbortedTransaction struct {\n\t\tProducerId int64\n\t\tFirstOffset int64\n\t}\n\tvar p struct {\n\t\tPartition int32\n\t\tErrorCode int16\n\t\tHighwaterMarkOffset int64\n\t\tLastStableOffset int64\n\t\tLogStartOffset int64\n\t}\n\tvar messageSetSize int32\n\tvar abortedTransactions []AbortedTransaction\n\n\tif remain, err = readInt32(r, size, &throttle); err != nil {\n\t\treturn\n\t}\n\n\tif remain, err = readInt16(r, remain, &errorCode); err != nil {\n\t\treturn\n\t}\n\tif errorCode != 0 {\n\t\terr = Error(errorCode)\n\t\treturn\n\t}\n\n\tif remain, err = discardInt32(r, remain); err != nil {\n\t\treturn\n\t}\n\n\tif remain, err = readInt32(r, remain, &n); err != nil {\n\t\treturn\n\t}\n\n\t// This error should never trigger, unless there's a bug in the kafka client\n\t// or server.\n\tif n != 1 {\n\t\terr = fmt.Errorf(\"1 kafka topic was expected in the fetch response but the client received %d\", n)\n\t\treturn\n\t}\n\n\t// We ignore the topic name because we've requests messages for a single\n\t// topic, unless there's a bug in the kafka server we will have received\n\t// the name of the topic that we requested.\n\tif remain, err = discardString(r, remain); err != nil {\n\t\treturn\n\t}\n\n\tif remain, err = readInt32(r, remain, &n); err != nil {\n\t\treturn\n\t}\n\n\t// This error should never trigger, unless there's a bug in the kafka client\n\t// or server.\n\tif n != 1 {\n\t\terr = fmt.Errorf(\"1 kafka partition was expected in the fetch response but the client received %d\", n)\n\t\treturn\n\t}\n\n\tif remain, err = read(r, remain, &p); err != nil {\n\t\treturn\n\t}\n\n\tvar abortedTransactionLen int\n\tif remain, err = readArrayLen(r, remain, &abortedTransactionLen); err != nil {\n\t\treturn\n\t}\n\n\tif abortedTransactionLen == -1 {\n\t\tabortedTransactions = nil\n\t} else {\n\t\tabortedTransactions = make([]AbortedTransaction, abortedTransactionLen)\n\t\tfor i := 0; i < abortedTransactionLen; i++ {\n\t\t\tif remain, err = read(r, remain, &abortedTransactions[i]); err != nil {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\n\tif p.ErrorCode != 0 {\n\t\terr = Error(p.ErrorCode)\n\t\treturn\n\t}\n\n\tremain, err = readInt32(r, remain, &messageSetSize)\n\tif err != nil {\n\t\treturn\n\t}\n\n\t// This error should never trigger, unless there's a bug in the kafka client\n\t// or server.\n\tif remain != int(messageSetSize) {\n\t\terr = fmt.Errorf(\"the size of the message set in a fetch response doesn't match the number of remaining bytes (message set size = %d, remaining bytes = %d)\", messageSetSize, remain)\n\t\treturn\n\t}\n\n\twatermark = p.HighwaterMarkOffset\n\treturn\n\n}\n" }, { "path": "read_test.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"errors\"\n\t\"io/ioutil\"\n\t\"math\"\n\t\"reflect\"\n\t\"testing\"\n)\n\ntype VarIntTestCase struct {\n\tv int64\n\tr int\n\ttc []byte\n}\n\nfunc TestReadVarInt(t *testing.T) {\n\ttestCases := []*VarIntTestCase{\n\t\t{v: 0, r: 3, tc: []byte{0, 1, 10, 0}},\n\t\t{v: -1, r: 3, tc: []byte{1, 1, 10, 0}},\n\t\t{v: 1, r: 3, tc: []byte{2, 1, 10, 0}},\n\t\t{v: -2, r: 3, tc: []byte{3, 1, 10, 0}},\n\t\t{v: 2, r: 3, tc: []byte{4, 1, 10, 0}},\n\t\t{v: 64, r: 2, tc: []byte{128, 1, 10, 0}},\n\t\t{v: -64, r: 3, tc: []byte{127, 1, 10, 0}},\n\t\t{v: -196, r: 2, tc: []byte{135, 3, 10, 0}},\n\t\t{v: -24772, r: 1, tc: []byte{135, 131, 3, 0}},\n\t}\n\n\tfor _, tc := range testCases {\n\t\tvar v int64\n\t\trd := bufio.NewReader(bytes.NewReader(tc.tc))\n\t\tremain, err := readVarInt(rd, len(tc.tc), &v)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"Failure during reading: %v\", err)\n\t\t}\n\t\tif v != tc.v {\n\t\t\tt.Errorf(\"Expected %v; got %v\", tc.v, v)\n\t\t}\n\t\tif remain != tc.r {\n\t\t\tt.Errorf(\"Expected remain %v; got %v\", tc.r, remain)\n\t\t}\n\t}\n}\n\nfunc TestReadVarIntFailing(t *testing.T) {\n\tvar v int64\n\ttestCase := []byte{135, 135}\n\trd := bufio.NewReader(bytes.NewReader(testCase))\n\t_, err := readVarInt(rd, len(testCase), &v)\n\tif !errors.Is(err, errShortRead) {\n\t\tt.Errorf(\"Expected error while parsing var int: %v\", err)\n\t}\n}\n\nfunc TestReadStringArray(t *testing.T) {\n\ttestCases := map[string]struct {\n\t\tValue []string\n\t}{\n\t\t\"nil\": {\n\t\t\tValue: nil,\n\t\t},\n\t\t\"multiple elements\": {\n\t\t\tValue: []string{\"a\", \"b\", \"c\"},\n\t\t},\n\t}\n\n\tfor label, test := range testCases {\n\t\tt.Run(label, func(t *testing.T) {\n\t\t\tb := bytes.NewBuffer(nil)\n\t\t\tw := &writeBuffer{w: b}\n\t\t\tw.writeStringArray(test.Value)\n\n\t\t\tvar actual []string\n\t\t\treadStringArray(bufio.NewReader(b), b.Len(), &actual)\n\t\t\tif !reflect.DeepEqual(test.Value, actual) {\n\t\t\t\tt.Errorf(\"expected %v; got %v\", test.Value, actual)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestReadMapStringInt32(t *testing.T) {\n\ttestCases := map[string]struct {\n\t\tData map[string][]int32\n\t}{\n\t\t\"empty\": {\n\t\t\tData: map[string][]int32{},\n\t\t},\n\t\t\"single element\": {\n\t\t\tData: map[string][]int32{\n\t\t\t\t\"key\": {0, 1, 2},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor label, test := range testCases {\n\t\tt.Run(label, func(t *testing.T) {\n\t\t\tb := bytes.NewBuffer(nil)\n\t\t\tw := &writeBuffer{w: b}\n\t\t\tw.writeInt32(int32(len(test.Data)))\n\n\t\t\tfor key, values := range test.Data {\n\t\t\t\tw.writeString(key)\n\t\t\t\tw.writeInt32Array(values)\n\t\t\t}\n\n\t\t\tvar actual map[string][]int32\n\t\t\treadMapStringInt32(bufio.NewReader(b), b.Len(), &actual)\n\t\t\tif !reflect.DeepEqual(test.Data, actual) {\n\t\t\t\tt.Errorf(\"expected %#v; got %#v\", test.Data, actual)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestReadNewBytes(t *testing.T) {\n\n\tt.Run(\"reads new bytes\", func(t *testing.T) {\n\t\tr := bufio.NewReader(bytes.NewReader([]byte(\"foobar\")))\n\n\t\tb, remain, err := readNewBytes(r, 6, 3)\n\t\tif string(b) != \"foo\" {\n\t\t\tt.Error(\"should have returned 3 bytes\")\n\t\t}\n\t\tif remain != 3 {\n\t\t\tt.Error(\"should have calculated remaining correctly\")\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Error(\"should not have errored\")\n\t\t}\n\n\t\tb, remain, err = readNewBytes(r, remain, 3)\n\t\tif string(b) != \"bar\" {\n\t\t\tt.Error(\"should have returned 3 bytes\")\n\t\t}\n\t\tif remain != 0 {\n\t\t\tt.Error(\"should have calculated remaining correctly\")\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Error(\"should not have errored\")\n\t\t}\n\n\t\tb, err = r.Peek(0)\n\t\tif len(b) > 0 {\n\t\t\tt.Error(\"not all bytes were consumed\")\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Error(\"should not have errored during peek\")\n\t\t}\n\t})\n\n\tt.Run(\"discards bytes when insufficient\", func(t *testing.T) {\n\t\tr := bufio.NewReader(bytes.NewReader([]byte(\"foo\")))\n\t\tb, remain, err := readNewBytes(bufio.NewReader(r), 3, 4)\n\t\tif string(b) != \"foo\" {\n\t\t\tt.Error(\"should have returned available bytes\")\n\t\t}\n\t\tif remain != 0 {\n\t\t\tt.Error(\"all bytes should have been consumed\")\n\t\t}\n\t\tif !errors.Is(err, errShortRead) {\n\t\t\tt.Error(\"should have returned errShortRead\")\n\t\t}\n\t\tb, err = r.Peek(0)\n\t\tif len(b) > 0 {\n\t\t\tt.Error(\"not all bytes were consumed\")\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Error(\"should not have errored during peek\")\n\t\t}\n\t})\n}\n\nfunc BenchmarkWriteVarInt(b *testing.B) {\n\twb := &writeBuffer{w: ioutil.Discard}\n\n\tfor i := 0; i < b.N; i++ {\n\t\twb.writeVarInt(math.MaxInt64)\n\t}\n}\n\nfunc BenchmarkReadVarInt(b *testing.B) {\n\tb1 := new(bytes.Buffer)\n\twb := &writeBuffer{w: b1}\n\n\tconst N = math.MaxInt64\n\twb.writeVarInt(N)\n\n\tb2 := bytes.NewReader(b1.Bytes())\n\trb := bufio.NewReader(b2)\n\tn := b1.Len()\n\n\tfor i := 0; i < b.N; i++ {\n\t\tv := int64(0)\n\t\tr, err := readVarInt(rb, n, &v)\n\n\t\tif err != nil {\n\t\t\tb.Fatalf(\"unexpected error reading a varint from the input: %v\", err)\n\t\t}\n\n\t\tif r != 0 {\n\t\t\tb.Fatalf(\"unexpected bytes remaining to be read in the input (%d B)\", r)\n\t\t}\n\n\t\tif v != N {\n\t\t\tb.Fatalf(\"value mismatch, expected %d but found %d\", N, v)\n\t\t}\n\n\t\tb2.Reset(b1.Bytes())\n\t\trb.Reset(b2)\n\t}\n}\n" }, { "path": "reader.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"sort\"\n\t\"strconv\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n)\n\nconst (\n\tLastOffset int64 = -1 // The most recent offset available for a partition.\n\tFirstOffset int64 = -2 // The least recent offset available for a partition.\n)\n\nconst (\n\t// defaultCommitRetries holds the number of commit attempts to make\n\t// before giving up.\n\tdefaultCommitRetries = 3\n)\n\nconst (\n\t// defaultFetchMinBytes of 1 byte means that fetch requests are answered as\n\t// soon as a single byte of data is available or the fetch request times out\n\t// waiting for data to arrive.\n\tdefaultFetchMinBytes = 1\n)\n\nvar (\n\terrOnlyAvailableWithGroup = errors.New(\"unavailable when GroupID is not set\")\n\terrNotAvailableWithGroup = errors.New(\"unavailable when GroupID is set\")\n)\n\nconst (\n\t// defaultReadBackoffMax/Min sets the boundaries for how long the reader wait before\n\t// polling for new messages.\n\tdefaultReadBackoffMin = 100 * time.Millisecond\n\tdefaultReadBackoffMax = 1 * time.Second\n)\n\n// Reader provides a high-level API for consuming messages from kafka.\n//\n// A Reader automatically manages reconnections to a kafka server, and\n// blocking methods have context support for asynchronous cancellations.\n//\n// Note that it is important to call `Close()` on a `Reader` when a process exits.\n// The kafka server needs a graceful disconnect to stop it from continuing to\n// attempt to send messages to the connected clients. The given example will not\n// call `Close()` if the process is terminated with SIGINT (ctrl-c at the shell) or\n// SIGTERM (as docker stop or a kubernetes restart does). This can result in a\n// delay when a new reader on the same topic connects (e.g. new process started\n// or new container running). Use a `signal.Notify` handler to close the reader on\n// process shutdown.\ntype Reader struct {\n\t// immutable fields of the reader\n\tconfig ReaderConfig\n\n\t// communication channels between the parent reader and its subreaders\n\tmsgs chan readerMessage\n\n\t// mutable fields of the reader (synchronized on the mutex)\n\tmutex sync.Mutex\n\tjoin sync.WaitGroup\n\tcancel context.CancelFunc\n\tstop context.CancelFunc\n\tdone chan struct{}\n\tcommits chan commitRequest\n\tversion int64 // version holds the generation of the spawned readers\n\toffset int64\n\tlag int64\n\tclosed bool\n\n\t// Without a group subscription (when Reader.config.GroupID == \"\"),\n\t// when errors occur, the Reader gets a synthetic readerMessage with\n\t// a non-nil err set. With group subscriptions however, when an error\n\t// occurs in Reader.run, there's no reader running (sic, cf. reader vs.\n\t// Reader) and there's no way to let the high-level methods like\n\t// FetchMessage know that an error indeed occurred. If an error in run\n\t// occurs, it will be non-block-sent to this unbuffered channel, where\n\t// the high-level methods can select{} on it and notify the caller.\n\trunError chan error\n\n\t// reader stats are all made of atomic values, no need for synchronization.\n\tonce uint32\n\tstctx context.Context\n\t// reader stats are all made of atomic values, no need for synchronization.\n\t// Use a pointer to ensure 64-bit alignment of the values.\n\tstats *readerStats\n}\n\n// useConsumerGroup indicates whether the Reader is part of a consumer group.\nfunc (r *Reader) useConsumerGroup() bool { return r.config.GroupID != \"\" }\n\nfunc (r *Reader) getTopics() []string {\n\tif len(r.config.GroupTopics) > 0 {\n\t\treturn r.config.GroupTopics[:]\n\t}\n\n\treturn []string{r.config.Topic}\n}\n\n// useSyncCommits indicates whether the Reader is configured to perform sync or\n// async commits.\nfunc (r *Reader) useSyncCommits() bool { return r.config.CommitInterval == 0 }\n\nfunc (r *Reader) unsubscribe() {\n\tr.cancel()\n\tr.join.Wait()\n\t// it would be interesting to drain the r.msgs channel at this point since\n\t// it will contain buffered messages for partitions that may not be\n\t// re-assigned to this reader in the next consumer group generation.\n\t// however, draining the channel could race with the client calling\n\t// ReadMessage, which could result in messages delivered and/or committed\n\t// with gaps in the offset. for now, we will err on the side of caution and\n\t// potentially have those messages be reprocessed in the next generation by\n\t// another consumer to avoid such a race.\n}\n\nfunc (r *Reader) subscribe(allAssignments map[string][]PartitionAssignment) {\n\toffsets := make(map[topicPartition]int64)\n\tfor topic, assignments := range allAssignments {\n\t\tfor _, assignment := range assignments {\n\t\t\tkey := topicPartition{\n\t\t\t\ttopic: topic,\n\t\t\t\tpartition: int32(assignment.ID),\n\t\t\t}\n\t\t\toffsets[key] = assignment.Offset\n\t\t}\n\t}\n\n\tr.mutex.Lock()\n\tr.start(offsets)\n\tr.mutex.Unlock()\n\n\tr.withLogger(func(l Logger) {\n\t\tl.Printf(\"subscribed to topics and partitions: %+v\", offsets)\n\t})\n}\n\n// commitOffsetsWithRetry attempts to commit the specified offsets and retries\n// up to the specified number of times.\nfunc (r *Reader) commitOffsetsWithRetry(gen *Generation, offsetStash offsetStash, retries int) (err error) {\n\tconst (\n\t\tbackoffDelayMin = 100 * time.Millisecond\n\t\tbackoffDelayMax = 5 * time.Second\n\t)\n\n\tfor attempt := 0; attempt < retries; attempt++ {\n\t\tif attempt != 0 {\n\t\t\tif !sleep(r.stctx, backoff(attempt, backoffDelayMin, backoffDelayMax)) {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\n\t\tif err = gen.CommitOffsets(offsetStash); err == nil {\n\t\t\treturn\n\t\t}\n\t}\n\n\treturn // err will not be nil\n}\n\n// offsetStash holds offsets by topic => partition => offset.\ntype offsetStash map[string]map[int]int64\n\n// merge updates the offsetStash with the offsets from the provided messages.\nfunc (o offsetStash) merge(commits []commit) {\n\tfor _, c := range commits {\n\t\toffsetsByPartition, ok := o[c.topic]\n\t\tif !ok {\n\t\t\toffsetsByPartition = map[int]int64{}\n\t\t\to[c.topic] = offsetsByPartition\n\t\t}\n\n\t\tif offset, ok := offsetsByPartition[c.partition]; !ok || c.offset > offset {\n\t\t\toffsetsByPartition[c.partition] = c.offset\n\t\t}\n\t}\n}\n\n// reset clears the contents of the offsetStash.\nfunc (o offsetStash) reset() {\n\tfor key := range o {\n\t\tdelete(o, key)\n\t}\n}\n\n// commitLoopImmediate handles each commit synchronously.\nfunc (r *Reader) commitLoopImmediate(ctx context.Context, gen *Generation) {\n\toffsets := offsetStash{}\n\n\tfor {\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\t// drain the commit channel and prepare a single, final commit.\n\t\t\t// the commit will combine any outstanding requests and the result\n\t\t\t// will be sent back to all the callers of CommitMessages so that\n\t\t\t// they can return.\n\t\t\tvar errchs []chan<- error\n\t\t\tfor hasCommits := true; hasCommits; {\n\t\t\t\tselect {\n\t\t\t\tcase req := <-r.commits:\n\t\t\t\t\toffsets.merge(req.commits)\n\t\t\t\t\terrchs = append(errchs, req.errch)\n\t\t\t\tdefault:\n\t\t\t\t\thasCommits = false\n\t\t\t\t}\n\t\t\t}\n\t\t\terr := r.commitOffsetsWithRetry(gen, offsets, defaultCommitRetries)\n\t\t\tfor _, errch := range errchs {\n\t\t\t\t// NOTE : this will be a buffered channel and will not block.\n\t\t\t\terrch <- err\n\t\t\t}\n\t\t\treturn\n\n\t\tcase req := <-r.commits:\n\t\t\toffsets.merge(req.commits)\n\t\t\treq.errch <- r.commitOffsetsWithRetry(gen, offsets, defaultCommitRetries)\n\t\t\toffsets.reset()\n\t\t}\n\t}\n}\n\n// commitLoopInterval handles each commit asynchronously with a period defined\n// by ReaderConfig.CommitInterval.\nfunc (r *Reader) commitLoopInterval(ctx context.Context, gen *Generation) {\n\tticker := time.NewTicker(r.config.CommitInterval)\n\tdefer ticker.Stop()\n\n\t// the offset stash should not survive rebalances b/c the consumer may\n\t// receive new assignments.\n\toffsets := offsetStash{}\n\n\tcommit := func() {\n\t\tif err := r.commitOffsetsWithRetry(gen, offsets, defaultCommitRetries); err != nil {\n\t\t\tr.withErrorLogger(func(l Logger) { l.Printf(\"%v\", err) })\n\t\t} else {\n\t\t\toffsets.reset()\n\t\t}\n\t}\n\n\tfor {\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\t// drain the commit channel in order to prepare the final commit.\n\t\t\tfor hasCommits := true; hasCommits; {\n\t\t\t\tselect {\n\t\t\t\tcase req := <-r.commits:\n\t\t\t\t\toffsets.merge(req.commits)\n\t\t\t\tdefault:\n\t\t\t\t\thasCommits = false\n\t\t\t\t}\n\t\t\t}\n\t\t\tcommit()\n\t\t\treturn\n\n\t\tcase <-ticker.C:\n\t\t\tcommit()\n\n\t\tcase req := <-r.commits:\n\t\t\toffsets.merge(req.commits)\n\t\t}\n\t}\n}\n\n// commitLoop processes commits off the commit chan.\nfunc (r *Reader) commitLoop(ctx context.Context, gen *Generation) {\n\tr.withLogger(func(l Logger) {\n\t\tl.Printf(\"started commit for group %s\\n\", r.config.GroupID)\n\t})\n\tdefer r.withLogger(func(l Logger) {\n\t\tl.Printf(\"stopped commit for group %s\\n\", r.config.GroupID)\n\t})\n\n\tif r.useSyncCommits() {\n\t\tr.commitLoopImmediate(ctx, gen)\n\t} else {\n\t\tr.commitLoopInterval(ctx, gen)\n\t}\n}\n\n// run provides the main consumer group management loop. Each iteration performs the\n// handshake to join the Reader to the consumer group.\n//\n// This function is responsible for closing the consumer group upon exit.\nfunc (r *Reader) run(cg *ConsumerGroup) {\n\tdefer close(r.done)\n\tdefer cg.Close()\n\n\tr.withLogger(func(l Logger) {\n\t\tl.Printf(\"entering loop for consumer group, %v\\n\", r.config.GroupID)\n\t})\n\n\tfor {\n\t\t// Limit the number of attempts at waiting for the next\n\t\t// consumer generation.\n\t\tvar err error\n\t\tvar gen *Generation\n\t\tfor attempt := 1; attempt <= r.config.MaxAttempts; attempt++ {\n\t\t\tgen, err = cg.Next(r.stctx)\n\t\t\tif err == nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\tif errors.Is(err, r.stctx.Err()) {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tr.stats.errors.observe(1)\n\t\t\tr.withErrorLogger(func(l Logger) {\n\t\t\t\tl.Printf(\"%v\", err)\n\t\t\t})\n\t\t\t// Continue with next attempt...\n\t\t}\n\t\tif err != nil {\n\t\t\t// All attempts have failed.\n\t\t\tselect {\n\t\t\tcase r.runError <- err:\n\t\t\t\t// If somebody's receiving on the runError, let\n\t\t\t\t// them know the error occurred.\n\t\t\tdefault:\n\t\t\t\t// Otherwise, don't block to allow healing.\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\t\tr.stats.rebalances.observe(1)\n\n\t\tr.subscribe(gen.Assignments)\n\n\t\tgen.Start(func(ctx context.Context) {\n\t\t\tr.commitLoop(ctx, gen)\n\t\t})\n\t\tgen.Start(func(ctx context.Context) {\n\t\t\t// wait for the generation to end and then unsubscribe.\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\t// continue to next generation\n\t\t\tcase <-r.stctx.Done():\n\t\t\t\t// this will be the last loop because the reader is closed.\n\t\t\t}\n\t\t\tr.unsubscribe()\n\t\t})\n\t}\n}\n\n// ReaderConfig is a configuration object used to create new instances of\n// Reader.\ntype ReaderConfig struct {\n\t// The list of broker addresses used to connect to the kafka cluster.\n\tBrokers []string\n\n\t// GroupID holds the optional consumer group id. If GroupID is specified, then\n\t// Partition should NOT be specified e.g. 0\n\tGroupID string\n\n\t// GroupTopics allows specifying multiple topics, but can only be used in\n\t// combination with GroupID, as it is a consumer-group feature. As such, if\n\t// GroupID is set, then either Topic or GroupTopics must be defined.\n\tGroupTopics []string\n\n\t// The topic to read messages from.\n\tTopic string\n\n\t// Partition to read messages from. Either Partition or GroupID may\n\t// be assigned, but not both\n\tPartition int\n\n\t// An dialer used to open connections to the kafka server. This field is\n\t// optional, if nil, the default dialer is used instead.\n\tDialer *Dialer\n\n\t// The capacity of the internal message queue, defaults to 100 if none is\n\t// set.\n\tQueueCapacity int\n\n\t// MinBytes indicates to the broker the minimum batch size that the consumer\n\t// will accept. Setting a high minimum when consuming from a low-volume topic\n\t// may result in delayed delivery when the broker does not have enough data to\n\t// satisfy the defined minimum.\n\t//\n\t// Default: 1\n\tMinBytes int\n\n\t// MaxBytes indicates to the broker the maximum batch size that the consumer\n\t// will accept. The broker will truncate a message to satisfy this maximum, so\n\t// choose a value that is high enough for your largest message size.\n\t//\n\t// Default: 1MB\n\tMaxBytes int\n\n\t// Maximum amount of time to wait for new data to come when fetching batches\n\t// of messages from kafka.\n\t//\n\t// Default: 10s\n\tMaxWait time.Duration\n\n\t// ReadBatchTimeout amount of time to wait to fetch message from kafka messages batch.\n\t//\n\t// Default: 10s\n\tReadBatchTimeout time.Duration\n\n\t// ReadLagInterval sets the frequency at which the reader lag is updated.\n\t// Setting this field to a negative value disables lag reporting.\n\tReadLagInterval time.Duration\n\n\t// GroupBalancers is the priority-ordered list of client-side consumer group\n\t// balancing strategies that will be offered to the coordinator. The first\n\t// strategy that all group members support will be chosen by the leader.\n\t//\n\t// Default: [Range, RoundRobin]\n\t//\n\t// Only used when GroupID is set\n\tGroupBalancers []GroupBalancer\n\n\t// HeartbeatInterval sets the optional frequency at which the reader sends the consumer\n\t// group heartbeat update.\n\t//\n\t// Default: 3s\n\t//\n\t// Only used when GroupID is set\n\tHeartbeatInterval time.Duration\n\n\t// CommitInterval indicates the interval at which offsets are committed to\n\t// the broker. If 0, commits will be handled synchronously.\n\t//\n\t// Default: 0\n\t//\n\t// Only used when GroupID is set\n\tCommitInterval time.Duration\n\n\t// PartitionWatchInterval indicates how often a reader checks for partition changes.\n\t// If a reader sees a partition change (such as a partition add) it will rebalance the group\n\t// picking up new partitions.\n\t//\n\t// Default: 5s\n\t//\n\t// Only used when GroupID is set and WatchPartitionChanges is set.\n\tPartitionWatchInterval time.Duration\n\n\t// WatchForPartitionChanges is used to inform kafka-go that a consumer group should be\n\t// polling the brokers and rebalancing if any partition changes happen to the topic.\n\tWatchPartitionChanges bool\n\n\t// SessionTimeout optionally sets the length of time that may pass without a heartbeat\n\t// before the coordinator considers the consumer dead and initiates a rebalance.\n\t//\n\t// Default: 30s\n\t//\n\t// Only used when GroupID is set\n\tSessionTimeout time.Duration\n\n\t// RebalanceTimeout optionally sets the length of time the coordinator will wait\n\t// for members to join as part of a rebalance. For kafka servers under higher\n\t// load, it may be useful to set this value higher.\n\t//\n\t// Default: 30s\n\t//\n\t// Only used when GroupID is set\n\tRebalanceTimeout time.Duration\n\n\t// JoinGroupBackoff optionally sets the length of time to wait between re-joining\n\t// the consumer group after an error.\n\t//\n\t// Default: 5s\n\tJoinGroupBackoff time.Duration\n\n\t// RetentionTime optionally sets the length of time the consumer group will be saved\n\t// by the broker. -1 will disable the setting and leave the\n\t// retention up to the broker's offsets.retention.minutes property. By\n\t// default, that setting is 1 day for kafka < 2.0 and 7 days for kafka >= 2.0.\n\t//\n\t// Default: -1\n\t//\n\t// Only used when GroupID is set\n\tRetentionTime time.Duration\n\n\t// StartOffset determines from whence the consumer group should begin\n\t// consuming when it finds a partition without a committed offset. If\n\t// non-zero, it must be set to one of FirstOffset or LastOffset.\n\t//\n\t// Default: FirstOffset\n\t//\n\t// Only used when GroupID is set\n\tStartOffset int64\n\n\t// BackoffDelayMin optionally sets the smallest amount of time the reader will wait before\n\t// polling for new messages\n\t//\n\t// Default: 100ms\n\tReadBackoffMin time.Duration\n\n\t// BackoffDelayMax optionally sets the maximum amount of time the reader will wait before\n\t// polling for new messages\n\t//\n\t// Default: 1s\n\tReadBackoffMax time.Duration\n\n\t// If not nil, specifies a logger used to report internal changes within the\n\t// reader.\n\tLogger Logger\n\n\t// ErrorLogger is the logger used to report errors. If nil, the reader falls\n\t// back to using Logger instead.\n\tErrorLogger Logger\n\n\t// IsolationLevel controls the visibility of transactional records.\n\t// ReadUncommitted makes all records visible. With ReadCommitted only\n\t// non-transactional and committed records are visible.\n\tIsolationLevel IsolationLevel\n\n\t// Limit of how many attempts to connect will be made before returning the error.\n\t//\n\t// The default is to try 3 times.\n\tMaxAttempts int\n\n\t// OffsetOutOfRangeError indicates that the reader should return an error in\n\t// the event of an OffsetOutOfRange error, rather than retrying indefinitely.\n\t// This flag is being added to retain backwards-compatibility, so it will be\n\t// removed in a future version of kafka-go.\n\tOffsetOutOfRangeError bool\n}\n\n// Validate method validates ReaderConfig properties.\nfunc (config *ReaderConfig) Validate() error {\n\tif len(config.Brokers) == 0 {\n\t\treturn errors.New(\"cannot create a new kafka reader with an empty list of broker addresses\")\n\t}\n\n\tif config.Partition < 0 || config.Partition >= math.MaxInt32 {\n\t\treturn fmt.Errorf(\"partition number out of bounds: %d\", config.Partition)\n\t}\n\n\tif config.MinBytes < 0 {\n\t\treturn fmt.Errorf(\"invalid negative minimum batch size (min = %d)\", config.MinBytes)\n\t}\n\n\tif config.MaxBytes < 0 {\n\t\treturn fmt.Errorf(\"invalid negative maximum batch size (max = %d)\", config.MaxBytes)\n\t}\n\n\tif config.GroupID != \"\" {\n\t\tif config.Partition != 0 {\n\t\t\treturn errors.New(\"either Partition or GroupID may be specified, but not both\")\n\t\t}\n\n\t\tif len(config.Topic) == 0 && len(config.GroupTopics) == 0 {\n\t\t\treturn errors.New(\"either Topic or GroupTopics must be specified with GroupID\")\n\t\t}\n\t} else if len(config.Topic) == 0 {\n\t\treturn errors.New(\"cannot create a new kafka reader with an empty topic\")\n\t}\n\n\tif config.MinBytes > config.MaxBytes {\n\t\treturn fmt.Errorf(\"minimum batch size greater than the maximum (min = %d, max = %d)\", config.MinBytes, config.MaxBytes)\n\t}\n\n\tif config.ReadBackoffMax < 0 {\n\t\treturn fmt.Errorf(\"ReadBackoffMax out of bounds: %d\", config.ReadBackoffMax)\n\t}\n\n\tif config.ReadBackoffMin < 0 {\n\t\treturn fmt.Errorf(\"ReadBackoffMin out of bounds: %d\", config.ReadBackoffMin)\n\t}\n\n\treturn nil\n}\n\n// ReaderStats is a data structure returned by a call to Reader.Stats that exposes\n// details about the behavior of the reader.\ntype ReaderStats struct {\n\tDials int64 `metric:\"kafka.reader.dial.count\" type:\"counter\"`\n\tFetches int64 `metric:\"kafka.reader.fetch.count\" type:\"counter\"`\n\tMessages int64 `metric:\"kafka.reader.message.count\" type:\"counter\"`\n\tBytes int64 `metric:\"kafka.reader.message.bytes\" type:\"counter\"`\n\tRebalances int64 `metric:\"kafka.reader.rebalance.count\" type:\"counter\"`\n\tTimeouts int64 `metric:\"kafka.reader.timeout.count\" type:\"counter\"`\n\tErrors int64 `metric:\"kafka.reader.error.count\" type:\"counter\"`\n\n\tDialTime DurationStats `metric:\"kafka.reader.dial.seconds\"`\n\tReadTime DurationStats `metric:\"kafka.reader.read.seconds\"`\n\tWaitTime DurationStats `metric:\"kafka.reader.wait.seconds\"`\n\tFetchSize SummaryStats `metric:\"kafka.reader.fetch.size\"`\n\tFetchBytes SummaryStats `metric:\"kafka.reader.fetch.bytes\"`\n\n\tOffset int64 `metric:\"kafka.reader.offset\" type:\"gauge\"`\n\tLag int64 `metric:\"kafka.reader.lag\" type:\"gauge\"`\n\tMinBytes int64 `metric:\"kafka.reader.fetch_bytes.min\" type:\"gauge\"`\n\tMaxBytes int64 `metric:\"kafka.reader.fetch_bytes.max\" type:\"gauge\"`\n\tMaxWait time.Duration `metric:\"kafka.reader.fetch_wait.max\" type:\"gauge\"`\n\tQueueLength int64 `metric:\"kafka.reader.queue.length\" type:\"gauge\"`\n\tQueueCapacity int64 `metric:\"kafka.reader.queue.capacity\" type:\"gauge\"`\n\n\tClientID string `tag:\"client_id\"`\n\tTopic string `tag:\"topic\"`\n\tPartition string `tag:\"partition\"`\n\n\t// The original `Fetches` field had a typo where the metric name was called\n\t// \"kafak...\" instead of \"kafka...\", in order to offer time to fix monitors\n\t// that may be relying on this mistake we are temporarily introducing this\n\t// field.\n\tDeprecatedFetchesWithTypo int64 `metric:\"kafak.reader.fetch.count\" type:\"counter\"`\n}\n\n// readerStats is a struct that contains statistics on a reader.\ntype readerStats struct {\n\tdials counter\n\tfetches counter\n\tmessages counter\n\tbytes counter\n\trebalances counter\n\ttimeouts counter\n\terrors counter\n\tdialTime summary\n\treadTime summary\n\twaitTime summary\n\tfetchSize summary\n\tfetchBytes summary\n\toffset gauge\n\tlag gauge\n\tpartition string\n}\n\n// NewReader creates and returns a new Reader configured with config.\n// The offset is initialized to FirstOffset.\nfunc NewReader(config ReaderConfig) *Reader {\n\tif err := config.Validate(); err != nil {\n\t\tpanic(err)\n\t}\n\n\tif config.GroupID != \"\" {\n\t\tif len(config.GroupBalancers) == 0 {\n\t\t\tconfig.GroupBalancers = []GroupBalancer{\n\t\t\t\tRangeGroupBalancer{},\n\t\t\t\tRoundRobinGroupBalancer{},\n\t\t\t}\n\t\t}\n\t}\n\n\tif config.Dialer == nil {\n\t\tconfig.Dialer = DefaultDialer\n\t}\n\n\tif config.MaxBytes == 0 {\n\t\tconfig.MaxBytes = 1e6 // 1 MB\n\t}\n\n\tif config.MinBytes == 0 {\n\t\tconfig.MinBytes = defaultFetchMinBytes\n\t}\n\n\tif config.MaxWait == 0 {\n\t\tconfig.MaxWait = 10 * time.Second\n\t}\n\n\tif config.ReadBatchTimeout == 0 {\n\t\tconfig.ReadBatchTimeout = 10 * time.Second\n\t}\n\n\tif config.ReadLagInterval == 0 {\n\t\tconfig.ReadLagInterval = 1 * time.Minute\n\t}\n\n\tif config.ReadBackoffMin == 0 {\n\t\tconfig.ReadBackoffMin = defaultReadBackoffMin\n\t}\n\n\tif config.ReadBackoffMax == 0 {\n\t\tconfig.ReadBackoffMax = defaultReadBackoffMax\n\t}\n\n\tif config.ReadBackoffMax < config.ReadBackoffMin {\n\t\tpanic(fmt.Errorf(\"ReadBackoffMax %d smaller than ReadBackoffMin %d\", config.ReadBackoffMax, config.ReadBackoffMin))\n\t}\n\n\tif config.QueueCapacity == 0 {\n\t\tconfig.QueueCapacity = 100\n\t}\n\n\tif config.MaxAttempts == 0 {\n\t\tconfig.MaxAttempts = 3\n\t}\n\n\t// when configured as a consumer group; stats should report a partition of -1\n\treaderStatsPartition := config.Partition\n\tif config.GroupID != \"\" {\n\t\treaderStatsPartition = -1\n\t}\n\n\t// when configured as a consume group, start version as 1 to ensure that only\n\t// the rebalance function will start readers\n\tversion := int64(0)\n\tif config.GroupID != \"\" {\n\t\tversion = 1\n\t}\n\n\tstctx, stop := context.WithCancel(context.Background())\n\tr := &Reader{\n\t\tconfig: config,\n\t\tmsgs: make(chan readerMessage, config.QueueCapacity),\n\t\tcancel: func() {},\n\t\tcommits: make(chan commitRequest, config.QueueCapacity),\n\t\tstop: stop,\n\t\toffset: FirstOffset,\n\t\tstctx: stctx,\n\t\tstats: &readerStats{\n\t\t\tdialTime: makeSummary(),\n\t\t\treadTime: makeSummary(),\n\t\t\twaitTime: makeSummary(),\n\t\t\tfetchSize: makeSummary(),\n\t\t\tfetchBytes: makeSummary(),\n\t\t\t// Generate the string representation of the partition number only\n\t\t\t// once when the reader is created.\n\t\t\tpartition: strconv.Itoa(readerStatsPartition),\n\t\t},\n\t\tversion: version,\n\t}\n\tif r.useConsumerGroup() {\n\t\tr.done = make(chan struct{})\n\t\tr.runError = make(chan error)\n\t\tcg, err := NewConsumerGroup(ConsumerGroupConfig{\n\t\t\tID: r.config.GroupID,\n\t\t\tBrokers: r.config.Brokers,\n\t\t\tDialer: r.config.Dialer,\n\t\t\tTopics: r.getTopics(),\n\t\t\tGroupBalancers: r.config.GroupBalancers,\n\t\t\tHeartbeatInterval: r.config.HeartbeatInterval,\n\t\t\tPartitionWatchInterval: r.config.PartitionWatchInterval,\n\t\t\tWatchPartitionChanges: r.config.WatchPartitionChanges,\n\t\t\tSessionTimeout: r.config.SessionTimeout,\n\t\t\tRebalanceTimeout: r.config.RebalanceTimeout,\n\t\t\tJoinGroupBackoff: r.config.JoinGroupBackoff,\n\t\t\tRetentionTime: r.config.RetentionTime,\n\t\t\tStartOffset: r.config.StartOffset,\n\t\t\tLogger: r.config.Logger,\n\t\t\tErrorLogger: r.config.ErrorLogger,\n\t\t})\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t\tgo r.run(cg)\n\t}\n\n\treturn r\n}\n\n// Config returns the reader's configuration.\nfunc (r *Reader) Config() ReaderConfig {\n\treturn r.config\n}\n\n// Close closes the stream, preventing the program from reading any more\n// messages from it.\nfunc (r *Reader) Close() error {\n\tatomic.StoreUint32(&r.once, 1)\n\n\tr.mutex.Lock()\n\tclosed := r.closed\n\tr.closed = true\n\tr.mutex.Unlock()\n\n\tr.cancel()\n\tr.stop()\n\tr.join.Wait()\n\n\tif r.done != nil {\n\t\t<-r.done\n\t}\n\n\tif !closed {\n\t\tclose(r.msgs)\n\t}\n\n\treturn nil\n}\n\n// ReadMessage reads and return the next message from the r. The method call\n// blocks until a message becomes available, or an error occurs. The program\n// may also specify a context to asynchronously cancel the blocking operation.\n//\n// The method returns io.EOF to indicate that the reader has been closed.\n//\n// If consumer groups are used, ReadMessage will automatically commit the\n// offset when called. Note that this could result in an offset being committed\n// before the message is fully processed.\n//\n// If more fine-grained control of when offsets are committed is required, it\n// is recommended to use FetchMessage with CommitMessages instead.\nfunc (r *Reader) ReadMessage(ctx context.Context) (Message, error) {\n\tm, err := r.FetchMessage(ctx)\n\tif err != nil {\n\t\treturn Message{}, fmt.Errorf(\"fetching message: %w\", err)\n\t}\n\n\tif r.useConsumerGroup() {\n\t\tif err := r.CommitMessages(ctx, m); err != nil {\n\t\t\treturn Message{}, fmt.Errorf(\"committing message: %w\", err)\n\t\t}\n\t}\n\n\treturn m, nil\n}\n\n// FetchMessage reads and return the next message from the r. The method call\n// blocks until a message becomes available, or an error occurs. The program\n// may also specify a context to asynchronously cancel the blocking operation.\n//\n// The method returns io.EOF to indicate that the reader has been closed.\n//\n// FetchMessage does not commit offsets automatically when using consumer groups.\n// Use CommitMessages to commit the offset.\nfunc (r *Reader) FetchMessage(ctx context.Context) (Message, error) {\n\tr.activateReadLag()\n\n\tfor {\n\t\tr.mutex.Lock()\n\n\t\tif !r.closed && r.version == 0 {\n\t\t\tr.start(r.getTopicPartitionOffset())\n\t\t}\n\n\t\tversion := r.version\n\t\tr.mutex.Unlock()\n\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\treturn Message{}, ctx.Err()\n\n\t\tcase err := <-r.runError:\n\t\t\treturn Message{}, err\n\n\t\tcase m, ok := <-r.msgs:\n\t\t\tif !ok {\n\t\t\t\treturn Message{}, io.EOF\n\t\t\t}\n\n\t\t\tif m.version >= version {\n\t\t\t\tr.mutex.Lock()\n\n\t\t\t\tswitch {\n\t\t\t\tcase m.error != nil:\n\t\t\t\tcase version == r.version:\n\t\t\t\t\tr.offset = m.message.Offset + 1\n\t\t\t\t\tr.lag = m.watermark - r.offset\n\t\t\t\t}\n\n\t\t\t\tr.mutex.Unlock()\n\n\t\t\t\tif errors.Is(m.error, io.EOF) {\n\t\t\t\t\t// io.EOF is used as a marker to indicate that the stream\n\t\t\t\t\t// has been closed, in case it was received from the inner\n\t\t\t\t\t// reader we don't want to confuse the program and replace\n\t\t\t\t\t// the error with io.ErrUnexpectedEOF.\n\t\t\t\t\tm.error = io.ErrUnexpectedEOF\n\t\t\t\t}\n\n\t\t\t\treturn m.message, m.error\n\t\t\t}\n\t\t}\n\t}\n}\n\n// CommitMessages commits the list of messages passed as argument. The program\n// may pass a context to asynchronously cancel the commit operation when it was\n// configured to be blocking.\n//\n// Because kafka consumer groups track a single offset per partition, the\n// highest message offset passed to CommitMessages will cause all previous\n// messages to be committed. Applications need to account for these Kafka\n// limitations when committing messages, and maintain message ordering if they\n// need strong delivery guarantees. This property makes it valid to pass only\n// the last message seen to CommitMessages in order to move the offset of the\n// topic/partition it belonged to forward, effectively committing all previous\n// messages in the partition.\nfunc (r *Reader) CommitMessages(ctx context.Context, msgs ...Message) error {\n\tif !r.useConsumerGroup() {\n\t\treturn errOnlyAvailableWithGroup\n\t}\n\n\tvar errch <-chan error\n\tcreq := commitRequest{\n\t\tcommits: makeCommits(msgs...),\n\t}\n\n\tif r.useSyncCommits() {\n\t\tch := make(chan error, 1)\n\t\terrch, creq.errch = ch, ch\n\t}\n\n\tselect {\n\tcase r.commits <- creq:\n\tcase <-ctx.Done():\n\t\treturn ctx.Err()\n\tcase <-r.stctx.Done():\n\t\t// This context is used to ensure we don't allow commits after the\n\t\t// reader was closed.\n\t\treturn io.ErrClosedPipe\n\t}\n\n\tif !r.useSyncCommits() {\n\t\treturn nil\n\t}\n\n\tselect {\n\tcase <-ctx.Done():\n\t\treturn ctx.Err()\n\tcase err := <-errch:\n\t\treturn err\n\t}\n}\n\n// ReadLag returns the current lag of the reader by fetching the last offset of\n// the topic and partition and computing the difference between that value and\n// the offset of the last message returned by ReadMessage.\n//\n// This method is intended to be used in cases where a program may be unable to\n// call ReadMessage to update the value returned by Lag, but still needs to get\n// an up to date estimation of how far behind the reader is. For example when\n// the consumer is not ready to process the next message.\n//\n// The function returns a lag of zero when the reader's current offset is\n// negative.\nfunc (r *Reader) ReadLag(ctx context.Context) (lag int64, err error) {\n\tif r.useConsumerGroup() {\n\t\treturn 0, errNotAvailableWithGroup\n\t}\n\n\ttype offsets struct {\n\t\tfirst int64\n\t\tlast int64\n\t}\n\n\toffch := make(chan offsets, 1)\n\terrch := make(chan error, 1)\n\n\tgo func() {\n\t\tvar off offsets\n\t\tvar err error\n\n\t\tfor _, broker := range r.config.Brokers {\n\t\t\tvar conn *Conn\n\n\t\t\tif conn, err = r.config.Dialer.DialLeader(ctx, \"tcp\", broker, r.config.Topic, r.config.Partition); err != nil {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tdeadline, _ := ctx.Deadline()\n\t\t\tconn.SetDeadline(deadline)\n\n\t\t\toff.first, off.last, err = conn.ReadOffsets()\n\t\t\tconn.Close()\n\n\t\t\tif err == nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\tif err != nil {\n\t\t\terrch <- err\n\t\t} else {\n\t\t\toffch <- off\n\t\t}\n\t}()\n\n\tselect {\n\tcase off := <-offch:\n\t\tswitch cur := r.Offset(); {\n\t\tcase cur == FirstOffset:\n\t\t\tlag = off.last - off.first\n\n\t\tcase cur == LastOffset:\n\t\t\tlag = 0\n\n\t\tdefault:\n\t\t\tlag = off.last - cur\n\t\t}\n\tcase err = <-errch:\n\tcase <-ctx.Done():\n\t\terr = ctx.Err()\n\t}\n\n\treturn\n}\n\n// Offset returns the current absolute offset of the reader, or -1\n// if r is backed by a consumer group.\nfunc (r *Reader) Offset() int64 {\n\tif r.useConsumerGroup() {\n\t\treturn -1\n\t}\n\n\tr.mutex.Lock()\n\toffset := r.offset\n\tr.mutex.Unlock()\n\tr.withLogger(func(log Logger) {\n\t\tlog.Printf(\"looking up offset of kafka reader for partition %d of %s: %s\", r.config.Partition, r.config.Topic, toHumanOffset(offset))\n\t})\n\treturn offset\n}\n\n// Lag returns the lag of the last message returned by ReadMessage, or -1\n// if r is backed by a consumer group.\nfunc (r *Reader) Lag() int64 {\n\tif r.useConsumerGroup() {\n\t\treturn -1\n\t}\n\n\tr.mutex.Lock()\n\tlag := r.lag\n\tr.mutex.Unlock()\n\treturn lag\n}\n\n// SetOffset changes the offset from which the next batch of messages will be\n// read. The method fails with io.ErrClosedPipe if the reader has already been closed.\n//\n// From version 0.2.0, FirstOffset and LastOffset can be used to indicate the first\n// or last available offset in the partition. Please note while -1 and -2 were accepted\n// to indicate the first or last offset in previous versions, the meanings of the numbers\n// were swapped in 0.2.0 to match the meanings in other libraries and the Kafka protocol\n// specification.\nfunc (r *Reader) SetOffset(offset int64) error {\n\tif r.useConsumerGroup() {\n\t\treturn errNotAvailableWithGroup\n\t}\n\n\tvar err error\n\tr.mutex.Lock()\n\n\tif r.closed {\n\t\terr = io.ErrClosedPipe\n\t} else if offset != r.offset {\n\t\tr.withLogger(func(log Logger) {\n\t\t\tlog.Printf(\"setting the offset of the kafka reader for partition %d of %s from %s to %s\",\n\t\t\t\tr.config.Partition, r.config.Topic, toHumanOffset(r.offset), toHumanOffset(offset))\n\t\t})\n\t\tr.offset = offset\n\n\t\tif r.version != 0 {\n\t\t\tr.start(r.getTopicPartitionOffset())\n\t\t}\n\n\t\tr.activateReadLag()\n\t}\n\n\tr.mutex.Unlock()\n\treturn err\n}\n\n// SetOffsetAt changes the offset from which the next batch of messages will be\n// read given the timestamp t.\n//\n// The method fails if the unable to connect partition leader, or unable to read the offset\n// given the ts, or if the reader has been closed.\nfunc (r *Reader) SetOffsetAt(ctx context.Context, t time.Time) error {\n\tr.mutex.Lock()\n\tif r.closed {\n\t\tr.mutex.Unlock()\n\t\treturn io.ErrClosedPipe\n\t}\n\tr.mutex.Unlock()\n\n\tif len(r.config.Brokers) < 1 {\n\t\treturn errors.New(\"no brokers in config\")\n\t}\n\tvar conn *Conn\n\tvar err error\n\tfor _, broker := range r.config.Brokers {\n\t\tconn, err = r.config.Dialer.DialLeader(ctx, \"tcp\", broker, r.config.Topic, r.config.Partition)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tdeadline, _ := ctx.Deadline()\n\t\tconn.SetDeadline(deadline)\n\t\toffset, err := conn.ReadOffset(t)\n\t\tconn.Close()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\treturn r.SetOffset(offset)\n\t}\n\treturn fmt.Errorf(\"error dialing all brokers, one of the errors: %w\", err)\n}\n\n// Stats returns a snapshot of the reader stats since the last time the method\n// was called, or since the reader was created if it is called for the first\n// time.\n//\n// A typical use of this method is to spawn a goroutine that will periodically\n// call Stats on a kafka reader and report the metrics to a stats collection\n// system.\nfunc (r *Reader) Stats() ReaderStats {\n\tstats := ReaderStats{\n\t\tDials: r.stats.dials.snapshot(),\n\t\tFetches: r.stats.fetches.snapshot(),\n\t\tMessages: r.stats.messages.snapshot(),\n\t\tBytes: r.stats.bytes.snapshot(),\n\t\tRebalances: r.stats.rebalances.snapshot(),\n\t\tTimeouts: r.stats.timeouts.snapshot(),\n\t\tErrors: r.stats.errors.snapshot(),\n\t\tDialTime: r.stats.dialTime.snapshotDuration(),\n\t\tReadTime: r.stats.readTime.snapshotDuration(),\n\t\tWaitTime: r.stats.waitTime.snapshotDuration(),\n\t\tFetchSize: r.stats.fetchSize.snapshot(),\n\t\tFetchBytes: r.stats.fetchBytes.snapshot(),\n\t\tOffset: r.stats.offset.snapshot(),\n\t\tLag: r.stats.lag.snapshot(),\n\t\tMinBytes: int64(r.config.MinBytes),\n\t\tMaxBytes: int64(r.config.MaxBytes),\n\t\tMaxWait: r.config.MaxWait,\n\t\tQueueLength: int64(len(r.msgs)),\n\t\tQueueCapacity: int64(cap(r.msgs)),\n\t\tClientID: r.config.Dialer.ClientID,\n\t\tTopic: r.config.Topic,\n\t\tPartition: r.stats.partition,\n\t}\n\t// TODO: remove when we get rid of the deprecated field.\n\tstats.DeprecatedFetchesWithTypo = stats.Fetches\n\treturn stats\n}\n\nfunc (r *Reader) getTopicPartitionOffset() map[topicPartition]int64 {\n\tkey := topicPartition{topic: r.config.Topic, partition: int32(r.config.Partition)}\n\treturn map[topicPartition]int64{key: r.offset}\n}\n\nfunc (r *Reader) withLogger(do func(Logger)) {\n\tif r.config.Logger != nil {\n\t\tdo(r.config.Logger)\n\t}\n}\n\nfunc (r *Reader) withErrorLogger(do func(Logger)) {\n\tif r.config.ErrorLogger != nil {\n\t\tdo(r.config.ErrorLogger)\n\t} else {\n\t\tr.withLogger(do)\n\t}\n}\n\nfunc (r *Reader) activateReadLag() {\n\tif r.config.ReadLagInterval > 0 && atomic.CompareAndSwapUint32(&r.once, 0, 1) {\n\t\t// read lag will only be calculated when not using consumer groups\n\t\t// todo discuss how capturing read lag should interact with rebalancing\n\t\tif !r.useConsumerGroup() {\n\t\t\tgo r.readLag(r.stctx)\n\t\t}\n\t}\n}\n\nfunc (r *Reader) readLag(ctx context.Context) {\n\tticker := time.NewTicker(r.config.ReadLagInterval)\n\tdefer ticker.Stop()\n\n\tfor {\n\t\ttimeout, cancel := context.WithTimeout(ctx, r.config.ReadLagInterval/2)\n\t\tlag, err := r.ReadLag(timeout)\n\t\tcancel()\n\n\t\tif err != nil {\n\t\t\tr.stats.errors.observe(1)\n\t\t\tr.withErrorLogger(func(log Logger) {\n\t\t\t\tlog.Printf(\"kafka reader failed to read lag of partition %d of %s: %s\", r.config.Partition, r.config.Topic, err)\n\t\t\t})\n\t\t} else {\n\t\t\tr.stats.lag.observe(lag)\n\t\t}\n\n\t\tselect {\n\t\tcase <-ticker.C:\n\t\tcase <-ctx.Done():\n\t\t\treturn\n\t\t}\n\t}\n}\n\nfunc (r *Reader) start(offsetsByPartition map[topicPartition]int64) {\n\tif r.closed {\n\t\t// don't start child reader if parent Reader is closed\n\t\treturn\n\t}\n\n\tctx, cancel := context.WithCancel(context.Background())\n\n\tr.cancel() // always cancel the previous reader\n\tr.cancel = cancel\n\tr.version++\n\n\tr.join.Add(len(offsetsByPartition))\n\tfor key, offset := range offsetsByPartition {\n\t\tgo func(ctx context.Context, key topicPartition, offset int64, join *sync.WaitGroup) {\n\t\t\tdefer join.Done()\n\n\t\t\t(&reader{\n\t\t\t\tdialer: r.config.Dialer,\n\t\t\t\tlogger: r.config.Logger,\n\t\t\t\terrorLogger: r.config.ErrorLogger,\n\t\t\t\tbrokers: r.config.Brokers,\n\t\t\t\ttopic: key.topic,\n\t\t\t\tpartition: int(key.partition),\n\t\t\t\tminBytes: r.config.MinBytes,\n\t\t\t\tmaxBytes: r.config.MaxBytes,\n\t\t\t\tmaxWait: r.config.MaxWait,\n\t\t\t\treadBatchTimeout: r.config.ReadBatchTimeout,\n\t\t\t\tbackoffDelayMin: r.config.ReadBackoffMin,\n\t\t\t\tbackoffDelayMax: r.config.ReadBackoffMax,\n\t\t\t\tversion: r.version,\n\t\t\t\tmsgs: r.msgs,\n\t\t\t\tstats: r.stats,\n\t\t\t\tisolationLevel: r.config.IsolationLevel,\n\t\t\t\tmaxAttempts: r.config.MaxAttempts,\n\n\t\t\t\t// backwards-compatibility flags\n\t\t\t\toffsetOutOfRangeError: r.config.OffsetOutOfRangeError,\n\t\t\t}).run(ctx, offset)\n\t\t}(ctx, key, offset, &r.join)\n\t}\n}\n\n// A reader reads messages from kafka and produces them on its channels, it's\n// used as a way to asynchronously fetch messages while the main program reads\n// them using the high level reader API.\ntype reader struct {\n\tdialer *Dialer\n\tlogger Logger\n\terrorLogger Logger\n\tbrokers []string\n\ttopic string\n\tpartition int\n\tminBytes int\n\tmaxBytes int\n\tmaxWait time.Duration\n\treadBatchTimeout time.Duration\n\tbackoffDelayMin time.Duration\n\tbackoffDelayMax time.Duration\n\tversion int64\n\tmsgs chan<- readerMessage\n\tstats *readerStats\n\tisolationLevel IsolationLevel\n\tmaxAttempts int\n\n\toffsetOutOfRangeError bool\n}\n\ntype readerMessage struct {\n\tversion int64\n\tmessage Message\n\twatermark int64\n\terror error\n}\n\nfunc (r *reader) run(ctx context.Context, offset int64) {\n\t// This is the reader's main loop, it only ends if the context is canceled\n\t// and will keep attempting to reader messages otherwise.\n\t//\n\t// Retrying indefinitely has the nice side effect of preventing Read calls\n\t// on the parent reader to block if connection to the kafka server fails,\n\t// the reader keeps reporting errors on the error channel which will then\n\t// be surfaced to the program.\n\t// If the reader wasn't retrying then the program would block indefinitely\n\t// on a Read call after reading the first error.\n\tfor attempt := 0; true; attempt++ {\n\t\tif attempt != 0 {\n\t\t\tif !sleep(ctx, backoff(attempt, r.backoffDelayMin, r.backoffDelayMax)) {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\n\t\tr.withLogger(func(log Logger) {\n\t\t\tlog.Printf(\"initializing kafka reader for partition %d of %s starting at offset %d\", r.partition, r.topic, toHumanOffset(offset))\n\t\t})\n\n\t\tconn, start, err := r.initialize(ctx, offset)\n\t\tif err != nil {\n\t\t\tif errors.Is(err, OffsetOutOfRange) {\n\t\t\t\tif r.offsetOutOfRangeError {\n\t\t\t\t\tr.sendError(ctx, err)\n\t\t\t\t\treturn\n\t\t\t\t}\n\n\t\t\t\t// This would happen if the requested offset is passed the last\n\t\t\t\t// offset on the partition leader. In that case we're just going\n\t\t\t\t// to retry later hoping that enough data has been produced.\n\t\t\t\tr.withErrorLogger(func(log Logger) {\n\t\t\t\t\tlog.Printf(\"error initializing the kafka reader for partition %d of %s: %s\", r.partition, r.topic, err)\n\t\t\t\t})\n\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\t// Perform a configured number of attempts before\n\t\t\t// reporting first errors, this helps mitigate\n\t\t\t// situations where the kafka server is temporarily\n\t\t\t// unavailable.\n\t\t\tif attempt >= r.maxAttempts {\n\t\t\t\tr.sendError(ctx, err)\n\t\t\t} else {\n\t\t\t\tr.stats.errors.observe(1)\n\t\t\t\tr.withErrorLogger(func(log Logger) {\n\t\t\t\t\tlog.Printf(\"error initializing the kafka reader for partition %d of %s: %s\", r.partition, r.topic, err)\n\t\t\t\t})\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\n\t\t// Resetting the attempt counter ensures that if a failure occurs after\n\t\t// a successful initialization we don't keep increasing the backoff\n\t\t// timeout.\n\t\tattempt = 0\n\n\t\t// Now we're sure to have an absolute offset number, may anything happen\n\t\t// to the connection we know we'll want to restart from this offset.\n\t\toffset = start\n\n\t\terrcount := 0\n\treadLoop:\n\t\tfor {\n\t\t\tif !sleep(ctx, backoff(errcount, r.backoffDelayMin, r.backoffDelayMax)) {\n\t\t\t\tconn.Close()\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\toffset, err = r.read(ctx, offset, conn)\n\t\t\tswitch {\n\t\t\tcase err == nil:\n\t\t\t\terrcount = 0\n\t\t\t\tcontinue\n\n\t\t\tcase errors.Is(err, io.EOF):\n\t\t\t\t// done with this batch of messages...carry on. note that this\n\t\t\t\t// block relies on the batch repackaging real io.EOF errors as\n\t\t\t\t// io.UnexpectedEOF. otherwise, we would end up swallowing real\n\t\t\t\t// errors here.\n\t\t\t\terrcount = 0\n\t\t\t\tcontinue\n\n\t\t\tcase errors.Is(err, io.ErrNoProgress):\n\t\t\t\t// This error is returned by the Conn when it believes the connection\n\t\t\t\t// has been corrupted, so we need to explicitly close it. Since we are\n\t\t\t\t// explicitly handling it and a retry will pick up, we can suppress the\n\t\t\t\t// error metrics and logs for this case.\n\t\t\t\tconn.Close()\n\t\t\t\tbreak readLoop\n\n\t\t\tcase errors.Is(err, UnknownTopicOrPartition):\n\t\t\t\tr.withErrorLogger(func(log Logger) {\n\t\t\t\t\tlog.Printf(\"failed to read from current broker %v for partition %d of %s at offset %d: %v\", r.brokers, r.partition, r.topic, toHumanOffset(offset), err)\n\t\t\t\t})\n\n\t\t\t\tconn.Close()\n\n\t\t\t\t// The next call to .initialize will re-establish a connection to the proper\n\t\t\t\t// topic/partition broker combo.\n\t\t\t\tr.stats.rebalances.observe(1)\n\t\t\t\tbreak readLoop\n\n\t\t\tcase errors.Is(err, NotLeaderForPartition):\n\t\t\t\tr.withErrorLogger(func(log Logger) {\n\t\t\t\t\tlog.Printf(\"failed to read from current broker for partition %d of %s at offset %d: %v\", r.partition, r.topic, toHumanOffset(offset), err)\n\t\t\t\t})\n\n\t\t\t\tconn.Close()\n\n\t\t\t\t// The next call to .initialize will re-establish a connection to the proper\n\t\t\t\t// partition leader.\n\t\t\t\tr.stats.rebalances.observe(1)\n\t\t\t\tbreak readLoop\n\n\t\t\tcase errors.Is(err, RequestTimedOut):\n\t\t\t\t// Timeout on the kafka side, this can be safely retried.\n\t\t\t\terrcount = 0\n\t\t\t\tr.withLogger(func(log Logger) {\n\t\t\t\t\tlog.Printf(\"no messages received from kafka within the allocated time for partition %d of %s at offset %d: %v\", r.partition, r.topic, toHumanOffset(offset), err)\n\t\t\t\t})\n\t\t\t\tr.stats.timeouts.observe(1)\n\t\t\t\tcontinue\n\n\t\t\tcase errors.Is(err, OffsetOutOfRange):\n\t\t\t\tfirst, last, err := r.readOffsets(conn)\n\t\t\t\tif err != nil {\n\t\t\t\t\tr.withErrorLogger(func(log Logger) {\n\t\t\t\t\t\tlog.Printf(\"the kafka reader got an error while attempting to determine whether it was reading before the first offset or after the last offset of partition %d of %s: %s\", r.partition, r.topic, err)\n\t\t\t\t\t})\n\t\t\t\t\tconn.Close()\n\t\t\t\t\tbreak readLoop\n\t\t\t\t}\n\n\t\t\t\tswitch {\n\t\t\t\tcase offset < first:\n\t\t\t\t\tr.withErrorLogger(func(log Logger) {\n\t\t\t\t\t\tlog.Printf(\"the kafka reader is reading before the first offset for partition %d of %s, skipping from offset %d to %d (%d messages)\", r.partition, r.topic, toHumanOffset(offset), first, first-offset)\n\t\t\t\t\t})\n\t\t\t\t\toffset, errcount = first, 0\n\t\t\t\t\tcontinue // retry immediately so we don't keep falling behind due to the backoff\n\n\t\t\t\tcase offset < last:\n\t\t\t\t\terrcount = 0\n\t\t\t\t\tcontinue // more messages have already become available, retry immediately\n\n\t\t\t\tdefault:\n\t\t\t\t\t// We may be reading past the last offset, will retry later.\n\t\t\t\t\tr.withErrorLogger(func(log Logger) {\n\t\t\t\t\t\tlog.Printf(\"the kafka reader is reading passed the last offset for partition %d of %s at offset %d\", r.partition, r.topic, toHumanOffset(offset))\n\t\t\t\t\t})\n\t\t\t\t}\n\n\t\t\tcase errors.Is(err, context.Canceled):\n\t\t\t\t// Another reader has taken over, we can safely quit.\n\t\t\t\tconn.Close()\n\t\t\t\treturn\n\n\t\t\tcase errors.Is(err, errUnknownCodec):\n\t\t\t\t// The compression codec is either unsupported or has not been\n\t\t\t\t// imported. This is a fatal error b/c the reader cannot\n\t\t\t\t// proceed.\n\t\t\t\tr.sendError(ctx, err)\n\t\t\t\tbreak readLoop\n\n\t\t\tdefault:\n\t\t\t\tvar kafkaError Error\n\t\t\t\tif errors.As(err, &kafkaError) {\n\t\t\t\t\tr.sendError(ctx, err)\n\t\t\t\t} else {\n\t\t\t\t\tr.withErrorLogger(func(log Logger) {\n\t\t\t\t\t\tlog.Printf(\"the kafka reader got an unknown error reading partition %d of %s at offset %d: %s\", r.partition, r.topic, toHumanOffset(offset), err)\n\t\t\t\t\t})\n\t\t\t\t\tr.stats.errors.observe(1)\n\t\t\t\t\tconn.Close()\n\t\t\t\t\tbreak readLoop\n\t\t\t\t}\n\t\t\t}\n\n\t\t\terrcount++\n\t\t}\n\t}\n}\n\nfunc (r *reader) initialize(ctx context.Context, offset int64) (conn *Conn, start int64, err error) {\n\tfor i := 0; i != len(r.brokers) && conn == nil; i++ {\n\t\tbroker := r.brokers[i]\n\t\tvar first, last int64\n\n\t\tt0 := time.Now()\n\t\tconn, err = r.dialer.DialLeader(ctx, \"tcp\", broker, r.topic, r.partition)\n\t\tt1 := time.Now()\n\t\tr.stats.dials.observe(1)\n\t\tr.stats.dialTime.observeDuration(t1.Sub(t0))\n\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\n\t\tif first, last, err = r.readOffsets(conn); err != nil {\n\t\t\tconn.Close()\n\t\t\tconn = nil\n\t\t\tbreak\n\t\t}\n\n\t\tswitch {\n\t\tcase offset == FirstOffset:\n\t\t\toffset = first\n\n\t\tcase offset == LastOffset:\n\t\t\toffset = last\n\n\t\tcase offset < first:\n\t\t\toffset = first\n\t\t}\n\n\t\tr.withLogger(func(log Logger) {\n\t\t\tlog.Printf(\"the kafka reader for partition %d of %s is seeking to offset %d\", r.partition, r.topic, toHumanOffset(offset))\n\t\t})\n\n\t\tif start, err = conn.Seek(offset, SeekAbsolute); err != nil {\n\t\t\tconn.Close()\n\t\t\tconn = nil\n\t\t\tbreak\n\t\t}\n\n\t\tconn.SetDeadline(time.Time{})\n\t}\n\n\treturn\n}\n\nfunc (r *reader) read(ctx context.Context, offset int64, conn *Conn) (int64, error) {\n\tr.stats.fetches.observe(1)\n\tr.stats.offset.observe(offset)\n\n\tt0 := time.Now()\n\tconn.SetReadDeadline(t0.Add(r.maxWait))\n\n\tbatch := conn.ReadBatchWith(ReadBatchConfig{\n\t\tMinBytes: r.minBytes,\n\t\tMaxBytes: r.maxBytes,\n\t\tIsolationLevel: r.isolationLevel,\n\t})\n\thighWaterMark := batch.HighWaterMark()\n\n\tt1 := time.Now()\n\tr.stats.waitTime.observeDuration(t1.Sub(t0))\n\n\tvar msg Message\n\tvar err error\n\tvar size int64\n\tvar bytes int64\n\n\tfor {\n\t\tconn.SetReadDeadline(time.Now().Add(r.readBatchTimeout))\n\n\t\tif msg, err = batch.ReadMessage(); err != nil {\n\t\t\tbatch.Close()\n\t\t\tbreak\n\t\t}\n\n\t\tn := int64(len(msg.Key) + len(msg.Value))\n\t\tr.stats.messages.observe(1)\n\t\tr.stats.bytes.observe(n)\n\n\t\tif err = r.sendMessage(ctx, msg, highWaterMark); err != nil {\n\t\t\tbatch.Close()\n\t\t\tbreak\n\t\t}\n\n\t\toffset = msg.Offset + 1\n\t\tr.stats.offset.observe(offset)\n\t\tr.stats.lag.observe(highWaterMark - offset)\n\n\t\tsize++\n\t\tbytes += n\n\t}\n\n\tconn.SetReadDeadline(time.Time{})\n\n\tt2 := time.Now()\n\tr.stats.readTime.observeDuration(t2.Sub(t1))\n\tr.stats.fetchSize.observe(size)\n\tr.stats.fetchBytes.observe(bytes)\n\treturn offset, err\n}\n\nfunc (r *reader) readOffsets(conn *Conn) (first, last int64, err error) {\n\tconn.SetDeadline(time.Now().Add(10 * time.Second))\n\treturn conn.ReadOffsets()\n}\n\nfunc (r *reader) sendMessage(ctx context.Context, msg Message, watermark int64) error {\n\tselect {\n\tcase r.msgs <- readerMessage{version: r.version, message: msg, watermark: watermark}:\n\t\treturn nil\n\tcase <-ctx.Done():\n\t\treturn ctx.Err()\n\t}\n}\n\nfunc (r *reader) sendError(ctx context.Context, err error) error {\n\tselect {\n\tcase r.msgs <- readerMessage{version: r.version, error: err}:\n\t\treturn nil\n\tcase <-ctx.Done():\n\t\treturn ctx.Err()\n\t}\n}\n\nfunc (r *reader) withLogger(do func(Logger)) {\n\tif r.logger != nil {\n\t\tdo(r.logger)\n\t}\n}\n\nfunc (r *reader) withErrorLogger(do func(Logger)) {\n\tif r.errorLogger != nil {\n\t\tdo(r.errorLogger)\n\t} else {\n\t\tr.withLogger(do)\n\t}\n}\n\n// extractTopics returns the unique list of topics represented by the set of\n// provided members.\nfunc extractTopics(members []GroupMember) []string {\n\tvisited := map[string]struct{}{}\n\tvar topics []string\n\n\tfor _, member := range members {\n\t\tfor _, topic := range member.Topics {\n\t\t\tif _, seen := visited[topic]; seen {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\ttopics = append(topics, topic)\n\t\t\tvisited[topic] = struct{}{}\n\t\t}\n\t}\n\n\tsort.Strings(topics)\n\n\treturn topics\n}\n\ntype humanOffset int64\n\nfunc toHumanOffset(v int64) humanOffset {\n\treturn humanOffset(v)\n}\n\nfunc (offset humanOffset) Format(w fmt.State, _ rune) {\n\tv := int64(offset)\n\tswitch v {\n\tcase FirstOffset:\n\t\tfmt.Fprint(w, \"first offset\")\n\tcase LastOffset:\n\t\tfmt.Fprint(w, \"last offset\")\n\tdefault:\n\t\tfmt.Fprint(w, strconv.FormatInt(v, 10))\n\t}\n}\n" }, { "path": "reader_test.go", "content": "package kafka\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math/rand\"\n\t\"net\"\n\t\"os\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestReader(t *testing.T) {\n\ttests := []struct {\n\t\tscenario string\n\t\tfunction func(*testing.T, context.Context, *Reader)\n\t}{\n\t\t{\n\t\t\tscenario: \"calling Read with a context that has been canceled returns an error\",\n\t\t\tfunction: testReaderReadCanceled,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"all messages of the stream are returned when calling ReadMessage repeatedly\",\n\t\t\tfunction: testReaderReadMessages,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"test special offsets -1 and -2\",\n\t\t\tfunction: testReaderSetSpecialOffsets,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"setting the offset to random values returns the expected messages when Read is called\",\n\t\t\tfunction: testReaderSetRandomOffset,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"setting the offset by TimeStamp\",\n\t\t\tfunction: testReaderSetOffsetAt,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"calling Lag returns the lag of the last message read from kafka\",\n\t\t\tfunction: testReaderLag,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"calling ReadLag returns the current lag of a reader\",\n\t\t\tfunction: testReaderReadLag,\n\t\t},\n\n\t\t{ // https://github.com/segmentio/kafka-go/issues/30\n\t\t\tscenario: \"reading from an out-of-range offset waits until the context is cancelled\",\n\t\t\tfunction: testReaderOutOfRangeGetsCanceled,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"topic being recreated will return an error\",\n\t\t\tfunction: testReaderTopicRecreated,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\ttestFunc := test.function\n\t\tt.Run(test.scenario, func(t *testing.T) {\n\t\t\tt.Parallel()\n\n\t\t\tctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)\n\t\t\tdefer cancel()\n\n\t\t\tr := NewReader(ReaderConfig{\n\t\t\t\tBrokers: []string{\"localhost:9092\"},\n\t\t\t\tTopic: makeTopic(),\n\t\t\t\tMinBytes: 1,\n\t\t\t\tMaxBytes: 10e6,\n\t\t\t\tMaxWait: 100 * time.Millisecond,\n\t\t\t\tLogger: newTestKafkaLogger(t, \"\"),\n\t\t\t})\n\t\t\tdefer r.Close()\n\t\t\ttestFunc(t, ctx, r)\n\t\t})\n\t}\n}\n\nfunc testReaderReadCanceled(t *testing.T, ctx context.Context, r *Reader) {\n\tctx, cancel := context.WithCancel(ctx)\n\tcancel()\n\n\tif _, err := r.ReadMessage(ctx); !errors.Is(err, context.Canceled) {\n\t\tt.Error(err)\n\t}\n}\n\nfunc testReaderReadMessages(t *testing.T, ctx context.Context, r *Reader) {\n\tconst N = 1000\n\tprepareReader(t, ctx, r, makeTestSequence(N)...)\n\n\tvar offset int64\n\n\tfor i := 0; i != N; i++ {\n\t\tm, err := r.ReadMessage(ctx)\n\t\tif err != nil {\n\t\t\tt.Error(\"reading message at offset\", offset, \"failed:\", err)\n\t\t\treturn\n\t\t}\n\t\toffset = m.Offset + 1\n\t\tv, _ := strconv.Atoi(string(m.Value))\n\t\tif v != i {\n\t\t\tt.Error(\"message at index\", i, \"has wrong value:\", v)\n\t\t\treturn\n\t\t}\n\t}\n}\n\nfunc testReaderSetSpecialOffsets(t *testing.T, ctx context.Context, r *Reader) {\n\tprepareReader(t, ctx, r, Message{Value: []byte(\"first\")})\n\tprepareReader(t, ctx, r, makeTestSequence(3)...)\n\n\tgo func() {\n\t\ttime.Sleep(1 * time.Second)\n\t\tprepareReader(t, ctx, r, Message{Value: []byte(\"last\")})\n\t}()\n\n\tfor _, test := range []struct {\n\t\toff, final int64\n\t\twant string\n\t}{\n\t\t{FirstOffset, 1, \"first\"},\n\t\t{LastOffset, 5, \"last\"},\n\t} {\n\t\toffset := test.off\n\t\tif err := r.SetOffset(offset); err != nil {\n\t\t\tt.Error(\"setting offset\", offset, \"failed:\", err)\n\t\t}\n\t\tm, err := r.ReadMessage(ctx)\n\t\tif err != nil {\n\t\t\tt.Error(\"reading at offset\", offset, \"failed:\", err)\n\t\t}\n\t\tif string(m.Value) != test.want {\n\t\t\tt.Error(\"message at offset\", offset, \"has wrong value:\", string(m.Value))\n\t\t}\n\t\tif off := r.Offset(); off != test.final {\n\t\t\tt.Errorf(\"bad final offset: got %d, want %d\", off, test.final)\n\t\t}\n\t}\n}\n\nfunc testReaderSetRandomOffset(t *testing.T, ctx context.Context, r *Reader) {\n\tconst N = 10\n\tprepareReader(t, ctx, r, makeTestSequence(N)...)\n\n\tfor i := 0; i != 2*N; i++ {\n\t\toffset := rand.Intn(N)\n\t\tr.SetOffset(int64(offset))\n\t\tm, err := r.ReadMessage(ctx)\n\t\tif err != nil {\n\t\t\tt.Error(\"seeking to offset\", offset, \"failed:\", err)\n\t\t\treturn\n\t\t}\n\t\tv, _ := strconv.Atoi(string(m.Value))\n\t\tif v != offset {\n\t\t\tt.Error(\"message at offset\", offset, \"has wrong value:\", v)\n\t\t\treturn\n\t\t}\n\t}\n}\n\nfunc testReaderSetOffsetAt(t *testing.T, ctx context.Context, r *Reader) {\n\t// We make 2 batches of messages here with a brief 2 second pause\n\t// to ensure messages 0...9 will be written a few seconds before messages 10...19\n\t// We'll then fetch the timestamp for message offset 10 and use that timestamp to set\n\t// our reader\n\tconst N = 10\n\tprepareReader(t, ctx, r, makeTestSequence(N)...)\n\ttime.Sleep(time.Second * 2)\n\tprepareReader(t, ctx, r, makeTestSequence(N)...)\n\n\tvar ts time.Time\n\tfor i := 0; i < N*2; i++ {\n\t\tm, err := r.ReadMessage(ctx)\n\t\tif err != nil {\n\t\t\tt.Error(\"error reading message\", err)\n\t\t}\n\t\t// grab the time for the 10th message\n\t\tif i == 10 {\n\t\t\tts = m.Time\n\t\t}\n\t}\n\n\terr := r.SetOffsetAt(ctx, ts)\n\tif err != nil {\n\t\tt.Fatal(\"error setting offset by timestamp\", err)\n\t}\n\n\tm, err := r.ReadMessage(context.Background())\n\tif err != nil {\n\t\tt.Fatal(\"error reading message\", err)\n\t}\n\n\tif m.Offset != 10 {\n\t\tt.Errorf(\"expected offset of 10, received offset %d\", m.Offset)\n\t}\n}\n\nfunc testReaderLag(t *testing.T, ctx context.Context, r *Reader) {\n\tconst N = 5\n\tprepareReader(t, ctx, r, makeTestSequence(N)...)\n\n\tif lag := r.Lag(); lag != 0 {\n\t\tt.Errorf(\"the initial lag value is %d but was expected to be 0\", lag)\n\t}\n\n\tfor i := 0; i != N; i++ {\n\t\tr.ReadMessage(ctx)\n\t\texpect := int64(N - (i + 1))\n\n\t\tif lag := r.Lag(); lag != expect {\n\t\t\tt.Errorf(\"the lag value at offset %d is %d but was expected to be %d\", i, lag, expect)\n\t\t}\n\t}\n}\n\nfunc testReaderReadLag(t *testing.T, ctx context.Context, r *Reader) {\n\tconst N = 5\n\tprepareReader(t, ctx, r, makeTestSequence(N)...)\n\n\tif lag, err := r.ReadLag(ctx); err != nil {\n\t\tt.Error(err)\n\t} else if lag != N {\n\t\tt.Errorf(\"the initial lag value is %d but was expected to be %d\", lag, N)\n\t}\n\n\tfor i := 0; i != N; i++ {\n\t\tr.ReadMessage(ctx)\n\t\texpect := int64(N - (i + 1))\n\n\t\tif lag, err := r.ReadLag(ctx); err != nil {\n\t\t\tt.Error(err)\n\t\t} else if lag != expect {\n\t\t\tt.Errorf(\"the lag value at offset %d is %d but was expected to be %d\", i, lag, expect)\n\t\t}\n\t}\n}\n\nfunc testReaderOutOfRangeGetsCanceled(t *testing.T, ctx context.Context, r *Reader) {\n\tprepareReader(t, ctx, r, makeTestSequence(10)...)\n\n\tconst D = 100 * time.Millisecond\n\tt0 := time.Now()\n\n\tctx, cancel := context.WithTimeout(ctx, D)\n\tdefer cancel()\n\n\tif err := r.SetOffset(42); err != nil {\n\t\tt.Error(err)\n\t}\n\n\t_, err := r.ReadMessage(ctx)\n\tif !errors.Is(err, context.DeadlineExceeded) {\n\t\tt.Error(\"bad error:\", err)\n\t}\n\n\tt1 := time.Now()\n\n\tif d := t1.Sub(t0); d < D {\n\t\tt.Error(\"ReadMessage returned too early after\", d)\n\t}\n}\n\nfunc createTopic(t *testing.T, topic string, partitions int) {\n\tt.Helper()\n\n\tt.Logf(\"createTopic(%s, %d)\", topic, partitions)\n\n\tconn, err := Dial(\"tcp\", \"localhost:9092\")\n\tif err != nil {\n\t\terr = fmt.Errorf(\"createTopic, Dial: %w\", err)\n\t\tt.Fatal(err)\n\t}\n\tdefer conn.Close()\n\n\tcontroller, err := conn.Controller()\n\tif err != nil {\n\t\terr = fmt.Errorf(\"createTopic, conn.Controller: %w\", err)\n\t\tt.Fatal(err)\n\t}\n\n\tconn, err = Dial(\"tcp\", net.JoinHostPort(controller.Host, strconv.Itoa(controller.Port)))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tconn.SetDeadline(time.Now().Add(10 * time.Second))\n\n\t_, err = conn.createTopics(createTopicsRequest{\n\t\tTopics: []createTopicsRequestV0Topic{\n\t\t\t{\n\t\t\t\tTopic: topic,\n\t\t\t\tNumPartitions: int32(partitions),\n\t\t\t\tReplicationFactor: 1,\n\t\t\t},\n\t\t},\n\t\tTimeout: milliseconds(5 * time.Second),\n\t})\n\tif err != nil {\n\t\tif !errors.Is(err, TopicAlreadyExists) {\n\t\t\terr = fmt.Errorf(\"createTopic, conn.createTopics: %w\", err)\n\t\t\tt.Error(err)\n\t\t\tt.FailNow()\n\t\t}\n\t}\n\n\tctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)\n\tdefer cancel()\n\n\twaitForTopic(ctx, t, topic)\n}\n\n// Block until topic exists.\nfunc waitForTopic(ctx context.Context, t *testing.T, topic string) {\n\tt.Helper()\n\n\tfor {\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\tt.Fatalf(\"reached deadline before verifying topic existence\")\n\t\tdefault:\n\t\t}\n\n\t\tcli := &Client{\n\t\t\tAddr: TCP(\"localhost:9092\"),\n\t\t\tTimeout: 5 * time.Second,\n\t\t}\n\n\t\tresponse, err := cli.Metadata(ctx, &MetadataRequest{\n\t\t\tAddr: cli.Addr,\n\t\t\tTopics: []string{topic},\n\t\t})\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"waitForTopic: error listing topics: %s\", err.Error())\n\t\t}\n\n\t\t// Find a topic which has at least 1 partition in the metadata response\n\t\tfor _, top := range response.Topics {\n\t\t\tif top.Name != topic {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tnumPartitions := len(top.Partitions)\n\t\t\tt.Logf(\"waitForTopic: found topic %q with %d partitions\",\n\t\t\t\ttopic, numPartitions)\n\n\t\t\tif numPartitions > 0 {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\n\t\tt.Logf(\"retrying after 100ms\")\n\t\ttime.Sleep(100 * time.Millisecond)\n\t\tcontinue\n\t}\n}\n\nfunc deleteTopic(t *testing.T, topic ...string) {\n\tt.Helper()\n\tconn, err := Dial(\"tcp\", \"localhost:9092\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer conn.Close()\n\n\tcontroller, err := conn.Controller()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tconn, err = Dial(\"tcp\", net.JoinHostPort(controller.Host, strconv.Itoa(controller.Port)))\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tconn.SetDeadline(time.Now().Add(10 * time.Second))\n\n\tif err := conn.DeleteTopics(topic...); err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestReaderOnNonZeroPartition(t *testing.T) {\n\ttests := []struct {\n\t\tscenario string\n\t\tfunction func(*testing.T, context.Context, *Reader)\n\t}{\n\t\t{\n\t\t\tscenario: \"topic and partition should now be included in header\",\n\t\t\tfunction: testReaderSetsTopicAndPartition,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\ttestFunc := test.function\n\t\tt.Run(test.scenario, func(t *testing.T) {\n\t\t\tt.Parallel()\n\n\t\t\ttopic := makeTopic()\n\t\t\tcreateTopic(t, topic, 2)\n\t\t\tdefer deleteTopic(t, topic)\n\n\t\t\tctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)\n\t\t\tdefer cancel()\n\n\t\t\tr := NewReader(ReaderConfig{\n\t\t\t\tBrokers: []string{\"localhost:9092\"},\n\t\t\t\tTopic: topic,\n\t\t\t\tPartition: 1,\n\t\t\t\tMinBytes: 1,\n\t\t\t\tMaxBytes: 10e6,\n\t\t\t\tMaxWait: 100 * time.Millisecond,\n\t\t\t})\n\t\t\tdefer r.Close()\n\t\t\ttestFunc(t, ctx, r)\n\t\t})\n\t}\n}\n\nfunc testReaderSetsTopicAndPartition(t *testing.T, ctx context.Context, r *Reader) {\n\tconst N = 3\n\tprepareReader(t, ctx, r, makeTestSequence(N)...)\n\n\tfor i := 0; i != N; i++ {\n\t\tm, err := r.ReadMessage(ctx)\n\t\tif err != nil {\n\t\t\tt.Error(\"reading message failed:\", err)\n\t\t\treturn\n\t\t}\n\n\t\tif m.Topic == \"\" {\n\t\t\tt.Error(\"expected topic to be set\")\n\t\t\treturn\n\t\t}\n\t\tif m.Topic != r.config.Topic {\n\t\t\tt.Errorf(\"expected message to contain topic, %v; got %v\", r.config.Topic, m.Topic)\n\t\t\treturn\n\t\t}\n\t\tif m.Partition != r.config.Partition {\n\t\t\tt.Errorf(\"expected partition to be set; expected 1, got %v\", m.Partition)\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// TestReadTruncatedMessages uses a configuration designed to get the Broker to\n// return truncated messages. It exercises the case where an earlier bug caused\n// reading to time out by attempting to read beyond the current response. This\n// test is not perfect, but it is pretty reliable about reproducing the issue.\n//\n// NOTE : it currently only succeeds against kafka 0.10.1.0, so it will be\n// skipped. It's here so that it can be manually run.\nfunc TestReadTruncatedMessages(t *testing.T) {\n\t// todo : it would be great to get it to work against 0.11.0.0 so we could\n\t// include it in CI unit tests.\n\tt.Skip()\n\n\tctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)\n\tdefer cancel()\n\tr := NewReader(ReaderConfig{\n\t\tBrokers: []string{\"localhost:9092\"},\n\t\tTopic: makeTopic(),\n\t\tMinBytes: 1,\n\t\tMaxBytes: 100,\n\t\tMaxWait: 100 * time.Millisecond,\n\t})\n\tdefer r.Close()\n\tn := 500\n\tprepareReader(t, ctx, r, makeTestSequence(n)...)\n\tfor i := 0; i < n; i++ {\n\t\tif _, err := r.ReadMessage(ctx); err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\t}\n}\n\nfunc makeTestSequence(n int) []Message {\n\tbase := time.Now()\n\tmsgs := make([]Message, n)\n\tfor i := 0; i != n; i++ {\n\t\tmsgs[i] = Message{\n\t\t\tTime: base.Add(time.Duration(i) * time.Millisecond).Truncate(time.Millisecond),\n\t\t\tValue: []byte(strconv.Itoa(i)),\n\t\t}\n\t}\n\treturn msgs\n}\n\nfunc prepareReader(t *testing.T, ctx context.Context, r *Reader, msgs ...Message) {\n\tconfig := r.Config()\n\tvar conn *Conn\n\tvar err error\n\n\tfor {\n\t\tif conn, err = DialLeader(ctx, \"tcp\", \"localhost:9092\", config.Topic, config.Partition); err == nil {\n\t\t\tbreak\n\t\t}\n\t\tselect {\n\t\tcase <-time.After(time.Second):\n\t\tcase <-ctx.Done():\n\t\t\tt.Fatal(ctx.Err())\n\t\t}\n\t}\n\n\tdefer conn.Close()\n\n\tif _, err := conn.WriteMessages(msgs...); err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nvar (\n\tbenchmarkReaderOnce sync.Once\n\tbenchmarkReaderTopic = makeTopic()\n\tbenchmarkReaderPayload = make([]byte, 2*1024)\n)\n\nfunc BenchmarkReader(b *testing.B) {\n\tconst broker = \"localhost:9092\"\n\tctx := context.Background()\n\n\tbenchmarkReaderOnce.Do(func() {\n\t\tconn, err := DialLeader(ctx, \"tcp\", broker, benchmarkReaderTopic, 0)\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t\tdefer conn.Close()\n\n\t\tmsgs := make([]Message, 1000)\n\t\tfor i := range msgs {\n\t\t\tmsgs[i].Value = benchmarkReaderPayload\n\t\t}\n\n\t\tfor i := 0; i != 10; i++ { // put 10K messages\n\t\t\tif _, err := conn.WriteMessages(msgs...); err != nil {\n\t\t\t\tb.Fatal(err)\n\t\t\t}\n\t\t}\n\n\t\tb.ResetTimer()\n\t})\n\n\tr := NewReader(ReaderConfig{\n\t\tBrokers: []string{broker},\n\t\tTopic: benchmarkReaderTopic,\n\t\tPartition: 0,\n\t\tMinBytes: 1e3,\n\t\tMaxBytes: 1e6,\n\t\tMaxWait: 100 * time.Millisecond,\n\t})\n\n\tfor i := 0; i < b.N; i++ {\n\t\tif (i % 10000) == 0 {\n\t\t\tr.SetOffset(-1)\n\t\t}\n\t\t_, err := r.ReadMessage(ctx)\n\t\tif err != nil {\n\t\t\tb.Fatal(err)\n\t\t}\n\t}\n\n\tr.Close()\n\tb.SetBytes(int64(len(benchmarkReaderPayload)))\n}\n\nfunc TestCloseLeavesGroup(t *testing.T) {\n\tif os.Getenv(\"KAFKA_VERSION\") == \"2.3.1\" {\n\t\t// There's a bug in 2.3.1 that causes the MemberMetadata to be in the wrong format and thus\n\t\t// leads to an error when decoding the DescribeGroupsResponse.\n\t\t//\n\t\t// See https://issues.apache.org/jira/browse/KAFKA-9150 for details.\n\t\tt.Skip(\"Skipping because kafka version is 2.3.1\")\n\t}\n\n\tctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)\n\tdefer cancel()\n\n\ttopic := makeTopic()\n\tcreateTopic(t, topic, 1)\n\tdefer deleteTopic(t, topic)\n\n\tgroupID := makeGroupID()\n\tr := NewReader(ReaderConfig{\n\t\tBrokers: []string{\"localhost:9092\"},\n\t\tTopic: topic,\n\t\tGroupID: groupID,\n\t\tMinBytes: 1,\n\t\tMaxBytes: 10e6,\n\t\tMaxWait: 100 * time.Millisecond,\n\t\tRebalanceTimeout: time.Second,\n\t})\n\tprepareReader(t, ctx, r, Message{Value: []byte(\"test\")})\n\n\tconn, err := Dial(\"tcp\", r.config.Brokers[0])\n\tif err != nil {\n\t\tt.Fatalf(\"error dialing: %v\", err)\n\t}\n\tdefer conn.Close()\n\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\tdescGroups := func() DescribeGroupsResponse {\n\t\tresp, err := client.DescribeGroups(\n\t\t\tctx,\n\t\t\t&DescribeGroupsRequest{\n\t\t\t\tGroupIDs: []string{groupID},\n\t\t\t},\n\t\t)\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"error from describeGroups %v\", err)\n\t\t}\n\t\treturn *resp\n\t}\n\n\t_, err = r.ReadMessage(ctx)\n\tif err != nil {\n\t\tt.Fatalf(\"our reader never joind its group or couldn't read a message: %v\", err)\n\t}\n\tresp := descGroups()\n\tif len(resp.Groups) != 1 {\n\t\tt.Fatalf(\"expected 1 group. got: %d\", len(resp.Groups))\n\t}\n\tif len(resp.Groups[0].Members) != 1 {\n\t\tt.Fatalf(\"expected group membership size of %d, but got %d\", 1, len(resp.Groups[0].Members))\n\t}\n\n\terr = r.Close()\n\tif err != nil {\n\t\tt.Fatalf(\"unexpected error closing reader: %s\", err.Error())\n\t}\n\tresp = descGroups()\n\tif len(resp.Groups) != 1 {\n\t\tt.Fatalf(\"expected 1 group. got: %d\", len(resp.Groups))\n\t}\n\tif len(resp.Groups[0].Members) != 0 {\n\t\tt.Fatalf(\"expected group membership size of %d, but got %d\", 0, len(resp.Groups[0].Members))\n\t}\n}\n\nfunc testConsumerGroupImmediateClose(t *testing.T, ctx context.Context, r *Reader) {\n\tif err := r.Close(); err != nil {\n\t\tt.Fatalf(\"bad err: %v\", err)\n\t}\n}\n\nfunc testConsumerGroupSimple(t *testing.T, ctx context.Context, r *Reader) {\n\tif err := r.Close(); err != nil {\n\t\tt.Fatalf(\"bad err: %v\", err)\n\t}\n}\n\nfunc TestReaderSetOffsetWhenConsumerGroupsEnabled(t *testing.T) {\n\tr := &Reader{config: ReaderConfig{GroupID: \"not-zero\"}}\n\tif err := r.SetOffset(LastOffset); !errors.Is(err, errNotAvailableWithGroup) {\n\t\tt.Fatalf(\"expected %v; got %v\", errNotAvailableWithGroup, err)\n\t}\n}\n\nfunc TestReaderOffsetWhenConsumerGroupsEnabled(t *testing.T) {\n\tr := &Reader{config: ReaderConfig{GroupID: \"not-zero\"}}\n\tif offset := r.Offset(); offset != -1 {\n\t\tt.Fatalf(\"expected -1; got %v\", offset)\n\t}\n}\n\nfunc TestReaderLagWhenConsumerGroupsEnabled(t *testing.T) {\n\tr := &Reader{config: ReaderConfig{GroupID: \"not-zero\"}}\n\tif offset := r.Lag(); offset != -1 {\n\t\tt.Fatalf(\"expected -1; got %v\", offset)\n\t}\n}\n\nfunc TestReaderReadLagReturnsZeroLagWhenConsumerGroupsEnabled(t *testing.T) {\n\tr := &Reader{config: ReaderConfig{GroupID: \"not-zero\"}}\n\tlag, err := r.ReadLag(context.Background())\n\n\tif !errors.Is(err, errNotAvailableWithGroup) {\n\t\tt.Fatalf(\"expected %v; got %v\", errNotAvailableWithGroup, err)\n\t}\n\n\tif lag != 0 {\n\t\tt.Fatalf(\"expected 0; got %d\", lag)\n\t}\n}\n\nfunc TestReaderPartitionWhenConsumerGroupsEnabled(t *testing.T) {\n\tinvoke := func() (boom bool) {\n\t\tdefer func() {\n\t\t\tif r := recover(); r != nil {\n\t\t\t\tboom = true\n\t\t\t}\n\t\t}()\n\n\t\tNewReader(ReaderConfig{\n\t\t\tGroupID: \"set\",\n\t\t\tPartition: 1,\n\t\t})\n\t\treturn false\n\t}\n\n\tif !invoke() {\n\t\tt.Fatalf(\"expected panic; but NewReader worked?!\")\n\t}\n}\n\nfunc TestExtractTopics(t *testing.T) {\n\ttestCases := map[string]struct {\n\t\tMembers []GroupMember\n\t\tTopics []string\n\t}{\n\t\t\"nil\": {},\n\t\t\"single member, single topic\": {\n\t\t\tMembers: []GroupMember{\n\t\t\t\t{\n\t\t\t\t\tID: \"a\",\n\t\t\t\t\tTopics: []string{\"topic\"},\n\t\t\t\t},\n\t\t\t},\n\t\t\tTopics: []string{\"topic\"},\n\t\t},\n\t\t\"two members, single topic\": {\n\t\t\tMembers: []GroupMember{\n\t\t\t\t{\n\t\t\t\t\tID: \"a\",\n\t\t\t\t\tTopics: []string{\"topic\"},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tID: \"b\",\n\t\t\t\t\tTopics: []string{\"topic\"},\n\t\t\t\t},\n\t\t\t},\n\t\t\tTopics: []string{\"topic\"},\n\t\t},\n\t\t\"two members, two topics\": {\n\t\t\tMembers: []GroupMember{\n\t\t\t\t{\n\t\t\t\t\tID: \"a\",\n\t\t\t\t\tTopics: []string{\"topic-1\"},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tID: \"b\",\n\t\t\t\t\tTopics: []string{\"topic-2\"},\n\t\t\t\t},\n\t\t\t},\n\t\t\tTopics: []string{\"topic-1\", \"topic-2\"},\n\t\t},\n\t\t\"three members, three shared topics\": {\n\t\t\tMembers: []GroupMember{\n\t\t\t\t{\n\t\t\t\t\tID: \"a\",\n\t\t\t\t\tTopics: []string{\"topic-1\", \"topic-2\"},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tID: \"b\",\n\t\t\t\t\tTopics: []string{\"topic-2\", \"topic-3\"},\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tID: \"c\",\n\t\t\t\t\tTopics: []string{\"topic-3\", \"topic-1\"},\n\t\t\t\t},\n\t\t\t},\n\t\t\tTopics: []string{\"topic-1\", \"topic-2\", \"topic-3\"},\n\t\t},\n\t}\n\n\tfor label, tc := range testCases {\n\t\tt.Run(label, func(t *testing.T) {\n\t\t\ttopics := extractTopics(tc.Members)\n\t\t\tif !reflect.DeepEqual(tc.Topics, topics) {\n\t\t\t\tt.Errorf(\"expected %v; got %v\", tc.Topics, topics)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestReaderConsumerGroup(t *testing.T) {\n\ttests := []struct {\n\t\tscenario string\n\t\tpartitions int\n\t\tcommitInterval time.Duration\n\t\tfunction func(*testing.T, context.Context, *Reader)\n\t}{\n\t\t{\n\t\t\tscenario: \"basic handshake\",\n\t\t\tpartitions: 1,\n\t\t\tfunction: testReaderConsumerGroupHandshake,\n\t\t},\n\t\t{\n\t\t\tscenario: \"verify offset committed\",\n\t\t\tpartitions: 1,\n\t\t\tfunction: testReaderConsumerGroupVerifyOffsetCommitted,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"verify offset committed when using interval committer\",\n\t\t\tpartitions: 1,\n\t\t\tcommitInterval: 400 * time.Millisecond,\n\t\t\tfunction: testReaderConsumerGroupVerifyPeriodicOffsetCommitter,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"rebalance across many partitions and consumers\",\n\t\t\tpartitions: 8,\n\t\t\tfunction: testReaderConsumerGroupRebalanceAcrossManyPartitionsAndConsumers,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"consumer group commits on close\",\n\t\t\tpartitions: 3,\n\t\t\tfunction: testReaderConsumerGroupVerifyCommitsOnClose,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"consumer group rebalance\",\n\t\t\tpartitions: 3,\n\t\t\tfunction: testReaderConsumerGroupRebalance,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"consumer group rebalance across topics\",\n\t\t\tpartitions: 3,\n\t\t\tfunction: testReaderConsumerGroupRebalanceAcrossTopics,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"consumer group reads content across partitions\",\n\t\t\tpartitions: 3,\n\t\t\tfunction: testReaderConsumerGroupReadContentAcrossPartitions,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"Close immediately after NewReader\",\n\t\t\tpartitions: 1,\n\t\t\tfunction: testConsumerGroupImmediateClose,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"Close immediately after NewReader\",\n\t\t\tpartitions: 1,\n\t\t\tfunction: testConsumerGroupSimple,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\tt.Run(test.scenario, func(t *testing.T) {\n\t\t\t// It appears that some of the tests depend on all these tests being\n\t\t\t// run concurrently to pass... this is brittle and should be fixed\n\t\t\t// at some point.\n\t\t\tt.Parallel()\n\n\t\t\ttopic := makeTopic()\n\t\t\tcreateTopic(t, topic, test.partitions)\n\t\t\tdefer deleteTopic(t, topic)\n\n\t\t\tgroupID := makeGroupID()\n\t\t\tr := NewReader(ReaderConfig{\n\t\t\t\tBrokers: []string{\"localhost:9092\"},\n\t\t\t\tTopic: topic,\n\t\t\t\tGroupID: groupID,\n\t\t\t\tHeartbeatInterval: 2 * time.Second,\n\t\t\t\tCommitInterval: test.commitInterval,\n\t\t\t\tRebalanceTimeout: 2 * time.Second,\n\t\t\t\tRetentionTime: time.Hour,\n\t\t\t\tMinBytes: 1,\n\t\t\t\tMaxBytes: 1e6,\n\t\t\t})\n\t\t\tdefer r.Close()\n\n\t\t\tctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)\n\t\t\tdefer cancel()\n\n\t\t\ttest.function(t, ctx, r)\n\t\t})\n\t}\n}\n\nfunc testReaderConsumerGroupHandshake(t *testing.T, ctx context.Context, r *Reader) {\n\tprepareReader(t, context.Background(), r, makeTestSequence(5)...)\n\n\tm, err := r.ReadMessage(ctx)\n\tif err != nil {\n\t\tt.Errorf(\"bad err: %v\", err)\n\t}\n\tif m.Topic != r.config.Topic {\n\t\tt.Errorf(\"topic not set\")\n\t}\n\tif m.Offset != 0 {\n\t\tt.Errorf(\"offset not set\")\n\t}\n\n\tm, err = r.ReadMessage(ctx)\n\tif err != nil {\n\t\tt.Errorf(\"bad err: %v\", err)\n\t}\n\tif m.Topic != r.config.Topic {\n\t\tt.Errorf(\"topic not set\")\n\t}\n\tif m.Offset != 1 {\n\t\tt.Errorf(\"offset not set\")\n\t}\n}\n\nfunc testReaderConsumerGroupVerifyOffsetCommitted(t *testing.T, ctx context.Context, r *Reader) {\n\tprepareReader(t, context.Background(), r, makeTestSequence(3)...)\n\n\tif _, err := r.FetchMessage(ctx); err != nil {\n\t\tt.Errorf(\"bad err: %v\", err) // skip the first message\n\t}\n\n\tm, err := r.FetchMessage(ctx)\n\tif err != nil {\n\t\tt.Errorf(\"bad err: %v\", err)\n\t}\n\n\tif err := r.CommitMessages(ctx, m); err != nil {\n\t\tt.Errorf(\"bad commit message: %v\", err)\n\t}\n\n\toffsets := getOffsets(t, r.config)\n\tif expected := map[int]int64{0: m.Offset + 1}; !reflect.DeepEqual(expected, offsets) {\n\t\tt.Errorf(\"expected %v; got %v\", expected, offsets)\n\t}\n}\n\nfunc testReaderConsumerGroupVerifyPeriodicOffsetCommitter(t *testing.T, ctx context.Context, r *Reader) {\n\tprepareReader(t, context.Background(), r, makeTestSequence(3)...)\n\n\tif _, err := r.FetchMessage(ctx); err != nil {\n\t\tt.Errorf(\"bad err: %v\", err) // skip the first message\n\t}\n\n\tm, err := r.FetchMessage(ctx)\n\tif err != nil {\n\t\tt.Errorf(\"bad err: %v\", err)\n\t}\n\n\tstarted := time.Now()\n\tif err := r.CommitMessages(ctx, m); err != nil {\n\t\tt.Errorf(\"bad commit message: %v\", err)\n\t}\n\tif elapsed := time.Since(started); elapsed > 10*time.Millisecond {\n\t\tt.Errorf(\"background commits should happen nearly instantly\")\n\t}\n\n\t// wait for committer to pick up the commits\n\ttime.Sleep(r.config.CommitInterval * 3)\n\n\toffsets := getOffsets(t, r.config)\n\tif expected := map[int]int64{0: m.Offset + 1}; !reflect.DeepEqual(expected, offsets) {\n\t\tt.Errorf(\"expected %v; got %v\", expected, offsets)\n\t}\n}\n\nfunc testReaderConsumerGroupVerifyCommitsOnClose(t *testing.T, ctx context.Context, r *Reader) {\n\tprepareReader(t, context.Background(), r, makeTestSequence(3)...)\n\n\tif _, err := r.FetchMessage(ctx); err != nil {\n\t\tt.Errorf(\"bad err: %v\", err) // skip the first message\n\t}\n\n\tm, err := r.FetchMessage(ctx)\n\tif err != nil {\n\t\tt.Errorf(\"bad err: %v\", err)\n\t}\n\n\tif err := r.CommitMessages(ctx, m); err != nil {\n\t\tt.Errorf(\"bad commit message: %v\", err)\n\t}\n\n\tif err := r.Close(); err != nil {\n\t\tt.Errorf(\"bad Close: %v\", err)\n\t}\n\n\tr2 := NewReader(r.config)\n\tdefer r2.Close()\n\n\toffsets := getOffsets(t, r2.config)\n\tif expected := map[int]int64{0: m.Offset + 1}; !reflect.DeepEqual(expected, offsets) {\n\t\tt.Errorf(\"expected %v; got %v\", expected, offsets)\n\t}\n}\n\nfunc testReaderConsumerGroupReadContentAcrossPartitions(t *testing.T, ctx context.Context, r *Reader) {\n\tconst N = 12\n\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\twriter := &Writer{\n\t\tAddr: TCP(r.config.Brokers...),\n\t\tTopic: r.config.Topic,\n\t\tBalancer: &RoundRobin{},\n\t\tBatchSize: 1,\n\t\tTransport: client.Transport,\n\t}\n\tif err := writer.WriteMessages(ctx, makeTestSequence(N)...); err != nil {\n\t\tt.Fatalf(\"bad write messages: %v\", err)\n\t}\n\tif err := writer.Close(); err != nil {\n\t\tt.Fatalf(\"bad write err: %v\", err)\n\t}\n\n\tpartitions := map[int]struct{}{}\n\tfor i := 0; i < N; i++ {\n\t\tm, err := r.FetchMessage(ctx)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"bad error: %s\", err)\n\t\t}\n\t\tpartitions[m.Partition] = struct{}{}\n\t}\n\n\tif v := len(partitions); v != 3 {\n\t\tt.Errorf(\"expected messages across 3 partitions; got messages across %v partitions\", v)\n\t}\n}\n\nfunc testReaderConsumerGroupRebalance(t *testing.T, ctx context.Context, r *Reader) {\n\tr2 := NewReader(r.config)\n\tdefer r.Close()\n\n\tconst (\n\t\tN = 12\n\t\tpartitions = 2\n\t)\n\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\t// rebalance should result in 12 message in each of the partitions\n\twriter := &Writer{\n\t\tAddr: TCP(r.config.Brokers...),\n\t\tTopic: r.config.Topic,\n\t\tBalancer: &RoundRobin{},\n\t\tBatchSize: 1,\n\t\tTransport: client.Transport,\n\t}\n\tif err := writer.WriteMessages(ctx, makeTestSequence(N*partitions)...); err != nil {\n\t\tt.Fatalf(\"bad write messages: %v\", err)\n\t}\n\tif err := writer.Close(); err != nil {\n\t\tt.Fatalf(\"bad write err: %v\", err)\n\t}\n\n\t// after rebalance, each reader should have a partition to itself\n\tfor i := 0; i < N; i++ {\n\t\tif _, err := r2.FetchMessage(ctx); err != nil {\n\t\t\tt.Errorf(\"expect to read from reader 2\")\n\t\t}\n\t\tif _, err := r.FetchMessage(ctx); err != nil {\n\t\t\tt.Errorf(\"expect to read from reader 1\")\n\t\t}\n\t}\n}\n\nfunc testReaderConsumerGroupRebalanceAcrossTopics(t *testing.T, ctx context.Context, r *Reader) {\n\t// create a second reader that shares the groupID, but reads from a different topic\n\tclient, topic2, shutdown := newLocalClientAndTopic()\n\tdefer shutdown()\n\n\tr2 := NewReader(ReaderConfig{\n\t\tBrokers: r.config.Brokers,\n\t\tTopic: topic2,\n\t\tGroupID: r.config.GroupID,\n\t\tHeartbeatInterval: r.config.HeartbeatInterval,\n\t\tSessionTimeout: r.config.SessionTimeout,\n\t\tRetentionTime: r.config.RetentionTime,\n\t\tMinBytes: r.config.MinBytes,\n\t\tMaxBytes: r.config.MaxBytes,\n\t\tLogger: r.config.Logger,\n\t})\n\tdefer r.Close()\n\tprepareReader(t, ctx, r2, makeTestSequence(1)...)\n\n\tconst (\n\t\tN = 12\n\t)\n\n\t// write messages across both partitions\n\twriter := &Writer{\n\t\tAddr: TCP(r.config.Brokers...),\n\t\tTopic: r.config.Topic,\n\t\tBalancer: &RoundRobin{},\n\t\tBatchSize: 1,\n\t\tTransport: client.Transport,\n\t}\n\tif err := writer.WriteMessages(ctx, makeTestSequence(N)...); err != nil {\n\t\tt.Fatalf(\"bad write messages: %v\", err)\n\t}\n\tif err := writer.Close(); err != nil {\n\t\tt.Fatalf(\"bad write err: %v\", err)\n\t}\n\n\t// after rebalance, r2 should read topic2 and r1 should read ALL of the original topic\n\tif _, err := r2.FetchMessage(ctx); err != nil {\n\t\tt.Errorf(\"expect to read from reader 2\")\n\t}\n\n\t// all N messages on the original topic should be read by the original reader\n\tfor i := 0; i < N; i++ {\n\t\tif _, err := r.FetchMessage(ctx); err != nil {\n\t\t\tt.Errorf(\"expect to read from reader 1\")\n\t\t}\n\t}\n}\n\nfunc testReaderConsumerGroupRebalanceAcrossManyPartitionsAndConsumers(t *testing.T, ctx context.Context, r *Reader) {\n\t// I've rebalanced up to 100 servers, but the rebalance can take upwards\n\t// of a minute and that seems too long for unit tests. Also, setting this\n\t// to a larger number seems to make the kafka broker unresponsive.\n\t// TODO research if there's a way to reduce rebalance time across many partitions\n\t// svls: the described behavior is due to the thundering herd of readers\n\t// hitting the rebalance timeout. introducing the 100ms sleep in the\n\t// loop below in order to give time for the sync group to finish has\n\t// greatly helped, though we still hit the timeout from time to time.\n\tconst N = 8\n\n\tvar readers []*Reader\n\n\tfor i := 0; i < N-1; i++ {\n\t\treader := NewReader(r.config)\n\t\treaders = append(readers, reader)\n\t\ttime.Sleep(100 * time.Millisecond)\n\t}\n\tdefer func() {\n\t\tfor _, r := range readers {\n\t\t\tr.Close()\n\t\t\ttime.Sleep(100 * time.Millisecond)\n\t\t}\n\t}()\n\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\n\t// write messages across both partitions\n\twriter := &Writer{\n\t\tAddr: TCP(r.config.Brokers...),\n\t\tTopic: r.config.Topic,\n\t\tBalancer: &RoundRobin{},\n\t\tBatchSize: 1,\n\t\tTransport: client.Transport,\n\t}\n\tif err := writer.WriteMessages(ctx, makeTestSequence(N*3)...); err != nil {\n\t\tt.Fatalf(\"bad write messages: %v\", err)\n\t}\n\tif err := writer.Close(); err != nil {\n\t\tt.Fatalf(\"bad write err: %v\", err)\n\t}\n\n\t// all N messages on the original topic should be read by the original reader\n\tfor i := 0; i < N-1; i++ {\n\t\tif _, err := readers[i].FetchMessage(ctx); err != nil {\n\t\t\tt.Errorf(\"reader %v expected to read 1 message\", i)\n\t\t}\n\t}\n\n\tif _, err := r.FetchMessage(ctx); err != nil {\n\t\tt.Errorf(\"expect to read from original reader\")\n\t}\n}\n\nfunc TestOffsetStash(t *testing.T) {\n\tconst topic = \"topic\"\n\n\tnewMessage := func(partition int, offset int64) Message {\n\t\treturn Message{\n\t\t\tTopic: topic,\n\t\t\tPartition: partition,\n\t\t\tOffset: offset,\n\t\t}\n\t}\n\n\ttests := map[string]struct {\n\t\tGiven offsetStash\n\t\tMessages []Message\n\t\tExpected offsetStash\n\t}{\n\t\t\"nil\": {},\n\t\t\"empty given, single message\": {\n\t\t\tGiven: offsetStash{},\n\t\t\tMessages: []Message{newMessage(0, 0)},\n\t\t\tExpected: offsetStash{\n\t\t\t\ttopic: {0: 1},\n\t\t\t},\n\t\t},\n\t\t\"ignores earlier offsets\": {\n\t\t\tGiven: offsetStash{\n\t\t\t\ttopic: {0: 2},\n\t\t\t},\n\t\t\tMessages: []Message{newMessage(0, 0)},\n\t\t\tExpected: offsetStash{\n\t\t\t\ttopic: {0: 2},\n\t\t\t},\n\t\t},\n\t\t\"uses latest offset\": {\n\t\t\tGiven: offsetStash{},\n\t\t\tMessages: []Message{\n\t\t\t\tnewMessage(0, 2),\n\t\t\t\tnewMessage(0, 3),\n\t\t\t\tnewMessage(0, 1),\n\t\t\t},\n\t\t\tExpected: offsetStash{\n\t\t\t\ttopic: {0: 4},\n\t\t\t},\n\t\t},\n\t\t\"uses latest offset, across multiple topics\": {\n\t\t\tGiven: offsetStash{},\n\t\t\tMessages: []Message{\n\t\t\t\tnewMessage(0, 2),\n\t\t\t\tnewMessage(0, 3),\n\t\t\t\tnewMessage(0, 1),\n\t\t\t\tnewMessage(1, 5),\n\t\t\t\tnewMessage(1, 6),\n\t\t\t},\n\t\t\tExpected: offsetStash{\n\t\t\t\ttopic: {\n\t\t\t\t\t0: 4,\n\t\t\t\t\t1: 7,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tfor label, test := range tests {\n\t\tt.Run(label, func(t *testing.T) {\n\t\t\ttest.Given.merge(makeCommits(test.Messages...))\n\t\t\tif !reflect.DeepEqual(test.Expected, test.Given) {\n\t\t\t\tt.Errorf(\"expected %v; got %v\", test.Expected, test.Given)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestValidateReader(t *testing.T) {\n\ttests := []struct {\n\t\tconfig ReaderConfig\n\t\terrorOccured bool\n\t}{\n\t\t{config: ReaderConfig{}, errorOccured: true},\n\t\t{config: ReaderConfig{Brokers: []string{\"broker1\"}}, errorOccured: true},\n\t\t{config: ReaderConfig{Brokers: []string{\"broker1\"}, Topic: \"topic1\"}, errorOccured: false},\n\t\t{config: ReaderConfig{Brokers: []string{\"broker1\"}, Topic: \"topic1\", Partition: -1}, errorOccured: true},\n\t\t{config: ReaderConfig{Brokers: []string{\"broker1\"}, Topic: \"topic1\", Partition: 1, MinBytes: -1}, errorOccured: true},\n\t\t{config: ReaderConfig{Brokers: []string{\"broker1\"}, Topic: \"topic1\", Partition: 1, MinBytes: 5, MaxBytes: -1}, errorOccured: true},\n\t\t{config: ReaderConfig{Brokers: []string{\"broker1\"}, Topic: \"topic1\", Partition: 1, MinBytes: 5, MaxBytes: 6}, errorOccured: false},\n\t}\n\tfor _, test := range tests {\n\t\terr := test.config.Validate()\n\t\tif test.errorOccured && err == nil {\n\t\t\tt.Fail()\n\t\t}\n\t\tif !test.errorOccured && err != nil {\n\t\t\tt.Fail()\n\t\t}\n\t}\n}\n\nfunc TestCommitLoopImmediateFlushOnGenerationEnd(t *testing.T) {\n\tt.Parallel()\n\tvar committedOffset int64\n\tvar commitCount int\n\tgen := &Generation{\n\t\tconn: mockCoordinator{\n\t\t\toffsetCommitFunc: func(r offsetCommitRequestV2) (offsetCommitResponseV2, error) {\n\t\t\t\tcommitCount++\n\t\t\t\tcommittedOffset = r.Topics[0].Partitions[0].Offset\n\t\t\t\treturn offsetCommitResponseV2{}, nil\n\t\t\t},\n\t\t},\n\t\tdone: make(chan struct{}),\n\t\tlog: func(func(Logger)) {},\n\t\tlogError: func(func(Logger)) {},\n\t\tjoined: make(chan struct{}),\n\t}\n\n\t// initialize commits so that the commitLoopImmediate select statement blocks\n\tr := &Reader{stctx: context.Background(), commits: make(chan commitRequest, 100)}\n\n\tfor i := 0; i < 100; i++ {\n\t\tcr := commitRequest{\n\t\t\tcommits: []commit{{\n\t\t\t\ttopic: \"topic\",\n\t\t\t\tpartition: 0,\n\t\t\t\toffset: int64(i) + 1,\n\t\t\t}},\n\t\t\terrch: make(chan<- error, 1),\n\t\t}\n\t\tr.commits <- cr\n\t}\n\n\tgen.Start(func(ctx context.Context) {\n\t\tr.commitLoopImmediate(ctx, gen)\n\t})\n\n\tgen.close()\n\n\tif committedOffset != 100 {\n\t\tt.Fatalf(\"expected commited offset to be 100 but got %d\", committedOffset)\n\t}\n\n\tif commitCount >= 100 {\n\t\tt.Fatalf(\"expected a single final commit on generation end got %d\", commitCount)\n\t}\n}\n\nfunc TestCommitOffsetsWithRetry(t *testing.T) {\n\toffsets := offsetStash{\"topic\": {0: 0}}\n\n\ttests := map[string]struct {\n\t\tFails int\n\t\tInvocations int\n\t\tHasError bool\n\t}{\n\t\t\"happy path\": {\n\t\t\tInvocations: 1,\n\t\t},\n\t\t\"1 retry\": {\n\t\t\tFails: 1,\n\t\t\tInvocations: 2,\n\t\t},\n\t\t\"out of retries\": {\n\t\t\tFails: defaultCommitRetries + 1,\n\t\t\tInvocations: defaultCommitRetries,\n\t\t\tHasError: true,\n\t\t},\n\t}\n\n\tfor label, test := range tests {\n\t\tt.Run(label, func(t *testing.T) {\n\t\t\tcount := 0\n\t\t\tgen := &Generation{\n\t\t\t\tconn: mockCoordinator{\n\t\t\t\t\toffsetCommitFunc: func(offsetCommitRequestV2) (offsetCommitResponseV2, error) {\n\t\t\t\t\t\tcount++\n\t\t\t\t\t\tif count <= test.Fails {\n\t\t\t\t\t\t\treturn offsetCommitResponseV2{}, io.EOF\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn offsetCommitResponseV2{}, nil\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t\tdone: make(chan struct{}),\n\t\t\t\tlog: func(func(Logger)) {},\n\t\t\t\tlogError: func(func(Logger)) {},\n\t\t\t}\n\n\t\t\tr := &Reader{stctx: context.Background()}\n\t\t\terr := r.commitOffsetsWithRetry(gen, offsets, defaultCommitRetries)\n\t\t\tswitch {\n\t\t\tcase test.HasError && err == nil:\n\t\t\t\tt.Error(\"bad err: expected not nil; got nil\")\n\t\t\tcase !test.HasError && err != nil:\n\t\t\t\tt.Errorf(\"bad err: expected nil; got %v\", err)\n\t\t\t}\n\t\t})\n\t}\n}\n\n// Test that a reader won't continually rebalance when there are more consumers\n// than partitions in a group.\n// https://github.com/segmentio/kafka-go/issues/200\nfunc TestRebalanceTooManyConsumers(t *testing.T) {\n\tctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)\n\tdefer cancel()\n\n\tconf := ReaderConfig{\n\t\tBrokers: []string{\"localhost:9092\"},\n\t\tGroupID: makeGroupID(),\n\t\tTopic: makeTopic(),\n\t\tMaxWait: time.Second,\n\t}\n\n\t// Create the first reader and wait for it to become the leader.\n\tr1 := NewReader(conf)\n\n\t// Give the reader some time to setup before reading a message\n\ttime.Sleep(1 * time.Second)\n\tprepareReader(t, ctx, r1, makeTestSequence(1)...)\n\n\t_, err := r1.ReadMessage(ctx)\n\tif err != nil {\n\t\tt.Fatalf(\"failed to read message: %v\", err)\n\t}\n\t// Clear the stats from the first rebalance.\n\tr1.Stats()\n\n\t// Second reader should cause one rebalance for each r1 and r2.\n\tr2 := NewReader(conf)\n\n\t// Wait for rebalances.\n\ttime.Sleep(5 * time.Second)\n\n\t// Before the fix, r2 would cause continuous rebalances,\n\t// as it tried to handshake() repeatedly.\n\trebalances := r1.Stats().Rebalances + r2.Stats().Rebalances\n\tif rebalances > 2 {\n\t\tt.Errorf(\"unexpected rebalances to first reader, got %d\", rebalances)\n\t}\n}\n\nfunc TestConsumerGroupWithMissingTopic(t *testing.T) {\n\tt.Skip(\"this test doesn't work when the cluster is configured to auto-create topics\")\n\n\tctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)\n\tdefer cancel()\n\n\tconf := ReaderConfig{\n\t\tBrokers: []string{\"localhost:9092\"},\n\t\tGroupID: makeGroupID(),\n\t\tTopic: makeTopic(),\n\t\tMaxWait: time.Second,\n\t\tPartitionWatchInterval: 100 * time.Millisecond,\n\t\tWatchPartitionChanges: true,\n\t}\n\n\tr := NewReader(conf)\n\tdefer r.Close()\n\n\trecvErr := make(chan error, 1)\n\tgo func() {\n\t\t_, err := r.ReadMessage(ctx)\n\t\trecvErr <- err\n\t}()\n\n\ttime.Sleep(time.Second)\n\tclient, shutdown := newLocalClientWithTopic(conf.Topic, 1)\n\tdefer shutdown()\n\n\tw := &Writer{\n\t\tAddr: TCP(r.config.Brokers...),\n\t\tTopic: r.config.Topic,\n\t\tBatchTimeout: 10 * time.Millisecond,\n\t\tBatchSize: 1,\n\t\tTransport: client.Transport,\n\t}\n\tdefer w.Close()\n\tif err := w.WriteMessages(ctx, Message{}); err != nil {\n\t\tt.Fatalf(\"write error: %+v\", err)\n\t}\n\n\tif err := <-recvErr; err != nil {\n\t\tt.Fatalf(\"read error: %+v\", err)\n\t}\n\n\tnMsgs := r.Stats().Messages\n\tif nMsgs != 1 {\n\t\tt.Fatalf(\"expected to receive one message, but got %d\", nMsgs)\n\t}\n}\n\nfunc TestConsumerGroupWithTopic(t *testing.T) {\n\tctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)\n\tdefer cancel()\n\n\tconf := ReaderConfig{\n\t\tBrokers: []string{\"localhost:9092\"},\n\t\tGroupID: makeGroupID(),\n\t\tTopic: makeTopic(),\n\t\tMaxWait: time.Second,\n\t\tPartitionWatchInterval: 100 * time.Millisecond,\n\t\tWatchPartitionChanges: true,\n\t\tLogger: newTestKafkaLogger(t, \"Reader:\"),\n\t}\n\n\tr := NewReader(conf)\n\tdefer r.Close()\n\n\trecvErr := make(chan error, len(conf.GroupTopics))\n\tgo func() {\n\t\tmsg, err := r.ReadMessage(ctx)\n\t\tt.Log(msg)\n\t\trecvErr <- err\n\t}()\n\n\ttime.Sleep(conf.MaxWait)\n\n\tclient, shutdown := newLocalClientWithTopic(conf.Topic, 1)\n\tdefer shutdown()\n\n\tw := &Writer{\n\t\tAddr: TCP(r.config.Brokers...),\n\t\tTopic: conf.Topic,\n\t\tBatchTimeout: 10 * time.Millisecond,\n\t\tBatchSize: 1,\n\t\tTransport: client.Transport,\n\t\tLogger: newTestKafkaLogger(t, \"Writer:\"),\n\t}\n\tdefer w.Close()\n\tif err := w.WriteMessages(ctx, Message{Value: []byte(conf.Topic)}); err != nil {\n\t\tt.Fatalf(\"write error: %+v\", err)\n\t}\n\n\tif err := <-recvErr; err != nil {\n\t\tt.Fatalf(\"read error: %+v\", err)\n\t}\n\n\tnMsgs := r.Stats().Messages\n\tif nMsgs != 1 {\n\t\tt.Fatalf(\"expected to receive 1 message, but got %d\", nMsgs)\n\t}\n}\n\nfunc TestConsumerGroupWithGroupTopicsSingle(t *testing.T) {\n\tctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)\n\tdefer cancel()\n\n\tconf := ReaderConfig{\n\t\tBrokers: []string{\"localhost:9092\"},\n\t\tGroupID: makeGroupID(),\n\t\tGroupTopics: []string{makeTopic()},\n\t\tMaxWait: time.Second,\n\t\tPartitionWatchInterval: 100 * time.Millisecond,\n\t\tWatchPartitionChanges: true,\n\t\tLogger: newTestKafkaLogger(t, \"Reader:\"),\n\t}\n\n\tr := NewReader(conf)\n\tdefer r.Close()\n\n\trecvErr := make(chan error, len(conf.GroupTopics))\n\tgo func() {\n\t\tmsg, err := r.ReadMessage(ctx)\n\t\tt.Log(msg)\n\t\trecvErr <- err\n\t}()\n\n\ttime.Sleep(conf.MaxWait)\n\n\tfor i, topic := range conf.GroupTopics {\n\t\tclient, shutdown := newLocalClientWithTopic(topic, 1)\n\t\tdefer shutdown()\n\n\t\tw := &Writer{\n\t\t\tAddr: TCP(r.config.Brokers...),\n\t\t\tTopic: topic,\n\t\t\tBatchTimeout: 10 * time.Millisecond,\n\t\t\tBatchSize: 1,\n\t\t\tTransport: client.Transport,\n\t\t\tLogger: newTestKafkaLogger(t, fmt.Sprintf(\"Writer(%d):\", i)),\n\t\t}\n\t\tdefer w.Close()\n\t\tif err := w.WriteMessages(ctx, Message{Value: []byte(topic)}); err != nil {\n\t\t\tt.Fatalf(\"write error: %+v\", err)\n\t\t}\n\t}\n\n\tif err := <-recvErr; err != nil {\n\t\tt.Fatalf(\"read error: %+v\", err)\n\t}\n\n\tnMsgs := r.Stats().Messages\n\tif nMsgs != int64(len(conf.GroupTopics)) {\n\t\tt.Fatalf(\"expected to receive %d messages, but got %d\", len(conf.GroupTopics), nMsgs)\n\t}\n}\n\nfunc TestConsumerGroupWithGroupTopicsMultiple(t *testing.T) {\n\tctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)\n\tdefer cancel()\n\n\tclient, shutdown := newLocalClient()\n\tdefer shutdown()\n\tt1 := makeTopic()\n\tcreateTopic(t, t1, 1)\n\tdefer deleteTopic(t, t1)\n\tt2 := makeTopic()\n\tcreateTopic(t, t2, 1)\n\tdefer deleteTopic(t, t2)\n\tconf := ReaderConfig{\n\t\tBrokers: []string{\"localhost:9092\"},\n\t\tGroupID: makeGroupID(),\n\t\tGroupTopics: []string{t1, t2},\n\t\tMaxWait: time.Second,\n\t\tPartitionWatchInterval: 100 * time.Millisecond,\n\t\tWatchPartitionChanges: true,\n\t\tLogger: newTestKafkaLogger(t, \"Reader:\"),\n\t}\n\n\tr := NewReader(conf)\n\n\tw := &Writer{\n\t\tAddr: TCP(r.config.Brokers...),\n\t\tBatchTimeout: 10 * time.Millisecond,\n\t\tBatchSize: 1,\n\t\tTransport: client.Transport,\n\t\tLogger: newTestKafkaLogger(t, \"Writer:\"),\n\t}\n\tdefer w.Close()\n\n\ttime.Sleep(time.Second)\n\n\tmsgs := make([]Message, 0, len(conf.GroupTopics))\n\tfor _, topic := range conf.GroupTopics {\n\t\tmsgs = append(msgs, Message{Topic: topic})\n\t}\n\tif err := w.WriteMessages(ctx, msgs...); err != nil {\n\t\tt.Logf(\"write error: %+v\", err)\n\t}\n\n\twg := new(sync.WaitGroup)\n\twg.Add(len(msgs))\n\n\tgo func() {\n\t\twg.Wait()\n\t\tt.Log(\"closing reader\")\n\t\tr.Close()\n\t}()\n\n\tfor {\n\t\tmsg, err := r.ReadMessage(ctx)\n\t\tif err != nil {\n\t\t\tif errors.Is(err, io.EOF) {\n\t\t\t\tt.Log(\"reader closed\")\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t\tt.Fatalf(\"read error: %+v\", err)\n\t\t} else {\n\t\t\tt.Logf(\"message read: %+v\", msg)\n\t\t\twg.Done()\n\t\t}\n\t}\n\n\tnMsgs := r.Stats().Messages\n\tif nMsgs != int64(len(conf.GroupTopics)) {\n\t\tt.Fatalf(\"expected to receive %d messages, but got %d\", len(conf.GroupTopics), nMsgs)\n\t}\n}\n\nfunc getOffsets(t *testing.T, config ReaderConfig) map[int]int64 {\n\t// minimal config required to lookup coordinator\n\tcg := ConsumerGroup{\n\t\tconfig: ConsumerGroupConfig{\n\t\t\tID: config.GroupID,\n\t\t\tBrokers: config.Brokers,\n\t\t\tDialer: config.Dialer,\n\t\t},\n\t}\n\n\tconn, err := cg.coordinator()\n\tif err != nil {\n\t\tt.Errorf(\"unable to connect to coordinator: %v\", err)\n\t}\n\tdefer conn.Close()\n\n\toffsets, err := conn.offsetFetch(offsetFetchRequestV1{\n\t\tGroupID: config.GroupID,\n\t\tTopics: []offsetFetchRequestV1Topic{{\n\t\t\tTopic: config.Topic,\n\t\t\tPartitions: []int32{0},\n\t\t}},\n\t})\n\tif err != nil {\n\t\tt.Errorf(\"bad fetchOffsets: %v\", err)\n\t}\n\n\tm := map[int]int64{}\n\n\tfor _, r := range offsets.Responses {\n\t\tif r.Topic == config.Topic {\n\t\t\tfor _, p := range r.PartitionResponses {\n\t\t\t\tm[int(p.Partition)] = p.Offset\n\t\t\t}\n\t\t}\n\t}\n\n\treturn m\n}\n\nconst (\n\tconnTO = 1 * time.Second\n\tconnTestTO = 2 * connTO\n)\n\nfunc TestErrorCannotConnect(t *testing.T) {\n\tr := NewReader(ReaderConfig{\n\t\tBrokers: []string{\"localhost:9093\"},\n\t\tDialer: &Dialer{Timeout: connTO},\n\t\tMaxAttempts: 1,\n\t\tTopic: makeTopic(),\n\t})\n\tctx, cancel := context.WithTimeout(context.Background(), connTestTO)\n\tdefer cancel()\n\n\t_, err := r.FetchMessage(ctx)\n\tif err == nil || ctx.Err() != nil {\n\t\tt.Errorf(\"Reader.FetchMessage must fail when it cannot \" +\n\t\t\t\"connect\")\n\t}\n}\n\nfunc TestErrorCannotConnectGroupSubscription(t *testing.T) {\n\tr := NewReader(ReaderConfig{\n\t\tBrokers: []string{\"localhost:9093\"},\n\t\tDialer: &Dialer{Timeout: 1 * time.Second},\n\t\tGroupID: \"foobar\",\n\t\tMaxAttempts: 1,\n\t\tTopic: makeTopic(),\n\t})\n\tctx, cancel := context.WithTimeout(context.Background(), connTestTO)\n\tdefer cancel()\n\n\t_, err := r.FetchMessage(ctx)\n\tif err == nil || ctx.Err() != nil {\n\t\tt.Errorf(\"Reader.FetchMessage with a group subscription \" +\n\t\t\t\"must fail when it cannot connect\")\n\t}\n}\n\n// Tests that the reader can handle messages where the response is truncated\n// due to reaching MaxBytes.\n//\n// If MaxBytes is too small to fit 1 record then it will never truncate, so\n// we start from a small message size and increase it until we are sure\n// truncation has happened at some point.\nfunc TestReaderTruncatedResponse(t *testing.T) {\n\ttopic := makeTopic()\n\tcreateTopic(t, topic, 1)\n\tdefer deleteTopic(t, topic)\n\n\treaderMaxBytes := 100\n\tbatchSize := 4\n\tmaxMsgPadding := 5\n\treadContextTimeout := 10 * time.Second\n\n\tvar msgs []Message\n\t// The key of each message\n\tn := 0\n\t// `i` is the amount of padding per message\n\tfor i := 0; i < maxMsgPadding; i++ {\n\t\tbb := bytes.Buffer{}\n\t\tfor x := 0; x < i; x++ {\n\t\t\t_, err := bb.WriteRune('0')\n\t\t\trequire.NoError(t, err)\n\t\t}\n\t\tpadding := bb.Bytes()\n\t\t// `j` is the number of times the message repeats\n\t\tfor j := 0; j < batchSize*4; j++ {\n\t\t\tmsgs = append(msgs, Message{\n\t\t\t\tKey: []byte(fmt.Sprintf(\"%05d\", n)),\n\t\t\t\tValue: padding,\n\t\t\t})\n\t\t\tn++\n\t\t}\n\t}\n\n\twr := NewWriter(WriterConfig{\n\t\tBrokers: []string{\"localhost:9092\"},\n\t\tBatchSize: batchSize,\n\t\tAsync: false,\n\t\tTopic: topic,\n\t\tBalancer: &LeastBytes{},\n\t})\n\terr := wr.WriteMessages(context.Background(), msgs...)\n\trequire.NoError(t, err)\n\n\tctx, cancel := context.WithTimeout(context.Background(), readContextTimeout)\n\tdefer cancel()\n\tr := NewReader(ReaderConfig{\n\t\tBrokers: []string{\"localhost:9092\"},\n\t\tTopic: topic,\n\t\tMinBytes: 1,\n\t\tMaxBytes: readerMaxBytes,\n\t\t// Speed up testing\n\t\tMaxWait: 100 * time.Millisecond,\n\t})\n\tdefer r.Close()\n\n\texpectedKeys := map[string]struct{}{}\n\tfor _, k := range msgs {\n\t\texpectedKeys[string(k.Key)] = struct{}{}\n\t}\n\tkeys := map[string]struct{}{}\n\tfor {\n\t\tm, err := r.FetchMessage(ctx)\n\t\trequire.NoError(t, err)\n\t\tkeys[string(m.Key)] = struct{}{}\n\n\t\tt.Logf(\"got key %s have %d keys expect %d\\n\", string(m.Key), len(keys), len(expectedKeys))\n\t\tif len(keys) == len(expectedKeys) {\n\t\t\trequire.Equal(t, expectedKeys, keys)\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// Tests that the reader can read record batches from log compacted topics\n// where the batch ends with compacted records.\n//\n// This test forces varying sized chunks of duplicated messages along with\n// configuring the topic with a minimal `segment.bytes` in order to\n// guarantee that at least 1 batch can be compacted down to 0 \"unread\" messages\n// with at least 1 \"old\" message otherwise the batch is skipped entirely.\nfunc TestReaderReadCompactedMessage(t *testing.T) {\n\ttopic := makeTopic()\n\tcreateTopicWithCompaction(t, topic, 1)\n\tdefer deleteTopic(t, topic)\n\n\tmsgs := makeTestDuplicateSequence()\n\n\twriteMessagesForCompactionCheck(t, topic, msgs)\n\n\texpectedKeys := map[string]int{}\n\tfor _, msg := range msgs {\n\t\texpectedKeys[string(msg.Key)] = 1\n\t}\n\n\t// kafka 2.0.1 is extra slow\n\tctx, cancel := context.WithTimeout(context.Background(), time.Second*120)\n\tdefer cancel()\n\tfor {\n\t\tsuccess := func() bool {\n\t\t\tr := NewReader(ReaderConfig{\n\t\t\t\tBrokers: []string{\"localhost:9092\"},\n\t\t\t\tTopic: topic,\n\t\t\t\tMinBytes: 200,\n\t\t\t\tMaxBytes: 200,\n\t\t\t\t// Speed up testing\n\t\t\t\tMaxWait: 100 * time.Millisecond,\n\t\t\t})\n\t\t\tdefer r.Close()\n\n\t\t\tkeys := map[string]int{}\n\t\t\tfor {\n\t\t\t\tm, err := r.FetchMessage(ctx)\n\t\t\t\tif err != nil {\n\t\t\t\t\tt.Logf(\"can't get message from compacted log: %v\", err)\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t\tkeys[string(m.Key)]++\n\n\t\t\t\tif len(keys) == countKeys(msgs) {\n\t\t\t\t\tt.Logf(\"got keys: %+v\", keys)\n\t\t\t\t\treturn reflect.DeepEqual(keys, expectedKeys)\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\t\tif success {\n\t\t\treturn\n\t\t}\n\t\tselect {\n\t\tcase <-ctx.Done():\n\t\t\tt.Fatal(ctx.Err())\n\t\tdefault:\n\t\t}\n\t}\n}\n\n// writeMessagesForCompactionCheck writes messages with specific writer configuration.\nfunc writeMessagesForCompactionCheck(t *testing.T, topic string, msgs []Message) {\n\tt.Helper()\n\n\twr := NewWriter(WriterConfig{\n\t\tBrokers: []string{\"localhost:9092\"},\n\t\t// Batch size must be large enough to have multiple compacted records\n\t\t// for testing more edge cases.\n\t\tBatchSize: 3,\n\t\tAsync: false,\n\t\tTopic: topic,\n\t\tBalancer: &LeastBytes{},\n\t})\n\terr := wr.WriteMessages(context.Background(), msgs...)\n\trequire.NoError(t, err)\n}\n\n// makeTestDuplicateSequence creates messages for compacted log testing\n//\n// All keys and values are 4 characters long to tightly control how many\n// messages are per log segment.\nfunc makeTestDuplicateSequence() []Message {\n\tvar msgs []Message\n\t// `n` is an increasing counter so it is never compacted.\n\tn := 0\n\t// `i` determines how many compacted records to create\n\tfor i := 0; i < 5; i++ {\n\t\t// `j` is how many times the current pattern repeats. We repeat because\n\t\t// as long as we have a pattern that is slightly larger/smaller than\n\t\t// the log segment size then if we repeat enough it will eventually\n\t\t// try all configurations.\n\t\tfor j := 0; j < 30; j++ {\n\t\t\tmsgs = append(msgs, Message{\n\t\t\t\tKey: []byte(fmt.Sprintf(\"%04d\", n)),\n\t\t\t\tValue: []byte(fmt.Sprintf(\"%04d\", n)),\n\t\t\t})\n\t\t\tn++\n\n\t\t\t// This produces the duplicated messages to compact.\n\t\t\tfor k := 0; k < i; k++ {\n\t\t\t\tmsgs = append(msgs, Message{\n\t\t\t\t\tKey: []byte(\"dup_\"),\n\t\t\t\t\tValue: []byte(\"dup_\"),\n\t\t\t\t})\n\t\t\t}\n\t\t}\n\t}\n\n\t// \"end markers\" to force duplicate message outside of the last segment of\n\t// the log so that they can all be compacted.\n\tfor i := 0; i < 10; i++ {\n\t\tmsgs = append(msgs, Message{\n\t\t\tKey: []byte(fmt.Sprintf(\"e-%02d\", i)),\n\t\t\tValue: []byte(fmt.Sprintf(\"e-%02d\", i)),\n\t\t})\n\t}\n\treturn msgs\n}\n\n// countKeys counts unique keys from given Message slice.\nfunc countKeys(msgs []Message) int {\n\tm := make(map[string]struct{})\n\tfor _, msg := range msgs {\n\t\tm[string(msg.Key)] = struct{}{}\n\t}\n\treturn len(m)\n}\n\nfunc createTopicWithCompaction(t *testing.T, topic string, partitions int) {\n\tt.Helper()\n\n\tt.Logf(\"createTopic(%s, %d)\", topic, partitions)\n\n\tconn, err := Dial(\"tcp\", \"localhost:9092\")\n\trequire.NoError(t, err)\n\tdefer conn.Close()\n\n\tcontroller, err := conn.Controller()\n\trequire.NoError(t, err)\n\n\tconn, err = Dial(\"tcp\", net.JoinHostPort(controller.Host, strconv.Itoa(controller.Port)))\n\trequire.NoError(t, err)\n\n\tconn.SetDeadline(time.Now().Add(10 * time.Second))\n\n\terr = conn.CreateTopics(TopicConfig{\n\t\tTopic: topic,\n\t\tNumPartitions: partitions,\n\t\tReplicationFactor: 1,\n\t\tConfigEntries: []ConfigEntry{\n\t\t\t{\n\t\t\t\tConfigName: \"cleanup.policy\",\n\t\t\t\tConfigValue: \"compact\",\n\t\t\t},\n\t\t\t{\n\t\t\t\tConfigName: \"segment.bytes\",\n\t\t\t\tConfigValue: \"200\",\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\tif !errors.Is(err, TopicAlreadyExists) {\n\t\t\trequire.NoError(t, err)\n\t\t}\n\t}\n\n\tctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)\n\tdefer cancel()\n\twaitForTopic(ctx, t, topic)\n}\n\n// The current behavior of the Reader is to retry OffsetOutOfRange errors\n// indefinitely, which results in programs hanging in the event of a topic being\n// re-created while a consumer is running. To retain backwards-compatibility,\n// ReaderConfig.OffsetOutOfRangeError is being used to instruct the Reader to\n// return an error in this case instead, allowing callers to react.\nfunc testReaderTopicRecreated(t *testing.T, ctx context.Context, r *Reader) {\n\tr.config.OffsetOutOfRangeError = true\n\n\ttopic := r.config.Topic\n\n\t// add 1 message to the topic\n\tprepareReader(t, ctx, r, makeTestSequence(1)...)\n\n\t// consume the message (moving the offset from 0 -> 1)\n\t_, err := r.ReadMessage(ctx)\n\trequire.NoError(t, err)\n\n\t// destroy the topic, then recreate it so the offset now becomes 0\n\tdeleteTopic(t, topic)\n\tcreateTopic(t, topic, 1)\n\n\t// expect an error, since the offset should now be out of range\n\t_, err = r.ReadMessage(ctx)\n\trequire.ErrorIs(t, err, OffsetOutOfRange)\n}\n" }, { "path": "record.go", "content": "package kafka\n\nimport (\n\t\"github.com/segmentio/kafka-go/protocol\"\n)\n\n// Header is a key/value pair type representing headers set on records.\ntype Header = protocol.Header\n\n// Bytes is an interface representing a sequence of bytes. This abstraction\n// makes it possible for programs to inject data into produce requests without\n// having to load in into an intermediary buffer, or read record keys and values\n// from a fetch response directly from internal buffers.\n//\n// Bytes are not safe to use concurrently from multiple goroutines.\ntype Bytes = protocol.Bytes\n\n// NewBytes constructs a Bytes value from a byte slice.\n//\n// If b is nil, nil is returned.\nfunc NewBytes(b []byte) Bytes { return protocol.NewBytes(b) }\n\n// ReadAll reads b into a byte slice.\nfunc ReadAll(b Bytes) ([]byte, error) { return protocol.ReadAll(b) }\n\n// Record is an interface representing a single kafka record.\n//\n// Record values are not safe to use concurrently from multiple goroutines.\ntype Record = protocol.Record\n\n// RecordReader is an interface representing a sequence of records. Record sets\n// are used in both produce and fetch requests to represent the sequence of\n// records that are sent to or receive from kafka brokers.\n//\n// RecordReader values are not safe to use concurrently from multiple goroutines.\ntype RecordReader = protocol.RecordReader\n\n// NewRecordReader reconstructs a RecordSet which exposes the sequence of records\n// passed as arguments.\nfunc NewRecordReader(records ...Record) RecordReader {\n\treturn protocol.NewRecordReader(records...)\n}\n" }, { "path": "recordbatch.go", "content": "package kafka\n\nimport (\n\t\"bytes\"\n\t\"time\"\n)\n\nconst recordBatchHeaderSize int32 = 0 +\n\t8 + // base offset\n\t4 + // batch length\n\t4 + // partition leader epoch\n\t1 + // magic\n\t4 + // crc\n\t2 + // attributes\n\t4 + // last offset delta\n\t8 + // first timestamp\n\t8 + // max timestamp\n\t8 + // producer id\n\t2 + // producer epoch\n\t4 + // base sequence\n\t4 // msg count\n\nfunc recordBatchSize(msgs ...Message) (size int32) {\n\tsize = recordBatchHeaderSize\n\tbaseTime := msgs[0].Time\n\n\tfor i := range msgs {\n\t\tmsg := &msgs[i]\n\t\tmsz := recordSize(msg, msg.Time.Sub(baseTime), int64(i))\n\t\tsize += int32(msz + varIntLen(int64(msz)))\n\t}\n\n\treturn\n}\n\nfunc compressRecordBatch(codec CompressionCodec, msgs ...Message) (compressed *bytes.Buffer, attributes int16, size int32, err error) {\n\tcompressed = acquireBuffer()\n\tcompressor := codec.NewWriter(compressed)\n\twb := &writeBuffer{w: compressor}\n\n\tfor i, msg := range msgs {\n\t\twb.writeRecord(0, msgs[0].Time, int64(i), msg)\n\t}\n\n\tif err = compressor.Close(); err != nil {\n\t\treleaseBuffer(compressed)\n\t\treturn\n\t}\n\n\tattributes = int16(codec.Code())\n\tsize = recordBatchHeaderSize + int32(compressed.Len())\n\treturn\n}\n\ntype recordBatch struct {\n\t// required input parameters\n\tcodec CompressionCodec\n\tattributes int16\n\tmsgs []Message\n\n\t// parameters calculated during init\n\tcompressed *bytes.Buffer\n\tsize int32\n}\n\nfunc newRecordBatch(codec CompressionCodec, msgs ...Message) (r *recordBatch, err error) {\n\tr = &recordBatch{\n\t\tcodec: codec,\n\t\tmsgs: msgs,\n\t}\n\tif r.codec == nil {\n\t\tr.size = recordBatchSize(r.msgs...)\n\t} else {\n\t\tr.compressed, r.attributes, r.size, err = compressRecordBatch(r.codec, r.msgs...)\n\t}\n\treturn\n}\n\nfunc (r *recordBatch) writeTo(wb *writeBuffer) {\n\twb.writeInt32(r.size)\n\n\tbaseTime := r.msgs[0].Time\n\tlastTime := r.msgs[len(r.msgs)-1].Time\n\tif r.compressed != nil {\n\t\twb.writeRecordBatch(r.attributes, r.size, len(r.msgs), baseTime, lastTime, func(wb *writeBuffer) {\n\t\t\twb.Write(r.compressed.Bytes())\n\t\t})\n\t\treleaseBuffer(r.compressed)\n\t} else {\n\t\twb.writeRecordBatch(r.attributes, r.size, len(r.msgs), baseTime, lastTime, func(wb *writeBuffer) {\n\t\t\tfor i, msg := range r.msgs {\n\t\t\t\twb.writeRecord(0, r.msgs[0].Time, int64(i), msg)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc recordSize(msg *Message, timestampDelta time.Duration, offsetDelta int64) int {\n\treturn 1 + // attributes\n\t\tvarIntLen(int64(milliseconds(timestampDelta))) +\n\t\tvarIntLen(offsetDelta) +\n\t\tvarBytesLen(msg.Key) +\n\t\tvarBytesLen(msg.Value) +\n\t\tvarArrayLen(len(msg.Headers), func(i int) int {\n\t\t\th := &msg.Headers[i]\n\t\t\treturn varStringLen(h.Key) + varBytesLen(h.Value)\n\t\t})\n}\n" }, { "path": "resolver.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"net\"\n)\n\n// The Resolver interface is used as an abstraction to provide service discovery\n// of the hosts of a kafka cluster.\ntype Resolver interface {\n\t// LookupHost looks up the given host using the local resolver.\n\t// It returns a slice of that host's addresses.\n\tLookupHost(ctx context.Context, host string) (addrs []string, err error)\n}\n\n// BrokerResolver is an interface implemented by types that translate host\n// names into a network address.\n//\n// This resolver is not intended to be a general purpose interface. Instead,\n// it is tailored to the particular needs of the kafka protocol, with the goal\n// being to provide a flexible mechanism for extending broker name resolution\n// while retaining context that is specific to interacting with a kafka cluster.\n//\n// Resolvers must be safe to use from multiple goroutines.\ntype BrokerResolver interface {\n\t// Returns the IP addresses of the broker passed as argument.\n\tLookupBrokerIPAddr(ctx context.Context, broker Broker) ([]net.IPAddr, error)\n}\n\n// NewBrokerResolver constructs a Resolver from r.\n//\n// If r is nil, net.DefaultResolver is used instead.\nfunc NewBrokerResolver(r *net.Resolver) BrokerResolver {\n\treturn brokerResolver{r}\n}\n\ntype brokerResolver struct {\n\t*net.Resolver\n}\n\nfunc (r brokerResolver) LookupBrokerIPAddr(ctx context.Context, broker Broker) ([]net.IPAddr, error) {\n\tipAddrs, err := r.LookupIPAddr(ctx, broker.Host)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tif len(ipAddrs) == 0 {\n\t\treturn nil, &net.DNSError{\n\t\t\tErr: \"no addresses were returned by the resolver\",\n\t\t\tName: broker.Host,\n\t\t\tIsTemporary: true,\n\t\t\tIsNotFound: true,\n\t\t}\n\t}\n\n\treturn ipAddrs, nil\n}\n" }, { "path": "resource.go", "content": "package kafka\n\nimport (\n\t\"fmt\"\n\t\"strings\"\n)\n\n// https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/ResourceType.java\ntype ResourceType int8\n\nconst (\n\tResourceTypeUnknown ResourceType = 0\n\tResourceTypeAny ResourceType = 1\n\tResourceTypeTopic ResourceType = 2\n\tResourceTypeGroup ResourceType = 3\n\t// See https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/config/ConfigResource.java#L36\n\tResourceTypeBroker ResourceType = 4\n\tResourceTypeCluster ResourceType = 4\n\tResourceTypeTransactionalID ResourceType = 5\n\tResourceTypeDelegationToken ResourceType = 6\n)\n\nfunc (rt ResourceType) String() string {\n\tmapping := map[ResourceType]string{\n\t\tResourceTypeUnknown: \"Unknown\",\n\t\tResourceTypeAny: \"Any\",\n\t\tResourceTypeTopic: \"Topic\",\n\t\tResourceTypeGroup: \"Group\",\n\t\t// Note that ResourceTypeBroker and ResourceTypeCluster have the same value.\n\t\t// A map cannot have duplicate values so we just use the same value for both.\n\t\tResourceTypeCluster: \"Cluster\",\n\t\tResourceTypeTransactionalID: \"Transactionalid\",\n\t\tResourceTypeDelegationToken: \"Delegationtoken\",\n\t}\n\ts, ok := mapping[rt]\n\tif !ok {\n\t\ts = mapping[ResourceTypeUnknown]\n\t}\n\treturn s\n}\n\nfunc (rt ResourceType) MarshalText() ([]byte, error) {\n\treturn []byte(rt.String()), nil\n}\n\nfunc (rt *ResourceType) UnmarshalText(text []byte) error {\n\tnormalized := strings.ToLower(string(text))\n\tmapping := map[string]ResourceType{\n\t\t\"unknown\": ResourceTypeUnknown,\n\t\t\"any\": ResourceTypeAny,\n\t\t\"topic\": ResourceTypeTopic,\n\t\t\"group\": ResourceTypeGroup,\n\t\t\"broker\": ResourceTypeBroker,\n\t\t\"cluster\": ResourceTypeCluster,\n\t\t\"transactionalid\": ResourceTypeTransactionalID,\n\t\t\"delegationtoken\": ResourceTypeDelegationToken,\n\t}\n\tparsed, ok := mapping[normalized]\n\tif !ok {\n\t\t*rt = ResourceTypeUnknown\n\t\treturn fmt.Errorf(\"cannot parse %s as a ResourceType\", normalized)\n\t}\n\t*rt = parsed\n\treturn nil\n}\n\n// https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/PatternType.java\ntype PatternType int8\n\nconst (\n\t// PatternTypeUnknown represents any PatternType which this client cannot\n\t// understand.\n\tPatternTypeUnknown PatternType = 0\n\t// PatternTypeAny matches any resource pattern type.\n\tPatternTypeAny PatternType = 1\n\t// PatternTypeMatch perform pattern matching.\n\tPatternTypeMatch PatternType = 2\n\t// PatternTypeLiteral represents a literal name.\n\t// A literal name defines the full name of a resource, e.g. topic with name\n\t// 'foo', or group with name 'bob'.\n\tPatternTypeLiteral PatternType = 3\n\t// PatternTypePrefixed represents a prefixed name.\n\t// A prefixed name defines a prefix for a resource, e.g. topics with names\n\t// that start with 'foo'.\n\tPatternTypePrefixed PatternType = 4\n)\n\nfunc (pt PatternType) String() string {\n\tmapping := map[PatternType]string{\n\t\tPatternTypeUnknown: \"Unknown\",\n\t\tPatternTypeAny: \"Any\",\n\t\tPatternTypeMatch: \"Match\",\n\t\tPatternTypeLiteral: \"Literal\",\n\t\tPatternTypePrefixed: \"Prefixed\",\n\t}\n\ts, ok := mapping[pt]\n\tif !ok {\n\t\ts = mapping[PatternTypeUnknown]\n\t}\n\treturn s\n}\n\nfunc (pt PatternType) MarshalText() ([]byte, error) {\n\treturn []byte(pt.String()), nil\n}\n\nfunc (pt *PatternType) UnmarshalText(text []byte) error {\n\tnormalized := strings.ToLower(string(text))\n\tmapping := map[string]PatternType{\n\t\t\"unknown\": PatternTypeUnknown,\n\t\t\"any\": PatternTypeAny,\n\t\t\"match\": PatternTypeMatch,\n\t\t\"literal\": PatternTypeLiteral,\n\t\t\"prefixed\": PatternTypePrefixed,\n\t}\n\tparsed, ok := mapping[normalized]\n\tif !ok {\n\t\t*pt = PatternTypeUnknown\n\t\treturn fmt.Errorf(\"cannot parse %s as a PatternType\", normalized)\n\t}\n\t*pt = parsed\n\treturn nil\n}\n" }, { "path": "resource_test.go", "content": "package kafka\n\nimport \"testing\"\n\nfunc TestResourceTypeMarshal(t *testing.T) {\n\tfor i := ResourceTypeUnknown; i <= ResourceTypeDelegationToken; i++ {\n\t\ttext, err := i.MarshalText()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"couldn't marshal %d to text: %s\", i, err)\n\t\t}\n\t\tvar got ResourceType\n\t\terr = got.UnmarshalText(text)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"couldn't unmarshal %s to ResourceType: %s\", text, err)\n\t\t}\n\t\tif got != i {\n\t\t\tt.Errorf(\"got %d, want %d\", got, i)\n\t\t}\n\t}\n}\n\n// Verify that the text version of ResourceTypeBroker is \"Cluster\".\n// This is added since ResourceTypeBroker and ResourceTypeCluster\n// have the same value.\nfunc TestResourceTypeBroker(t *testing.T) {\n\ttext, err := ResourceTypeBroker.MarshalText()\n\tif err != nil {\n\t\tt.Errorf(\"couldn't marshal %d to text: %s\", ResourceTypeBroker, err)\n\t}\n\tif string(text) != \"Cluster\" {\n\t\tt.Errorf(\"got %s, want %s\", string(text), \"Cluster\")\n\t}\n\tvar got ResourceType\n\terr = got.UnmarshalText(text)\n\tif err != nil {\n\t\tt.Errorf(\"couldn't unmarshal %s to ResourceType: %s\", text, err)\n\t}\n\tif got != ResourceTypeBroker {\n\t\tt.Errorf(\"got %d, want %d\", got, ResourceTypeBroker)\n\t}\n}\n\nfunc TestPatternTypeMarshal(t *testing.T) {\n\tfor i := PatternTypeUnknown; i <= PatternTypePrefixed; i++ {\n\t\ttext, err := i.MarshalText()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"couldn't marshal %d to text: %s\", i, err)\n\t\t}\n\t\tvar got PatternType\n\t\terr = got.UnmarshalText(text)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"couldn't unmarshal %s to PatternType: %s\", text, err)\n\t\t}\n\t\tif got != i {\n\t\t\tt.Errorf(\"got %d, want %d\", got, i)\n\t\t}\n\t}\n}\n" }, { "path": "sasl/aws_msk_iam/go.mod", "content": "module github.com/segmentio/kafka-go/sasl/aws_msk_iam\n\ngo 1.15\n\nrequire (\n\tgithub.com/aws/aws-sdk-go v1.41.3\n\tgithub.com/segmentio/kafka-go v0.4.28\n)\n" }, { "path": "sasl/aws_msk_iam/go.sum", "content": "github.com/aws/aws-sdk-go v1.41.3 h1:deglLZ1jjHdhkd6Rbad1MZM4gL+1pfnTfjuFk6CGJFM=\ngithub.com/aws/aws-sdk-go v1.41.3/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=\ngithub.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=\ngithub.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=\ngithub.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=\ngithub.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY=\ngithub.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=\ngithub.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=\ngithub.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=\ngithub.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=\ngithub.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=\ngithub.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=\ngithub.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=\ngithub.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=\ngithub.com/klauspost/compress v1.9.8 h1:VMAMUUOh+gaxKTMk+zqbjsSjsIcUcL/LF4o63i82QyA=\ngithub.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=\ngithub.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=\ngithub.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=\ngithub.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=\ngithub.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=\ngithub.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=\ngithub.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A=\ngithub.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=\ngithub.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=\ngithub.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/segmentio/kafka-go v0.4.28 h1:ATYbyenAlsoFxnV+VpIJMF87bvRuRsX7fezHNfpwkdM=\ngithub.com/segmentio/kafka-go v0.4.28/go.mod h1:XzMcoMjSzDGHcIwpWUI7GB43iKZ2fTVmryPSGLf/MPg=\ngithub.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=\ngithub.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=\ngithub.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=\ngithub.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0=\ngithub.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=\ngolang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=\ngolang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284 h1:rlLehGeYg6jfoyz/eDqDU1iRXLKfR42nnNh57ytKEWo=\ngolang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=\ngolang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=\ngolang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=\ngolang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=\ngolang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=\ngolang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=\ngopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\n" }, { "path": "sasl/aws_msk_iam/msk_iam.go", "content": "package aws_msk_iam\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"fmt\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"runtime\"\n\t\"strings\"\n\t\"time\"\n\n\tsigv4 \"github.com/aws/aws-sdk-go/aws/signer/v4\"\n\t\"github.com/segmentio/kafka-go/sasl\"\n)\n\nconst (\n\t// These constants come from https://github.com/aws/aws-msk-iam-auth#details and\n\t// https://github.com/aws/aws-msk-iam-auth/blob/main/src/main/java/software/amazon/msk/auth/iam/internals/AWS4SignedPayloadGenerator.java.\n\tsignVersion = \"2020_10_22\"\n\tsignService = \"kafka-cluster\"\n\tsignAction = \"kafka-cluster:Connect\"\n\tsignVersionKey = \"version\"\n\tsignHostKey = \"host\"\n\tsignUserAgentKey = \"user-agent\"\n\tsignActionKey = \"action\"\n\tqueryActionKey = \"Action\"\n)\n\nvar signUserAgent = fmt.Sprintf(\"kafka-go/sasl/aws_msk_iam/%s\", runtime.Version())\n\n// Mechanism implements sasl.Mechanism for the AWS_MSK_IAM mechanism, based on the official java implementation:\n// https://github.com/aws/aws-msk-iam-auth\ntype Mechanism struct {\n\t// The sigv4.Signer to use when signing the request. Required.\n\tSigner *sigv4.Signer\n\t// The region where the msk cluster is hosted, e.g. \"us-east-1\". Required.\n\tRegion string\n\t// The time the request is planned for. Optional, defaults to time.Now() at time of authentication.\n\tSignTime time.Time\n\t// The duration for which the presigned request is active. Optional, defaults to 5 minutes.\n\tExpiry time.Duration\n}\n\nfunc (m *Mechanism) Name() string {\n\treturn \"AWS_MSK_IAM\"\n}\n\n// Start produces the authentication values required for AWS_MSK_IAM. It produces the following json as a byte array,\n// making use of the aws-sdk to produce the signed output.\n// \t{\n// \t \"version\" : \"2020_10_22\",\n// \t \"host\" : \"\",\n// \t \"user-agent\": \"\",\n// \t \"action\": \"kafka-cluster:Connect\",\n// \t \"x-amz-algorithm\" : \"\",\n// \t \"x-amz-credential\" : \"///kafka-cluster/aws4_request\",\n// \t \"x-amz-date\" : \"\",\n// \t \"x-amz-security-token\" : \"\",\n// \t \"x-amz-signedheaders\" : \"host\",\n// \t \"x-amz-expires\" : \"\",\n// \t \"x-amz-signature\" : \"\"\n// \t}\nfunc (m *Mechanism) Start(ctx context.Context) (sess sasl.StateMachine, ir []byte, err error) {\n\tsaslMeta := sasl.MetadataFromContext(ctx)\n\tif saslMeta == nil {\n\t\treturn nil, nil, errors.New(\"missing sasl metadata\")\n\t}\n\n\tquery := url.Values{\n\t\tqueryActionKey: {signAction},\n\t}\n\n\tsignUrl := url.URL{\n\t\tScheme: \"kafka\",\n\t\tHost: saslMeta.Host,\n\t\tPath: \"/\",\n\t\tRawQuery: query.Encode(),\n\t}\n\n\treq, err := http.NewRequest(\"GET\", signUrl.String(), nil)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tsignTime := m.SignTime\n\tif signTime.IsZero() {\n\t\tsignTime = time.Now()\n\t}\n\n\texpiry := m.Expiry\n\tif expiry == 0 {\n\t\texpiry = 5 * time.Minute\n\t}\n\n\theader, err := m.Signer.Presign(req, nil, signService, m.Region, expiry, signTime)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\tsignedMap := map[string]string{\n\t\tsignVersionKey: signVersion,\n\t\tsignHostKey: signUrl.Host,\n\t\tsignUserAgentKey: signUserAgent,\n\t\tsignActionKey: signAction,\n\t}\n\t// The protocol requires lowercase keys.\n\tfor key, vals := range header {\n\t\tsignedMap[strings.ToLower(key)] = vals[0]\n\t}\n\tfor key, vals := range req.URL.Query() {\n\t\tsignedMap[strings.ToLower(key)] = vals[0]\n\t}\n\n\tsignedJson, err := json.Marshal(signedMap)\n\treturn m, signedJson, err\n}\n\nfunc (m *Mechanism) Next(ctx context.Context, challenge []byte) (bool, []byte, error) {\n\t// After the initial step, the authentication is complete\n\t// kafka will return error if it rejected the credentials, so we'll only\n\t// arrive here on success.\n\treturn true, nil, nil\n}\n" }, { "path": "sasl/aws_msk_iam/msk_iam_test.go", "content": "package aws_msk_iam\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"encoding/json\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/sasl\"\n\n\t\"github.com/aws/aws-sdk-go/aws/credentials\"\n\tsigv4 \"github.com/aws/aws-sdk-go/aws/signer/v4\"\n)\n\nconst (\n\taccessKeyId = \"ACCESS_KEY\"\n\tsecretAccessKey = \"SECRET_KEY\"\n)\n\n// using a fixed time allows the signature to be verifiable in a test\nvar signTime = time.Date(2021, 10, 14, 13, 5, 0, 0, time.UTC)\n\nfunc TestAwsMskIamMechanism(t *testing.T) {\n\ttests := []struct {\n\t\tdescription string\n\t\tctx func() context.Context\n\t\tshouldFail bool\n\t}{\n\t\t{\n\t\t\tdescription: \"with metadata\",\n\t\t\tctx: func() context.Context {\n\t\t\t\treturn sasl.WithMetadata(context.Background(), &sasl.Metadata{\n\t\t\t\t\tHost: \"localhost\",\n\t\t\t\t\tPort: 9092,\n\t\t\t\t})\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tdescription: \"without metadata\",\n\t\t\tctx: func() context.Context {\n\t\t\t\treturn context.Background()\n\t\t\t},\n\t\t\tshouldFail: true,\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.description, func(t *testing.T) {\n\t\t\tctx := tt.ctx()\n\n\t\t\tcreds := credentials.NewStaticCredentials(accessKeyId, secretAccessKey, \"\")\n\t\t\tmskMechanism := &Mechanism{\n\t\t\t\tSigner: sigv4.NewSigner(creds),\n\t\t\t\tRegion: \"us-east-1\",\n\t\t\t\tSignTime: signTime,\n\t\t\t}\n\n\t\t\tsess, auth, err := mskMechanism.Start(ctx)\n\t\t\tif tt.shouldFail { // if error is expected\n\t\t\t\tif err == nil { // but we don't find one\n\t\t\t\t\tt.Fatal(\"error expected\")\n\t\t\t\t} else { // but we do find one\n\t\t\t\t\treturn // return early since the remaining assertions are irrelevant\n\t\t\t\t}\n\t\t\t} else { // if error is not expected (typical)\n\t\t\t\tif err != nil { // but we do find one\n\t\t\t\t\tt.Fatal(err)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif sess != mskMechanism {\n\t\t\t\tt.Error(\n\t\t\t\t\t\"Unexpected session\",\n\t\t\t\t\t\"expected\", mskMechanism,\n\t\t\t\t\t\"got\", sess,\n\t\t\t\t)\n\t\t\t}\n\n\t\t\texpectedMap := map[string]string{\n\t\t\t\t\"version\": \"2020_10_22\",\n\t\t\t\t\"action\": \"kafka-cluster:Connect\",\n\t\t\t\t\"host\": \"localhost\",\n\t\t\t\t\"user-agent\": signUserAgent,\n\t\t\t\t\"x-amz-algorithm\": \"AWS4-HMAC-SHA256\",\n\t\t\t\t\"x-amz-credential\": \"ACCESS_KEY/20211014/us-east-1/kafka-cluster/aws4_request\",\n\t\t\t\t\"x-amz-date\": \"20211014T130500Z\",\n\t\t\t\t\"x-amz-expires\": \"300\",\n\t\t\t\t\"x-amz-signedheaders\": \"host\",\n\t\t\t\t\"x-amz-signature\": \"6b8d25f9b45b9c7db9da855a49112d80379224153a27fd279c305a5b7940d1a7\",\n\t\t\t}\n\t\t\texpectedAuth, err := json.Marshal(expectedMap)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\tif !bytes.Equal(expectedAuth, auth) {\n\t\t\t\tt.Error(\"Unexpected authentication\",\n\t\t\t\t\t\"expected\", expectedAuth,\n\t\t\t\t\t\"got\", auth,\n\t\t\t\t)\n\t\t\t}\n\t\t})\n\t}\n}\n" }, { "path": "sasl/aws_msk_iam_v2/README.md", "content": "# AWS MSK IAM V2\n\nThis extension provides a capability to get authenticated with [AWS Managed Apache Kafka](https://aws.amazon.com/msk/)\nthrough AWS IAM.\n\n## How to use\n\nThis module is an extension for MSK users and thus this is isolated from `kafka-go` module.\nYou can add this module to your dependency by running the command below.\n\n```shell\ngo get github.com/segmentio/kafka-go/sasl/aws_msk_iam_v2\n```\n\nPlease find the sample code in [example_test.go](./example_test.go), you can use the `Mechanism` for SASL authentication of `Reader` and `Writer`.\n" }, { "path": "sasl/aws_msk_iam_v2/example_test.go", "content": "package aws_msk_iam_v2_test\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go-v2/config\"\n\t\"github.com/segmentio/kafka-go\"\n\t\"github.com/segmentio/kafka-go/sasl/aws_msk_iam_v2\"\n)\n\nfunc main() {\n\tcfg, err := config.LoadDefaultConfig(context.TODO())\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\tmechanism := aws_msk_iam_v2.NewMechanism(cfg)\n\t_ = kafka.ReaderConfig{\n\t\tBrokers: []string{\"https://localhost\"},\n\t\tGroupID: \"some-consumer-group\",\n\t\tGroupTopics: []string{\"some-topic\"},\n\t\tDialer: &kafka.Dialer{\n\t\t\tTimeout: 10 * time.Second,\n\t\t\tDualStack: true,\n\t\t\tSASLMechanism: mechanism,\n\t\t\tTLS: &tls.Config{},\n\t\t},\n\t}\n}\n" }, { "path": "sasl/aws_msk_iam_v2/go.mod", "content": "module github.com/segmentio/kafka-go/sasl/aws_msk_iam_v2\n\ngo 1.15\n\nrequire (\n\tgithub.com/aws/aws-sdk-go-v2 v1.16.12\n\tgithub.com/aws/aws-sdk-go-v2/config v1.17.2\n\tgithub.com/aws/aws-sdk-go-v2/credentials v1.12.15\n\tgithub.com/segmentio/kafka-go v0.4.34\n\tgithub.com/stretchr/testify v1.8.0\n)\n" }, { "path": "sasl/aws_msk_iam_v2/go.sum", "content": "github.com/aws/aws-sdk-go-v2 v1.16.12 h1:wbMYa2PlFysFx2GLIQojr6FJV5+OWCM/BwyHXARxETA=\ngithub.com/aws/aws-sdk-go-v2 v1.16.12/go.mod h1:C+Ym0ag2LIghJbXhfXZ0YEEp49rBWowxKzJLUoob0ts=\ngithub.com/aws/aws-sdk-go-v2/config v1.17.2 h1:V96WPd2a1H/MXGZjk4zto+KpYnwZI2kdIdy/cI8kYnQ=\ngithub.com/aws/aws-sdk-go-v2/config v1.17.2/go.mod h1:jumS/AMwul4WaG8vyXsF6kUndG9zndR+yfYBwl4i9ds=\ngithub.com/aws/aws-sdk-go-v2/credentials v1.12.15 h1:6DONxG9cR3pAuISj1Irh5u2SRqCfIJwyHNyDDes7SZw=\ngithub.com/aws/aws-sdk-go-v2/credentials v1.12.15/go.mod h1:41zTC6U/78fUD7ZCa5NymTJANDjfqySg5YEAYVFl2Ic=\ngithub.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.13 h1:+uferi8SUDZtMloCDt24Zenyy/i71C/ua5mjUCpbpN0=\ngithub.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.13/go.mod h1:y0eXmsNBFIVjUE8ZBjES8myOHlMsXDz7qGT93+MVdjk=\ngithub.com/aws/aws-sdk-go-v2/internal/configsources v1.1.19 h1:gC5mudiFrWGhzcdoWj1iCGUfrzCpQG0MQIQf0CXFFQQ=\ngithub.com/aws/aws-sdk-go-v2/internal/configsources v1.1.19/go.mod h1:llxE6bwUZhuCas0K7qGiu5OgMis3N7kdWtFSxoHmJ7E=\ngithub.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.13 h1:qezY57na06d6kSE7uuB0N7XEflu914AXx/hg2L8Ykcw=\ngithub.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.13/go.mod h1:lB12mkZqCSo5PsdBFLNqc2M/OOYgNAy8UtaktyuWvE8=\ngithub.com/aws/aws-sdk-go-v2/internal/ini v1.3.20 h1:GvszACAU8GSV3+Tant5GutW6smY8WavrP8ZuRS9Ku4Q=\ngithub.com/aws/aws-sdk-go-v2/internal/ini v1.3.20/go.mod h1:bfTcsThj5a9P5pIGRy0QudJ8k4+issxXX+O6Djnd5Cs=\ngithub.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.13 h1:ObfthqDyhe7rMAOa7pqft6974VHIk8BAJB7kYdoIfTA=\ngithub.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.13/go.mod h1:V390DK4MQxLpDdXxFqizyz8KUxuWImkW/xzgXMz0yyk=\ngithub.com/aws/aws-sdk-go-v2/service/sso v1.11.18 h1:gTn1a/FbcOXK5LQS88dD5k+PKwyjVvhAEEwyN4c6eW8=\ngithub.com/aws/aws-sdk-go-v2/service/sso v1.11.18/go.mod h1:ytmEi5+qwcSNcV2pVA8PIb1DnKT/0Bu/K4nfJHwoM6c=\ngithub.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.1 h1:p48IfndYbRk3iDsoQAmVXdCKEM5+7Y50JAPikjwk8gI=\ngithub.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.1/go.mod h1:NY+G+8PW0ISyJ7/6t5mgOe6qpJiwZa9Jix05WPscJjg=\ngithub.com/aws/aws-sdk-go-v2/service/sts v1.16.14 h1:7kxso8VZLQ86Jg27QRBw6fjrQhQ8CMNMZ7SB0w7RQiA=\ngithub.com/aws/aws-sdk-go-v2/service/sts v1.16.14/go.mod h1:Y+BUV19q3OmQVqNUlbZ40zVi3NM6Biuxwkx/qdSD/CY=\ngithub.com/aws/smithy-go v1.13.0 h1:YfyEmSJLo7fAv8FbuDK4R8F9aAmi9DZ88Zb/KJJmUl0=\ngithub.com/aws/smithy-go v1.13.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=\ngithub.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=\ngithub.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=\ngithub.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=\ngithub.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=\ngithub.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=\ngithub.com/klauspost/compress v1.15.7 h1:7cgTQxJCU/vy+oP/E3B9RGbQTgbiVzIJWIKOLoAsPok=\ngithub.com/klauspost/compress v1.15.7/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=\ngithub.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=\ngithub.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=\ngithub.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=\ngithub.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/segmentio/kafka-go v0.4.34 h1:Dm6YlLMiVSiwwav20KY0AoY63s661FXevwJ3CVHUERo=\ngithub.com/segmentio/kafka-go v0.4.34/go.mod h1:GAjxBQJdQMB5zfNA21AhpaqOB2Mu+w3De4ni3Gbm8y0=\ngithub.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=\ngithub.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=\ngithub.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=\ngithub.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw=\ngithub.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=\ngithub.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4=\ngithub.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=\ngolang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=\ngolang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=\ngolang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=\ngolang.org/x/net v0.0.0-20220706163947-c90051bbdb60 h1:8NSylCMxLW4JvserAndSgFL7aPli6A68yf0bYFTcWCM=\ngolang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=\ngolang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=\ngolang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=\ngolang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=\ngolang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=\ngolang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\ngopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=\ngopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\n" }, { "path": "sasl/aws_msk_iam_v2/msk_iam.go", "content": "package aws_msk_iam_v2\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\tsigner \"github.com/aws/aws-sdk-go-v2/aws/signer/v4\"\n\t\"github.com/segmentio/kafka-go/sasl\"\n)\n\nconst (\n\t// These constants come from https://github.com/aws/aws-msk-iam-auth#details and\n\t// https://github.com/aws/aws-msk-iam-auth/blob/main/src/main/java/software/amazon/msk/auth/iam/internals/AWS4SignedPayloadGenerator.java.\n\tsignAction = \"kafka-cluster:Connect\"\n\tsignPayload = \"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\" // the hex encoded SHA-256 of an empty string\n\tsignService = \"kafka-cluster\"\n\tsignVersion = \"2020_10_22\"\n\tsignActionKey = \"action\"\n\tsignHostKey = \"host\"\n\tsignUserAgentKey = \"user-agent\"\n\tsignVersionKey = \"version\"\n\tqueryActionKey = \"Action\"\n\tqueryExpiryKey = \"X-Amz-Expires\"\n)\n\nvar signUserAgent = \"kafka-go/sasl/aws_msk_iam_v2/\" + runtime.Version()\n\n// Mechanism implements sasl.Mechanism for the AWS_MSK_IAM mechanism, based on the official java implementation:\n// https://github.com/aws/aws-msk-iam-auth\ntype Mechanism struct {\n\t// The sigv4.Signer of aws-sdk-go-v2 to use when signing the request. Required.\n\tSigner *signer.Signer\n\t// The aws.Config.Credentials or config.CredentialsProvider of aws-sdk-go-v2. Required.\n\tCredentials aws.CredentialsProvider\n\t// The region where the msk cluster is hosted, e.g. \"us-east-1\". Required.\n\tRegion string\n\t// The time the request is planned for. Optional, defaults to time.Now() at time of authentication.\n\tSignTime time.Time\n\t// The duration for which the presigned request is active. Optional, defaults to 5 minutes.\n\tExpiry time.Duration\n}\n\nfunc (m *Mechanism) Name() string {\n\treturn \"AWS_MSK_IAM\"\n}\n\nfunc (m *Mechanism) Next(ctx context.Context, challenge []byte) (bool, []byte, error) {\n\t// After the initial step, the authentication is complete\n\t// kafka will return error if it rejected the credentials, so we'll only\n\t// arrive here on success.\n\treturn true, nil, nil\n}\n\n// Start produces the authentication values required for AWS_MSK_IAM. It produces the following json as a byte array,\n// making use of the aws-sdk to produce the signed output.\n//\n//\t{\n//\t \"version\" : \"2020_10_22\",\n//\t \"host\" : \"\",\n//\t \"user-agent\": \"\",\n//\t \"action\": \"kafka-cluster:Connect\",\n//\t \"x-amz-algorithm\" : \"\",\n//\t \"x-amz-credential\" : \"///kafka-cluster/aws4_request\",\n//\t \"x-amz-date\" : \"\",\n//\t \"x-amz-security-token\" : \"\",\n//\t \"x-amz-signedheaders\" : \"host\",\n//\t \"x-amz-expires\" : \"\",\n//\t \"x-amz-signature\" : \"\"\n//\t}\nfunc (m *Mechanism) Start(ctx context.Context) (sess sasl.StateMachine, ir []byte, err error) {\n\tsignedMap, err := m.preSign(ctx)\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\n\tsignedJson, err := json.Marshal(signedMap)\n\treturn m, signedJson, err\n}\n\n// preSign produces the authentication values required for AWS_MSK_IAM.\nfunc (m *Mechanism) preSign(ctx context.Context) (map[string]string, error) {\n\treq, err := buildReq(ctx, defaultExpiry(m.Expiry))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tcreds, err := m.Credentials.Retrieve(ctx)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tsignedUrl, header, err := m.Signer.PresignHTTP(ctx, creds, req, signPayload, signService, m.Region, defaultSignTime(m.SignTime))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tu, err := url.Parse(signedUrl)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn buildSignedMap(u, header), nil\n}\n\n// buildReq builds http.Request for aws PreSign.\nfunc buildReq(ctx context.Context, expiry time.Duration) (*http.Request, error) {\n\tquery := url.Values{\n\t\tqueryActionKey: {signAction},\n\t\tqueryExpiryKey: {strconv.FormatInt(int64(expiry/time.Second), 10)},\n\t}\n\tsaslMeta := sasl.MetadataFromContext(ctx)\n\tif saslMeta == nil {\n\t\treturn nil, errors.New(\"missing sasl metadata\")\n\t}\n\n\tsignUrl := url.URL{\n\t\tScheme: \"kafka\",\n\t\tHost: saslMeta.Host,\n\t\tPath: \"/\",\n\t\tRawQuery: query.Encode(),\n\t}\n\n\treq, err := http.NewRequest(http.MethodGet, signUrl.String(), nil)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn req, nil\n}\n\n// buildSignedMap builds signed string map which will be used to authenticate with MSK.\nfunc buildSignedMap(u *url.URL, header http.Header) map[string]string {\n\tsignedMap := map[string]string{\n\t\tsignVersionKey: signVersion,\n\t\tsignHostKey: u.Host,\n\t\tsignUserAgentKey: signUserAgent,\n\t\tsignActionKey: signAction,\n\t}\n\t// The protocol requires lowercase keys.\n\tfor key, vals := range header {\n\t\tsignedMap[strings.ToLower(key)] = vals[0]\n\t}\n\tfor key, vals := range u.Query() {\n\t\tsignedMap[strings.ToLower(key)] = vals[0]\n\t}\n\n\treturn signedMap\n}\n\n// defaultExpiry set default expiration time if user doesn't define Mechanism.Expiry.\nfunc defaultExpiry(v time.Duration) time.Duration {\n\tif v == 0 {\n\t\treturn 5 * time.Minute\n\t}\n\treturn v\n}\n\n// defaultSignTime set default sign time if user doesn't define Mechanism.SignTime.\nfunc defaultSignTime(v time.Time) time.Time {\n\tif v.IsZero() {\n\t\treturn time.Now()\n\t}\n\treturn v\n}\n\n// NewMechanism provides\nfunc NewMechanism(awsCfg aws.Config) *Mechanism {\n\treturn &Mechanism{\n\t\tSigner: signer.NewSigner(),\n\t\tCredentials: awsCfg.Credentials,\n\t\tRegion: awsCfg.Region,\n\t}\n}\n" }, { "path": "sasl/aws_msk_iam_v2/msk_iam_test.go", "content": "package aws_msk_iam_v2\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"encoding/json\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go-v2/aws\"\n\t\"github.com/segmentio/kafka-go/sasl\"\n\t\"github.com/stretchr/testify/assert\"\n\n\tsigner \"github.com/aws/aws-sdk-go-v2/aws/signer/v4\"\n\t\"github.com/aws/aws-sdk-go-v2/credentials\"\n)\n\nconst (\n\taccessKeyId = \"ACCESS_KEY\"\n\tsecretAccessKey = \"SECRET_KEY\"\n)\n\n// using a fixed time allows the signature to be verifiable in a test\nvar signTime = time.Date(2021, 10, 14, 13, 5, 0, 0, time.UTC)\n\nfunc TestAwsMskIamMechanism(t *testing.T) {\n\tcreds := credentials.NewStaticCredentialsProvider(accessKeyId, secretAccessKey, \"\")\n\tctxWithMetadata := func() context.Context {\n\t\treturn sasl.WithMetadata(context.Background(), &sasl.Metadata{\n\t\t\tHost: \"localhost\",\n\t\t\tPort: 9092,\n\t\t})\n\t}\n\n\ttests := []struct {\n\t\tdescription string\n\t\tctx func() context.Context\n\t\tshouldFail bool\n\t}{\n\t\t{\n\t\t\tdescription: \"with metadata\",\n\t\t\tctx: ctxWithMetadata,\n\t\t},\n\t\t{\n\t\t\tdescription: \"without metadata\",\n\t\t\tctx: func() context.Context {\n\t\t\t\treturn context.Background()\n\t\t\t},\n\t\t\tshouldFail: true,\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tt.Run(tt.description, func(t *testing.T) {\n\t\t\tctx := tt.ctx()\n\n\t\t\tmskMechanism := &Mechanism{\n\t\t\t\tSigner: signer.NewSigner(),\n\t\t\t\tCredentials: creds,\n\t\t\t\tRegion: \"us-east-1\",\n\t\t\t\tSignTime: signTime,\n\t\t\t}\n\t\t\tsess, auth, err := mskMechanism.Start(ctx)\n\t\t\tif tt.shouldFail { // if error is expected\n\t\t\t\tif err == nil { // but we don't find one\n\t\t\t\t\tt.Fatal(\"error expected\")\n\t\t\t\t} else { // but we do find one\n\t\t\t\t\treturn // return early since the remaining assertions are irrelevant\n\t\t\t\t}\n\t\t\t} else { // if error is not expected (typical)\n\t\t\t\tif err != nil { // but we do find one\n\t\t\t\t\tt.Fatal(err)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif sess != mskMechanism {\n\t\t\t\tt.Error(\n\t\t\t\t\t\"Unexpected session\",\n\t\t\t\t\t\"expected\", mskMechanism,\n\t\t\t\t\t\"got\", sess,\n\t\t\t\t)\n\t\t\t}\n\n\t\t\texpectedMap := map[string]string{\n\t\t\t\t\"version\": \"2020_10_22\",\n\t\t\t\t\"action\": \"kafka-cluster:Connect\",\n\t\t\t\t\"host\": \"localhost\",\n\t\t\t\t\"user-agent\": signUserAgent,\n\t\t\t\t\"x-amz-algorithm\": \"AWS4-HMAC-SHA256\",\n\t\t\t\t\"x-amz-credential\": \"ACCESS_KEY/20211014/us-east-1/kafka-cluster/aws4_request\",\n\t\t\t\t\"x-amz-date\": \"20211014T130500Z\",\n\t\t\t\t\"x-amz-expires\": \"300\",\n\t\t\t\t\"x-amz-signedheaders\": \"host\",\n\t\t\t\t\"x-amz-signature\": \"6b8d25f9b45b9c7db9da855a49112d80379224153a27fd279c305a5b7940d1a7\",\n\t\t\t}\n\t\t\texpectedAuth, err := json.Marshal(expectedMap)\n\t\t\tif err != nil {\n\t\t\t\tt.Fatal(err)\n\t\t\t}\n\n\t\t\tif !bytes.Equal(expectedAuth, auth) {\n\t\t\t\tt.Error(\"Unexpected authentication\",\n\t\t\t\t\t\"expected\", expectedAuth,\n\t\t\t\t\t\"got\", auth,\n\t\t\t\t)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestDefaultExpiry(t *testing.T) {\n\texpiry := time.Second * 5\n\ttestCases := map[string]struct {\n\t\tExpiry time.Duration\n\t}{\n\t\t\"with default\": {Expiry: expiry},\n\t\t\"without default\": {},\n\t}\n\n\tfor name, testCase := range testCases {\n\t\tt.Run(name, func(t *testing.T) {\n\t\t\tactual := defaultExpiry(testCase.Expiry)\n\t\t\tif testCase.Expiry == 0 {\n\t\t\t\tassert.Equal(t, time.Minute*5, actual)\n\t\t\t} else {\n\t\t\t\tassert.Equal(t, expiry, actual)\n\t\t\t}\n\n\t\t})\n\t}\n}\n\nfunc TestDefaultSignTime(t *testing.T) {\n\ttestCases := map[string]struct {\n\t\tSignTime time.Time\n\t}{\n\t\t\"with default\": {SignTime: signTime},\n\t\t\"without default\": {},\n\t}\n\n\tfor name, testCase := range testCases {\n\t\tt.Run(name, func(t *testing.T) {\n\t\t\tactual := defaultSignTime(testCase.SignTime)\n\t\t\tif testCase.SignTime.IsZero() {\n\t\t\t\tassert.True(t, actual.After(signTime))\n\t\t\t} else {\n\t\t\t\tassert.Equal(t, signTime, actual)\n\t\t\t}\n\t\t})\n\t}\n}\n\nfunc TestNewMechanism(t *testing.T) {\n\tregion := \"us-east-1\"\n\tcreds := credentials.StaticCredentialsProvider{}\n\tawsCfg := aws.Config{\n\t\tRegion: region,\n\t\tCredentials: creds,\n\t}\n\tm := NewMechanism(awsCfg)\n\tassert.Equal(t, m.Region, region)\n\tassert.Equal(t, m.Credentials, creds)\n}\n" }, { "path": "sasl/plain/plain.go", "content": "package plain\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\n\t\"github.com/segmentio/kafka-go/sasl\"\n)\n\n// Mechanism implements the PLAIN mechanism and passes the credentials in clear\n// text.\ntype Mechanism struct {\n\tUsername string\n\tPassword string\n}\n\nfunc (Mechanism) Name() string {\n\treturn \"PLAIN\"\n}\n\nfunc (m Mechanism) Start(ctx context.Context) (sasl.StateMachine, []byte, error) {\n\t// Mechanism is stateless, so it can also implement sasl.Session\n\treturn m, []byte(fmt.Sprintf(\"\\x00%s\\x00%s\", m.Username, m.Password)), nil\n}\n\nfunc (m Mechanism) Next(ctx context.Context, challenge []byte) (bool, []byte, error) {\n\t// kafka will return error if it rejected the credentials, so we'd only\n\t// arrive here on success.\n\treturn true, nil, nil\n}\n" }, { "path": "sasl/sasl.go", "content": "package sasl\n\nimport \"context\"\n\ntype ctxKey struct{}\n\n// Mechanism implements the SASL state machine for a particular mode of\n// authentication. It is used by the kafka.Dialer to perform the SASL\n// handshake.\n//\n// A Mechanism must be re-usable and safe for concurrent access by multiple\n// goroutines.\ntype Mechanism interface {\n\t// Name returns the identifier for this SASL mechanism. This string will be\n\t// passed to the SASL handshake request and much match one of the mechanisms\n\t// supported by Kafka.\n\tName() string\n\n\t// Start begins SASL authentication. It returns an authentication state\n\t// machine and \"initial response\" data (if required by the selected\n\t// mechanism). A non-nil error causes the client to abort the authentication\n\t// attempt.\n\t//\n\t// A nil ir value is different from a zero-length value. The nil value\n\t// indicates that the selected mechanism does not use an initial response,\n\t// while a zero-length value indicates an empty initial response, which must\n\t// be sent to the server.\n\tStart(ctx context.Context) (sess StateMachine, ir []byte, err error)\n}\n\n// StateMachine implements the SASL challenge/response flow for a single SASL\n// handshake. A StateMachine will be created by the Mechanism per connection,\n// so it does not need to be safe for concurrent access by multiple goroutines.\n//\n// Once the StateMachine is created by the Mechanism, the caller loops by\n// passing the server's response into Next and then sending Next's returned\n// bytes to the server. Eventually either Next will indicate that the\n// authentication has been successfully completed via the done return value, or\n// it will indicate that the authentication failed by returning a non-nil error.\ntype StateMachine interface {\n\t// Next continues challenge-response authentication. A non-nil error\n\t// indicates that the client should abort the authentication attempt. If\n\t// the client has been successfully authenticated, then the done return\n\t// value will be true.\n\tNext(ctx context.Context, challenge []byte) (done bool, response []byte, err error)\n}\n\n// Metadata contains additional data for performing SASL authentication.\ntype Metadata struct {\n\t// Host is the address of the broker the authentication will be\n\t// performed on.\n\tHost string\n\tPort int\n}\n\n// WithMetadata returns a copy of the context with associated Metadata.\nfunc WithMetadata(ctx context.Context, m *Metadata) context.Context {\n\treturn context.WithValue(ctx, ctxKey{}, m)\n}\n\n// MetadataFromContext retrieves the Metadata from the context.\nfunc MetadataFromContext(ctx context.Context) *Metadata {\n\tm, _ := ctx.Value(ctxKey{}).(*Metadata)\n\treturn m\n}\n" }, { "path": "sasl/sasl_test.go", "content": "package sasl_test\n\nimport (\n\t\"context\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go\"\n\t\"github.com/segmentio/kafka-go/sasl\"\n\t\"github.com/segmentio/kafka-go/sasl/plain\"\n\t\"github.com/segmentio/kafka-go/sasl/scram\"\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nconst (\n\tsaslTestConnect = \"localhost:9093\" // connect to sasl listener\n\tsaslTestTopic = \"test-writer-0\" // this topic is guaranteed to exist.\n)\n\nfunc TestSASL(t *testing.T) {\n\tscramUsers := map[scram.Algorithm]string{scram.SHA256: \"adminscram\", scram.SHA512: \"adminscram\"}\n\t// kafka 4.0.0 test environment supports only different users for different scram algorithms.\n\tif ktesting.KafkaIsAtLeast(\"4.0.0\") {\n\t\tscramUsers = map[scram.Algorithm]string{scram.SHA256: \"adminscram256\", scram.SHA512: \"adminscram512\"}\n\t}\n\ttests := []struct {\n\t\tvalid func() sasl.Mechanism\n\t\tinvalid func() sasl.Mechanism\n\t\tminKafka string\n\t}{\n\t\t{\n\t\t\tvalid: func() sasl.Mechanism {\n\t\t\t\treturn plain.Mechanism{\n\t\t\t\t\tUsername: \"adminplain\",\n\t\t\t\t\tPassword: \"admin-secret\",\n\t\t\t\t}\n\t\t\t},\n\t\t\tinvalid: func() sasl.Mechanism {\n\t\t\t\treturn plain.Mechanism{\n\t\t\t\t\tUsername: \"adminplain\",\n\t\t\t\t\tPassword: \"badpassword\",\n\t\t\t\t}\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tvalid: func() sasl.Mechanism {\n\t\t\t\tmech, _ := scram.Mechanism(scram.SHA256, scramUsers[scram.SHA256], \"admin-secret-256\")\n\t\t\t\treturn mech\n\t\t\t},\n\t\t\tinvalid: func() sasl.Mechanism {\n\t\t\t\tmech, _ := scram.Mechanism(scram.SHA256, scramUsers[scram.SHA256], \"badpassword\")\n\t\t\t\treturn mech\n\t\t\t},\n\t\t\tminKafka: \"0.10.2.0\",\n\t\t},\n\t\t{\n\t\t\tvalid: func() sasl.Mechanism {\n\t\t\t\tmech, _ := scram.Mechanism(scram.SHA512, scramUsers[scram.SHA512], \"admin-secret-512\")\n\t\t\t\treturn mech\n\t\t\t},\n\t\t\tinvalid: func() sasl.Mechanism {\n\t\t\t\tmech, _ := scram.Mechanism(scram.SHA512, scramUsers[scram.SHA512], \"badpassword\")\n\t\t\t\treturn mech\n\t\t\t},\n\t\t\tminKafka: \"0.10.2.0\",\n\t\t},\n\t}\n\n\tfor _, tt := range tests {\n\t\tmech := tt.valid()\n\t\tif !ktesting.KafkaIsAtLeast(tt.minKafka) {\n\t\t\tt.Skip(\"requires min kafka version \" + tt.minKafka)\n\t\t}\n\n\t\tt.Run(mech.Name()+\" success\", func(t *testing.T) {\n\t\t\ttestConnect(t, tt.valid(), true)\n\t\t})\n\t\tt.Run(mech.Name()+\" failure\", func(t *testing.T) {\n\t\t\ttestConnect(t, tt.invalid(), false)\n\t\t})\n\t\tt.Run(mech.Name()+\" is reusable\", func(t *testing.T) {\n\t\t\tmech := tt.valid()\n\t\t\ttestConnect(t, mech, true)\n\t\t\ttestConnect(t, mech, true)\n\t\t\ttestConnect(t, mech, true)\n\t\t})\n\n\t}\n}\n\nfunc testConnect(t *testing.T, mechanism sasl.Mechanism, success bool) {\n\tctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)\n\tdefer cancel()\n\n\td := kafka.Dialer{\n\t\tSASLMechanism: mechanism,\n\t}\n\t_, err := d.DialLeader(ctx, \"tcp\", saslTestConnect, saslTestTopic, 0)\n\tif success && err != nil {\n\t\tt.Errorf(\"should have logged in correctly, got err: %v\", err)\n\t} else if !success && err == nil {\n\t\tt.Errorf(\"should not have logged in correctly\")\n\t}\n}\n" }, { "path": "sasl/scram/scram.go", "content": "package scram\n\nimport (\n\t\"context\"\n\t\"crypto/sha256\"\n\t\"crypto/sha512\"\n\t\"hash\"\n\n\t\"github.com/segmentio/kafka-go/sasl\"\n\t\"github.com/xdg-go/scram\"\n)\n\n// Algorithm determines the hash function used by SCRAM to protect the user's\n// credentials.\ntype Algorithm interface {\n\t// Name returns the algorithm's name, e.g. \"SCRAM-SHA-256\"\n\tName() string\n\n\t// Hash returns a new hash.Hash.\n\tHash() hash.Hash\n}\n\ntype sha256Algo struct{}\n\nfunc (sha256Algo) Name() string {\n\treturn \"SCRAM-SHA-256\"\n}\n\nfunc (sha256Algo) Hash() hash.Hash {\n\treturn sha256.New()\n}\n\ntype sha512Algo struct{}\n\nfunc (sha512Algo) Name() string {\n\treturn \"SCRAM-SHA-512\"\n}\n\nfunc (sha512Algo) Hash() hash.Hash {\n\treturn sha512.New()\n}\n\nvar (\n\tSHA256 Algorithm = sha256Algo{}\n\tSHA512 Algorithm = sha512Algo{}\n)\n\ntype mechanism struct {\n\talgo Algorithm\n\tclient *scram.Client\n}\n\ntype session struct {\n\tconvo *scram.ClientConversation\n}\n\n// Mechanism returns a new sasl.Mechanism that will use SCRAM with the provided\n// Algorithm to securely transmit the provided credentials to Kafka.\n//\n// SCRAM-SHA-256 and SCRAM-SHA-512 were added to Kafka in 0.10.2.0. These\n// mechanisms will not work with older versions.\nfunc Mechanism(algo Algorithm, username, password string) (sasl.Mechanism, error) {\n\thashGen := scram.HashGeneratorFcn(algo.Hash)\n\tclient, err := hashGen.NewClient(username, password, \"\")\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &mechanism{\n\t\talgo: algo,\n\t\tclient: client,\n\t}, nil\n}\n\nfunc (m *mechanism) Name() string {\n\treturn m.algo.Name()\n}\n\nfunc (m *mechanism) Start(ctx context.Context) (sasl.StateMachine, []byte, error) {\n\tconvo := m.client.NewConversation()\n\tstr, err := convo.Step(\"\")\n\tif err != nil {\n\t\treturn nil, nil, err\n\t}\n\treturn &session{convo: convo}, []byte(str), nil\n}\n\nfunc (s *session) Next(ctx context.Context, challenge []byte) (bool, []byte, error) {\n\tstr, err := s.convo.Step(string(challenge))\n\treturn s.convo.Done(), []byte(str), err\n}\n" }, { "path": "saslauthenticate.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n)\n\ntype saslAuthenticateRequestV0 struct {\n\t// Data holds the SASL payload\n\tData []byte\n}\n\nfunc (t saslAuthenticateRequestV0) size() int32 {\n\treturn sizeofBytes(t.Data)\n}\n\nfunc (t *saslAuthenticateRequestV0) readFrom(r *bufio.Reader, sz int) (remain int, err error) {\n\treturn readBytes(r, sz, &t.Data)\n}\n\nfunc (t saslAuthenticateRequestV0) writeTo(wb *writeBuffer) {\n\twb.writeBytes(t.Data)\n}\n\ntype saslAuthenticateResponseV0 struct {\n\t// ErrorCode holds response error code\n\tErrorCode int16\n\n\tErrorMessage string\n\n\tData []byte\n}\n\nfunc (t saslAuthenticateResponseV0) size() int32 {\n\treturn sizeofInt16(t.ErrorCode) + sizeofString(t.ErrorMessage) + sizeofBytes(t.Data)\n}\n\nfunc (t saslAuthenticateResponseV0) writeTo(wb *writeBuffer) {\n\twb.writeInt16(t.ErrorCode)\n\twb.writeString(t.ErrorMessage)\n\twb.writeBytes(t.Data)\n}\n\nfunc (t *saslAuthenticateResponseV0) readFrom(r *bufio.Reader, sz int) (remain int, err error) {\n\tif remain, err = readInt16(r, sz, &t.ErrorCode); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readString(r, remain, &t.ErrorMessage); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readBytes(r, remain, &t.Data); err != nil {\n\t\treturn\n\t}\n\treturn\n}\n" }, { "path": "saslauthenticate_test.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"reflect\"\n\t\"testing\"\n)\n\nfunc TestSASLAuthenticateRequestV0(t *testing.T) {\n\titem := saslAuthenticateRequestV0{\n\t\tData: []byte(\"\\x00user\\x00pass\"),\n\t}\n\n\tb := bytes.NewBuffer(nil)\n\tw := &writeBuffer{w: b}\n\titem.writeTo(w)\n\n\tvar found saslAuthenticateRequestV0\n\tremain, err := (&found).readFrom(bufio.NewReader(b), b.Len())\n\tif err != nil {\n\t\tt.Error(err)\n\t\tt.FailNow()\n\t}\n\tif remain != 0 {\n\t\tt.Errorf(\"expected 0 remain, got %v\", remain)\n\t\tt.FailNow()\n\t}\n\tif !reflect.DeepEqual(item, found) {\n\t\tt.Error(\"expected item and found to be the same\")\n\t\tt.FailNow()\n\t}\n}\n\nfunc TestSASLAuthenticateResponseV0(t *testing.T) {\n\titem := saslAuthenticateResponseV0{\n\t\tErrorCode: 2,\n\t\tErrorMessage: \"Message\",\n\t\tData: []byte(\"bytes\"),\n\t}\n\n\tb := bytes.NewBuffer(nil)\n\tw := &writeBuffer{w: b}\n\titem.writeTo(w)\n\n\tvar found saslAuthenticateResponseV0\n\tremain, err := (&found).readFrom(bufio.NewReader(b), b.Len())\n\tif err != nil {\n\t\tt.Error(err)\n\t\tt.FailNow()\n\t}\n\tif remain != 0 {\n\t\tt.Errorf(\"expected 0 remain, got %v\", remain)\n\t\tt.FailNow()\n\t}\n\tif !reflect.DeepEqual(item, found) {\n\t\tt.Error(\"expected item and found to be the same\")\n\t\tt.FailNow()\n\t}\n}\n" }, { "path": "saslhandshake.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n)\n\n// saslHandshakeRequestV0 implements the format for V0 and V1 SASL\n// requests (they are identical).\ntype saslHandshakeRequestV0 struct {\n\t// Mechanism holds the SASL Mechanism chosen by the client.\n\tMechanism string\n}\n\nfunc (t saslHandshakeRequestV0) size() int32 {\n\treturn sizeofString(t.Mechanism)\n}\n\nfunc (t *saslHandshakeRequestV0) readFrom(r *bufio.Reader, sz int) (remain int, err error) {\n\treturn readString(r, sz, &t.Mechanism)\n}\n\nfunc (t saslHandshakeRequestV0) writeTo(wb *writeBuffer) {\n\twb.writeString(t.Mechanism)\n}\n\n// saslHandshakeResponseV0 implements the format for V0 and V1 SASL\n// responses (they are identical).\ntype saslHandshakeResponseV0 struct {\n\t// ErrorCode holds response error code\n\tErrorCode int16\n\n\t// Array of mechanisms enabled in the server\n\tEnabledMechanisms []string\n}\n\nfunc (t saslHandshakeResponseV0) size() int32 {\n\treturn sizeofInt16(t.ErrorCode) + sizeofStringArray(t.EnabledMechanisms)\n}\n\nfunc (t saslHandshakeResponseV0) writeTo(wb *writeBuffer) {\n\twb.writeInt16(t.ErrorCode)\n\twb.writeStringArray(t.EnabledMechanisms)\n}\n\nfunc (t *saslHandshakeResponseV0) readFrom(r *bufio.Reader, sz int) (remain int, err error) {\n\tif remain, err = readInt16(r, sz, &t.ErrorCode); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readStringArray(r, remain, &t.EnabledMechanisms); err != nil {\n\t\treturn\n\t}\n\treturn\n}\n" }, { "path": "saslhandshake_test.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"reflect\"\n\t\"testing\"\n)\n\nfunc TestSASLHandshakeRequestV0(t *testing.T) {\n\titem := saslHandshakeRequestV0{\n\t\tMechanism: \"SCRAM-SHA-512\",\n\t}\n\n\tb := bytes.NewBuffer(nil)\n\tw := &writeBuffer{w: b}\n\titem.writeTo(w)\n\n\tvar found saslHandshakeRequestV0\n\tremain, err := (&found).readFrom(bufio.NewReader(b), b.Len())\n\tif err != nil {\n\t\tt.Error(err)\n\t\tt.FailNow()\n\t}\n\tif remain != 0 {\n\t\tt.Errorf(\"expected 0 remain, got %v\", remain)\n\t\tt.FailNow()\n\t}\n\tif !reflect.DeepEqual(item, found) {\n\t\tt.Error(\"expected item and found to be the same\")\n\t\tt.FailNow()\n\t}\n}\n\nfunc TestSASLHandshakeResponseV0(t *testing.T) {\n\titem := saslHandshakeResponseV0{\n\t\tErrorCode: 2,\n\t\tEnabledMechanisms: []string{\"PLAIN\", \"SCRAM-SHA-512\"},\n\t}\n\n\tb := bytes.NewBuffer(nil)\n\tw := &writeBuffer{w: b}\n\titem.writeTo(w)\n\n\tvar found saslHandshakeResponseV0\n\tremain, err := (&found).readFrom(bufio.NewReader(b), b.Len())\n\tif err != nil {\n\t\tt.Error(err)\n\t\tt.FailNow()\n\t}\n\tif remain != 0 {\n\t\tt.Errorf(\"expected 0 remain, got %v\", remain)\n\t\tt.FailNow()\n\t}\n\tif !reflect.DeepEqual(item, found) {\n\t\tt.Error(\"expected item and found to be the same\")\n\t\tt.FailNow()\n\t}\n}\n" }, { "path": "scripts/wait-for-kafka.sh", "content": "#/bin/bash\n\nCOUNTER=0; \necho foo | nc localhost 9092\nSTATUS=$?\nATTEMPTS=60\nuntil [ ${STATUS} -eq 0 ] || [ \"$COUNTER\" -ge \"${ATTEMPTS}\" ];\ndo\n let COUNTER=$COUNTER+1;\n sleep 1;\n echo \"[$COUNTER] waiting for 9092 port to be open\";\n echo foo | nc localhost 9092\n STATUS=$?\ndone\n\nif [ \"${COUNTER}\" -gt \"${ATTEMPTS}\" ];\nthen\n echo \"Kafka is not running, failing\"\n exit 1\nfi" }, { "path": "sizeof.go", "content": "package kafka\n\nimport \"fmt\"\n\ntype sizable interface {\n\tsize() int32\n}\n\nfunc sizeof(a interface{}) int32 {\n\tswitch v := a.(type) {\n\tcase int8:\n\t\treturn 1\n\tcase int16:\n\t\treturn 2\n\tcase int32:\n\t\treturn 4\n\tcase int64:\n\t\treturn 8\n\tcase string:\n\t\treturn sizeofString(v)\n\tcase bool:\n\t\treturn 1\n\tcase []byte:\n\t\treturn sizeofBytes(v)\n\tcase sizable:\n\t\treturn v.size()\n\t}\n\tpanic(fmt.Sprintf(\"unsupported type: %T\", a))\n}\n\nfunc sizeofInt16(_ int16) int32 {\n\treturn 2\n}\n\nfunc sizeofInt32(_ int32) int32 {\n\treturn 4\n}\n\nfunc sizeofInt64(_ int64) int32 {\n\treturn 8\n}\n\nfunc sizeofString(s string) int32 {\n\treturn 2 + int32(len(s))\n}\n\nfunc sizeofNullableString(s *string) int32 {\n\tif s == nil {\n\t\treturn 2\n\t}\n\treturn sizeofString(*s)\n}\n\nfunc sizeofBytes(b []byte) int32 {\n\treturn 4 + int32(len(b))\n}\n\nfunc sizeofArray(n int, f func(int) int32) int32 {\n\ts := int32(4)\n\tfor i := 0; i != n; i++ {\n\t\ts += f(i)\n\t}\n\treturn s\n}\n\nfunc sizeofInt32Array(a []int32) int32 {\n\treturn 4 + (4 * int32(len(a)))\n}\n\nfunc sizeofStringArray(a []string) int32 {\n\treturn sizeofArray(len(a), func(i int) int32 { return sizeofString(a[i]) })\n}\n" }, { "path": "snappy/snappy.go", "content": "// Package snappy does nothing, it's kept for backward compatibility to avoid\n// breaking the majority of programs that imported it to install the compression\n// codec, which is now always included.\npackage snappy\n\nimport \"github.com/segmentio/kafka-go/compress/snappy\"\n\ntype CompressionCodec = snappy.Codec\n\ntype Framing = snappy.Framing\n\nconst (\n\tCode = 2\n\tFramed = snappy.Framed\n\tUnframed = snappy.Unframed\n)\n\nfunc NewCompressionCodec() *CompressionCodec {\n\treturn NewCompressionCodecFraming(Framed)\n}\n\nfunc NewCompressionCodecFraming(framing Framing) *CompressionCodec {\n\treturn &CompressionCodec{Framing: framing}\n}\n" }, { "path": "stats.go", "content": "package kafka\n\nimport (\n\t\"sync/atomic\"\n\t\"time\"\n)\n\n// SummaryStats is a data structure that carries a summary of observed values.\ntype SummaryStats struct {\n\tAvg int64 `metric:\"avg\" type:\"gauge\"`\n\tMin int64 `metric:\"min\" type:\"gauge\"`\n\tMax int64 `metric:\"max\" type:\"gauge\"`\n\tCount int64 `metric:\"count\" type:\"counter\"`\n\tSum int64 `metric:\"sum\" type:\"counter\"`\n}\n\n// DurationStats is a data structure that carries a summary of observed duration values.\ntype DurationStats struct {\n\tAvg time.Duration `metric:\"avg\" type:\"gauge\"`\n\tMin time.Duration `metric:\"min\" type:\"gauge\"`\n\tMax time.Duration `metric:\"max\" type:\"gauge\"`\n\tCount int64 `metric:\"count\" type:\"counter\"`\n\tSum time.Duration `metric:\"sum\" type:\"counter\"`\n}\n\n// counter is an atomic incrementing counter which gets reset on snapshot.\n//\n// Since atomic is used to mutate the statistic the value must be 64-bit aligned.\n// See https://golang.org/pkg/sync/atomic/#pkg-note-BUG\ntype counter int64\n\nfunc (c *counter) ptr() *int64 {\n\treturn (*int64)(c)\n}\n\nfunc (c *counter) observe(v int64) {\n\tatomic.AddInt64(c.ptr(), v)\n}\n\nfunc (c *counter) snapshot() int64 {\n\treturn atomic.SwapInt64(c.ptr(), 0)\n}\n\n// gauge is an atomic integer that may be set to any arbitrary value, the value\n// does not change after a snapshot.\n//\n// Since atomic is used to mutate the statistic the value must be 64-bit aligned.\n// See https://golang.org/pkg/sync/atomic/#pkg-note-BUG\ntype gauge int64\n\nfunc (g *gauge) ptr() *int64 {\n\treturn (*int64)(g)\n}\n\nfunc (g *gauge) observe(v int64) {\n\tatomic.StoreInt64(g.ptr(), v)\n}\n\nfunc (g *gauge) snapshot() int64 {\n\treturn atomic.LoadInt64(g.ptr())\n}\n\n// minimum is an atomic integral type that keeps track of the minimum of all\n// values that it observed between snapshots.\n//\n// Since atomic is used to mutate the statistic the value must be 64-bit aligned.\n// See https://golang.org/pkg/sync/atomic/#pkg-note-BUG\ntype minimum int64\n\nfunc (m *minimum) ptr() *int64 {\n\treturn (*int64)(m)\n}\n\nfunc (m *minimum) observe(v int64) {\n\tfor {\n\t\tptr := m.ptr()\n\t\tmin := atomic.LoadInt64(ptr)\n\n\t\tif min >= 0 && min <= v {\n\t\t\tbreak\n\t\t}\n\n\t\tif atomic.CompareAndSwapInt64(ptr, min, v) {\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nfunc (m *minimum) snapshot() int64 {\n\tp := m.ptr()\n\tv := atomic.LoadInt64(p)\n\tatomic.CompareAndSwapInt64(p, v, -1)\n\tif v < 0 {\n\t\tv = 0\n\t}\n\treturn v\n}\n\n// maximum is an atomic integral type that keeps track of the maximum of all\n// values that it observed between snapshots.\n//\n// Since atomic is used to mutate the statistic the value must be 64-bit aligned.\n// See https://golang.org/pkg/sync/atomic/#pkg-note-BUG\ntype maximum int64\n\nfunc (m *maximum) ptr() *int64 {\n\treturn (*int64)(m)\n}\n\nfunc (m *maximum) observe(v int64) {\n\tfor {\n\t\tptr := m.ptr()\n\t\tmax := atomic.LoadInt64(ptr)\n\n\t\tif max >= 0 && max >= v {\n\t\t\tbreak\n\t\t}\n\n\t\tif atomic.CompareAndSwapInt64(ptr, max, v) {\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nfunc (m *maximum) snapshot() int64 {\n\tp := m.ptr()\n\tv := atomic.LoadInt64(p)\n\tatomic.CompareAndSwapInt64(p, v, -1)\n\tif v < 0 {\n\t\tv = 0\n\t}\n\treturn v\n}\n\ntype summary struct {\n\tmin minimum\n\tmax maximum\n\tsum counter\n\tcount counter\n}\n\nfunc makeSummary() summary {\n\treturn summary{\n\t\tmin: -1,\n\t\tmax: -1,\n\t}\n}\n\nfunc (s *summary) observe(v int64) {\n\ts.min.observe(v)\n\ts.max.observe(v)\n\ts.sum.observe(v)\n\ts.count.observe(1)\n}\n\nfunc (s *summary) observeDuration(v time.Duration) {\n\ts.observe(int64(v))\n}\n\nfunc (s *summary) snapshot() SummaryStats {\n\tavg := int64(0)\n\tmin := s.min.snapshot()\n\tmax := s.max.snapshot()\n\tsum := s.sum.snapshot()\n\tcount := s.count.snapshot()\n\n\tif count != 0 {\n\t\tavg = int64(float64(sum) / float64(count))\n\t}\n\n\treturn SummaryStats{\n\t\tAvg: avg,\n\t\tMin: min,\n\t\tMax: max,\n\t\tCount: count,\n\t\tSum: sum,\n\t}\n}\n\nfunc (s *summary) snapshotDuration() DurationStats {\n\tsummary := s.snapshot()\n\treturn DurationStats{\n\t\tAvg: time.Duration(summary.Avg),\n\t\tMin: time.Duration(summary.Min),\n\t\tMax: time.Duration(summary.Max),\n\t\tCount: summary.Count,\n\t\tSum: time.Duration(summary.Sum),\n\t}\n}\n" }, { "path": "syncgroup.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n\t\"github.com/segmentio/kafka-go/protocol/consumer\"\n\t\"github.com/segmentio/kafka-go/protocol/syncgroup\"\n)\n\n// SyncGroupRequest is the request structure for the SyncGroup function.\ntype SyncGroupRequest struct {\n\t// Address of the kafka broker to sent he request to.\n\tAddr net.Addr\n\n\t// GroupID of the group to sync.\n\tGroupID string\n\n\t// The generation of the group.\n\tGenerationID int\n\n\t// The member ID assigned by the group.\n\tMemberID string\n\n\t// The unique identifier for the consumer instance.\n\tGroupInstanceID string\n\n\t// The name for the class of protocols implemented by the group being joined.\n\tProtocolType string\n\n\t// The group protocol name.\n\tProtocolName string\n\n\t// The group member assignments.\n\tAssignments []SyncGroupRequestAssignment\n}\n\n// SyncGroupRequestAssignment represents an assignement for a goroup memeber.\ntype SyncGroupRequestAssignment struct {\n\t// The ID of the member to assign.\n\tMemberID string\n\n\t// The member assignment.\n\tAssignment GroupProtocolAssignment\n}\n\n// SyncGroupResponse is the response structure for the SyncGroup function.\ntype SyncGroupResponse struct {\n\t// An error that may have occurred when attempting to sync the group.\n\t//\n\t// The errors contain the kafka error code. Programs may use the standard\n\t// errors.Is function to test the error against kafka error codes.\n\tError error\n\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// The group protocol type.\n\tProtocolType string\n\n\t// The group protocol name.\n\tProtocolName string\n\n\t// The member assignment.\n\tAssignment GroupProtocolAssignment\n}\n\n// GroupProtocolAssignment represents an assignment of topics and partitions for a group memeber.\ntype GroupProtocolAssignment struct {\n\t// The topics and partitions assigned to the group memeber.\n\tAssignedPartitions map[string][]int\n\n\t// UserData for the assignemnt.\n\tUserData []byte\n}\n\n// SyncGroup sends a sync group request to the coordinator and returns the response.\nfunc (c *Client) SyncGroup(ctx context.Context, req *SyncGroupRequest) (*SyncGroupResponse, error) {\n\tsyncGroup := syncgroup.Request{\n\t\tGroupID: req.GroupID,\n\t\tGenerationID: int32(req.GenerationID),\n\t\tMemberID: req.MemberID,\n\t\tGroupInstanceID: req.GroupInstanceID,\n\t\tProtocolType: req.ProtocolType,\n\t\tProtocolName: req.ProtocolName,\n\t\tAssignments: make([]syncgroup.RequestAssignment, 0, len(req.Assignments)),\n\t}\n\n\tfor _, assignment := range req.Assignments {\n\t\tassign := consumer.Assignment{\n\t\t\tVersion: consumer.MaxVersionSupported,\n\t\t\tAssignedPartitions: make([]consumer.TopicPartition, 0, len(assignment.Assignment.AssignedPartitions)),\n\t\t\tUserData: assignment.Assignment.UserData,\n\t\t}\n\n\t\tfor topic, partitions := range assignment.Assignment.AssignedPartitions {\n\t\t\ttp := consumer.TopicPartition{\n\t\t\t\tTopic: topic,\n\t\t\t\tPartitions: make([]int32, 0, len(partitions)),\n\t\t\t}\n\t\t\tfor _, partition := range partitions {\n\t\t\t\ttp.Partitions = append(tp.Partitions, int32(partition))\n\t\t\t}\n\t\t\tassign.AssignedPartitions = append(assign.AssignedPartitions, tp)\n\t\t}\n\n\t\tassignBytes, err := protocol.Marshal(consumer.MaxVersionSupported, assign)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"kafka.(*Client).SyncGroup: %w\", err)\n\t\t}\n\n\t\tsyncGroup.Assignments = append(syncGroup.Assignments, syncgroup.RequestAssignment{\n\t\t\tMemberID: assignment.MemberID,\n\t\t\tAssignment: assignBytes,\n\t\t})\n\t}\n\n\tm, err := c.roundTrip(ctx, req.Addr, &syncGroup)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).SyncGroup: %w\", err)\n\t}\n\n\tr := m.(*syncgroup.Response)\n\n\tvar assignment consumer.Assignment\n\terr = protocol.Unmarshal(r.Assignments, consumer.MaxVersionSupported, &assignment)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).SyncGroup: %w\", err)\n\t}\n\n\tres := &SyncGroupResponse{\n\t\tThrottle: makeDuration(r.ThrottleTimeMS),\n\t\tError: makeError(r.ErrorCode, \"\"),\n\t\tProtocolType: r.ProtocolType,\n\t\tProtocolName: r.ProtocolName,\n\t\tAssignment: GroupProtocolAssignment{\n\t\t\tAssignedPartitions: make(map[string][]int, len(assignment.AssignedPartitions)),\n\t\t\tUserData: assignment.UserData,\n\t\t},\n\t}\n\tpartitions := map[string][]int{}\n\tfor _, topicPartition := range assignment.AssignedPartitions {\n\t\tfor _, partition := range topicPartition.Partitions {\n\t\t\tpartitions[topicPartition.Topic] = append(partitions[topicPartition.Topic], int(partition))\n\t\t}\n\t}\n\tres.Assignment.AssignedPartitions = partitions\n\n\treturn res, nil\n}\n\ntype groupAssignment struct {\n\tVersion int16\n\tTopics map[string][]int32\n\tUserData []byte\n}\n\nfunc (t groupAssignment) size() int32 {\n\tsz := sizeofInt16(t.Version) + sizeofInt16(int16(len(t.Topics)))\n\n\tfor topic, partitions := range t.Topics {\n\t\tsz += sizeofString(topic) + sizeofInt32Array(partitions)\n\t}\n\n\treturn sz + sizeofBytes(t.UserData)\n}\n\nfunc (t groupAssignment) writeTo(wb *writeBuffer) {\n\twb.writeInt16(t.Version)\n\twb.writeInt32(int32(len(t.Topics)))\n\n\tfor topic, partitions := range t.Topics {\n\t\twb.writeString(topic)\n\t\twb.writeInt32Array(partitions)\n\t}\n\n\twb.writeBytes(t.UserData)\n}\n\nfunc (t *groupAssignment) readFrom(r *bufio.Reader, size int) (remain int, err error) {\n\t// I came across this case when testing for compatibility with bsm/sarama-cluster. It\n\t// appears in some cases, sarama-cluster can send a nil array entry. Admittedly, I\n\t// didn't look too closely at it.\n\tif size == 0 {\n\t\tt.Topics = map[string][]int32{}\n\t\treturn 0, nil\n\t}\n\n\tif remain, err = readInt16(r, size, &t.Version); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readMapStringInt32(r, remain, &t.Topics); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readBytes(r, remain, &t.UserData); err != nil {\n\t\treturn\n\t}\n\n\treturn\n}\n\nfunc (t groupAssignment) bytes() []byte {\n\tbuf := bytes.NewBuffer(nil)\n\tt.writeTo(&writeBuffer{w: buf})\n\treturn buf.Bytes()\n}\n\ntype syncGroupRequestGroupAssignmentV0 struct {\n\t// MemberID assigned by the group coordinator\n\tMemberID string\n\n\t// MemberAssignments holds client encoded assignments\n\t//\n\t// See consumer groups section of https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol\n\tMemberAssignments []byte\n}\n\nfunc (t syncGroupRequestGroupAssignmentV0) size() int32 {\n\treturn sizeofString(t.MemberID) +\n\t\tsizeofBytes(t.MemberAssignments)\n}\n\nfunc (t syncGroupRequestGroupAssignmentV0) writeTo(wb *writeBuffer) {\n\twb.writeString(t.MemberID)\n\twb.writeBytes(t.MemberAssignments)\n}\n\ntype syncGroupRequestV0 struct {\n\t// GroupID holds the unique group identifier\n\tGroupID string\n\n\t// GenerationID holds the generation of the group.\n\tGenerationID int32\n\n\t// MemberID assigned by the group coordinator\n\tMemberID string\n\n\tGroupAssignments []syncGroupRequestGroupAssignmentV0\n}\n\nfunc (t syncGroupRequestV0) size() int32 {\n\treturn sizeofString(t.GroupID) +\n\t\tsizeofInt32(t.GenerationID) +\n\t\tsizeofString(t.MemberID) +\n\t\tsizeofArray(len(t.GroupAssignments), func(i int) int32 { return t.GroupAssignments[i].size() })\n}\n\nfunc (t syncGroupRequestV0) writeTo(wb *writeBuffer) {\n\twb.writeString(t.GroupID)\n\twb.writeInt32(t.GenerationID)\n\twb.writeString(t.MemberID)\n\twb.writeArray(len(t.GroupAssignments), func(i int) { t.GroupAssignments[i].writeTo(wb) })\n}\n\ntype syncGroupResponseV0 struct {\n\t// ErrorCode holds response error code\n\tErrorCode int16\n\n\t// MemberAssignments holds client encoded assignments\n\t//\n\t// See consumer groups section of https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol\n\tMemberAssignments []byte\n}\n\nfunc (t syncGroupResponseV0) size() int32 {\n\treturn sizeofInt16(t.ErrorCode) +\n\t\tsizeofBytes(t.MemberAssignments)\n}\n\nfunc (t syncGroupResponseV0) writeTo(wb *writeBuffer) {\n\twb.writeInt16(t.ErrorCode)\n\twb.writeBytes(t.MemberAssignments)\n}\n\nfunc (t *syncGroupResponseV0) readFrom(r *bufio.Reader, sz int) (remain int, err error) {\n\tif remain, err = readInt16(r, sz, &t.ErrorCode); err != nil {\n\t\treturn\n\t}\n\tif remain, err = readBytes(r, remain, &t.MemberAssignments); err != nil {\n\t\treturn\n\t}\n\treturn\n}\n" }, { "path": "syncgroup_test.go", "content": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"io\"\n\t\"reflect\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestClientSyncGroup(t *testing.T) {\n\t// In order to get to a sync group call we need to first\n\t// join a group.\n\ttopic := makeTopic()\n\tclient, shutdown := newLocalClient()\n\tclient.Timeout = time.Minute\n\t// Although at higher api versions ClientID is nullable\n\t// for some reason the SyncGroup API call errors\n\t// when ClientID is null.\n\t// The Java Kafka Consumer generates a ClientID if one is not\n\t// present or if the provided ClientID is empty.\n\tclient.Transport.(*Transport).ClientID = \"test-client\"\n\tdefer shutdown()\n\n\terr := clientCreateTopic(client, topic, 3)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tgroupID := makeGroupID()\n\n\tctx, cancel := context.WithTimeout(context.Background(), time.Minute)\n\tdefer cancel()\n\trespc, err := waitForCoordinatorIndefinitely(ctx, client, &FindCoordinatorRequest{\n\t\tAddr: client.Addr,\n\t\tKey: groupID,\n\t\tKeyType: CoordinatorKeyTypeConsumer,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif respc.Error != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tgroupInstanceID := \"group-instance-id\"\n\tuserData := \"user-data\"\n\n\tvar rrGroupBalancer RoundRobinGroupBalancer\n\n\treq := &JoinGroupRequest{\n\t\tGroupID: groupID,\n\t\tGroupInstanceID: groupInstanceID,\n\t\tProtocolType: \"consumer\",\n\t\tSessionTimeout: time.Minute,\n\t\tRebalanceTimeout: time.Minute,\n\t\tProtocols: []GroupProtocol{\n\t\t\t{\n\t\t\t\tName: rrGroupBalancer.ProtocolName(),\n\t\t\t\tMetadata: GroupProtocolSubscription{\n\t\t\t\t\tTopics: []string{topic},\n\t\t\t\t\tUserData: []byte(userData),\n\t\t\t\t\tOwnedPartitions: map[string][]int{\n\t\t\t\t\t\ttopic: {0, 1, 2},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\n\tvar resp *JoinGroupResponse\n\n\tfor {\n\t\tresp, err = client.JoinGroup(ctx, req)\n\t\tif err != nil {\n\t\t\tt.Fatal(err)\n\t\t}\n\n\t\tif errors.Is(resp.Error, MemberIDRequired) {\n\t\t\treq.MemberID = resp.MemberID\n\t\t\ttime.Sleep(time.Second)\n\t\t\tcontinue\n\t\t}\n\n\t\tif resp.Error != nil {\n\t\t\tt.Fatal(resp.Error)\n\t\t}\n\t\tbreak\n\t}\n\n\tif resp.MemberID != resp.LeaderID {\n\t\tt.Fatalf(\"expected to be group leader %s got %s\", resp.MemberID, resp.LeaderID)\n\t}\n\n\tgroupMembers := make([]GroupMember, 0, len(resp.Members))\n\tgroupUserDataLookup := make(map[string]GroupMember)\n\tfor _, member := range resp.Members {\n\t\tgm := GroupMember{\n\t\t\tID: member.ID,\n\t\t\tTopics: member.Metadata.Topics,\n\t\t\tUserData: member.Metadata.UserData,\n\t\t}\n\t\tgroupMembers = append(groupMembers, gm)\n\t\tgroupUserDataLookup[member.ID] = gm\n\t}\n\n\tmetaResp, err := client.Metadata(ctx, &MetadataRequest{\n\t\tTopics: []string{topic},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tassignments := rrGroupBalancer.AssignGroups(groupMembers, metaResp.Topics[0].Partitions)\n\n\tsgRequest := &SyncGroupRequest{\n\t\tGroupID: groupID,\n\t\tGenerationID: resp.GenerationID,\n\t\tMemberID: resp.MemberID,\n\t\tGroupInstanceID: groupInstanceID,\n\t\tProtocolType: \"consumer\",\n\t\tProtocolName: rrGroupBalancer.ProtocolName(),\n\t}\n\n\tfor member, assignment := range assignments {\n\t\tsgRequest.Assignments = append(sgRequest.Assignments, SyncGroupRequestAssignment{\n\t\t\tMemberID: member,\n\t\t\tAssignment: GroupProtocolAssignment{\n\t\t\t\tAssignedPartitions: assignment,\n\t\t\t\tUserData: groupUserDataLookup[member].UserData,\n\t\t\t},\n\t\t})\n\t}\n\tsgResp, err := client.SyncGroup(ctx, sgRequest)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif sgResp.Error != nil {\n\t\tt.Fatal(sgResp.Error)\n\t}\n\n\texpectedAssignment := GroupProtocolAssignment{\n\t\tAssignedPartitions: map[string][]int{\n\t\t\ttopic: {0, 1, 2},\n\t\t},\n\t\tUserData: []byte(userData),\n\t}\n\n\tif !reflect.DeepEqual(sgResp.Assignment, expectedAssignment) {\n\t\tt.Fatalf(\"\\nexpected assignment to be \\n%#v \\ngot\\n%#v\", expectedAssignment, sgResp.Assignment)\n\t}\n}\n\nfunc TestGroupAssignment(t *testing.T) {\n\titem := groupAssignment{\n\t\tVersion: 1,\n\t\tTopics: map[string][]int32{\n\t\t\t\"a\": {1, 2, 3},\n\t\t\t\"b\": {4, 5},\n\t\t},\n\t\tUserData: []byte(`blah`),\n\t}\n\n\tb := bytes.NewBuffer(nil)\n\tw := &writeBuffer{w: b}\n\titem.writeTo(w)\n\n\tvar found groupAssignment\n\tremain, err := (&found).readFrom(bufio.NewReader(b), b.Len())\n\tif err != nil {\n\t\tt.Error(err)\n\t\tt.FailNow()\n\t}\n\tif remain != 0 {\n\t\tt.Errorf(\"expected 0 remain, got %v\", remain)\n\t\tt.FailNow()\n\t}\n\tif !reflect.DeepEqual(item, found) {\n\t\tt.Error(\"expected item and found to be the same\")\n\t\tt.FailNow()\n\t}\n}\n\nfunc TestGroupAssignmentReadsFromZeroSize(t *testing.T) {\n\tvar item groupAssignment\n\tremain, err := (&item).readFrom(bufio.NewReader(bytes.NewReader(nil)), 0)\n\tif err != nil {\n\t\tt.Error(err)\n\t\tt.FailNow()\n\t}\n\tif remain != 0 {\n\t\tt.Errorf(\"expected 0 remain, got %v\", remain)\n\t\tt.FailNow()\n\t}\n\tif item.Topics == nil {\n\t\tt.Error(\"expected non nil Topics to be assigned\")\n\t}\n}\n\nfunc TestSyncGroupResponseV0(t *testing.T) {\n\titem := syncGroupResponseV0{\n\t\tErrorCode: 2,\n\t\tMemberAssignments: []byte(`blah`),\n\t}\n\n\tb := bytes.NewBuffer(nil)\n\tw := &writeBuffer{w: b}\n\titem.writeTo(w)\n\n\tvar found syncGroupResponseV0\n\tremain, err := (&found).readFrom(bufio.NewReader(b), b.Len())\n\tif err != nil {\n\t\tt.Error(err)\n\t\tt.FailNow()\n\t}\n\tif remain != 0 {\n\t\tt.Errorf(\"expected 0 remain, got %v\", remain)\n\t\tt.FailNow()\n\t}\n\tif !reflect.DeepEqual(item, found) {\n\t\tt.Error(\"expected item and found to be the same\")\n\t\tt.FailNow()\n\t}\n}\n\nfunc BenchmarkSyncGroupResponseV0(t *testing.B) {\n\titem := syncGroupResponseV0{\n\t\tErrorCode: 2,\n\t\tMemberAssignments: []byte(`blah`),\n\t}\n\n\tb := bytes.NewBuffer(nil)\n\tw := &writeBuffer{w: b}\n\titem.writeTo(w)\n\n\tr := bytes.NewReader(b.Bytes())\n\treader := bufio.NewReader(r)\n\tsize := b.Len()\n\n\tfor i := 0; i < t.N; i++ {\n\t\tr.Seek(0, io.SeekStart)\n\t\tvar found syncGroupResponseV0\n\t\tremain, err := (&found).readFrom(reader, size)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t\tt.FailNow()\n\t\t}\n\t\tif remain != 0 {\n\t\t\tt.Errorf(\"expected 0 remain, got %v\", remain)\n\t\t\tt.FailNow()\n\t\t}\n\t}\n}\n" }, { "path": "testing/conn.go", "content": "package testing\n\nimport (\n\t\"context\"\n\t\"net\"\n\t\"sync\"\n)\n\ntype ConnWaitGroup struct {\n\tDialFunc func(context.Context, string, string) (net.Conn, error)\n\tsync.WaitGroup\n}\n\nfunc (g *ConnWaitGroup) Dial(ctx context.Context, network, address string) (net.Conn, error) {\n\tc, err := g.DialFunc(ctx, network, address)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tg.Add(1)\n\treturn &groupConn{Conn: c, group: g}, nil\n}\n\ntype groupConn struct {\n\tnet.Conn\n\tgroup *ConnWaitGroup\n\tonce sync.Once\n}\n\nfunc (c *groupConn) Close() error {\n\tdefer c.once.Do(c.group.Done)\n\treturn c.Conn.Close()\n}\n" }, { "path": "testing/version.go", "content": "package testing\n\nimport (\n\t\"os\"\n\t\"strconv\"\n\t\"strings\"\n)\n\ntype semver []int\n\nfunc (v semver) atLeast(other semver) bool {\n\tfor i := range v {\n\t\tif i >= len(other) {\n\t\t\tbreak\n\t\t}\n\t\tif v[i] < other[i] {\n\t\t\treturn false\n\t\t}\n\t\tif v[i] > other[i] {\n\t\t\treturn true\n\t\t}\n\t}\n\tfor i := len(v); i < len(other); i++ {\n\t\tif other[i] > 0 {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// kafkaVersion is set in the circle config. It can also be provided on the\n// command line in order to target a particular kafka version.\nvar kafkaVersion = parseVersion(os.Getenv(\"KAFKA_VERSION\"))\n\n// KafkaIsAtLeast returns true when the test broker is running a protocol\n// version that is semver or newer. It determines the broker's version using\n// the `KAFKA_VERSION` environment variable. If the var is unset, then this\n// function will return true.\nfunc KafkaIsAtLeast(semver string) bool {\n\treturn kafkaVersion.atLeast(parseVersion(semver))\n}\n\nfunc parseVersion(semver string) semver {\n\tif semver == \"\" {\n\t\treturn nil\n\t}\n\tparts := strings.Split(semver, \".\")\n\tversion := make([]int, len(parts))\n\tfor i := range version {\n\t\tv, err := strconv.Atoi(parts[i])\n\t\tif err != nil {\n\t\t\t// panic-ing because tests should be using hard-coded version values\n\t\t\tpanic(\"invalid version string: \" + semver)\n\t\t}\n\t\tversion[i] = v\n\t}\n\treturn version\n}\n" }, { "path": "testing/version_test.go", "content": "package testing\n\nimport (\n\t\"testing\"\n)\n\nfunc TestSemVersionAtLeastEmpty(t *testing.T) {\n\tresult := semver([]int{}).atLeast(semver([]int{1, 2}))\n\tif result {\n\t\tt.Errorf(\"Empty version can't be at least 1.2\")\n\t}\n}\n\nfunc TestSemVersionAtLeastShorter(t *testing.T) {\n\tresult := semver([]int{1, 1}).atLeast(semver([]int{1, 1, 2}))\n\tif result {\n\t\tt.Errorf(\"Version 1.1 version can't be at least 1.1.2\")\n\t}\n}\n" }, { "path": "time.go", "content": "package kafka\n\nimport (\n\t\"math\"\n\t\"time\"\n)\n\nconst (\n\tmaxTimeout = time.Duration(math.MaxInt32) * time.Millisecond\n\tminTimeout = time.Duration(math.MinInt32) * time.Millisecond\n\tdefaultRTT = 1 * time.Second\n)\n\nfunc makeTime(t int64) time.Time {\n\tif t <= 0 {\n\t\treturn time.Time{}\n\t}\n\treturn time.Unix(t/1000, (t%1000)*int64(time.Millisecond)).UTC()\n}\n\nfunc timestamp(t time.Time) int64 {\n\tif t.IsZero() {\n\t\treturn 0\n\t}\n\treturn t.UnixNano() / int64(time.Millisecond)\n}\n\nfunc makeDuration(ms int32) time.Duration {\n\treturn time.Duration(ms) * time.Millisecond\n}\n\nfunc milliseconds(d time.Duration) int32 {\n\tswitch {\n\tcase d > maxTimeout:\n\t\td = maxTimeout\n\tcase d < minTimeout:\n\t\td = minTimeout\n\t}\n\treturn int32(d / time.Millisecond)\n}\n\nfunc deadlineToTimeout(deadline time.Time, now time.Time) time.Duration {\n\tif deadline.IsZero() {\n\t\treturn maxTimeout\n\t}\n\treturn deadline.Sub(now)\n}\n\nfunc adjustDeadlineForRTT(deadline time.Time, now time.Time, rtt time.Duration) time.Time {\n\tif !deadline.IsZero() {\n\t\ttimeout := deadline.Sub(now)\n\t\tif timeout < rtt {\n\t\t\trtt = timeout / 4\n\t\t}\n\t\tdeadline = deadline.Add(-rtt)\n\t}\n\treturn deadline\n}\n" }, { "path": "topics/list_topics.go", "content": "// Package topics is an experimental package that provides additional tooling\n// around Kafka Topics. This package does not make any promises around\n// backwards compatibility.\npackage topics\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"regexp\"\n\n\t\"github.com/segmentio/kafka-go\"\n)\n\n// List returns a slice of all the Topics.\nfunc List(ctx context.Context, client *kafka.Client) (topics []kafka.Topic, err error) {\n\tif client == nil {\n\t\treturn nil, errors.New(\"client is required\")\n\t}\n\tresponse, err := client.Metadata(ctx, &kafka.MetadataRequest{\n\t\tAddr: client.Addr,\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn response.Topics, nil\n}\n\n// ListRe returns a slice of Topics that match a regex.\nfunc ListRe(ctx context.Context, cli *kafka.Client, re *regexp.Regexp) (topics []kafka.Topic, err error) {\n\talltopics, err := List(ctx, cli)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tfor _, val := range alltopics {\n\t\tif re.MatchString(val.Name) {\n\t\t\ttopics = append(topics, val)\n\t\t}\n\t}\n\treturn topics, nil\n}\n" }, { "path": "topics/list_topics_test.go", "content": "package topics\n\nimport (\n\t\"context\"\n\t\"net\"\n\t\"regexp\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go\"\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc TestListReNil(t *testing.T) {\n\t_, err := ListRe(context.Background(), nil, nil)\n\tif err == nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc TestListRe(t *testing.T) {\n\tclient, shutdown := newLocalClientWithTopic(\"TestTopics-A\", 1)\n\tdefer shutdown()\n\tclientCreateTopic(client, \"TestTopics-B\", 1)\n\n\tallRegex := regexp.MustCompile(\"TestTopics-.*\")\n\tfooRegex := regexp.MustCompile(\"TestTopics-B\")\n\n\t// Get all the topics\n\ttopics, err := ListRe(context.Background(), client, allRegex)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(topics) != 2 {\n\t\tt.Error(\"the wrong number of topics were returned. \", len(topics))\n\t}\n\n\t// Get one topic\n\ttopics, err = ListRe(context.Background(), client, fooRegex)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif len(topics) != 1 {\n\t\tt.Error(\"the wrong number of topics were returned. \", len(topics))\n\t}\n}\n\nfunc newLocalClientWithTopic(topic string, partitions int) (*kafka.Client, func()) {\n\tclient, shutdown := newLocalClient()\n\tif err := clientCreateTopic(client, topic, partitions); err != nil {\n\t\tshutdown()\n\t\tpanic(err)\n\t}\n\treturn client, func() {\n\t\tclient.DeleteTopics(context.Background(), &kafka.DeleteTopicsRequest{\n\t\t\tTopics: []string{topic},\n\t\t})\n\t\tshutdown()\n\t}\n}\n\nfunc clientCreateTopic(client *kafka.Client, topic string, partitions int) error {\n\t_, err := client.CreateTopics(context.Background(), &kafka.CreateTopicsRequest{\n\t\tTopics: []kafka.TopicConfig{{\n\t\t\tTopic: topic,\n\t\t\tNumPartitions: partitions,\n\t\t\tReplicationFactor: 1,\n\t\t}},\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// Topic creation seems to be asynchronous. Metadata for the topic partition\n\t// layout in the cluster is available in the controller before being synced\n\t// with the other brokers, which causes \"Error:[3] Unknown Topic Or Partition\"\n\t// when sending requests to the partition leaders.\n\t//\n\t// This loop will wait up to 2 seconds polling the cluster until no errors\n\t// are returned.\n\tfor i := 0; i < 20; i++ {\n\t\tr, err := client.Fetch(context.Background(), &kafka.FetchRequest{\n\t\t\tTopic: topic,\n\t\t\tPartition: 0,\n\t\t\tOffset: 0,\n\t\t})\n\t\tif err == nil && r.Error == nil {\n\t\t\tbreak\n\t\t}\n\t\ttime.Sleep(100 * time.Millisecond)\n\t}\n\n\treturn nil\n}\n\nfunc newLocalClient() (*kafka.Client, func()) {\n\treturn newClient(kafka.TCP(\"localhost\"))\n}\n\nfunc newClient(addr net.Addr) (*kafka.Client, func()) {\n\tconns := &ktesting.ConnWaitGroup{\n\t\tDialFunc: (&net.Dialer{}).DialContext,\n\t}\n\n\ttransport := &kafka.Transport{\n\t\tDial: conns.Dial,\n\t\tResolver: kafka.NewBrokerResolver(nil),\n\t}\n\n\tclient := &kafka.Client{\n\t\tAddr: addr,\n\t\tTimeout: 5 * time.Second,\n\t\tTransport: transport,\n\t}\n\n\treturn client, func() { transport.CloseIdleConnections(); conns.Wait() }\n}\n" }, { "path": "transport.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math/rand\"\n\t\"net\"\n\t\"runtime/pprof\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n\t\"github.com/segmentio/kafka-go/protocol/apiversions\"\n\t\"github.com/segmentio/kafka-go/protocol/createtopics\"\n\t\"github.com/segmentio/kafka-go/protocol/findcoordinator\"\n\tmeta \"github.com/segmentio/kafka-go/protocol/metadata\"\n\t\"github.com/segmentio/kafka-go/protocol/saslauthenticate\"\n\t\"github.com/segmentio/kafka-go/protocol/saslhandshake\"\n\t\"github.com/segmentio/kafka-go/sasl\"\n)\n\n// Request is an interface implemented by types that represent messages sent\n// from kafka clients to brokers.\ntype Request = protocol.Message\n\n// Response is an interface implemented by types that represent messages sent\n// from kafka brokers in response to client requests.\ntype Response = protocol.Message\n\n// RoundTripper is an interface implemented by types which support interacting\n// with kafka brokers.\ntype RoundTripper interface {\n\t// RoundTrip sends a request to a kafka broker and returns the response that\n\t// was received, or a non-nil error.\n\t//\n\t// The context passed as first argument can be used to asynchronnously abort\n\t// the call if needed.\n\tRoundTrip(context.Context, net.Addr, Request) (Response, error)\n}\n\n// Transport is an implementation of the RoundTripper interface.\n//\n// Transport values manage a pool of connections and automatically discovers the\n// clusters layout to route requests to the appropriate brokers.\n//\n// Transport values are safe to use concurrently from multiple goroutines.\n//\n// Note: The intent is for the Transport to become the underlying layer of the\n// kafka.Reader and kafka.Writer types.\ntype Transport struct {\n\t// A function used to establish connections to the kafka cluster.\n\tDial func(context.Context, string, string) (net.Conn, error)\n\n\t// Time limit set for establishing connections to the kafka cluster. This\n\t// limit includes all round trips done to establish the connections (TLS\n\t// handshake, SASL negotiation, etc...).\n\t//\n\t// Defaults to 5s.\n\tDialTimeout time.Duration\n\n\t// Maximum amount of time that connections will remain open and unused.\n\t// The transport will manage to automatically close connections that have\n\t// been idle for too long, and re-open them on demand when the transport is\n\t// used again.\n\t//\n\t// Defaults to 30s.\n\tIdleTimeout time.Duration\n\n\t// TTL for the metadata cached by this transport. Note that the value\n\t// configured here is an upper bound, the transport randomizes the TTLs to\n\t// avoid getting into states where multiple clients end up synchronized and\n\t// cause bursts of requests to the kafka broker.\n\t//\n\t// Default to 6s.\n\tMetadataTTL time.Duration\n\n\t// Topic names for the metadata cached by this transport. If this field is left blank,\n\t// metadata information of all topics in the cluster will be retrieved.\n\tMetadataTopics []string\n\n\t// Unique identifier that the transport communicates to the brokers when it\n\t// sends requests.\n\tClientID string\n\n\t// An optional configuration for TLS connections established by this\n\t// transport.\n\t//\n\t// If the Server\n\tTLS *tls.Config\n\n\t// SASL configures the Transfer to use SASL authentication.\n\tSASL sasl.Mechanism\n\n\t// An optional resolver used to translate broker host names into network\n\t// addresses.\n\t//\n\t// The resolver will be called for every request (not every connection),\n\t// making it possible to implement ACL policies by validating that the\n\t// program is allowed to connect to the kafka broker. This also means that\n\t// the resolver should probably provide a caching layer to avoid storming\n\t// the service discovery backend with requests.\n\t//\n\t// When set, the Dial function is not responsible for performing name\n\t// resolution, and is always called with a pre-resolved address.\n\tResolver BrokerResolver\n\n\t// The background context used to control goroutines started internally by\n\t// the transport.\n\t//\n\t// If nil, context.Background() is used instead.\n\tContext context.Context\n\n\tmutex sync.RWMutex\n\tpools map[networkAddress]*connPool\n}\n\n// DefaultTransport is the default transport used by kafka clients in this\n// package.\nvar DefaultTransport RoundTripper = &Transport{\n\tDial: (&net.Dialer{\n\t\tTimeout: 3 * time.Second,\n\t\tDualStack: true,\n\t}).DialContext,\n}\n\n// CloseIdleConnections closes all idle connections immediately, and marks all\n// connections that are in use to be closed when they become idle again.\nfunc (t *Transport) CloseIdleConnections() {\n\tt.mutex.Lock()\n\tdefer t.mutex.Unlock()\n\n\tfor _, pool := range t.pools {\n\t\tpool.unref()\n\t}\n\n\tfor k := range t.pools {\n\t\tdelete(t.pools, k)\n\t}\n}\n\n// RoundTrip sends a request to a kafka cluster and returns the response, or an\n// error if no responses were received.\n//\n// Message types are available in sub-packages of the protocol package. Each\n// kafka API is implemented in a different sub-package. For example, the request\n// and response types for the Fetch API are available in the protocol/fetch\n// package.\n//\n// The type of the response message will match the type of the request. For\n// example, if RoundTrip was called with a *fetch.Request as argument, the value\n// returned will be of type *fetch.Response. It is safe for the program to do a\n// type assertion after checking that no error was returned.\n//\n// This example illustrates the way this method is expected to be used:\n//\n//\tr, err := transport.RoundTrip(ctx, addr, &fetch.Request{ ... })\n//\tif err != nil {\n//\t\t...\n//\t} else {\n//\t\tres := r.(*fetch.Response)\n//\t\t...\n//\t}\n//\n// The transport automatically selects the highest version of the API that is\n// supported by both the kafka-go package and the kafka broker. The negotiation\n// happens transparently once when connections are established.\n//\n// This API was introduced in version 0.4 as a way to leverage the lower-level\n// features of the kafka protocol, but also provide a more efficient way of\n// managing connections to kafka brokers.\nfunc (t *Transport) RoundTrip(ctx context.Context, addr net.Addr, req Request) (Response, error) {\n\tp := t.grabPool(addr)\n\tdefer p.unref()\n\treturn p.roundTrip(ctx, req)\n}\n\nfunc (t *Transport) dial() func(context.Context, string, string) (net.Conn, error) {\n\tif t.Dial != nil {\n\t\treturn t.Dial\n\t}\n\treturn defaultDialer.DialContext\n}\n\nfunc (t *Transport) dialTimeout() time.Duration {\n\tif t.DialTimeout > 0 {\n\t\treturn t.DialTimeout\n\t}\n\treturn 5 * time.Second\n}\n\nfunc (t *Transport) idleTimeout() time.Duration {\n\tif t.IdleTimeout > 0 {\n\t\treturn t.IdleTimeout\n\t}\n\treturn 30 * time.Second\n}\n\nfunc (t *Transport) metadataTTL() time.Duration {\n\tif t.MetadataTTL > 0 {\n\t\treturn t.MetadataTTL\n\t}\n\treturn 6 * time.Second\n}\n\nfunc (t *Transport) grabPool(addr net.Addr) *connPool {\n\tk := networkAddress{\n\t\tnetwork: addr.Network(),\n\t\taddress: addr.String(),\n\t}\n\n\tt.mutex.RLock()\n\tp := t.pools[k]\n\tif p != nil {\n\t\tp.ref()\n\t}\n\tt.mutex.RUnlock()\n\n\tif p != nil {\n\t\treturn p\n\t}\n\n\tt.mutex.Lock()\n\tdefer t.mutex.Unlock()\n\n\tif p := t.pools[k]; p != nil {\n\t\tp.ref()\n\t\treturn p\n\t}\n\n\tctx, cancel := context.WithCancel(t.context())\n\n\tp = &connPool{\n\t\trefc: 2,\n\n\t\tdial: t.dial(),\n\t\tdialTimeout: t.dialTimeout(),\n\t\tidleTimeout: t.idleTimeout(),\n\t\tmetadataTTL: t.metadataTTL(),\n\t\tmetadataTopics: t.MetadataTopics,\n\t\tclientID: t.ClientID,\n\t\ttls: t.TLS,\n\t\tsasl: t.SASL,\n\t\tresolver: t.Resolver,\n\n\t\tready: make(event),\n\t\twake: make(chan event),\n\t\tconns: make(map[int32]*connGroup),\n\t\tcancel: cancel,\n\t}\n\n\tp.ctrl = p.newConnGroup(addr)\n\tgo p.discover(ctx, p.wake)\n\n\tif t.pools == nil {\n\t\tt.pools = make(map[networkAddress]*connPool)\n\t}\n\tt.pools[k] = p\n\treturn p\n}\n\nfunc (t *Transport) context() context.Context {\n\tif t.Context != nil {\n\t\treturn t.Context\n\t}\n\treturn context.Background()\n}\n\ntype event chan struct{}\n\nfunc (e event) trigger() { close(e) }\n\ntype connPool struct {\n\trefc uintptr\n\t// Immutable fields of the connection pool. Connections access these field\n\t// on their parent pool in a ready-only fashion, so no synchronization is\n\t// required.\n\tdial func(context.Context, string, string) (net.Conn, error)\n\tdialTimeout time.Duration\n\tidleTimeout time.Duration\n\tmetadataTTL time.Duration\n\tmetadataTopics []string\n\tclientID string\n\ttls *tls.Config\n\tsasl sasl.Mechanism\n\tresolver BrokerResolver\n\t// Signaling mechanisms to orchestrate communications between the pool and\n\t// the rest of the program.\n\tonce sync.Once // ensure that `ready` is triggered only once\n\tready event // triggered after the first metadata update\n\twake chan event // used to force metadata updates\n\tcancel context.CancelFunc\n\t// Mutable fields of the connection pool, access must be synchronized.\n\tmutex sync.RWMutex\n\tconns map[int32]*connGroup // data connections used for produce/fetch/etc...\n\tctrl *connGroup // control connections used for metadata requests\n\tstate atomic.Value // cached cluster state\n}\n\ntype connPoolState struct {\n\tmetadata *meta.Response // last metadata response seen by the pool\n\terr error // last error from metadata requests\n\tlayout protocol.Cluster // cluster layout built from metadata response\n}\n\nfunc (p *connPool) grabState() connPoolState {\n\tstate, _ := p.state.Load().(connPoolState)\n\treturn state\n}\n\nfunc (p *connPool) setState(state connPoolState) {\n\tp.state.Store(state)\n}\n\nfunc (p *connPool) ref() {\n\tatomic.AddUintptr(&p.refc, +1)\n}\n\nfunc (p *connPool) unref() {\n\tif atomic.AddUintptr(&p.refc, ^uintptr(0)) == 0 {\n\t\tp.mutex.Lock()\n\t\tdefer p.mutex.Unlock()\n\n\t\tfor _, conns := range p.conns {\n\t\t\tconns.closeIdleConns()\n\t\t}\n\n\t\tp.ctrl.closeIdleConns()\n\t\tp.cancel()\n\t}\n}\n\nfunc (p *connPool) roundTrip(ctx context.Context, req Request) (Response, error) {\n\t// This first select should never block after the first metadata response\n\t// that would mark the pool as `ready`.\n\tselect {\n\tcase <-p.ready:\n\tcase <-ctx.Done():\n\t\treturn nil, ctx.Err()\n\t}\n\n\tstate := p.grabState()\n\tvar response promise\n\n\tswitch m := req.(type) {\n\tcase *meta.Request:\n\t\t// We serve metadata requests directly from the transport cache unless\n\t\t// we would like to auto create a topic that isn't in our cache.\n\t\t//\n\t\t// This reduces the number of round trips to kafka brokers while keeping\n\t\t// the logic simple when applying partitioning strategies.\n\t\tif state.err != nil {\n\t\t\treturn nil, state.err\n\t\t}\n\n\t\tcachedMeta := filterMetadataResponse(m, state.metadata)\n\t\t// requestNeeded indicates if we need to send this metadata request to the server.\n\t\t// It's true when we want to auto-create topics and we don't have the topic in our\n\t\t// cache.\n\t\tvar requestNeeded bool\n\t\tif m.AllowAutoTopicCreation {\n\t\t\tfor _, topic := range cachedMeta.Topics {\n\t\t\t\tif topic.ErrorCode == int16(UnknownTopicOrPartition) {\n\t\t\t\t\trequestNeeded = true\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif !requestNeeded {\n\t\t\treturn cachedMeta, nil\n\t\t}\n\n\tcase protocol.Splitter:\n\t\t// Messages that implement the Splitter interface trigger the creation of\n\t\t// multiple requests that are all merged back into a single results by\n\t\t// a merger.\n\t\tmessages, merger, err := m.Split(state.layout)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tpromises := make([]promise, len(messages))\n\t\tfor i, m := range messages {\n\t\t\tpromises[i] = p.sendRequest(ctx, m, state)\n\t\t}\n\t\tresponse = join(promises, messages, merger)\n\t}\n\n\tif response == nil {\n\t\tresponse = p.sendRequest(ctx, req, state)\n\t}\n\n\tr, err := response.await(ctx)\n\tif err != nil {\n\t\treturn r, err\n\t}\n\n\tswitch resp := r.(type) {\n\tcase *createtopics.Response:\n\t\t// Force an update of the metadata when adding topics,\n\t\t// otherwise the cached state would get out of sync.\n\t\ttopicsToRefresh := make([]string, 0, len(resp.Topics))\n\t\tfor _, topic := range resp.Topics {\n\t\t\t// fixes issue 672: don't refresh topics that failed to create, it causes the library to hang indefinitely\n\t\t\tif topic.ErrorCode != 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\ttopicsToRefresh = append(topicsToRefresh, topic.Name)\n\t\t}\n\n\t\tp.refreshMetadata(ctx, topicsToRefresh)\n\tcase *meta.Response:\n\t\tm := req.(*meta.Request)\n\t\t// If we get here with allow auto topic creation then\n\t\t// we didn't have that topic in our cache, so we should update\n\t\t// the cache.\n\t\tif m.AllowAutoTopicCreation {\n\t\t\ttopicsToRefresh := make([]string, 0, len(resp.Topics))\n\t\t\tfor _, topic := range resp.Topics {\n\t\t\t\t// Don't refresh topics that failed to create, since that may\n\t\t\t\t// mean that enable automatic topic creation is not enabled.\n\t\t\t\t// That causes the library to hang indefinitely, same as\n\t\t\t\t// don't refresh topics that failed to create,\n\t\t\t\t// createtopics process. Fixes issue 806.\n\t\t\t\tif topic.ErrorCode != 0 {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\ttopicsToRefresh = append(topicsToRefresh, topic.Name)\n\t\t\t}\n\t\t\tp.refreshMetadata(ctx, topicsToRefresh)\n\t\t}\n\t}\n\n\treturn r, nil\n}\n\n// refreshMetadata forces an update of the cached cluster metadata, and waits\n// for the given list of topics to appear. This waiting mechanism is necessary\n// to account for the fact that topic creation is asynchronous in kafka, and\n// causes subsequent requests to fail while the cluster state is propagated to\n// all the brokers.\nfunc (p *connPool) refreshMetadata(ctx context.Context, expectTopics []string) {\n\tminBackoff := 100 * time.Millisecond\n\tmaxBackoff := 2 * time.Second\n\tcancel := ctx.Done()\n\n\tfor ctx.Err() == nil {\n\t\tnotify := make(event)\n\t\tselect {\n\t\tcase <-cancel:\n\t\t\treturn\n\t\tcase p.wake <- notify:\n\t\t\tselect {\n\t\t\tcase <-notify:\n\t\t\tcase <-cancel:\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\n\t\tstate := p.grabState()\n\t\tfound := 0\n\n\t\tfor _, topic := range expectTopics {\n\t\t\tif _, ok := state.layout.Topics[topic]; ok {\n\t\t\t\tfound++\n\t\t\t}\n\t\t}\n\n\t\tif found == len(expectTopics) {\n\t\t\treturn\n\t\t}\n\n\t\tif delay := time.Duration(rand.Int63n(int64(minBackoff))); delay > 0 {\n\t\t\ttimer := time.NewTimer(minBackoff)\n\t\t\tselect {\n\t\t\tcase <-cancel:\n\t\t\tcase <-timer.C:\n\t\t\t}\n\t\t\ttimer.Stop()\n\n\t\t\tif minBackoff *= 2; minBackoff > maxBackoff {\n\t\t\t\tminBackoff = maxBackoff\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (p *connPool) setReady() {\n\tp.once.Do(p.ready.trigger)\n}\n\n// update is called periodically by the goroutine running the discover method\n// to refresh the cluster layout information used by the transport to route\n// requests to brokers.\nfunc (p *connPool) update(ctx context.Context, metadata *meta.Response, err error) {\n\tvar layout protocol.Cluster\n\n\tif metadata != nil {\n\t\tmetadata.ThrottleTimeMs = 0\n\n\t\t// Normalize the lists so we can apply binary search on them.\n\t\tsortMetadataBrokers(metadata.Brokers)\n\t\tsortMetadataTopics(metadata.Topics)\n\n\t\tfor i := range metadata.Topics {\n\t\t\tt := &metadata.Topics[i]\n\t\t\tsortMetadataPartitions(t.Partitions)\n\t\t}\n\n\t\tlayout = makeLayout(metadata)\n\t}\n\n\tstate := p.grabState()\n\taddBrokers := make(map[int32]struct{})\n\tdelBrokers := make(map[int32]struct{})\n\n\tif err != nil {\n\t\t// Only update the error on the transport if the cluster layout was\n\t\t// unknown. This ensures that we prioritize a previously known state\n\t\t// of the cluster to reduce the impact of transient failures.\n\t\tif state.metadata != nil {\n\t\t\treturn\n\t\t}\n\t\tstate.err = err\n\t} else {\n\t\tfor id, b2 := range layout.Brokers {\n\t\t\tif b1, ok := state.layout.Brokers[id]; !ok {\n\t\t\t\taddBrokers[id] = struct{}{}\n\t\t\t} else if b1 != b2 {\n\t\t\t\taddBrokers[id] = struct{}{}\n\t\t\t\tdelBrokers[id] = struct{}{}\n\t\t\t}\n\t\t}\n\n\t\tfor id := range state.layout.Brokers {\n\t\t\tif _, ok := layout.Brokers[id]; !ok {\n\t\t\t\tdelBrokers[id] = struct{}{}\n\t\t\t}\n\t\t}\n\n\t\tstate.metadata, state.layout = metadata, layout\n\t\tstate.err = nil\n\t}\n\n\tdefer p.setReady()\n\tdefer p.setState(state)\n\n\tif len(addBrokers) != 0 || len(delBrokers) != 0 {\n\t\t// Only acquire the lock when there is a change of layout. This is an\n\t\t// infrequent event so we don't risk introducing regular contention on\n\t\t// the mutex if we were to lock it on every update.\n\t\tp.mutex.Lock()\n\t\tdefer p.mutex.Unlock()\n\n\t\tif ctx.Err() != nil {\n\t\t\treturn // the pool has been closed, no need to update\n\t\t}\n\n\t\tfor id := range delBrokers {\n\t\t\tif broker := p.conns[id]; broker != nil {\n\t\t\t\tbroker.closeIdleConns()\n\t\t\t\tdelete(p.conns, id)\n\t\t\t}\n\t\t}\n\n\t\tfor id := range addBrokers {\n\t\t\tbroker := layout.Brokers[id]\n\t\t\tp.conns[id] = p.newBrokerConnGroup(Broker{\n\t\t\t\tRack: broker.Rack,\n\t\t\t\tHost: broker.Host,\n\t\t\t\tPort: int(broker.Port),\n\t\t\t\tID: int(broker.ID),\n\t\t\t})\n\t\t}\n\t}\n}\n\n// discover is the entry point of an internal goroutine for the transport which\n// periodically requests updates of the cluster metadata and refreshes the\n// transport cached cluster layout.\nfunc (p *connPool) discover(ctx context.Context, wake <-chan event) {\n\tprng := rand.New(rand.NewSource(time.Now().UnixNano()))\n\tmetadataTTL := func() time.Duration {\n\t\treturn time.Duration(prng.Int63n(int64(p.metadataTTL)))\n\t}\n\n\ttimer := time.NewTimer(metadataTTL())\n\tdefer timer.Stop()\n\n\tvar notify event\n\tdone := ctx.Done()\n\n\treq := &meta.Request{\n\t\tTopicNames: p.metadataTopics,\n\t}\n\n\tfor {\n\t\tc, err := p.grabClusterConn(ctx)\n\t\tif err != nil {\n\t\t\tp.update(ctx, nil, err)\n\t\t} else {\n\t\t\tres := make(async, 1)\n\t\t\tdeadline, cancel := context.WithTimeout(ctx, p.metadataTTL)\n\t\t\tc.reqs <- connRequest{\n\t\t\t\tctx: deadline,\n\t\t\t\treq: req,\n\t\t\t\tres: res,\n\t\t\t}\n\t\t\tr, err := res.await(deadline)\n\t\t\tcancel()\n\t\t\tif err != nil && errors.Is(err, ctx.Err()) {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tret, _ := r.(*meta.Response)\n\t\t\tp.update(ctx, ret, err)\n\t\t}\n\n\t\tif notify != nil {\n\t\t\tnotify.trigger()\n\t\t\tnotify = nil\n\t\t}\n\n\t\tselect {\n\t\tcase <-timer.C:\n\t\t\ttimer.Reset(metadataTTL())\n\t\tcase <-done:\n\t\t\treturn\n\t\tcase notify = <-wake:\n\t\t}\n\t}\n}\n\n// grabBrokerConn returns a connection to a specific broker represented by the\n// broker id passed as argument. If the broker id was not known, an error is\n// returned.\nfunc (p *connPool) grabBrokerConn(ctx context.Context, brokerID int32) (*conn, error) {\n\tp.mutex.RLock()\n\tg := p.conns[brokerID]\n\tp.mutex.RUnlock()\n\tif g == nil {\n\t\treturn nil, BrokerNotAvailable\n\t}\n\treturn g.grabConnOrConnect(ctx)\n}\n\n// grabClusterConn returns the connection to the kafka cluster that the pool is\n// configured to connect to.\n//\n// The transport uses a shared `control` connection to the cluster for any\n// requests that aren't supposed to be sent to specific brokers (e.g. Fetch or\n// Produce requests). Requests intended to be routed to specific brokers are\n// dispatched on a separate pool of connections that the transport maintains.\n// This split help avoid head-of-line blocking situations where control requests\n// like Metadata would be queued behind large responses from Fetch requests for\n// example.\n//\n// In either cases, the requests are multiplexed so we can keep a minimal number\n// of connections open (N+1, where N is the number of brokers in the cluster).\nfunc (p *connPool) grabClusterConn(ctx context.Context) (*conn, error) {\n\treturn p.ctrl.grabConnOrConnect(ctx)\n}\n\nfunc (p *connPool) sendRequest(ctx context.Context, req Request, state connPoolState) promise {\n\tbrokerID := int32(-1)\n\n\tswitch m := req.(type) {\n\tcase protocol.BrokerMessage:\n\t\t// Some requests are supposed to be sent to specific brokers (e.g. the\n\t\t// partition leaders). They implement the BrokerMessage interface to\n\t\t// delegate the routing decision to each message type.\n\t\tbroker, err := m.Broker(state.layout)\n\t\tif err != nil {\n\t\t\treturn reject(err)\n\t\t}\n\t\tbrokerID = broker.ID\n\n\tcase protocol.GroupMessage:\n\t\t// Some requests are supposed to be sent to a group coordinator,\n\t\t// look up which broker is currently the coordinator for the group\n\t\t// so we can get a connection to that broker.\n\t\t//\n\t\t// TODO: should we cache the coordinator info?\n\t\tp := p.sendRequest(ctx, &findcoordinator.Request{Key: m.Group()}, state)\n\t\tr, err := p.await(ctx)\n\t\tif err != nil {\n\t\t\treturn reject(err)\n\t\t}\n\t\tbrokerID = r.(*findcoordinator.Response).NodeID\n\tcase protocol.TransactionalMessage:\n\t\tp := p.sendRequest(ctx, &findcoordinator.Request{\n\t\t\tKey: m.Transaction(),\n\t\t\tKeyType: int8(CoordinatorKeyTypeTransaction),\n\t\t}, state)\n\t\tr, err := p.await(ctx)\n\t\tif err != nil {\n\t\t\treturn reject(err)\n\t\t}\n\t\tbrokerID = r.(*findcoordinator.Response).NodeID\n\t}\n\n\tvar c *conn\n\tvar err error\n\tif brokerID >= 0 {\n\t\tc, err = p.grabBrokerConn(ctx, brokerID)\n\t} else {\n\t\tc, err = p.grabClusterConn(ctx)\n\t}\n\tif err != nil {\n\t\treturn reject(err)\n\t}\n\n\tres := make(async, 1)\n\n\tc.reqs <- connRequest{\n\t\tctx: ctx,\n\t\treq: req,\n\t\tres: res,\n\t}\n\n\treturn res\n}\n\nfunc filterMetadataResponse(req *meta.Request, res *meta.Response) *meta.Response {\n\tret := *res\n\n\tif req.TopicNames != nil {\n\t\tret.Topics = make([]meta.ResponseTopic, len(req.TopicNames))\n\n\t\tfor i, topicName := range req.TopicNames {\n\t\t\tj, ok := findMetadataTopic(res.Topics, topicName)\n\t\t\tif ok {\n\t\t\t\tret.Topics[i] = res.Topics[j]\n\t\t\t} else {\n\t\t\t\tret.Topics[i] = meta.ResponseTopic{\n\t\t\t\t\tErrorCode: int16(UnknownTopicOrPartition),\n\t\t\t\t\tName: topicName,\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn &ret\n}\n\nfunc findMetadataTopic(topics []meta.ResponseTopic, topicName string) (int, bool) {\n\ti := sort.Search(len(topics), func(i int) bool {\n\t\treturn topics[i].Name >= topicName\n\t})\n\treturn i, i >= 0 && i < len(topics) && topics[i].Name == topicName\n}\n\nfunc sortMetadataBrokers(brokers []meta.ResponseBroker) {\n\tsort.Slice(brokers, func(i, j int) bool {\n\t\treturn brokers[i].NodeID < brokers[j].NodeID\n\t})\n}\n\nfunc sortMetadataTopics(topics []meta.ResponseTopic) {\n\tsort.Slice(topics, func(i, j int) bool {\n\t\treturn topics[i].Name < topics[j].Name\n\t})\n}\n\nfunc sortMetadataPartitions(partitions []meta.ResponsePartition) {\n\tsort.Slice(partitions, func(i, j int) bool {\n\t\treturn partitions[i].PartitionIndex < partitions[j].PartitionIndex\n\t})\n}\n\nfunc makeLayout(metadataResponse *meta.Response) protocol.Cluster {\n\tlayout := protocol.Cluster{\n\t\tController: metadataResponse.ControllerID,\n\t\tBrokers: make(map[int32]protocol.Broker),\n\t\tTopics: make(map[string]protocol.Topic),\n\t}\n\n\tfor _, broker := range metadataResponse.Brokers {\n\t\tlayout.Brokers[broker.NodeID] = protocol.Broker{\n\t\t\tRack: broker.Rack,\n\t\t\tHost: broker.Host,\n\t\t\tPort: broker.Port,\n\t\t\tID: broker.NodeID,\n\t\t}\n\t}\n\n\tfor _, topic := range metadataResponse.Topics {\n\t\tif topic.IsInternal {\n\t\t\tcontinue // TODO: do we need to expose those?\n\t\t}\n\t\tlayout.Topics[topic.Name] = protocol.Topic{\n\t\t\tName: topic.Name,\n\t\t\tError: topic.ErrorCode,\n\t\t\tPartitions: makePartitions(topic.Partitions),\n\t\t}\n\t}\n\n\treturn layout\n}\n\nfunc makePartitions(metadataPartitions []meta.ResponsePartition) map[int32]protocol.Partition {\n\tprotocolPartitions := make(map[int32]protocol.Partition, len(metadataPartitions))\n\tnumBrokerIDs := 0\n\n\tfor _, p := range metadataPartitions {\n\t\tnumBrokerIDs += len(p.ReplicaNodes) + len(p.IsrNodes) + len(p.OfflineReplicas)\n\t}\n\n\t// Reduce the memory footprint a bit by allocating a single buffer to write\n\t// all broker ids.\n\tbrokerIDs := make([]int32, 0, numBrokerIDs)\n\n\tfor _, p := range metadataPartitions {\n\t\tvar rep, isr, off []int32\n\t\tbrokerIDs, rep = appendBrokerIDs(brokerIDs, p.ReplicaNodes)\n\t\tbrokerIDs, isr = appendBrokerIDs(brokerIDs, p.IsrNodes)\n\t\tbrokerIDs, off = appendBrokerIDs(brokerIDs, p.OfflineReplicas)\n\n\t\tprotocolPartitions[p.PartitionIndex] = protocol.Partition{\n\t\t\tID: p.PartitionIndex,\n\t\t\tError: p.ErrorCode,\n\t\t\tLeader: p.LeaderID,\n\t\t\tReplicas: rep,\n\t\t\tISR: isr,\n\t\t\tOffline: off,\n\t\t}\n\t}\n\n\treturn protocolPartitions\n}\n\nfunc appendBrokerIDs(ids, brokers []int32) ([]int32, []int32) {\n\ti := len(ids)\n\tids = append(ids, brokers...)\n\treturn ids, ids[i:len(ids):len(ids)]\n}\n\nfunc (p *connPool) newConnGroup(a net.Addr) *connGroup {\n\treturn &connGroup{\n\t\taddr: a,\n\t\tpool: p,\n\t\tbroker: Broker{\n\t\t\tID: -1,\n\t\t},\n\t}\n}\n\nfunc (p *connPool) newBrokerConnGroup(broker Broker) *connGroup {\n\treturn &connGroup{\n\t\taddr: &networkAddress{\n\t\t\tnetwork: \"tcp\",\n\t\t\taddress: net.JoinHostPort(broker.Host, strconv.Itoa(broker.Port)),\n\t\t},\n\t\tpool: p,\n\t\tbroker: broker,\n\t}\n}\n\ntype connRequest struct {\n\tctx context.Context\n\treq Request\n\tres async\n}\n\n// The promise interface is used as a message passing abstraction to coordinate\n// between goroutines that handle requests and responses.\ntype promise interface {\n\t// Waits until the promise is resolved, rejected, or the context canceled.\n\tawait(context.Context) (Response, error)\n}\n\n// async is an implementation of the promise interface which supports resolving\n// or rejecting the await call asynchronously.\ntype async chan interface{}\n\nfunc (p async) await(ctx context.Context) (Response, error) {\n\tselect {\n\tcase x := <-p:\n\t\tswitch v := x.(type) {\n\t\tcase nil:\n\t\t\treturn nil, nil // A nil response is ok (e.g. when RequiredAcks is None)\n\t\tcase Response:\n\t\t\treturn v, nil\n\t\tcase error:\n\t\t\treturn nil, v\n\t\tdefault:\n\t\t\tpanic(fmt.Errorf(\"BUG: promise resolved with impossible value of type %T\", v))\n\t\t}\n\tcase <-ctx.Done():\n\t\treturn nil, ctx.Err()\n\t}\n}\n\nfunc (p async) resolve(res Response) { p <- res }\n\nfunc (p async) reject(err error) { p <- err }\n\n// rejected is an implementation of the promise interface which is always\n// returns an error. Values of this type are constructed using the reject\n// function.\ntype rejected struct{ err error }\n\nfunc reject(err error) promise { return &rejected{err: err} }\n\nfunc (p *rejected) await(ctx context.Context) (Response, error) {\n\treturn nil, p.err\n}\n\n// joined is an implementation of the promise interface which merges results\n// from multiple promises into one await call using a merger.\ntype joined struct {\n\tpromises []promise\n\trequests []Request\n\tmerger protocol.Merger\n}\n\nfunc join(promises []promise, requests []Request, merger protocol.Merger) promise {\n\treturn &joined{\n\t\tpromises: promises,\n\t\trequests: requests,\n\t\tmerger: merger,\n\t}\n}\n\nfunc (p *joined) await(ctx context.Context) (Response, error) {\n\tresults := make([]interface{}, len(p.promises))\n\n\tfor i, sub := range p.promises {\n\t\tm, err := sub.await(ctx)\n\t\tif err != nil {\n\t\t\tresults[i] = err\n\t\t} else {\n\t\t\tresults[i] = m\n\t\t}\n\t}\n\n\treturn p.merger.Merge(p.requests, results)\n}\n\n// Default dialer used by the transport connections when no Dial function\n// was configured by the program.\nvar defaultDialer = net.Dialer{\n\tTimeout: 3 * time.Second,\n\tDualStack: true,\n}\n\n// connGroup represents a logical connection group to a kafka broker. The\n// actual network connections are lazily open before sending requests, and\n// closed if they are unused for longer than the idle timeout.\ntype connGroup struct {\n\taddr net.Addr\n\tbroker Broker\n\t// Immutable state of the connection.\n\tpool *connPool\n\t// Shared state of the connection, this is synchronized on the mutex through\n\t// calls to the synchronized method. Both goroutines of the connection share\n\t// the state maintained in these fields.\n\tmutex sync.Mutex\n\tclosed bool\n\tidleConns []*conn // stack of idle connections\n}\n\nfunc (g *connGroup) closeIdleConns() {\n\tg.mutex.Lock()\n\tconns := g.idleConns\n\tg.idleConns = nil\n\tg.closed = true\n\tg.mutex.Unlock()\n\n\tfor _, c := range conns {\n\t\tc.close()\n\t}\n}\n\nfunc (g *connGroup) grabConnOrConnect(ctx context.Context) (*conn, error) {\n\trslv := g.pool.resolver\n\taddr := g.addr\n\tvar c *conn\n\n\tif rslv == nil {\n\t\tc = g.grabConn()\n\t} else {\n\t\tvar err error\n\t\tbroker := g.broker\n\n\t\tif broker.ID < 0 {\n\t\t\thost, port, err := splitHostPortNumber(addr.String())\n\t\t\tif err != nil {\n\t\t\t\treturn nil, err\n\t\t\t}\n\t\t\tbroker.Host = host\n\t\t\tbroker.Port = port\n\t\t}\n\n\t\tipAddrs, err := rslv.LookupBrokerIPAddr(ctx, broker)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\n\t\tfor _, ipAddr := range ipAddrs {\n\t\t\tnetwork := addr.Network()\n\t\t\taddress := net.JoinHostPort(ipAddr.String(), strconv.Itoa(broker.Port))\n\n\t\t\tif c = g.grabConnTo(network, address); c != nil {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\n\tif c == nil {\n\t\tconnChan := make(chan *conn)\n\t\terrChan := make(chan error)\n\n\t\tgo func() {\n\t\t\tc, err := g.connect(ctx, addr)\n\t\t\tif err != nil {\n\t\t\t\tselect {\n\t\t\t\tcase errChan <- err:\n\t\t\t\tcase <-ctx.Done():\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tselect {\n\t\t\t\tcase connChan <- c:\n\t\t\t\tcase <-ctx.Done():\n\t\t\t\t\tif !g.releaseConn(c) {\n\t\t\t\t\t\tc.close()\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}()\n\n\t\tselect {\n\t\tcase c = <-connChan:\n\t\tcase err := <-errChan:\n\t\t\treturn nil, err\n\t\tcase <-ctx.Done():\n\t\t\treturn nil, ctx.Err()\n\t\t}\n\t}\n\n\treturn c, nil\n}\n\nfunc (g *connGroup) grabConnTo(network, address string) *conn {\n\tg.mutex.Lock()\n\tdefer g.mutex.Unlock()\n\n\tfor i := len(g.idleConns) - 1; i >= 0; i-- {\n\t\tc := g.idleConns[i]\n\n\t\tif c.network == network && c.address == address {\n\t\t\tcopy(g.idleConns[i:], g.idleConns[i+1:])\n\t\t\tn := len(g.idleConns) - 1\n\t\t\tg.idleConns[n] = nil\n\t\t\tg.idleConns = g.idleConns[:n]\n\n\t\t\tif c.timer != nil {\n\t\t\t\tc.timer.Stop()\n\t\t\t}\n\n\t\t\treturn c\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (g *connGroup) grabConn() *conn {\n\tg.mutex.Lock()\n\tdefer g.mutex.Unlock()\n\n\tif len(g.idleConns) == 0 {\n\t\treturn nil\n\t}\n\n\tn := len(g.idleConns) - 1\n\tc := g.idleConns[n]\n\tg.idleConns[n] = nil\n\tg.idleConns = g.idleConns[:n]\n\n\tif c.timer != nil {\n\t\tc.timer.Stop()\n\t}\n\n\treturn c\n}\n\nfunc (g *connGroup) removeConn(c *conn) bool {\n\tg.mutex.Lock()\n\tdefer g.mutex.Unlock()\n\n\tif c.timer != nil {\n\t\tc.timer.Stop()\n\t}\n\n\tfor i, x := range g.idleConns {\n\t\tif x == c {\n\t\t\tcopy(g.idleConns[i:], g.idleConns[i+1:])\n\t\t\tn := len(g.idleConns) - 1\n\t\t\tg.idleConns[n] = nil\n\t\t\tg.idleConns = g.idleConns[:n]\n\t\t\treturn true\n\t\t}\n\t}\n\n\treturn false\n}\n\nfunc (g *connGroup) releaseConn(c *conn) bool {\n\tidleTimeout := g.pool.idleTimeout\n\n\tg.mutex.Lock()\n\tdefer g.mutex.Unlock()\n\n\tif g.closed {\n\t\treturn false\n\t}\n\n\tif c.timer != nil {\n\t\tc.timer.Reset(idleTimeout)\n\t} else {\n\t\tc.timer = time.AfterFunc(idleTimeout, func() {\n\t\t\tif g.removeConn(c) {\n\t\t\t\tc.close()\n\t\t\t}\n\t\t})\n\t}\n\n\tg.idleConns = append(g.idleConns, c)\n\treturn true\n}\n\nfunc (g *connGroup) connect(ctx context.Context, addr net.Addr) (*conn, error) {\n\tdeadline := time.Now().Add(g.pool.dialTimeout)\n\n\tctx, cancel := context.WithDeadline(ctx, deadline)\n\tdefer cancel()\n\n\tnetwork := strings.Split(addr.Network(), \",\")\n\taddress := strings.Split(addr.String(), \",\")\n\tvar netConn net.Conn\n\tvar netAddr net.Addr\n\tvar err error\n\n\tif len(address) > 1 {\n\t\t// Shuffle the list of addresses to randomize the order in which\n\t\t// connections are attempted. This prevents routing all connections\n\t\t// to the first broker (which will usually succeed).\n\t\trand.Shuffle(len(address), func(i, j int) {\n\t\t\tnetwork[i], network[j] = network[j], network[i]\n\t\t\taddress[i], address[j] = address[j], address[i]\n\t\t})\n\t}\n\n\tfor i := range address {\n\t\tnetConn, err = g.pool.dial(ctx, network[i], address[i])\n\t\tif err == nil {\n\t\t\tnetAddr = &networkAddress{\n\t\t\t\tnetwork: network[i],\n\t\t\t\taddress: address[i],\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t}\n\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tdefer func() {\n\t\tif netConn != nil {\n\t\t\tnetConn.Close()\n\t\t}\n\t}()\n\n\tif tlsConfig := g.pool.tls; tlsConfig != nil {\n\t\tif tlsConfig.ServerName == \"\" {\n\t\t\thost, _ := splitHostPort(netAddr.String())\n\t\t\ttlsConfig = tlsConfig.Clone()\n\t\t\ttlsConfig.ServerName = host\n\t\t}\n\t\tnetConn = tls.Client(netConn, tlsConfig)\n\t}\n\n\tpc := protocol.NewConn(netConn, g.pool.clientID)\n\tpc.SetDeadline(deadline)\n\n\tr, err := pc.RoundTrip(new(apiversions.Request))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tres := r.(*apiversions.Response)\n\tver := make(map[protocol.ApiKey]int16, len(res.ApiKeys))\n\n\tif res.ErrorCode != 0 {\n\t\treturn nil, fmt.Errorf(\"negotating API versions with kafka broker at %s: %w\", g.addr, Error(res.ErrorCode))\n\t}\n\n\tfor _, r := range res.ApiKeys {\n\t\tapiKey := protocol.ApiKey(r.ApiKey)\n\t\tver[apiKey] = apiKey.SelectVersion(r.MinVersion, r.MaxVersion)\n\t}\n\n\tpc.SetVersions(ver)\n\tpc.SetDeadline(time.Time{})\n\n\tif g.pool.sasl != nil {\n\t\thost, port, err := splitHostPortNumber(netAddr.String())\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tmetadata := &sasl.Metadata{\n\t\t\tHost: host,\n\t\t\tPort: port,\n\t\t}\n\t\tif err := authenticateSASL(sasl.WithMetadata(ctx, metadata), pc, g.pool.sasl); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\treqs := make(chan connRequest)\n\tc := &conn{\n\t\tnetwork: netAddr.Network(),\n\t\taddress: netAddr.String(),\n\t\treqs: reqs,\n\t\tgroup: g,\n\t}\n\tgo c.run(pc, reqs)\n\n\tnetConn = nil\n\treturn c, nil\n}\n\ntype conn struct {\n\treqs chan<- connRequest\n\tnetwork string\n\taddress string\n\tonce sync.Once\n\tgroup *connGroup\n\ttimer *time.Timer\n}\n\nfunc (c *conn) close() {\n\tc.once.Do(func() { close(c.reqs) })\n}\n\nfunc (c *conn) run(pc *protocol.Conn, reqs <-chan connRequest) {\n\tdefer pc.Close()\n\n\tfor cr := range reqs {\n\t\tr, err := c.roundTrip(cr.ctx, pc, cr.req)\n\t\tif err != nil {\n\t\t\tcr.res.reject(err)\n\t\t\tif !errors.Is(err, protocol.ErrNoRecord) {\n\t\t\t\tbreak\n\t\t\t}\n\t\t} else {\n\t\t\tcr.res.resolve(r)\n\t\t}\n\t\tif !c.group.releaseConn(c) {\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nfunc (c *conn) roundTrip(ctx context.Context, pc *protocol.Conn, req Request) (Response, error) {\n\tpprof.SetGoroutineLabels(ctx)\n\tdefer pprof.SetGoroutineLabels(context.Background())\n\n\tif deadline, hasDeadline := ctx.Deadline(); hasDeadline {\n\t\tpc.SetDeadline(deadline)\n\t\tdefer pc.SetDeadline(time.Time{})\n\t}\n\n\treturn pc.RoundTrip(req)\n}\n\n// authenticateSASL performs all of the required requests to authenticate this\n// connection. If any step fails, this function returns with an error. A nil\n// error indicates successful authentication.\nfunc authenticateSASL(ctx context.Context, pc *protocol.Conn, mechanism sasl.Mechanism) error {\n\tif err := saslHandshakeRoundTrip(pc, mechanism.Name()); err != nil {\n\t\treturn err\n\t}\n\n\tsess, state, err := mechanism.Start(ctx)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfor completed := false; !completed; {\n\t\tchallenge, err := saslAuthenticateRoundTrip(pc, state)\n\t\tif err != nil {\n\t\t\tif errors.Is(err, io.EOF) {\n\t\t\t\t// the broker may communicate a failed exchange by closing the\n\t\t\t\t// connection (esp. in the case where we're passing opaque sasl\n\t\t\t\t// data over the wire since there's no protocol info).\n\t\t\t\treturn SASLAuthenticationFailed\n\t\t\t}\n\n\t\t\treturn err\n\t\t}\n\n\t\tcompleted, state, err = sess.Next(ctx, challenge)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// saslHandshake sends the SASL handshake message. This will determine whether\n// the Mechanism is supported by the cluster. If it's not, this function will\n// error out with UnsupportedSASLMechanism.\n//\n// If the mechanism is unsupported, the handshake request will reply with the\n// list of the cluster's configured mechanisms, which could potentially be used\n// to facilitate negotiation. At the moment, we are not negotiating the\n// mechanism as we believe that brokers are usually known to the client, and\n// therefore the client should already know which mechanisms are supported.\n//\n// See http://kafka.apache.org/protocol.html#The_Messages_SaslHandshake\nfunc saslHandshakeRoundTrip(pc *protocol.Conn, mechanism string) error {\n\tmsg, err := pc.RoundTrip(&saslhandshake.Request{\n\t\tMechanism: mechanism,\n\t})\n\tif err != nil {\n\t\treturn err\n\t}\n\tres := msg.(*saslhandshake.Response)\n\tif res.ErrorCode != 0 {\n\t\terr = Error(res.ErrorCode)\n\t}\n\treturn err\n}\n\n// saslAuthenticate sends the SASL authenticate message. This function must\n// be immediately preceded by a successful saslHandshake.\n//\n// See http://kafka.apache.org/protocol.html#The_Messages_SaslAuthenticate\nfunc saslAuthenticateRoundTrip(pc *protocol.Conn, data []byte) ([]byte, error) {\n\tmsg, err := pc.RoundTrip(&saslauthenticate.Request{\n\t\tAuthBytes: data,\n\t})\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tres := msg.(*saslauthenticate.Response)\n\tif res.ErrorCode != 0 {\n\t\terr = makeError(res.ErrorCode, res.ErrorMessage)\n\t}\n\treturn res.AuthBytes, err\n}\n\nvar _ RoundTripper = (*Transport)(nil)\n" }, { "path": "transport_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"errors\"\n\t\"net\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n\t\"github.com/segmentio/kafka-go/protocol/createtopics\"\n\tmeta \"github.com/segmentio/kafka-go/protocol/metadata\"\n)\n\nfunc TestIssue477(t *testing.T) {\n\t// This test verifies that a connection attempt with a minimal TLS\n\t// configuration does not panic.\n\tl, err := net.Listen(\"tcp\", \"127.0.0.1:0\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer l.Close()\n\n\tcg := connGroup{\n\t\taddr: l.Addr(),\n\t\tpool: &connPool{\n\t\t\tdial: defaultDialer.DialContext,\n\t\t\ttls: &tls.Config{},\n\t\t},\n\t}\n\n\tif _, err := cg.connect(context.Background(), cg.addr); err != nil {\n\t\t// An error is expected here because we are not actually establishing\n\t\t// a TLS connection to a kafka broker.\n\t\tt.Log(err)\n\t} else {\n\t\tt.Error(\"no error was reported when attempting to establish a TLS connection to a non-TLS endpoint\")\n\t}\n}\n\nfunc TestIssue672(t *testing.T) {\n\t// ensure the test times out if the bug is re-introduced\n\tctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)\n\tdefer cancel()\n\n\t// we'll simulate a situation with one good topic and one bad topic (bad configuration)\n\tconst brokenTopicName = \"bad-topic\"\n\tconst okTopicName = \"good-topic\"\n\n\t// make the connection pool think it's immediately ready to send\n\tready := make(chan struct{})\n\tclose(ready)\n\n\t// allow the system to wake as much as it wants\n\twake := make(chan event)\n\tdefer close(wake)\n\tgo func() {\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn\n\t\t\tcase e := <-wake:\n\t\t\t\tif e == nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\te.trigger()\n\t\t\t}\n\t\t}\n\t}()\n\n\t// handle requests by immediately resolving them with a create topics response,\n\t// the \"bad topic\" will have an error value\n\trequests := make(chan connRequest, 1)\n\tdefer close(requests)\n\tgo func() {\n\t\trequest := <-requests\n\t\trequest.res.resolve(&createtopics.Response{\n\t\t\tThrottleTimeMs: 0,\n\t\t\tTopics: []createtopics.ResponseTopic{\n\t\t\t\t{\n\t\t\t\t\tName: brokenTopicName,\n\t\t\t\t\tErrorCode: int16(InvalidPartitionNumber),\n\t\t\t\t\tErrorMessage: InvalidPartitionNumber.Description(),\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tName: okTopicName,\n\t\t\t\t\tNumPartitions: 1,\n\t\t\t\t\tReplicationFactor: 1,\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}()\n\n\tpool := &connPool{\n\t\tready: ready,\n\t\twake: wake,\n\t\tconns: map[int32]*connGroup{},\n\t}\n\n\t// configure the state so it can find the good topic, but not the one that fails to create\n\tpool.setState(connPoolState{\n\t\tlayout: protocol.Cluster{\n\t\t\tTopics: map[string]protocol.Topic{\n\t\t\t\tokTopicName: {\n\t\t\t\t\tName: okTopicName,\n\t\t\t\t\tPartitions: map[int32]protocol.Partition{\n\t\t\t\t\t\t0: {},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\t// trick the connection pool into thinking it has a valid connection to a broker\n\tpool.conns[0] = &connGroup{\n\t\tpool: pool,\n\t\tbroker: Broker{},\n\t\tidleConns: []*conn{\n\t\t\t{\n\t\t\t\treqs: requests,\n\t\t\t},\n\t\t},\n\t}\n\n\t// perform the round trip:\n\t// - if the issue is presenting this will hang waiting for metadata to arrive that will\n\t// never arrive, causing a deadline timeout.\n\t// - if the issue is fixed this will resolve almost instantaneously\n\tr, err := pool.roundTrip(ctx, &createtopics.Request{\n\t\tTopics: []createtopics.RequestTopic{\n\t\t\t{\n\t\t\t\tName: brokenTopicName,\n\t\t\t\tNumPartitions: 0,\n\t\t\t\tReplicationFactor: 1,\n\t\t\t},\n\t\t\t{\n\t\t\t\tName: okTopicName,\n\t\t\t\tNumPartitions: 1,\n\t\t\t\tReplicationFactor: 1,\n\t\t\t},\n\t\t},\n\t})\n\t// detect if the issue is presenting using the context timeout (note that checking the err return value\n\t// isn't good enough as the original implementation didn't return the context cancellation error due to\n\t// being run in a defer)\n\tif errors.Is(ctx.Err(), context.DeadlineExceeded) {\n\t\tt.Fatalf(\"issue 672 is presenting! roundTrip should not have timed out\")\n\t}\n\n\t// ancillary assertions as general house-keeping, not directly related to the issue:\n\n\t// we're not expecting any errors in this test\n\tif err != nil {\n\t\tt.Fatalf(\"unexpected error provoking connection pool roundTrip: %v\", err)\n\t}\n\n\t// we expect a response containing the errors from the broker\n\tif r == nil {\n\t\tt.Fatal(\"expected a non-nil response\")\n\t}\n\n\t// we expect to have the create topic response with created earlier\n\t_, ok := r.(*createtopics.Response)\n\tif !ok {\n\t\tt.Fatalf(\"expected a createtopics.Response but got %T\", r)\n\t}\n}\n\nfunc TestIssue806(t *testing.T) {\n\t// ensure the test times out if the bug is re-introduced\n\tctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)\n\tdefer cancel()\n\n\t// simulate unknown topic want auto create with unknownTopicName,\n\tconst unknownTopicName = \"unknown-topic\"\n\tconst okTopicName = \"good-topic\"\n\n\t// make the connection pool think it's immediately ready to send\n\tready := make(chan struct{})\n\tclose(ready)\n\n\t// allow the system to wake as much as it wants\n\twake := make(chan event)\n\tdefer close(wake)\n\tgo func() {\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn\n\t\t\tcase e := <-wake:\n\t\t\t\tif e == nil {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t\te.trigger()\n\t\t\t}\n\t\t}\n\t}()\n\n\t// handle requests by immediately resolving them with a create topics response,\n\t// the \"unknown topic\" will have err UNKNOWN_TOPIC_OR_PARTITION\n\trequests := make(chan connRequest, 1)\n\tdefer close(requests)\n\tgo func() {\n\t\trequest := <-requests\n\t\trequest.res.resolve(&meta.Response{\n\t\t\tTopics: []meta.ResponseTopic{\n\t\t\t\t{\n\t\t\t\t\tName: unknownTopicName,\n\t\t\t\t\tErrorCode: int16(UnknownTopicOrPartition),\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\tName: okTopicName,\n\t\t\t\t\tPartitions: []meta.ResponsePartition{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartitionIndex: 0,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t})\n\t}()\n\n\tpool := &connPool{\n\t\tready: ready,\n\t\twake: wake,\n\t\tconns: map[int32]*connGroup{},\n\t}\n\n\t// configure the state,\n\t//\n\t// set cached metadata only have good topic,\n\t// so it need to request metadata,\n\t// caused by unknown topic cannot find in cached metadata\n\t//\n\t// set layout only have good topic,\n\t// so it can find the good topic, but not the one that fails to create\n\tpool.setState(connPoolState{\n\t\tmetadata: &meta.Response{\n\t\t\tTopics: []meta.ResponseTopic{\n\t\t\t\t{\n\t\t\t\t\tName: okTopicName,\n\t\t\t\t\tPartitions: []meta.ResponsePartition{\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartitionIndex: 0,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\tlayout: protocol.Cluster{\n\t\t\tTopics: map[string]protocol.Topic{\n\t\t\t\tokTopicName: {\n\t\t\t\t\tName: okTopicName,\n\t\t\t\t\tPartitions: map[int32]protocol.Partition{\n\t\t\t\t\t\t0: {},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\n\t// trick the connection pool into thinking it has a valid connection to request metadata\n\tpool.ctrl = &connGroup{\n\t\tpool: pool,\n\t\tbroker: Broker{},\n\t\tidleConns: []*conn{\n\t\t\t{\n\t\t\t\treqs: requests,\n\t\t\t},\n\t\t},\n\t}\n\n\t// perform the round trip:\n\t// - if the issue is presenting this will hang waiting for metadata to arrive that will\n\t// never arrive, causing a deadline timeout.\n\t// - if the issue is fixed this will resolve almost instantaneously\n\tr, err := pool.roundTrip(ctx, &meta.Request{\n\t\tTopicNames: []string{unknownTopicName},\n\t\tAllowAutoTopicCreation: true,\n\t})\n\t// detect if the issue is presenting using the context timeout (note that checking the err return value\n\t// isn't good enough as the original implementation didn't return the context cancellation error due to\n\t// being run in a defer)\n\tif errors.Is(ctx.Err(), context.DeadlineExceeded) {\n\t\tt.Fatalf(\"issue 806 is presenting! roundTrip should not have timed out\")\n\t}\n\n\t// ancillary assertions as general house-keeping, not directly related to the issue:\n\n\t// we're not expecting any errors in this test\n\tif err != nil {\n\t\tt.Fatalf(\"unexpected error provoking connection pool roundTrip: %v\", err)\n\t}\n\n\t// we expect a response containing the errors from the broker\n\tif r == nil {\n\t\tt.Fatal(\"expected a non-nil response\")\n\t}\n\n\t// we expect to have the create topic response with created earlier\n\t_, ok := r.(*meta.Response)\n\tif !ok {\n\t\tt.Fatalf(\"expected a meta.Response but got %T\", r)\n\t}\n}\n" }, { "path": "txnoffsetcommit.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/txnoffsetcommit\"\n)\n\n// TxnOffsetCommitRequest represents a request sent to a kafka broker to commit\n// offsets for a partition within a transaction.\ntype TxnOffsetCommitRequest struct {\n\t// Address of the kafka broker to send the request to.\n\tAddr net.Addr\n\n\t// The transactional id key.\n\tTransactionalID string\n\n\t// ID of the consumer group to publish the offsets for.\n\tGroupID string\n\n\t// The Producer ID (PID) for the current producer session;\n\t// received from an InitProducerID request.\n\tProducerID int\n\n\t// The epoch associated with the current producer session for the given PID\n\tProducerEpoch int\n\n\t// GenerationID is the current generation for the group.\n\tGenerationID int\n\n\t// ID of the group member submitting the offsets.\n\tMemberID string\n\n\t// GroupInstanceID is a unique identifier for the consumer.\n\tGroupInstanceID string\n\n\t// Set of topic partitions to publish the offsets for.\n\t//\n\t// Not that offset commits need to be submitted to the broker acting as the\n\t// group coordinator. This will be automatically resolved by the transport.\n\tTopics map[string][]TxnOffsetCommit\n}\n\n// TxnOffsetCommit represent the commit of an offset to a partition within a transaction.\n//\n// The extra metadata is opaque to the kafka protocol, it is intended to hold\n// information like an identifier for the process that committed the offset,\n// or the time at which the commit was made.\ntype TxnOffsetCommit struct {\n\tPartition int\n\tOffset int64\n\tMetadata string\n}\n\n// TxnOffsetFetchResponse represents a response from a kafka broker to an offset\n// commit request within a transaction.\ntype TxnOffsetCommitResponse struct {\n\t// The amount of time that the broker throttled the request.\n\tThrottle time.Duration\n\n\t// Set of topic partitions that the kafka broker has accepted offset commits\n\t// for.\n\tTopics map[string][]TxnOffsetCommitPartition\n}\n\n// TxnOffsetFetchPartition represents the state of a single partition in responses\n// to committing offsets within a transaction.\ntype TxnOffsetCommitPartition struct {\n\t// ID of the partition.\n\tPartition int\n\n\t// An error that may have occurred while attempting to publish consumer\n\t// group offsets for this partition.\n\t//\n\t// The error contains both the kafka error code, and an error message\n\t// returned by the kafka broker. Programs may use the standard errors.Is\n\t// function to test the error against kafka error codes.\n\tError error\n}\n\n// TxnOffsetCommit sends an txn offset commit request to a kafka broker and returns the\n// response.\nfunc (c *Client) TxnOffsetCommit(\n\tctx context.Context,\n\treq *TxnOffsetCommitRequest,\n) (*TxnOffsetCommitResponse, error) {\n\tprotoReq := &txnoffsetcommit.Request{\n\t\tTransactionalID: req.TransactionalID,\n\t\tGroupID: req.GroupID,\n\t\tProducerID: int64(req.ProducerID),\n\t\tProducerEpoch: int16(req.ProducerEpoch),\n\t\tGenerationID: int32(req.GenerationID),\n\t\tMemberID: req.MemberID,\n\t\tGroupInstanceID: req.GroupInstanceID,\n\t\tTopics: make([]txnoffsetcommit.RequestTopic, 0, len(req.Topics)),\n\t}\n\n\tfor topic, partitions := range req.Topics {\n\t\tparts := make([]txnoffsetcommit.RequestPartition, len(partitions))\n\t\tfor i, partition := range partitions {\n\t\t\tparts[i] = txnoffsetcommit.RequestPartition{\n\t\t\t\tPartition: int32(partition.Partition),\n\t\t\t\tCommittedOffset: int64(partition.Offset),\n\t\t\t\tCommittedMetadata: partition.Metadata,\n\t\t\t}\n\t\t}\n\t\tt := txnoffsetcommit.RequestTopic{\n\t\t\tName: topic,\n\t\t\tPartitions: parts,\n\t\t}\n\n\t\tprotoReq.Topics = append(protoReq.Topics, t)\n\t}\n\n\tm, err := c.roundTrip(ctx, req.Addr, protoReq)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"kafka.(*Client).TxnOffsetCommit: %w\", err)\n\t}\n\n\tr := m.(*txnoffsetcommit.Response)\n\n\tres := &TxnOffsetCommitResponse{\n\t\tThrottle: makeDuration(r.ThrottleTimeMs),\n\t\tTopics: make(map[string][]TxnOffsetCommitPartition, len(r.Topics)),\n\t}\n\n\tfor _, topic := range r.Topics {\n\t\tpartitions := make([]TxnOffsetCommitPartition, 0, len(topic.Partitions))\n\t\tfor _, partition := range topic.Partitions {\n\t\t\tpartitions = append(partitions, TxnOffsetCommitPartition{\n\t\t\t\tPartition: int(partition.Partition),\n\t\t\t\tError: makeError(partition.ErrorCode, \"\"),\n\t\t\t})\n\t\t}\n\t\tres.Topics[topic.Name] = partitions\n\t}\n\n\treturn res, nil\n}\n" }, { "path": "txnoffsetcommit_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"log\"\n\t\"os\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc TestClientTxnOffsetCommit(t *testing.T) {\n\tif !ktesting.KafkaIsAtLeast(\"0.11.0\") {\n\t\tt.Skip(\"Skipping test because kafka version is not high enough.\")\n\t}\n\n\t// TODO: look into why this test fails on Kafka 3.0.0 and higher when transactional support\n\t// work is revisited.\n\tif ktesting.KafkaIsAtLeast(\"3.0.0\") {\n\t\tt.Skip(\"Skipping test because it fails on Kafka version 3.0.0 or higher.\")\n\t}\n\n\ttransactionalID := makeTransactionalID()\n\ttopic := makeTopic()\n\n\tclient, shutdown := newLocalClientWithTopic(topic, 1)\n\tdefer shutdown()\n\twaitForTopic(context.TODO(), t, topic)\n\tdefer deleteTopic(t, topic)\n\n\tnow := time.Now()\n\n\tconst N = 10\n\trecords := make([]Record, 0, N)\n\tfor i := 0; i < N; i++ {\n\t\trecords = append(records, Record{\n\t\t\tTime: now,\n\t\t\tValue: NewBytes([]byte(\"test-message-\" + strconv.Itoa(i))),\n\t\t})\n\t}\n\tctx, cancel := context.WithTimeout(context.Background(), time.Second*30)\n\tdefer cancel()\n\tres, err := client.Produce(ctx, &ProduceRequest{\n\t\tTopic: topic,\n\t\tRequiredAcks: RequireAll,\n\t\tRecords: NewRecordReader(records...),\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif res.Error != nil {\n\t\tt.Error(res.Error)\n\t}\n\n\tfor index, err := range res.RecordErrors {\n\t\tt.Fatalf(\"record at index %d produced an error: %v\", index, err)\n\t}\n\n\tctx, cancel = context.WithTimeout(context.Background(), time.Second*30)\n\tdefer cancel()\n\trespc, err := waitForCoordinatorIndefinitely(ctx, client, &FindCoordinatorRequest{\n\t\tAddr: client.Addr,\n\t\tKey: transactionalID,\n\t\tKeyType: CoordinatorKeyTypeTransaction,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif respc.Error != nil {\n\t\tt.Fatal(respc.Error)\n\t}\n\n\tctx, cancel = context.WithTimeout(context.Background(), time.Second*30)\n\tdefer cancel()\n\trespc, err = waitForCoordinatorIndefinitely(ctx, client, &FindCoordinatorRequest{\n\t\tAddr: client.Addr,\n\t\tKey: transactionalID,\n\t\tKeyType: CoordinatorKeyTypeConsumer,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif respc.Error != nil {\n\t\tt.Fatal(respc.Error)\n\t}\n\n\tctx, cancel = context.WithTimeout(context.Background(), time.Second*30)\n\tdefer cancel()\n\tipResp, err := client.InitProducerID(ctx, &InitProducerIDRequest{\n\t\tTransactionalID: transactionalID,\n\t\tTransactionTimeoutMs: 10000,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif ipResp.Error != nil {\n\t\tt.Fatal(ipResp.Error)\n\t}\n\n\tgroupID := makeGroupID()\n\n\tgroup, err := NewConsumerGroup(ConsumerGroupConfig{\n\t\tID: groupID,\n\t\tTopics: []string{topic},\n\t\tBrokers: []string{\"localhost:9092\"},\n\t\tHeartbeatInterval: 2 * time.Second,\n\t\tRebalanceTimeout: 2 * time.Second,\n\t\tRetentionTime: time.Hour,\n\t\tLogger: log.New(os.Stdout, \"cg-test: \", 0),\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer group.Close()\n\n\tctx, cancel = context.WithTimeout(context.Background(), time.Second*30)\n\tdefer cancel()\n\tgen, err := group.Next(ctx)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tapresp, err := client.AddPartitionsToTxn(ctx, &AddPartitionsToTxnRequest{\n\t\tTransactionalID: transactionalID,\n\t\tProducerID: ipResp.Producer.ProducerID,\n\t\tProducerEpoch: ipResp.Producer.ProducerEpoch,\n\t\tTopics: map[string][]AddPartitionToTxn{\n\t\t\ttopic: {\n\t\t\t\t{\n\t\t\t\t\tPartition: 0,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tappartition := apresp.Topics[topic]\n\tif len(appartition) != 1 {\n\t\tt.Fatalf(\"unexpected partition count; expected: 1, got: %d\", len(appartition))\n\t}\n\n\tfor _, partition := range appartition {\n\t\tif partition.Error != nil {\n\t\t\tt.Fatal(partition.Error)\n\t\t}\n\t}\n\n\tclient.AddOffsetsToTxn(ctx, &AddOffsetsToTxnRequest{\n\t\tTransactionalID: transactionalID,\n\t\tProducerID: ipResp.Producer.ProducerID,\n\t\tProducerEpoch: ipResp.Producer.ProducerEpoch,\n\t\tGroupID: groupID,\n\t})\n\n\tctx, cancel = context.WithTimeout(context.Background(), time.Second*30)\n\tdefer cancel()\n\tresp, err := client.TxnOffsetCommit(ctx, &TxnOffsetCommitRequest{\n\t\tTransactionalID: transactionalID,\n\t\tGroupID: groupID,\n\t\tMemberID: gen.MemberID,\n\t\tProducerID: ipResp.Producer.ProducerID,\n\t\tProducerEpoch: ipResp.Producer.ProducerEpoch,\n\t\tGenerationID: int(gen.ID),\n\t\tGroupInstanceID: groupID,\n\t\tTopics: map[string][]TxnOffsetCommit{\n\t\t\ttopic: {\n\t\t\t\t{\n\t\t\t\t\tPartition: 0,\n\t\t\t\t\tOffset: 10,\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tpartitions := resp.Topics[topic]\n\n\tif len(partitions) != 1 {\n\t\tt.Fatalf(\"unexpected partition count; expected: 1, got: %d\", len(partitions))\n\t}\n\n\tfor _, partition := range partitions {\n\t\tif partition.Error != nil {\n\t\t\tt.Fatal(partition.Error)\n\t\t}\n\t}\n\n\terr = clientEndTxn(client, &EndTxnRequest{\n\t\tTransactionalID: transactionalID,\n\t\tProducerID: ipResp.Producer.ProducerID,\n\t\tProducerEpoch: ipResp.Producer.ProducerEpoch,\n\t\tCommitted: true,\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\t// seems like external visibility of the commit isn't\n\t// synchronous with the EndTxn request. This seems\n\t// to give enough time for the commit to become consistently visible.\n\t<-time.After(time.Second)\n\n\tofr, err := client.OffsetFetch(ctx, &OffsetFetchRequest{\n\t\tGroupID: groupID,\n\t\tTopics: map[string][]int{topic: {0}},\n\t})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif ofr.Error != nil {\n\t\tt.Error(ofr.Error)\n\t}\n\n\tfetresps := ofr.Topics[topic]\n\tif len(fetresps) != 1 {\n\t\tt.Fatalf(\"unexpected 1 offsetfetchpartition responses; got %d\", len(fetresps))\n\t}\n\n\tfor _, r := range fetresps {\n\t\tif r.Error != nil {\n\t\t\tt.Fatal(r.Error)\n\t\t}\n\n\t\tif r.CommittedOffset != 10 {\n\t\t\tt.Fatalf(\"expected committed offset to be 10; got: %v for partition: %v\", r.CommittedOffset, r.Partition)\n\t\t}\n\t}\n}\n" }, { "path": "write.go", "content": "package kafka\n\nimport (\n\t\"bytes\"\n\t\"encoding/binary\"\n\t\"fmt\"\n\t\"hash/crc32\"\n\t\"io\"\n\t\"time\"\n)\n\ntype writeBuffer struct {\n\tw io.Writer\n\tb [16]byte\n}\n\nfunc (wb *writeBuffer) writeInt8(i int8) {\n\twb.b[0] = byte(i)\n\twb.Write(wb.b[:1])\n}\n\nfunc (wb *writeBuffer) writeInt16(i int16) {\n\tbinary.BigEndian.PutUint16(wb.b[:2], uint16(i))\n\twb.Write(wb.b[:2])\n}\n\nfunc (wb *writeBuffer) writeInt32(i int32) {\n\tbinary.BigEndian.PutUint32(wb.b[:4], uint32(i))\n\twb.Write(wb.b[:4])\n}\n\nfunc (wb *writeBuffer) writeInt64(i int64) {\n\tbinary.BigEndian.PutUint64(wb.b[:8], uint64(i))\n\twb.Write(wb.b[:8])\n}\n\nfunc (wb *writeBuffer) writeVarInt(i int64) {\n\tu := uint64((i << 1) ^ (i >> 63))\n\tn := 0\n\n\tfor u >= 0x80 && n < len(wb.b) {\n\t\twb.b[n] = byte(u) | 0x80\n\t\tu >>= 7\n\t\tn++\n\t}\n\n\tif n < len(wb.b) {\n\t\twb.b[n] = byte(u)\n\t\tn++\n\t}\n\n\twb.Write(wb.b[:n])\n}\n\nfunc (wb *writeBuffer) writeString(s string) {\n\twb.writeInt16(int16(len(s)))\n\twb.WriteString(s)\n}\n\nfunc (wb *writeBuffer) writeVarString(s string) {\n\twb.writeVarInt(int64(len(s)))\n\twb.WriteString(s)\n}\n\nfunc (wb *writeBuffer) writeNullableString(s *string) {\n\tif s == nil {\n\t\twb.writeInt16(-1)\n\t} else {\n\t\twb.writeString(*s)\n\t}\n}\n\nfunc (wb *writeBuffer) writeBytes(b []byte) {\n\tn := len(b)\n\tif b == nil {\n\t\tn = -1\n\t}\n\twb.writeInt32(int32(n))\n\twb.Write(b)\n}\n\nfunc (wb *writeBuffer) writeVarBytes(b []byte) {\n\tif b != nil {\n\t\twb.writeVarInt(int64(len(b)))\n\t\twb.Write(b)\n\t} else {\n\t\t//-1 is used to indicate nil key\n\t\twb.writeVarInt(-1)\n\t}\n}\n\nfunc (wb *writeBuffer) writeBool(b bool) {\n\tv := int8(0)\n\tif b {\n\t\tv = 1\n\t}\n\twb.writeInt8(v)\n}\n\nfunc (wb *writeBuffer) writeArrayLen(n int) {\n\twb.writeInt32(int32(n))\n}\n\nfunc (wb *writeBuffer) writeArray(n int, f func(int)) {\n\twb.writeArrayLen(n)\n\tfor i := 0; i < n; i++ {\n\t\tf(i)\n\t}\n}\n\nfunc (wb *writeBuffer) writeVarArray(n int, f func(int)) {\n\twb.writeVarInt(int64(n))\n\tfor i := 0; i < n; i++ {\n\t\tf(i)\n\t}\n}\n\nfunc (wb *writeBuffer) writeStringArray(a []string) {\n\twb.writeArray(len(a), func(i int) { wb.writeString(a[i]) })\n}\n\nfunc (wb *writeBuffer) writeInt32Array(a []int32) {\n\twb.writeArray(len(a), func(i int) { wb.writeInt32(a[i]) })\n}\n\nfunc (wb *writeBuffer) write(a interface{}) {\n\tswitch v := a.(type) {\n\tcase int8:\n\t\twb.writeInt8(v)\n\tcase int16:\n\t\twb.writeInt16(v)\n\tcase int32:\n\t\twb.writeInt32(v)\n\tcase int64:\n\t\twb.writeInt64(v)\n\tcase string:\n\t\twb.writeString(v)\n\tcase []byte:\n\t\twb.writeBytes(v)\n\tcase bool:\n\t\twb.writeBool(v)\n\tcase writable:\n\t\tv.writeTo(wb)\n\tdefault:\n\t\tpanic(fmt.Sprintf(\"unsupported type: %T\", a))\n\t}\n}\n\nfunc (wb *writeBuffer) Write(b []byte) (int, error) {\n\treturn wb.w.Write(b)\n}\n\nfunc (wb *writeBuffer) WriteString(s string) (int, error) {\n\treturn io.WriteString(wb.w, s)\n}\n\nfunc (wb *writeBuffer) Flush() error {\n\tif x, ok := wb.w.(interface{ Flush() error }); ok {\n\t\treturn x.Flush()\n\t}\n\treturn nil\n}\n\ntype writable interface {\n\twriteTo(*writeBuffer)\n}\n\nfunc (wb *writeBuffer) writeFetchRequestV2(correlationID int32, clientID, topic string, partition int32, offset int64, minBytes, maxBytes int, maxWait time.Duration) error {\n\th := requestHeader{\n\t\tApiKey: int16(fetch),\n\t\tApiVersion: int16(v2),\n\t\tCorrelationID: correlationID,\n\t\tClientID: clientID,\n\t}\n\th.Size = (h.size() - 4) +\n\t\t4 + // replica ID\n\t\t4 + // max wait time\n\t\t4 + // min bytes\n\t\t4 + // topic array length\n\t\tsizeofString(topic) +\n\t\t4 + // partition array length\n\t\t4 + // partition\n\t\t8 + // offset\n\t\t4 // max bytes\n\n\th.writeTo(wb)\n\twb.writeInt32(-1) // replica ID\n\twb.writeInt32(milliseconds(maxWait))\n\twb.writeInt32(int32(minBytes))\n\n\t// topic array\n\twb.writeArrayLen(1)\n\twb.writeString(topic)\n\n\t// partition array\n\twb.writeArrayLen(1)\n\twb.writeInt32(partition)\n\twb.writeInt64(offset)\n\twb.writeInt32(int32(maxBytes))\n\n\treturn wb.Flush()\n}\n\nfunc (wb *writeBuffer) writeFetchRequestV5(correlationID int32, clientID, topic string, partition int32, offset int64, minBytes, maxBytes int, maxWait time.Duration, isolationLevel int8) error {\n\th := requestHeader{\n\t\tApiKey: int16(fetch),\n\t\tApiVersion: int16(v5),\n\t\tCorrelationID: correlationID,\n\t\tClientID: clientID,\n\t}\n\th.Size = (h.size() - 4) +\n\t\t4 + // replica ID\n\t\t4 + // max wait time\n\t\t4 + // min bytes\n\t\t4 + // max bytes\n\t\t1 + // isolation level\n\t\t4 + // topic array length\n\t\tsizeofString(topic) +\n\t\t4 + // partition array length\n\t\t4 + // partition\n\t\t8 + // offset\n\t\t8 + // log start offset\n\t\t4 // max bytes\n\n\th.writeTo(wb)\n\twb.writeInt32(-1) // replica ID\n\twb.writeInt32(milliseconds(maxWait))\n\twb.writeInt32(int32(minBytes))\n\twb.writeInt32(int32(maxBytes))\n\twb.writeInt8(isolationLevel) // isolation level 0 - read uncommitted\n\n\t// topic array\n\twb.writeArrayLen(1)\n\twb.writeString(topic)\n\n\t// partition array\n\twb.writeArrayLen(1)\n\twb.writeInt32(partition)\n\twb.writeInt64(offset)\n\twb.writeInt64(int64(0)) // log start offset only used when is sent by follower\n\twb.writeInt32(int32(maxBytes))\n\n\treturn wb.Flush()\n}\n\nfunc (wb *writeBuffer) writeFetchRequestV10(correlationID int32, clientID, topic string, partition int32, offset int64, minBytes, maxBytes int, maxWait time.Duration, isolationLevel int8) error {\n\th := requestHeader{\n\t\tApiKey: int16(fetch),\n\t\tApiVersion: int16(v10),\n\t\tCorrelationID: correlationID,\n\t\tClientID: clientID,\n\t}\n\th.Size = (h.size() - 4) +\n\t\t4 + // replica ID\n\t\t4 + // max wait time\n\t\t4 + // min bytes\n\t\t4 + // max bytes\n\t\t1 + // isolation level\n\t\t4 + // session ID\n\t\t4 + // session epoch\n\t\t4 + // topic array length\n\t\tsizeofString(topic) +\n\t\t4 + // partition array length\n\t\t4 + // partition\n\t\t4 + // current leader epoch\n\t\t8 + // fetch offset\n\t\t8 + // log start offset\n\t\t4 + // partition max bytes\n\t\t4 // forgotten topics data\n\n\th.writeTo(wb)\n\twb.writeInt32(-1) // replica ID\n\twb.writeInt32(milliseconds(maxWait))\n\twb.writeInt32(int32(minBytes))\n\twb.writeInt32(int32(maxBytes))\n\twb.writeInt8(isolationLevel) // isolation level 0 - read uncommitted\n\twb.writeInt32(0) //FIXME\n\twb.writeInt32(-1) //FIXME\n\n\t// topic array\n\twb.writeArrayLen(1)\n\twb.writeString(topic)\n\n\t// partition array\n\twb.writeArrayLen(1)\n\twb.writeInt32(partition)\n\twb.writeInt32(-1) //FIXME\n\twb.writeInt64(offset)\n\twb.writeInt64(int64(0)) // log start offset only used when is sent by follower\n\twb.writeInt32(int32(maxBytes))\n\n\t// forgotten topics array\n\twb.writeArrayLen(0) // forgotten topics not supported yet\n\n\treturn wb.Flush()\n}\n\nfunc (wb *writeBuffer) writeListOffsetRequestV1(correlationID int32, clientID, topic string, partition int32, time int64) error {\n\th := requestHeader{\n\t\tApiKey: int16(listOffsets),\n\t\tApiVersion: int16(v1),\n\t\tCorrelationID: correlationID,\n\t\tClientID: clientID,\n\t}\n\th.Size = (h.size() - 4) +\n\t\t4 + // replica ID\n\t\t4 + // topic array length\n\t\tsizeofString(topic) + // topic\n\t\t4 + // partition array length\n\t\t4 + // partition\n\t\t8 // time\n\n\th.writeTo(wb)\n\twb.writeInt32(-1) // replica ID\n\n\t// topic array\n\twb.writeArrayLen(1)\n\twb.writeString(topic)\n\n\t// partition array\n\twb.writeArrayLen(1)\n\twb.writeInt32(partition)\n\twb.writeInt64(time)\n\n\treturn wb.Flush()\n}\n\nfunc (wb *writeBuffer) writeProduceRequestV2(codec CompressionCodec, correlationID int32, clientID, topic string, partition int32, timeout time.Duration, requiredAcks int16, msgs ...Message) (err error) {\n\tvar size int32\n\tvar attributes int8\n\tvar compressed *bytes.Buffer\n\n\tif codec == nil {\n\t\tsize = messageSetSize(msgs...)\n\t} else {\n\t\tcompressed, attributes, size, err = compressMessageSet(codec, msgs...)\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\tmsgs = []Message{{Value: compressed.Bytes()}}\n\t}\n\n\th := requestHeader{\n\t\tApiKey: int16(produce),\n\t\tApiVersion: int16(v2),\n\t\tCorrelationID: correlationID,\n\t\tClientID: clientID,\n\t}\n\th.Size = (h.size() - 4) +\n\t\t2 + // required acks\n\t\t4 + // timeout\n\t\t4 + // topic array length\n\t\tsizeofString(topic) + // topic\n\t\t4 + // partition array length\n\t\t4 + // partition\n\t\t4 + // message set size\n\t\tsize\n\n\th.writeTo(wb)\n\twb.writeInt16(requiredAcks) // required acks\n\twb.writeInt32(milliseconds(timeout))\n\n\t// topic array\n\twb.writeArrayLen(1)\n\twb.writeString(topic)\n\n\t// partition array\n\twb.writeArrayLen(1)\n\twb.writeInt32(partition)\n\n\twb.writeInt32(size)\n\tcw := &crc32Writer{table: crc32.IEEETable}\n\n\tfor _, msg := range msgs {\n\t\twb.writeMessage(msg.Offset, attributes, msg.Time, msg.Key, msg.Value, cw)\n\t}\n\n\treleaseBuffer(compressed)\n\treturn wb.Flush()\n}\n\nfunc (wb *writeBuffer) writeProduceRequestV3(correlationID int32, clientID, topic string, partition int32, timeout time.Duration, requiredAcks int16, transactionalID *string, recordBatch *recordBatch) (err error) {\n\n\th := requestHeader{\n\t\tApiKey: int16(produce),\n\t\tApiVersion: int16(v3),\n\t\tCorrelationID: correlationID,\n\t\tClientID: clientID,\n\t}\n\n\th.Size = (h.size() - 4) +\n\t\tsizeofNullableString(transactionalID) +\n\t\t2 + // required acks\n\t\t4 + // timeout\n\t\t4 + // topic array length\n\t\tsizeofString(topic) + // topic\n\t\t4 + // partition array length\n\t\t4 + // partition\n\t\t4 + // message set size\n\t\trecordBatch.size\n\n\th.writeTo(wb)\n\twb.writeNullableString(transactionalID)\n\twb.writeInt16(requiredAcks) // required acks\n\twb.writeInt32(milliseconds(timeout))\n\n\t// topic array\n\twb.writeArrayLen(1)\n\twb.writeString(topic)\n\n\t// partition array\n\twb.writeArrayLen(1)\n\twb.writeInt32(partition)\n\n\trecordBatch.writeTo(wb)\n\n\treturn wb.Flush()\n}\n\nfunc (wb *writeBuffer) writeProduceRequestV7(correlationID int32, clientID, topic string, partition int32, timeout time.Duration, requiredAcks int16, transactionalID *string, recordBatch *recordBatch) (err error) {\n\n\th := requestHeader{\n\t\tApiKey: int16(produce),\n\t\tApiVersion: int16(v7),\n\t\tCorrelationID: correlationID,\n\t\tClientID: clientID,\n\t}\n\th.Size = (h.size() - 4) +\n\t\tsizeofNullableString(transactionalID) +\n\t\t2 + // required acks\n\t\t4 + // timeout\n\t\t4 + // topic array length\n\t\tsizeofString(topic) + // topic\n\t\t4 + // partition array length\n\t\t4 + // partition\n\t\t4 + // message set size\n\t\trecordBatch.size\n\n\th.writeTo(wb)\n\twb.writeNullableString(transactionalID)\n\twb.writeInt16(requiredAcks) // required acks\n\twb.writeInt32(milliseconds(timeout))\n\n\t// topic array\n\twb.writeArrayLen(1)\n\twb.writeString(topic)\n\n\t// partition array\n\twb.writeArrayLen(1)\n\twb.writeInt32(partition)\n\n\trecordBatch.writeTo(wb)\n\n\treturn wb.Flush()\n}\n\nfunc (wb *writeBuffer) writeRecordBatch(attributes int16, size int32, count int, baseTime, lastTime time.Time, write func(*writeBuffer)) {\n\tvar (\n\t\tbaseTimestamp = timestamp(baseTime)\n\t\tlastTimestamp = timestamp(lastTime)\n\t\tlastOffsetDelta = int32(count - 1)\n\t\tproducerID = int64(-1) // default producer id for now\n\t\tproducerEpoch = int16(-1) // default producer epoch for now\n\t\tbaseSequence = int32(-1) // default base sequence\n\t\trecordCount = int32(count) // record count\n\t\twriterBackup = wb.w\n\t)\n\n\t// dry run to compute the checksum\n\tcw := &crc32Writer{table: crc32.MakeTable(crc32.Castagnoli)}\n\twb.w = cw\n\tcw.writeInt16(attributes) // attributes, timestamp type 0 - create time, not part of a transaction, no control messages\n\tcw.writeInt32(lastOffsetDelta)\n\tcw.writeInt64(baseTimestamp)\n\tcw.writeInt64(lastTimestamp)\n\tcw.writeInt64(producerID)\n\tcw.writeInt16(producerEpoch)\n\tcw.writeInt32(baseSequence)\n\tcw.writeInt32(recordCount)\n\twrite(wb)\n\twb.w = writerBackup\n\n\t// actual write to the output buffer\n\twb.writeInt64(int64(0))\n\twb.writeInt32(int32(size - 12)) // 12 = batch length + base offset sizes\n\twb.writeInt32(-1) // partition leader epoch\n\twb.writeInt8(2) // magic byte\n\twb.writeInt32(int32(cw.crc32))\n\n\twb.writeInt16(attributes)\n\twb.writeInt32(lastOffsetDelta)\n\twb.writeInt64(baseTimestamp)\n\twb.writeInt64(lastTimestamp)\n\twb.writeInt64(producerID)\n\twb.writeInt16(producerEpoch)\n\twb.writeInt32(baseSequence)\n\twb.writeInt32(recordCount)\n\twrite(wb)\n}\n\nfunc compressMessageSet(codec CompressionCodec, msgs ...Message) (compressed *bytes.Buffer, attributes int8, size int32, err error) {\n\tcompressed = acquireBuffer()\n\tcompressor := codec.NewWriter(compressed)\n\twb := &writeBuffer{w: compressor}\n\tcw := &crc32Writer{table: crc32.IEEETable}\n\n\tfor offset, msg := range msgs {\n\t\twb.writeMessage(int64(offset), 0, msg.Time, msg.Key, msg.Value, cw)\n\t}\n\n\tif err = compressor.Close(); err != nil {\n\t\treleaseBuffer(compressed)\n\t\treturn\n\t}\n\n\tattributes = codec.Code()\n\tsize = messageSetSize(Message{Value: compressed.Bytes()})\n\treturn\n}\n\nfunc (wb *writeBuffer) writeMessage(offset int64, attributes int8, time time.Time, key, value []byte, cw *crc32Writer) {\n\tconst magicByte = 1 // compatible with kafka 0.10.0.0+\n\n\ttimestamp := timestamp(time)\n\tsize := messageSize(key, value)\n\n\t// dry run to compute the checksum\n\tcw.crc32 = 0\n\tcw.writeInt8(magicByte)\n\tcw.writeInt8(attributes)\n\tcw.writeInt64(timestamp)\n\tcw.writeBytes(key)\n\tcw.writeBytes(value)\n\n\t// actual write to the output buffer\n\twb.writeInt64(offset)\n\twb.writeInt32(size)\n\twb.writeInt32(int32(cw.crc32))\n\twb.writeInt8(magicByte)\n\twb.writeInt8(attributes)\n\twb.writeInt64(timestamp)\n\twb.writeBytes(key)\n\twb.writeBytes(value)\n}\n\n// Messages with magic >2 are called records. This method writes messages using message format 2.\nfunc (wb *writeBuffer) writeRecord(attributes int8, baseTime time.Time, offset int64, msg Message) {\n\ttimestampDelta := msg.Time.Sub(baseTime)\n\toffsetDelta := int64(offset)\n\n\twb.writeVarInt(int64(recordSize(&msg, timestampDelta, offsetDelta)))\n\twb.writeInt8(attributes)\n\twb.writeVarInt(int64(milliseconds(timestampDelta)))\n\twb.writeVarInt(offsetDelta)\n\n\twb.writeVarBytes(msg.Key)\n\twb.writeVarBytes(msg.Value)\n\twb.writeVarArray(len(msg.Headers), func(i int) {\n\t\th := &msg.Headers[i]\n\t\twb.writeVarString(h.Key)\n\t\twb.writeVarBytes(h.Value)\n\t})\n}\n\nfunc varIntLen(i int64) int {\n\tu := uint64((i << 1) ^ (i >> 63)) // zig-zag encoding\n\tn := 0\n\n\tfor u >= 0x80 {\n\t\tu >>= 7\n\t\tn++\n\t}\n\n\treturn n + 1\n}\n\nfunc varBytesLen(b []byte) int {\n\treturn varIntLen(int64(len(b))) + len(b)\n}\n\nfunc varStringLen(s string) int {\n\treturn varIntLen(int64(len(s))) + len(s)\n}\n\nfunc varArrayLen(n int, f func(int) int) int {\n\tsize := varIntLen(int64(n))\n\tfor i := 0; i < n; i++ {\n\t\tsize += f(i)\n\t}\n\treturn size\n}\n\nfunc messageSize(key, value []byte) int32 {\n\treturn 4 + // crc\n\t\t1 + // magic byte\n\t\t1 + // attributes\n\t\t8 + // timestamp\n\t\tsizeofBytes(key) +\n\t\tsizeofBytes(value)\n}\n\nfunc messageSetSize(msgs ...Message) (size int32) {\n\tfor _, msg := range msgs {\n\t\tsize += 8 + // offset\n\t\t\t4 + // message size\n\t\t\t4 + // crc\n\t\t\t1 + // magic byte\n\t\t\t1 + // attributes\n\t\t\t8 + // timestamp\n\t\t\tsizeofBytes(msg.Key) +\n\t\t\tsizeofBytes(msg.Value)\n\t}\n\treturn\n}\n" }, { "path": "write_test.go", "content": "package kafka\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"hash/crc32\"\n\t\"testing\"\n\t\"time\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nconst (\n\ttestCorrelationID = 1\n\ttestClientID = \"localhost\"\n\ttestTopic = \"topic\"\n\ttestPartition = 42\n)\n\ntype WriteVarIntTestCase struct {\n\tv []byte\n\ttc int64\n}\n\nfunc TestWriteVarInt(t *testing.T) {\n\ttestCases := []*WriteVarIntTestCase{\n\t\t{v: []byte{0}, tc: 0},\n\t\t{v: []byte{2}, tc: 1},\n\t\t{v: []byte{1}, tc: -1},\n\t\t{v: []byte{3}, tc: -2},\n\t\t{v: []byte{128, 2}, tc: 128},\n\t\t{v: []byte{254, 1}, tc: 127},\n\t\t{v: []byte{142, 6}, tc: 391},\n\t\t{v: []byte{142, 134, 6}, tc: 49543},\n\t}\n\n\tfor _, tc := range testCases {\n\t\tb := &bytes.Buffer{}\n\t\tw := &writeBuffer{w: b}\n\t\tw.writeVarInt(tc.tc)\n\n\t\tif !bytes.Equal(b.Bytes(), tc.v) {\n\t\t\tt.Errorf(\"Expected %v; got %v\", tc.v, b.Bytes())\n\t\t}\n\t}\n}\n\nfunc TestWriteOptimizations(t *testing.T) {\n\tt.Run(\"writeFetchRequestV2\", testWriteFetchRequestV2)\n\tt.Run(\"writeListOffsetRequestV1\", testWriteListOffsetRequestV1)\n\tt.Run(\"writeProduceRequestV2\", testWriteProduceRequestV2)\n}\n\nfunc testWriteFetchRequestV2(t *testing.T) {\n\tconst offset = 42\n\tconst minBytes = 10\n\tconst maxBytes = 1000\n\tconst maxWait = 100 * time.Millisecond\n\ttestWriteOptimization(t,\n\t\trequestHeader{\n\t\t\tApiKey: int16(fetch),\n\t\t\tApiVersion: int16(v2),\n\t\t\tCorrelationID: testCorrelationID,\n\t\t\tClientID: testClientID,\n\t\t},\n\t\tfetchRequestV2{\n\t\t\tReplicaID: -1,\n\t\t\tMaxWaitTime: milliseconds(maxWait),\n\t\t\tMinBytes: minBytes,\n\t\t\tTopics: []fetchRequestTopicV2{{\n\t\t\t\tTopicName: testTopic,\n\t\t\t\tPartitions: []fetchRequestPartitionV2{{\n\t\t\t\t\tPartition: testPartition,\n\t\t\t\t\tFetchOffset: offset,\n\t\t\t\t\tMaxBytes: maxBytes,\n\t\t\t\t}},\n\t\t\t}},\n\t\t},\n\t\tfunc(w *writeBuffer) {\n\t\t\tw.writeFetchRequestV2(testCorrelationID, testClientID, testTopic, testPartition, offset, minBytes, maxBytes, maxWait)\n\t\t},\n\t)\n}\n\nfunc testWriteListOffsetRequestV1(t *testing.T) {\n\tconst time = -1\n\ttestWriteOptimization(t,\n\t\trequestHeader{\n\t\t\tApiKey: int16(listOffsets),\n\t\t\tApiVersion: int16(v1),\n\t\t\tCorrelationID: testCorrelationID,\n\t\t\tClientID: testClientID,\n\t\t},\n\t\tlistOffsetRequestV1{\n\t\t\tReplicaID: -1,\n\t\t\tTopics: []listOffsetRequestTopicV1{{\n\t\t\t\tTopicName: testTopic,\n\t\t\t\tPartitions: []listOffsetRequestPartitionV1{{\n\t\t\t\t\tPartition: testPartition,\n\t\t\t\t\tTime: time,\n\t\t\t\t}},\n\t\t\t}},\n\t\t},\n\t\tfunc(w *writeBuffer) {\n\t\t\tw.writeListOffsetRequestV1(testCorrelationID, testClientID, testTopic, testPartition, time)\n\t\t},\n\t)\n}\n\nfunc testWriteProduceRequestV2(t *testing.T) {\n\tkey := []byte(nil)\n\tval := []byte(\"Hello World!\")\n\n\tmsg := messageSetItem{\n\t\tOffset: 10,\n\t\tMessage: message{\n\t\t\tMagicByte: 1,\n\t\t\tAttributes: 0,\n\t\t\tKey: key,\n\t\t\tValue: val,\n\t\t},\n\t}\n\tmsg.MessageSize = msg.Message.size()\n\tmsg.Message.CRC = msg.Message.crc32(&crc32Writer{\n\t\ttable: crc32.IEEETable,\n\t})\n\n\tconst timeout = 100\n\ttestWriteOptimization(t,\n\t\trequestHeader{\n\t\t\tApiKey: int16(produce),\n\t\t\tApiVersion: int16(v2),\n\t\t\tCorrelationID: testCorrelationID,\n\t\t\tClientID: testClientID,\n\t\t},\n\t\tproduceRequestV2{\n\t\t\tRequiredAcks: -1,\n\t\t\tTimeout: timeout,\n\t\t\tTopics: []produceRequestTopicV2{{\n\t\t\t\tTopicName: testTopic,\n\t\t\t\tPartitions: []produceRequestPartitionV2{{\n\t\t\t\t\tPartition: testPartition,\n\t\t\t\t\tMessageSetSize: msg.size(), MessageSet: messageSet{msg},\n\t\t\t\t}},\n\t\t\t}},\n\t\t},\n\t\tfunc(w *writeBuffer) {\n\t\t\tw.writeProduceRequestV2(nil, testCorrelationID, testClientID, testTopic, testPartition, timeout*time.Millisecond, -1, Message{\n\t\t\t\tOffset: 10,\n\t\t\t\tKey: key,\n\t\t\t\tValue: val,\n\t\t\t})\n\t\t},\n\t)\n}\n\nfunc testWriteOptimization(t *testing.T, h requestHeader, r request, f func(*writeBuffer)) {\n\tb1 := &bytes.Buffer{}\n\tw1 := &writeBuffer{w: b1}\n\n\tb2 := &bytes.Buffer{}\n\tw2 := &writeBuffer{w: b2}\n\n\th.Size = (h.size() + r.size()) - 4\n\th.writeTo(w1)\n\tr.writeTo(w1)\n\n\tf(w2)\n\n\tc1 := b1.Bytes()\n\tc2 := b2.Bytes()\n\n\tif !bytes.Equal(c1, c2) {\n\t\tt.Error(\"content differs\")\n\n\t\tn1 := len(c1)\n\t\tn2 := len(c2)\n\n\t\tif n1 != n2 {\n\t\t\tt.Log(\"content length 1 =\", n1)\n\t\t\tt.Log(\"content length 2 =\", n2)\n\t\t} else {\n\t\t\tfor i := 0; i != n1; i++ {\n\t\t\t\tif c1[i] != c2[i] {\n\t\t\t\t\tt.Logf(\"byte at offset %d/%d: %#x != %#x\", i, n1, c1[i], c2[i])\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc TestWriteV2RecordBatch(t *testing.T) {\n\tif !ktesting.KafkaIsAtLeast(\"0.11.0\") {\n\t\tt.Skip(\"RecordBatch was added in kafka 0.11.0\")\n\t\treturn\n\t}\n\n\tclient, topic, shutdown := newLocalClientAndTopic()\n\tdefer shutdown()\n\n\tmsgs := make([]Message, 15)\n\tfor i := range msgs {\n\t\tvalue := fmt.Sprintf(\"Sample message content: %d!\", i)\n\t\tmsgs[i] = Message{Key: []byte(\"Key\"), Value: []byte(value), Headers: []Header{{Key: \"hk\", Value: []byte(\"hv\")}}}\n\t}\n\n\tw := &Writer{\n\t\tAddr: TCP(\"localhost:9092\"),\n\t\tTopic: topic,\n\t\tBatchTimeout: 100 * time.Millisecond,\n\t\tBatchSize: 5,\n\t\tTransport: client.Transport,\n\t}\n\n\tctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)\n\tdefer cancel()\n\n\tif err := w.WriteMessages(ctx, msgs...); err != nil {\n\t\tt.Errorf(\"Failed to write v2 messages to kafka: %v\", err)\n\t\treturn\n\t}\n\tw.Close()\n\n\tr := NewReader(ReaderConfig{\n\t\tBrokers: []string{\"localhost:9092\"},\n\t\tTopic: topic,\n\t\tMaxWait: 100 * time.Millisecond,\n\t})\n\tdefer r.Close()\n\n\tmsg, err := r.ReadMessage(context.Background())\n\tif err != nil {\n\t\tt.Error(\"Failed to read message\")\n\t\treturn\n\t}\n\n\tif string(msg.Key) != \"Key\" {\n\t\tt.Error(\"Received message's key doesn't match\")\n\t\treturn\n\t}\n\tif msg.Headers[0].Key != \"hk\" {\n\t\tt.Error(\"Received message header's key doesn't match\")\n\t\treturn\n\t}\n}\n" }, { "path": "writer.go", "content": "package kafka\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"io\"\n\t\"net\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\tmetadataAPI \"github.com/segmentio/kafka-go/protocol/metadata\"\n)\n\n// The Writer type provides the implementation of a producer of kafka messages\n// that automatically distributes messages across partitions of a single topic\n// using a configurable balancing policy.\n//\n// Writes manage the dispatch of messages across partitions of the topic they\n// are configured to write to using a Balancer, and aggregate batches to\n// optimize the writes to kafka.\n//\n// Writers may be configured to be used synchronously or asynchronously. When\n// use synchronously, calls to WriteMessages block until the messages have been\n// written to kafka. In this mode, the program should inspect the error returned\n// by the function and test if it an instance of kafka.WriteErrors in order to\n// identify which messages have succeeded or failed, for example:\n//\n//\t\t// Construct a synchronous writer (the default mode).\n//\t\tw := &kafka.Writer{\n//\t\t\tAddr: kafka.TCP(\"localhost:9092\", \"localhost:9093\", \"localhost:9094\"),\n//\t\t\tTopic: \"topic-A\",\n//\t\t\tRequiredAcks: kafka.RequireAll,\n//\t\t}\n//\n//\t\t...\n//\n//\t // Passing a context can prevent the operation from blocking indefinitely.\n//\t\tswitch err := w.WriteMessages(ctx, msgs...).(type) {\n//\t\tcase nil:\n//\t\tcase kafka.WriteErrors:\n//\t\t\tfor i := range msgs {\n//\t\t\t\tif err[i] != nil {\n//\t\t\t\t\t// handle the error writing msgs[i]\n//\t\t\t\t\t...\n//\t\t\t\t}\n//\t\t\t}\n//\t\tdefault:\n//\t\t\t// handle other errors\n//\t\t\t...\n//\t\t}\n//\n// In asynchronous mode, the program may configure a completion handler on the\n// writer to receive notifications of messages being written to kafka:\n//\n//\tw := &kafka.Writer{\n//\t\tAddr: kafka.TCP(\"localhost:9092\", \"localhost:9093\", \"localhost:9094\"),\n//\t\tTopic: \"topic-A\",\n//\t\tRequiredAcks: kafka.RequireAll,\n//\t\tAsync: true, // make the writer asynchronous\n//\t\tCompletion: func(messages []kafka.Message, err error) {\n//\t\t\t...\n//\t\t},\n//\t}\n//\n//\t...\n//\n//\t// Because the writer is asynchronous, there is no need for the context to\n//\t// be cancelled, the call will never block.\n//\tif err := w.WriteMessages(context.Background(), msgs...); err != nil {\n//\t\t// Only validation errors would be reported in this case.\n//\t\t...\n//\t}\n//\n// Methods of Writer are safe to use concurrently from multiple goroutines,\n// however the writer configuration should not be modified after first use.\ntype Writer struct {\n\t// Address of the kafka cluster that this writer is configured to send\n\t// messages to.\n\t//\n\t// This field is required, attempting to write messages to a writer with a\n\t// nil address will error.\n\tAddr net.Addr\n\n\t// Topic is the name of the topic that the writer will produce messages to.\n\t//\n\t// Setting this field or not is a mutually exclusive option. If you set Topic\n\t// here, you must not set Topic for any produced Message. Otherwise, if you\tdo\n\t// not set Topic, every Message must have Topic specified.\n\tTopic string\n\n\t// The balancer used to distribute messages across partitions.\n\t//\n\t// The default is to use a round-robin distribution.\n\tBalancer Balancer\n\n\t// Limit on how many attempts will be made to deliver a message.\n\t//\n\t// The default is to try at most 10 times.\n\tMaxAttempts int\n\n\t// WriteBackoffMin optionally sets the smallest amount of time the writer waits before\n\t// it attempts to write a batch of messages\n\t//\n\t// Default: 100ms\n\tWriteBackoffMin time.Duration\n\n\t// WriteBackoffMax optionally sets the maximum amount of time the writer waits before\n\t// it attempts to write a batch of messages\n\t//\n\t// Default: 1s\n\tWriteBackoffMax time.Duration\n\n\t// Limit on how many messages will be buffered before being sent to a\n\t// partition.\n\t//\n\t// The default is to use a target batch size of 100 messages.\n\tBatchSize int\n\n\t// Limit the maximum size of a request in bytes before being sent to\n\t// a partition.\n\t//\n\t// The default is to use a kafka default value of 1048576.\n\tBatchBytes int64\n\n\t// Time limit on how often incomplete message batches will be flushed to\n\t// kafka.\n\t//\n\t// The default is to flush at least every second.\n\tBatchTimeout time.Duration\n\n\t// Timeout for read operations performed by the Writer.\n\t//\n\t// Defaults to 10 seconds.\n\tReadTimeout time.Duration\n\n\t// Timeout for write operation performed by the Writer.\n\t//\n\t// Defaults to 10 seconds.\n\tWriteTimeout time.Duration\n\n\t// Number of acknowledges from partition replicas required before receiving\n\t// a response to a produce request, the following values are supported:\n\t//\n\t// RequireNone (0) fire-and-forget, do not wait for acknowledgements from the\n\t// RequireOne (1) wait for the leader to acknowledge the writes\n\t// RequireAll (-1) wait for the full ISR to acknowledge the writes\n\t//\n\t// Defaults to RequireNone.\n\tRequiredAcks RequiredAcks\n\n\t// Setting this flag to true causes the WriteMessages method to never block.\n\t// It also means that errors are ignored since the caller will not receive\n\t// the returned value. Use this only if you don't care about guarantees of\n\t// whether the messages were written to kafka.\n\t//\n\t// Defaults to false.\n\tAsync bool\n\n\t// An optional function called when the writer succeeds or fails the\n\t// delivery of messages to a kafka partition. When writing the messages\n\t// fails, the `err` parameter will be non-nil.\n\t//\n\t// The messages that the Completion function is called with have their\n\t// topic, partition, offset, and time set based on the Produce responses\n\t// received from kafka. All messages passed to a call to the function have\n\t// been written to the same partition. The keys and values of messages are\n\t// referencing the original byte slices carried by messages in the calls to\n\t// WriteMessages.\n\t//\n\t// The function is called from goroutines started by the writer. Calls to\n\t// Close will block on the Completion function calls. When the Writer is\n\t// not writing asynchronously, the WriteMessages call will also block on\n\t// Completion function, which is a useful guarantee if the byte slices\n\t// for the message keys and values are intended to be reused after the\n\t// WriteMessages call returned.\n\t//\n\t// If a completion function panics, the program terminates because the\n\t// panic is not recovered by the writer and bubbles up to the top of the\n\t// goroutine's call stack.\n\tCompletion func(messages []Message, err error)\n\n\t// Compression set the compression codec to be used to compress messages.\n\tCompression Compression\n\n\t// If not nil, specifies a logger used to report internal changes within the\n\t// writer.\n\tLogger Logger\n\n\t// ErrorLogger is the logger used to report errors. If nil, the writer falls\n\t// back to using Logger instead.\n\tErrorLogger Logger\n\n\t// A transport used to send messages to kafka clusters.\n\t//\n\t// If nil, DefaultTransport is used.\n\tTransport RoundTripper\n\n\t// AllowAutoTopicCreation notifies writer to create topic if missing.\n\tAllowAutoTopicCreation bool\n\n\t// Manages the current set of partition-topic writers.\n\tgroup sync.WaitGroup\n\tmutex sync.Mutex\n\tclosed bool\n\twriters map[topicPartition]*partitionWriter\n\n\t// writer stats are all made of atomic values, no need for synchronization.\n\t// Use a pointer to ensure 64-bit alignment of the values. The once value is\n\t// used to lazily create the value when first used, allowing programs to use\n\t// the zero-value value of Writer.\n\tonce sync.Once\n\t*writerStats\n\n\t// If no balancer is configured, the writer uses this one. RoundRobin values\n\t// are safe to use concurrently from multiple goroutines, there is no need\n\t// for extra synchronization to access this field.\n\troundRobin RoundRobin\n\n\t// non-nil when a transport was created by NewWriter, remove in 1.0.\n\ttransport *Transport\n}\n\n// WriterConfig is a configuration type used to create new instances of Writer.\n//\n// DEPRECATED: writer values should be configured directly by assigning their\n// exported fields. This type is kept for backward compatibility, and will be\n// removed in version 1.0.\ntype WriterConfig struct {\n\t// The list of brokers used to discover the partitions available on the\n\t// kafka cluster.\n\t//\n\t// This field is required, attempting to create a writer with an empty list\n\t// of brokers will panic.\n\tBrokers []string\n\n\t// The topic that the writer will produce messages to.\n\t//\n\t// If provided, this will be used to set the topic for all produced messages.\n\t// If not provided, each Message must specify a topic for itself. This must be\n\t// mutually exclusive, otherwise the Writer will return an error.\n\tTopic string\n\n\t// The dialer used by the writer to establish connections to the kafka\n\t// cluster.\n\t//\n\t// If nil, the default dialer is used instead.\n\tDialer *Dialer\n\n\t// The balancer used to distribute messages across partitions.\n\t//\n\t// The default is to use a round-robin distribution.\n\tBalancer Balancer\n\n\t// Limit on how many attempts will be made to deliver a message.\n\t//\n\t// The default is to try at most 10 times.\n\tMaxAttempts int\n\n\t// DEPRECATED: in versions prior to 0.4, the writer used channels internally\n\t// to dispatch messages to partitions. This has been replaced by an in-memory\n\t// aggregation of batches which uses shared state instead of message passing,\n\t// making this option unnecessary.\n\tQueueCapacity int\n\n\t// Limit on how many messages will be buffered before being sent to a\n\t// partition.\n\t//\n\t// The default is to use a target batch size of 100 messages.\n\tBatchSize int\n\n\t// Limit the maximum size of a request in bytes before being sent to\n\t// a partition.\n\t//\n\t// The default is to use a kafka default value of 1048576.\n\tBatchBytes int\n\n\t// Time limit on how often incomplete message batches will be flushed to\n\t// kafka.\n\t//\n\t// The default is to flush at least every second.\n\tBatchTimeout time.Duration\n\n\t// Timeout for read operations performed by the Writer.\n\t//\n\t// Defaults to 10 seconds.\n\tReadTimeout time.Duration\n\n\t// Timeout for write operation performed by the Writer.\n\t//\n\t// Defaults to 10 seconds.\n\tWriteTimeout time.Duration\n\n\t// DEPRECATED: in versions prior to 0.4, the writer used to maintain a cache\n\t// the topic layout. With the change to use a transport to manage connections,\n\t// the responsibility of syncing the cluster layout has been delegated to the\n\t// transport.\n\tRebalanceInterval time.Duration\n\n\t// DEPRECATED: in versions prior to 0.4, the writer used to manage connections\n\t// to the kafka cluster directly. With the change to use a transport to manage\n\t// connections, the writer has no connections to manage directly anymore.\n\tIdleConnTimeout time.Duration\n\n\t// Number of acknowledges from partition replicas required before receiving\n\t// a response to a produce request. The default is -1, which means to wait for\n\t// all replicas, and a value above 0 is required to indicate how many replicas\n\t// should acknowledge a message to be considered successful.\n\tRequiredAcks int\n\n\t// Setting this flag to true causes the WriteMessages method to never block.\n\t// It also means that errors are ignored since the caller will not receive\n\t// the returned value. Use this only if you don't care about guarantees of\n\t// whether the messages were written to kafka.\n\tAsync bool\n\n\t// CompressionCodec set the codec to be used to compress Kafka messages.\n\tCompressionCodec\n\n\t// If not nil, specifies a logger used to report internal changes within the\n\t// writer.\n\tLogger Logger\n\n\t// ErrorLogger is the logger used to report errors. If nil, the writer falls\n\t// back to using Logger instead.\n\tErrorLogger Logger\n}\n\ntype topicPartition struct {\n\ttopic string\n\tpartition int32\n}\n\n// Validate method validates WriterConfig properties.\nfunc (config *WriterConfig) Validate() error {\n\tif len(config.Brokers) == 0 {\n\t\treturn errors.New(\"cannot create a kafka writer with an empty list of brokers\")\n\t}\n\treturn nil\n}\n\n// WriterStats is a data structure returned by a call to Writer.Stats that\n// exposes details about the behavior of the writer.\ntype WriterStats struct {\n\tWrites int64 `metric:\"kafka.writer.write.count\" type:\"counter\"`\n\tMessages int64 `metric:\"kafka.writer.message.count\" type:\"counter\"`\n\tBytes int64 `metric:\"kafka.writer.message.bytes\" type:\"counter\"`\n\tErrors int64 `metric:\"kafka.writer.error.count\" type:\"counter\"`\n\n\tBatchTime DurationStats `metric:\"kafka.writer.batch.seconds\"`\n\tBatchQueueTime DurationStats `metric:\"kafka.writer.batch.queue.seconds\"`\n\tWriteTime DurationStats `metric:\"kafka.writer.write.seconds\"`\n\tWaitTime DurationStats `metric:\"kafka.writer.wait.seconds\"`\n\tRetries int64 `metric:\"kafka.writer.retries.count\" type:\"counter\"`\n\tBatchSize SummaryStats `metric:\"kafka.writer.batch.size\"`\n\tBatchBytes SummaryStats `metric:\"kafka.writer.batch.bytes\"`\n\n\tMaxAttempts int64 `metric:\"kafka.writer.attempts.max\" type:\"gauge\"`\n\tWriteBackoffMin time.Duration `metric:\"kafka.writer.backoff.min\" type:\"gauge\"`\n\tWriteBackoffMax time.Duration `metric:\"kafka.writer.backoff.max\" type:\"gauge\"`\n\tMaxBatchSize int64 `metric:\"kafka.writer.batch.max\" type:\"gauge\"`\n\tBatchTimeout time.Duration `metric:\"kafka.writer.batch.timeout\" type:\"gauge\"`\n\tReadTimeout time.Duration `metric:\"kafka.writer.read.timeout\" type:\"gauge\"`\n\tWriteTimeout time.Duration `metric:\"kafka.writer.write.timeout\" type:\"gauge\"`\n\tRequiredAcks int64 `metric:\"kafka.writer.acks.required\" type:\"gauge\"`\n\tAsync bool `metric:\"kafka.writer.async\" type:\"gauge\"`\n\n\tTopic string `tag:\"topic\"`\n\n\t// DEPRECATED: these fields will only be reported for backward compatibility\n\t// if the Writer was constructed with NewWriter.\n\tDials int64 `metric:\"kafka.writer.dial.count\" type:\"counter\"`\n\tDialTime DurationStats `metric:\"kafka.writer.dial.seconds\"`\n\n\t// DEPRECATED: these fields were meaningful prior to kafka-go 0.4, changes\n\t// to the internal implementation and the introduction of the transport type\n\t// made them unnecessary.\n\t//\n\t// The values will be zero but are left for backward compatibility to avoid\n\t// breaking programs that used these fields.\n\tRebalances int64\n\tRebalanceInterval time.Duration\n\tQueueLength int64\n\tQueueCapacity int64\n\tClientID string\n}\n\n// writerStats is a struct that contains statistics on a writer.\n//\n// Since atomic is used to mutate the statistics the values must be 64-bit aligned.\n// This is easily accomplished by always allocating this struct directly, (i.e. using a pointer to the struct).\n// See https://golang.org/pkg/sync/atomic/#pkg-note-BUG\ntype writerStats struct {\n\tdials counter\n\twrites counter\n\tmessages counter\n\tbytes counter\n\terrors counter\n\tdialTime summary\n\tbatchTime summary\n\tbatchQueueTime summary\n\twriteTime summary\n\twaitTime summary\n\tretries counter\n\tbatchSize summary\n\tbatchSizeBytes summary\n}\n\n// NewWriter creates and returns a new Writer configured with config.\n//\n// DEPRECATED: Writer value can be instantiated and configured directly,\n// this function is retained for backward compatibility and will be removed\n// in version 1.0.\nfunc NewWriter(config WriterConfig) *Writer {\n\tif err := config.Validate(); err != nil {\n\t\tpanic(err)\n\t}\n\n\tif config.Dialer == nil {\n\t\tconfig.Dialer = DefaultDialer\n\t}\n\n\tif config.Balancer == nil {\n\t\tconfig.Balancer = &RoundRobin{}\n\t}\n\n\t// Converts the pre-0.4 Dialer API into a Transport.\n\tkafkaDialer := DefaultDialer\n\tif config.Dialer != nil {\n\t\tkafkaDialer = config.Dialer\n\t}\n\n\tdialer := (&net.Dialer{\n\t\tTimeout: kafkaDialer.Timeout,\n\t\tDeadline: kafkaDialer.Deadline,\n\t\tLocalAddr: kafkaDialer.LocalAddr,\n\t\tDualStack: kafkaDialer.DualStack,\n\t\tFallbackDelay: kafkaDialer.FallbackDelay,\n\t\tKeepAlive: kafkaDialer.KeepAlive,\n\t})\n\n\tvar resolver Resolver\n\tif r, ok := kafkaDialer.Resolver.(*net.Resolver); ok {\n\t\tdialer.Resolver = r\n\t} else {\n\t\tresolver = kafkaDialer.Resolver\n\t}\n\n\tstats := new(writerStats)\n\t// For backward compatibility with the pre-0.4 APIs, support custom\n\t// resolvers by wrapping the dial function.\n\tdial := func(ctx context.Context, network, addr string) (net.Conn, error) {\n\t\tstart := time.Now()\n\t\tdefer func() {\n\t\t\tstats.dials.observe(1)\n\t\t\tstats.dialTime.observe(int64(time.Since(start)))\n\t\t}()\n\t\taddress, err := lookupHost(ctx, addr, resolver)\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\treturn dialer.DialContext(ctx, network, address)\n\t}\n\n\tidleTimeout := config.IdleConnTimeout\n\tif idleTimeout == 0 {\n\t\t// Historical default value of WriterConfig.IdleTimeout, 9 minutes seems\n\t\t// like it is way too long when there is no ping mechanism in the kafka\n\t\t// protocol.\n\t\tidleTimeout = 9 * time.Minute\n\t}\n\n\tmetadataTTL := config.RebalanceInterval\n\tif metadataTTL == 0 {\n\t\t// Historical default value of WriterConfig.RebalanceInterval.\n\t\tmetadataTTL = 15 * time.Second\n\t}\n\n\ttransport := &Transport{\n\t\tDial: dial,\n\t\tSASL: kafkaDialer.SASLMechanism,\n\t\tTLS: kafkaDialer.TLS,\n\t\tClientID: kafkaDialer.ClientID,\n\t\tIdleTimeout: idleTimeout,\n\t\tMetadataTTL: metadataTTL,\n\t}\n\n\tw := &Writer{\n\t\tAddr: TCP(config.Brokers...),\n\t\tTopic: config.Topic,\n\t\tMaxAttempts: config.MaxAttempts,\n\t\tBatchSize: config.BatchSize,\n\t\tBalancer: config.Balancer,\n\t\tBatchBytes: int64(config.BatchBytes),\n\t\tBatchTimeout: config.BatchTimeout,\n\t\tReadTimeout: config.ReadTimeout,\n\t\tWriteTimeout: config.WriteTimeout,\n\t\tRequiredAcks: RequiredAcks(config.RequiredAcks),\n\t\tAsync: config.Async,\n\t\tLogger: config.Logger,\n\t\tErrorLogger: config.ErrorLogger,\n\t\tTransport: transport,\n\t\ttransport: transport,\n\t\twriterStats: stats,\n\t}\n\n\tif config.RequiredAcks == 0 {\n\t\t// Historically the writers created by NewWriter have used \"all\" as the\n\t\t// default value when 0 was specified.\n\t\tw.RequiredAcks = RequireAll\n\t}\n\n\tif config.CompressionCodec != nil {\n\t\tw.Compression = Compression(config.CompressionCodec.Code())\n\t}\n\n\treturn w\n}\n\n// enter is called by WriteMessages to indicate that a new inflight operation\n// has started, which helps synchronize with Close and ensure that the method\n// does not return until all inflight operations were completed.\nfunc (w *Writer) enter() bool {\n\tw.mutex.Lock()\n\tdefer w.mutex.Unlock()\n\tif w.closed {\n\t\treturn false\n\t}\n\tw.group.Add(1)\n\treturn true\n}\n\n// leave is called by WriteMessages to indicate that the inflight operation has\n// completed.\nfunc (w *Writer) leave() { w.group.Done() }\n\n// spawn starts a new asynchronous operation on the writer. This method is used\n// instead of starting goroutines inline to help manage the state of the\n// writer's wait group. The wait group is used to block Close calls until all\n// inflight operations have completed, therefore automatically including those\n// started with calls to spawn.\nfunc (w *Writer) spawn(f func()) {\n\tw.group.Add(1)\n\tgo func() {\n\t\tdefer w.group.Done()\n\t\tf()\n\t}()\n}\n\n// Close flushes pending writes, and waits for all writes to complete before\n// returning. Calling Close also prevents new writes from being submitted to\n// the writer, further calls to WriteMessages and the like will fail with\n// io.ErrClosedPipe.\nfunc (w *Writer) Close() error {\n\tw.mutex.Lock()\n\t// Marking the writer as closed here causes future calls to WriteMessages to\n\t// fail with io.ErrClosedPipe. Mutation of this field is synchronized on the\n\t// writer's mutex to ensure that no more increments of the wait group are\n\t// performed afterwards (which could otherwise race with the Wait below).\n\tw.closed = true\n\n\t// close all writers to trigger any pending batches\n\tfor _, writer := range w.writers {\n\t\twriter.close()\n\t}\n\n\tfor partition := range w.writers {\n\t\tdelete(w.writers, partition)\n\t}\n\n\tw.mutex.Unlock()\n\tw.group.Wait()\n\n\tif w.transport != nil {\n\t\tw.transport.CloseIdleConnections()\n\t}\n\n\treturn nil\n}\n\n// WriteMessages writes a batch of messages to the kafka topic configured on this\n// writer.\n//\n// Unless the writer was configured to write messages asynchronously, the method\n// blocks until all messages have been written, or until the maximum number of\n// attempts was reached.\n//\n// When sending synchronously and the writer's batch size is configured to be\n// greater than 1, this method blocks until either a full batch can be assembled\n// or the batch timeout is reached. The batch size and timeouts are evaluated\n// per partition, so the choice of Balancer can also influence the flushing\n// behavior. For example, the Hash balancer will require on average N * batch\n// size messages to trigger a flush where N is the number of partitions. The\n// best way to achieve good batching behavior is to share one Writer amongst\n// multiple go routines.\n//\n// When the method returns an error, it may be of type kafka.WriteError to allow\n// the caller to determine the status of each message.\n//\n// The context passed as first argument may also be used to asynchronously\n// cancel the operation. Note that in this case there are no guarantees made on\n// whether messages were written to kafka, they might also still be written\n// after this method has already returned, therefore it is important to not\n// modify byte slices of passed messages if WriteMessages returned early due\n// to a canceled context.\n// The program should assume that the whole batch failed and re-write the\n// messages later (which could then cause duplicates).\nfunc (w *Writer) WriteMessages(ctx context.Context, msgs ...Message) error {\n\tif w.Addr == nil {\n\t\treturn errors.New(\"kafka.(*Writer).WriteMessages: cannot create a kafka writer with a nil address\")\n\t}\n\n\tif !w.enter() {\n\t\treturn io.ErrClosedPipe\n\t}\n\tdefer w.leave()\n\n\tif len(msgs) == 0 {\n\t\treturn nil\n\t}\n\n\tbalancer := w.balancer()\n\tbatchBytes := w.batchBytes()\n\n\tfor i := range msgs {\n\t\tn := int64(msgs[i].totalSize())\n\t\tif n > batchBytes {\n\t\t\t// This error is left for backward compatibility with historical\n\t\t\t// behavior, but it can yield O(N^2) behaviors. The expectations\n\t\t\t// are that the program will check if WriteMessages returned a\n\t\t\t// MessageTooLargeError, discard the message that was exceeding\n\t\t\t// the maximum size, and try again.\n\t\t\treturn messageTooLarge(msgs, i)\n\t\t}\n\t}\n\n\t// We use int32 here to halve the memory footprint (compared to using int\n\t// on 64 bits architectures). We map lists of the message indexes instead\n\t// of the message values for the same reason, int32 is 4 bytes, vs a full\n\t// Message value which is 100+ bytes and contains pointers and contributes\n\t// to increasing GC work.\n\tassignments := make(map[topicPartition][]int32)\n\n\tfor i, msg := range msgs {\n\t\ttopic, err := w.chooseTopic(msg)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tnumPartitions, err := w.partitions(ctx, topic)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tpartition := balancer.Balance(msg, loadCachedPartitions(numPartitions)...)\n\n\t\tkey := topicPartition{\n\t\t\ttopic: topic,\n\t\t\tpartition: int32(partition),\n\t\t}\n\n\t\tassignments[key] = append(assignments[key], int32(i))\n\t}\n\n\tbatches := w.batchMessages(msgs, assignments)\n\tif w.Async {\n\t\treturn nil\n\t}\n\n\tdone := ctx.Done()\n\thasErrors := false\n\tfor batch := range batches {\n\t\tselect {\n\t\tcase <-done:\n\t\t\treturn ctx.Err()\n\t\tcase <-batch.done:\n\t\t\tif batch.err != nil {\n\t\t\t\thasErrors = true\n\t\t\t}\n\t\t}\n\t}\n\n\tif !hasErrors {\n\t\treturn nil\n\t}\n\n\twerr := make(WriteErrors, len(msgs))\n\n\tfor batch, indexes := range batches {\n\t\tfor _, i := range indexes {\n\t\t\twerr[i] = batch.err\n\t\t}\n\t}\n\treturn werr\n}\n\nfunc (w *Writer) batchMessages(messages []Message, assignments map[topicPartition][]int32) map[*writeBatch][]int32 {\n\tvar batches map[*writeBatch][]int32\n\tif !w.Async {\n\t\tbatches = make(map[*writeBatch][]int32, len(assignments))\n\t}\n\n\tw.mutex.Lock()\n\tdefer w.mutex.Unlock()\n\n\tif w.writers == nil {\n\t\tw.writers = map[topicPartition]*partitionWriter{}\n\t}\n\n\tfor key, indexes := range assignments {\n\t\twriter := w.writers[key]\n\t\tif writer == nil {\n\t\t\twriter = newPartitionWriter(w, key)\n\t\t\tw.writers[key] = writer\n\t\t}\n\t\twbatches := writer.writeMessages(messages, indexes)\n\n\t\tfor batch, idxs := range wbatches {\n\t\t\tbatches[batch] = idxs\n\t\t}\n\t}\n\n\treturn batches\n}\n\nfunc (w *Writer) produce(key topicPartition, batch *writeBatch) (*ProduceResponse, error) {\n\ttimeout := w.writeTimeout()\n\n\tctx, cancel := context.WithTimeout(context.Background(), timeout)\n\tdefer cancel()\n\n\treturn w.client(timeout).Produce(ctx, &ProduceRequest{\n\t\tPartition: int(key.partition),\n\t\tTopic: key.topic,\n\t\tRequiredAcks: w.RequiredAcks,\n\t\tCompression: w.Compression,\n\t\tRecords: &writerRecords{\n\t\t\tmsgs: batch.msgs,\n\t\t},\n\t})\n}\n\nfunc (w *Writer) partitions(ctx context.Context, topic string) (int, error) {\n\tclient := w.client(w.readTimeout())\n\t// Here we use the transport directly as an optimization to avoid the\n\t// construction of temporary request and response objects made by the\n\t// (*Client).Metadata API.\n\t//\n\t// It is expected that the transport will optimize this request by\n\t// caching recent results (the kafka.Transport types does).\n\tr, err := client.transport().RoundTrip(ctx, client.Addr, &metadataAPI.Request{\n\t\tTopicNames: []string{topic},\n\t\tAllowAutoTopicCreation: w.AllowAutoTopicCreation,\n\t})\n\tif err != nil {\n\t\treturn 0, err\n\t}\n\tfor _, t := range r.(*metadataAPI.Response).Topics {\n\t\tif t.Name == topic {\n\t\t\t// This should always hit, unless kafka has a bug.\n\t\t\tif t.ErrorCode != 0 {\n\t\t\t\treturn 0, Error(t.ErrorCode)\n\t\t\t}\n\t\t\treturn len(t.Partitions), nil\n\t\t}\n\t}\n\treturn 0, UnknownTopicOrPartition\n}\n\nfunc (w *Writer) client(timeout time.Duration) *Client {\n\treturn &Client{\n\t\tAddr: w.Addr,\n\t\tTransport: w.Transport,\n\t\tTimeout: timeout,\n\t}\n}\n\nfunc (w *Writer) balancer() Balancer {\n\tif w.Balancer != nil {\n\t\treturn w.Balancer\n\t}\n\treturn &w.roundRobin\n}\n\nfunc (w *Writer) maxAttempts() int {\n\tif w.MaxAttempts > 0 {\n\t\treturn w.MaxAttempts\n\t}\n\t// TODO: this is a very high default, if something has failed 9 times it\n\t// seems unlikely it will succeed on the 10th attempt. However, it does\n\t// carry the risk to greatly increase the volume of requests sent to the\n\t// kafka cluster. We should consider reducing this default (3?).\n\treturn 10\n}\n\nfunc (w *Writer) writeBackoffMin() time.Duration {\n\tif w.WriteBackoffMin > 0 {\n\t\treturn w.WriteBackoffMin\n\t}\n\treturn 100 * time.Millisecond\n}\n\nfunc (w *Writer) writeBackoffMax() time.Duration {\n\tif w.WriteBackoffMax > 0 {\n\t\treturn w.WriteBackoffMax\n\t}\n\treturn 1 * time.Second\n}\n\nfunc (w *Writer) batchSize() int {\n\tif w.BatchSize > 0 {\n\t\treturn w.BatchSize\n\t}\n\treturn 100\n}\n\nfunc (w *Writer) batchBytes() int64 {\n\tif w.BatchBytes > 0 {\n\t\treturn w.BatchBytes\n\t}\n\treturn 1048576\n}\n\nfunc (w *Writer) batchTimeout() time.Duration {\n\tif w.BatchTimeout > 0 {\n\t\treturn w.BatchTimeout\n\t}\n\treturn 1 * time.Second\n}\n\nfunc (w *Writer) readTimeout() time.Duration {\n\tif w.ReadTimeout > 0 {\n\t\treturn w.ReadTimeout\n\t}\n\treturn 10 * time.Second\n}\n\nfunc (w *Writer) writeTimeout() time.Duration {\n\tif w.WriteTimeout > 0 {\n\t\treturn w.WriteTimeout\n\t}\n\treturn 10 * time.Second\n}\n\nfunc (w *Writer) withLogger(do func(Logger)) {\n\tif w.Logger != nil {\n\t\tdo(w.Logger)\n\t}\n}\n\nfunc (w *Writer) withErrorLogger(do func(Logger)) {\n\tif w.ErrorLogger != nil {\n\t\tdo(w.ErrorLogger)\n\t} else {\n\t\tw.withLogger(do)\n\t}\n}\n\nfunc (w *Writer) stats() *writerStats {\n\tw.once.Do(func() {\n\t\t// This field is not nil when the writer was constructed with NewWriter\n\t\t// to share the value with the dial function and count dials.\n\t\tif w.writerStats == nil {\n\t\t\tw.writerStats = new(writerStats)\n\t\t}\n\t})\n\treturn w.writerStats\n}\n\n// Stats returns a snapshot of the writer stats since the last time the method\n// was called, or since the writer was created if it is called for the first\n// time.\n//\n// A typical use of this method is to spawn a goroutine that will periodically\n// call Stats on a kafka writer and report the metrics to a stats collection\n// system.\nfunc (w *Writer) Stats() WriterStats {\n\tstats := w.stats()\n\treturn WriterStats{\n\t\tDials: stats.dials.snapshot(),\n\t\tWrites: stats.writes.snapshot(),\n\t\tMessages: stats.messages.snapshot(),\n\t\tBytes: stats.bytes.snapshot(),\n\t\tErrors: stats.errors.snapshot(),\n\t\tDialTime: stats.dialTime.snapshotDuration(),\n\t\tBatchTime: stats.batchTime.snapshotDuration(),\n\t\tBatchQueueTime: stats.batchQueueTime.snapshotDuration(),\n\t\tWriteTime: stats.writeTime.snapshotDuration(),\n\t\tWaitTime: stats.waitTime.snapshotDuration(),\n\t\tRetries: stats.retries.snapshot(),\n\t\tBatchSize: stats.batchSize.snapshot(),\n\t\tBatchBytes: stats.batchSizeBytes.snapshot(),\n\t\tMaxAttempts: int64(w.maxAttempts()),\n\t\tWriteBackoffMin: w.writeBackoffMin(),\n\t\tWriteBackoffMax: w.writeBackoffMax(),\n\t\tMaxBatchSize: int64(w.batchSize()),\n\t\tBatchTimeout: w.batchTimeout(),\n\t\tReadTimeout: w.readTimeout(),\n\t\tWriteTimeout: w.writeTimeout(),\n\t\tRequiredAcks: int64(w.RequiredAcks),\n\t\tAsync: w.Async,\n\t\tTopic: w.Topic,\n\t}\n}\n\nfunc (w *Writer) chooseTopic(msg Message) (string, error) {\n\t// w.Topic and msg.Topic are mutually exclusive, meaning only 1 must be set\n\t// otherwise we will return an error.\n\tif w.Topic != \"\" && msg.Topic != \"\" {\n\t\treturn \"\", errors.New(\"kafka.(*Writer): Topic must not be specified for both Writer and Message\")\n\t} else if w.Topic == \"\" && msg.Topic == \"\" {\n\t\treturn \"\", errors.New(\"kafka.(*Writer): Topic must be specified for Writer or Message\")\n\t}\n\n\t// now we choose the topic, depending on which one is not empty\n\tif msg.Topic != \"\" {\n\t\treturn msg.Topic, nil\n\t}\n\n\treturn w.Topic, nil\n}\n\ntype batchQueue struct {\n\tqueue []*writeBatch\n\n\t// Pointers are used here to make `go vet` happy, and avoid copying mutexes.\n\t// It may be better to revert these to non-pointers and avoid the copies in\n\t// a different way.\n\tmutex *sync.Mutex\n\tcond *sync.Cond\n\n\tclosed bool\n}\n\nfunc (b *batchQueue) Put(batch *writeBatch) bool {\n\tb.cond.L.Lock()\n\tdefer b.cond.L.Unlock()\n\tdefer b.cond.Broadcast()\n\n\tif b.closed {\n\t\treturn false\n\t}\n\tb.queue = append(b.queue, batch)\n\treturn true\n}\n\nfunc (b *batchQueue) Get() *writeBatch {\n\tb.cond.L.Lock()\n\tdefer b.cond.L.Unlock()\n\n\tfor len(b.queue) == 0 && !b.closed {\n\t\tb.cond.Wait()\n\t}\n\n\tif len(b.queue) == 0 {\n\t\treturn nil\n\t}\n\n\tbatch := b.queue[0]\n\tb.queue[0] = nil\n\tb.queue = b.queue[1:]\n\n\treturn batch\n}\n\nfunc (b *batchQueue) Close() {\n\tb.cond.L.Lock()\n\tdefer b.cond.L.Unlock()\n\tdefer b.cond.Broadcast()\n\n\tb.closed = true\n}\n\nfunc newBatchQueue(initialSize int) batchQueue {\n\tbq := batchQueue{\n\t\tqueue: make([]*writeBatch, 0, initialSize),\n\t\tmutex: &sync.Mutex{},\n\t\tcond: &sync.Cond{},\n\t}\n\n\tbq.cond.L = bq.mutex\n\n\treturn bq\n}\n\n// partitionWriter is a writer for a topic-partion pair. It maintains messaging order\n// across batches of messages.\ntype partitionWriter struct {\n\tmeta topicPartition\n\tqueue batchQueue\n\n\tmutex sync.Mutex\n\tcurrBatch *writeBatch\n\n\t// reference to the writer that owns this batch. Used for the produce logic\n\t// as well as stat tracking\n\tw *Writer\n}\n\nfunc newPartitionWriter(w *Writer, key topicPartition) *partitionWriter {\n\twriter := &partitionWriter{\n\t\tmeta: key,\n\t\tqueue: newBatchQueue(10),\n\t\tw: w,\n\t}\n\tw.spawn(writer.writeBatches)\n\treturn writer\n}\n\nfunc (ptw *partitionWriter) writeBatches() {\n\tfor {\n\t\tbatch := ptw.queue.Get()\n\n\t\t// The only time we can return nil is when the queue is closed\n\t\t// and empty. If the queue is closed that means\n\t\t// the Writer is closed so once we're here it's time to exit.\n\t\tif batch == nil {\n\t\t\treturn\n\t\t}\n\n\t\tptw.writeBatch(batch)\n\t}\n}\n\nfunc (ptw *partitionWriter) writeMessages(msgs []Message, indexes []int32) map[*writeBatch][]int32 {\n\tptw.mutex.Lock()\n\tdefer ptw.mutex.Unlock()\n\n\tbatchSize := ptw.w.batchSize()\n\tbatchBytes := ptw.w.batchBytes()\n\n\tvar batches map[*writeBatch][]int32\n\tif !ptw.w.Async {\n\t\tbatches = make(map[*writeBatch][]int32, 1)\n\t}\n\n\tfor _, i := range indexes {\n\tassignMessage:\n\t\tbatch := ptw.currBatch\n\t\tif batch == nil {\n\t\t\tbatch = ptw.newWriteBatch()\n\t\t\tptw.currBatch = batch\n\t\t}\n\t\tif !batch.add(msgs[i], batchSize, batchBytes) {\n\t\t\tbatch.trigger()\n\t\t\tptw.queue.Put(batch)\n\t\t\tptw.currBatch = nil\n\t\t\tgoto assignMessage\n\t\t}\n\n\t\tif batch.full(batchSize, batchBytes) {\n\t\t\tbatch.trigger()\n\t\t\tptw.queue.Put(batch)\n\t\t\tptw.currBatch = nil\n\t\t}\n\n\t\tif !ptw.w.Async {\n\t\t\tbatches[batch] = append(batches[batch], i)\n\t\t}\n\t}\n\treturn batches\n}\n\n// ptw.w can be accessed here because this is called with the lock ptw.mutex already held.\nfunc (ptw *partitionWriter) newWriteBatch() *writeBatch {\n\tbatch := newWriteBatch(time.Now(), ptw.w.batchTimeout())\n\tptw.w.spawn(func() { ptw.awaitBatch(batch) })\n\treturn batch\n}\n\n// awaitBatch waits for a batch to either fill up or time out.\n// If the batch is full it only stops the timer, if the timer\n// expires it will queue the batch for writing if needed.\nfunc (ptw *partitionWriter) awaitBatch(batch *writeBatch) {\n\tselect {\n\tcase <-batch.timer.C:\n\t\tptw.mutex.Lock()\n\t\t// detach the batch from the writer if we're still attached\n\t\t// and queue for writing.\n\t\t// Only the current batch can expire, all previous batches were already written to the queue.\n\t\t// If writeMesseages locks pw.mutex after the timer fires but before this goroutine\n\t\t// can lock pw.mutex it will either have filled the batch and enqueued it which will mean\n\t\t// pw.currBatch != batch so we just move on.\n\t\t// Otherwise, we detach the batch from the ptWriter and enqueue it for writing.\n\t\tif ptw.currBatch == batch {\n\t\t\tptw.queue.Put(batch)\n\t\t\tptw.currBatch = nil\n\t\t}\n\t\tptw.mutex.Unlock()\n\tcase <-batch.ready:\n\t\t// The batch became full, it was removed from the ptwriter and its\n\t\t// ready channel was closed. We need to close the timer to avoid\n\t\t// having it leak until it expires.\n\t\tbatch.timer.Stop()\n\t}\n\tstats := ptw.w.stats()\n\tstats.batchQueueTime.observe(int64(time.Since(batch.time)))\n}\n\nfunc (ptw *partitionWriter) writeBatch(batch *writeBatch) {\n\tstats := ptw.w.stats()\n\tstats.batchTime.observe(int64(time.Since(batch.time)))\n\tstats.batchSize.observe(int64(len(batch.msgs)))\n\tstats.batchSizeBytes.observe(batch.bytes)\n\n\tvar res *ProduceResponse\n\tvar err error\n\tkey := ptw.meta\n\tfor attempt, maxAttempts := 0, ptw.w.maxAttempts(); attempt < maxAttempts; attempt++ {\n\t\tif attempt != 0 {\n\t\t\tstats.retries.observe(1)\n\t\t\t// TODO: should there be a way to asynchronously cancel this\n\t\t\t// operation?\n\t\t\t//\n\t\t\t// * If all goroutines that added message to this batch have stopped\n\t\t\t// waiting for it, should we abort?\n\t\t\t//\n\t\t\t// * If the writer has been closed? It reduces the durability\n\t\t\t// guarantees to abort, but may be better to avoid long wait times\n\t\t\t// on close.\n\t\t\t//\n\t\t\tdelay := backoff(attempt, ptw.w.writeBackoffMin(), ptw.w.writeBackoffMax())\n\t\t\tptw.w.withLogger(func(log Logger) {\n\t\t\t\tlog.Printf(\"backing off %s writing %d messages to %s (partition: %d)\", delay, len(batch.msgs), key.topic, key.partition)\n\t\t\t})\n\t\t\ttime.Sleep(delay)\n\t\t}\n\n\t\tptw.w.withLogger(func(log Logger) {\n\t\t\tlog.Printf(\"writing %d messages to %s (partition: %d)\", len(batch.msgs), key.topic, key.partition)\n\t\t})\n\n\t\tstart := time.Now()\n\t\tres, err = ptw.w.produce(key, batch)\n\n\t\tstats.writes.observe(1)\n\t\tstats.messages.observe(int64(len(batch.msgs)))\n\t\tstats.bytes.observe(batch.bytes)\n\t\t// stats.writeTime used to report the duration of WriteMessages, but the\n\t\t// implementation was broken and reporting values in the nanoseconds\n\t\t// range. In kafka-go 0.4, we recylced this value to instead report the\n\t\t// duration of produce requests, and changed the stats.waitTime value to\n\t\t// report the time that kafka has throttled the requests for.\n\t\tstats.writeTime.observe(int64(time.Since(start)))\n\n\t\tif res != nil {\n\t\t\terr = res.Error\n\t\t\tstats.waitTime.observe(int64(res.Throttle))\n\t\t}\n\n\t\tif err == nil {\n\t\t\tbreak\n\t\t}\n\n\t\tstats.errors.observe(1)\n\n\t\tptw.w.withErrorLogger(func(log Logger) {\n\t\t\tlog.Printf(\"error writing messages to %s (partition %d, attempt %d): %s\", key.topic, key.partition, attempt, err)\n\t\t})\n\n\t\tif !isTemporary(err) && !isTransientNetworkError(err) {\n\t\t\tbreak\n\t\t}\n\t}\n\n\tif res != nil {\n\t\tfor i := range batch.msgs {\n\t\t\tm := &batch.msgs[i]\n\t\t\tm.Topic = key.topic\n\t\t\tm.Partition = int(key.partition)\n\t\t\tm.Offset = res.BaseOffset + int64(i)\n\n\t\t\tif m.Time.IsZero() {\n\t\t\t\tm.Time = res.LogAppendTime\n\t\t\t}\n\t\t}\n\t}\n\n\tif ptw.w.Completion != nil {\n\t\tptw.w.Completion(batch.msgs, err)\n\t}\n\n\tbatch.complete(err)\n}\n\nfunc (ptw *partitionWriter) close() {\n\tptw.mutex.Lock()\n\tdefer ptw.mutex.Unlock()\n\n\tif ptw.currBatch != nil {\n\t\tbatch := ptw.currBatch\n\t\tptw.queue.Put(batch)\n\t\tptw.currBatch = nil\n\t\tbatch.trigger()\n\t}\n\n\tptw.queue.Close()\n}\n\ntype writeBatch struct {\n\ttime time.Time\n\tmsgs []Message\n\tsize int\n\tbytes int64\n\tready chan struct{}\n\tdone chan struct{}\n\ttimer *time.Timer\n\terr error // result of the batch completion\n}\n\nfunc newWriteBatch(now time.Time, timeout time.Duration) *writeBatch {\n\treturn &writeBatch{\n\t\ttime: now,\n\t\tready: make(chan struct{}),\n\t\tdone: make(chan struct{}),\n\t\ttimer: time.NewTimer(timeout),\n\t}\n}\n\nfunc (b *writeBatch) add(msg Message, maxSize int, maxBytes int64) bool {\n\tbytes := int64(msg.totalSize())\n\n\tif b.size > 0 && (b.bytes+bytes) > maxBytes {\n\t\treturn false\n\t}\n\n\tif cap(b.msgs) == 0 {\n\t\tb.msgs = make([]Message, 0, maxSize)\n\t}\n\n\tb.msgs = append(b.msgs, msg)\n\tb.size++\n\tb.bytes += bytes\n\treturn true\n}\n\nfunc (b *writeBatch) full(maxSize int, maxBytes int64) bool {\n\treturn b.size >= maxSize || b.bytes >= maxBytes\n}\n\nfunc (b *writeBatch) trigger() {\n\tclose(b.ready)\n}\n\nfunc (b *writeBatch) complete(err error) {\n\tb.err = err\n\tclose(b.done)\n}\n\ntype writerRecords struct {\n\tmsgs []Message\n\tindex int\n\trecord Record\n\tkey bytesReadCloser\n\tvalue bytesReadCloser\n}\n\nfunc (r *writerRecords) ReadRecord() (*Record, error) {\n\tif r.index >= 0 && r.index < len(r.msgs) {\n\t\tm := &r.msgs[r.index]\n\t\tr.index++\n\t\tr.record = Record{\n\t\t\tTime: m.Time,\n\t\t\tHeaders: m.Headers,\n\t\t}\n\t\tif m.Key != nil {\n\t\t\tr.key.Reset(m.Key)\n\t\t\tr.record.Key = &r.key\n\t\t}\n\t\tif m.Value != nil {\n\t\t\tr.value.Reset(m.Value)\n\t\t\tr.record.Value = &r.value\n\t\t}\n\t\treturn &r.record, nil\n\t}\n\treturn nil, io.EOF\n}\n\ntype bytesReadCloser struct{ bytes.Reader }\n\nfunc (*bytesReadCloser) Close() error { return nil }\n\n// A cache of []int values passed to balancers of writers, used to amortize the\n// heap allocation of the partition index lists.\n//\n// With hindsight, the use of `...int` to pass the partition list to Balancers\n// was not the best design choice: kafka partition numbers are monotonically\n// increasing, we could have simply passed the number of partitions instead.\n// If we ever revisit this API, we can hopefully remove this cache.\nvar partitionsCache atomic.Value\n\nfunc loadCachedPartitions(numPartitions int) []int {\n\tpartitions, ok := partitionsCache.Load().([]int)\n\tif ok && len(partitions) >= numPartitions {\n\t\treturn partitions[:numPartitions]\n\t}\n\n\tconst alignment = 128\n\tn := ((numPartitions / alignment) + 1) * alignment\n\n\tpartitions = make([]int, n)\n\tfor i := range partitions {\n\t\tpartitions[i] = i\n\t}\n\n\tpartitionsCache.Store(partitions)\n\treturn partitions[:numPartitions]\n}\n" }, { "path": "writer_test.go", "content": "package kafka\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/sasl/plain\"\n)\n\nfunc TestBatchQueue(t *testing.T) {\n\ttests := []struct {\n\t\tscenario string\n\t\tfunction func(*testing.T)\n\t}{\n\t\t{\n\t\t\tscenario: \"the remaining items in a queue can be gotten after closing\",\n\t\t\tfunction: testBatchQueueGetWorksAfterClose,\n\t\t},\n\t\t{\n\t\t\tscenario: \"putting into a closed queue fails\",\n\t\t\tfunction: testBatchQueuePutAfterCloseFails,\n\t\t},\n\t\t{\n\t\t\tscenario: \"putting into a queue awakes a goroutine in a get call\",\n\t\t\tfunction: testBatchQueuePutWakesSleepingGetter,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\ttestFunc := test.function\n\t\tt.Run(test.scenario, func(t *testing.T) {\n\t\t\tt.Parallel()\n\t\t\ttestFunc(t)\n\t\t})\n\t}\n}\n\nfunc testBatchQueuePutWakesSleepingGetter(t *testing.T) {\n\tbq := newBatchQueue(10)\n\tvar wg sync.WaitGroup\n\tready := make(chan struct{})\n\tvar batch *writeBatch\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tclose(ready)\n\t\tbatch = bq.Get()\n\t}()\n\t<-ready\n\tbq.Put(newWriteBatch(time.Now(), time.Hour*100))\n\twg.Wait()\n\tif batch == nil {\n\t\tt.Fatal(\"got nil batch\")\n\t}\n}\n\nfunc testBatchQueuePutAfterCloseFails(t *testing.T) {\n\tbq := newBatchQueue(10)\n\tbq.Close()\n\tif put := bq.Put(newWriteBatch(time.Now(), time.Hour*100)); put {\n\t\tt.Fatal(\"put batch into closed queue\")\n\t}\n}\n\nfunc testBatchQueueGetWorksAfterClose(t *testing.T) {\n\tbq := newBatchQueue(10)\n\tenqueueBatches := []*writeBatch{\n\t\tnewWriteBatch(time.Now(), time.Hour*100),\n\t\tnewWriteBatch(time.Now(), time.Hour*100),\n\t}\n\n\tfor _, batch := range enqueueBatches {\n\t\tput := bq.Put(batch)\n\t\tif !put {\n\t\t\tt.Fatal(\"failed to put batch into queue\")\n\t\t}\n\t}\n\n\tbq.Close()\n\n\tbatchesGotten := 0\n\tfor batchesGotten != 2 {\n\t\tdequeueBatch := bq.Get()\n\t\tif dequeueBatch == nil {\n\t\t\tt.Fatalf(\"no batch returned from get\")\n\t\t}\n\t\tbatchesGotten++\n\t}\n}\n\nfunc TestWriter(t *testing.T) {\n\ttests := []struct {\n\t\tscenario string\n\t\tfunction func(*testing.T)\n\t}{\n\t\t{\n\t\t\tscenario: \"closing a writer right after creating it returns promptly with no error\",\n\t\t\tfunction: testWriterClose,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"writing 1 message through a writer using round-robin balancing produces 1 message to the first partition\",\n\t\t\tfunction: testWriterRoundRobin1,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"running out of max attempts should return an error\",\n\t\t\tfunction: testWriterMaxAttemptsErr,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"writing a message larger then the max bytes should return an error\",\n\t\t\tfunction: testWriterMaxBytes,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"writing a batch of message based on batch byte size\",\n\t\t\tfunction: testWriterBatchBytes,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"writing a batch of messages\",\n\t\t\tfunction: testWriterBatchSize,\n\t\t},\n\n\t\t{\n\t\t\tscenario: \"writing messages with a small batch byte size\",\n\t\t\tfunction: testWriterSmallBatchBytes,\n\t\t},\n\t\t{\n\t\t\tscenario: \"writing messages with headers\",\n\t\t\tfunction: testWriterBatchBytesHeaders,\n\t\t},\n\t\t{\n\t\t\tscenario: \"setting a non default balancer on the writer\",\n\t\t\tfunction: testWriterSetsRightBalancer,\n\t\t},\n\t\t{\n\t\t\tscenario: \"setting RequiredAcks to None in Writer does not cause a panic\",\n\t\t\tfunction: testWriterRequiredAcksNone,\n\t\t},\n\t\t{\n\t\t\tscenario: \"writing messages to multiple topics\",\n\t\t\tfunction: testWriterMultipleTopics,\n\t\t},\n\t\t{\n\t\t\tscenario: \"writing messages without specifying a topic\",\n\t\t\tfunction: testWriterMissingTopic,\n\t\t},\n\t\t{\n\t\t\tscenario: \"specifying topic for message when already set for writer\",\n\t\t\tfunction: testWriterUnexpectedMessageTopic,\n\t\t},\n\t\t{\n\t\t\tscenario: \"writing a message to an invalid partition\",\n\t\t\tfunction: testWriterInvalidPartition,\n\t\t},\n\t\t{\n\t\t\tscenario: \"writing a message to a non-existent topic creates the topic\",\n\t\t\tfunction: testWriterAutoCreateTopic,\n\t\t},\n\t\t{\n\t\t\tscenario: \"terminates on an attempt to write a message to a nonexistent topic\",\n\t\t\tfunction: testWriterTerminateMissingTopic,\n\t\t},\n\t\t{\n\t\t\tscenario: \"writing a message with SASL Plain authentication\",\n\t\t\tfunction: testWriterSasl,\n\t\t},\n\t\t{\n\t\t\tscenario: \"test default configuration values\",\n\t\t\tfunction: testWriterDefaults,\n\t\t},\n\t\t{\n\t\t\tscenario: \"test default stats values\",\n\t\t\tfunction: testWriterDefaultStats,\n\t\t},\n\t\t{\n\t\t\tscenario: \"test stats values with override config\",\n\t\t\tfunction: testWriterOverrideConfigStats,\n\t\t},\n\t\t{\n\t\t\tscenario: \"test write message with writer data\",\n\t\t\tfunction: testWriteMessageWithWriterData,\n\t\t},\n\t}\n\n\tfor _, test := range tests {\n\t\ttestFunc := test.function\n\t\tt.Run(test.scenario, func(t *testing.T) {\n\t\t\tt.Parallel()\n\t\t\ttestFunc(t)\n\t\t})\n\t}\n}\n\nfunc newTestWriter(config WriterConfig) *Writer {\n\tif len(config.Brokers) == 0 {\n\t\tconfig.Brokers = []string{\"localhost:9092\"}\n\t}\n\treturn NewWriter(config)\n}\n\nfunc testWriterClose(t *testing.T) {\n\tconst topic = \"test-writer-0\"\n\tcreateTopic(t, topic, 1)\n\tdefer deleteTopic(t, topic)\n\n\tw := newTestWriter(WriterConfig{\n\t\tTopic: topic,\n\t})\n\n\tif err := w.Close(); err != nil {\n\t\tt.Error(err)\n\t}\n}\n\nfunc testWriterRequiredAcksNone(t *testing.T) {\n\ttopic := makeTopic()\n\tcreateTopic(t, topic, 1)\n\tdefer deleteTopic(t, topic)\n\n\ttransport := &Transport{}\n\tdefer transport.CloseIdleConnections()\n\n\twriter := &Writer{\n\t\tAddr: TCP(\"localhost:9092\"),\n\t\tTopic: topic,\n\t\tBalancer: &RoundRobin{},\n\t\tRequiredAcks: RequireNone,\n\t\tTransport: transport,\n\t}\n\tdefer writer.Close()\n\n\tmsg := Message{\n\t\tKey: []byte(\"ThisIsAKey\"),\n\t\tValue: []byte(\"Test message for required acks test\"),\n\t}\n\n\terr := writer.WriteMessages(context.Background(), msg)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n}\n\nfunc testWriterSetsRightBalancer(t *testing.T) {\n\tconst topic = \"test-writer-1\"\n\tbalancer := &CRC32Balancer{}\n\tw := newTestWriter(WriterConfig{\n\t\tTopic: topic,\n\t\tBalancer: balancer,\n\t})\n\tdefer w.Close()\n\n\tif w.Balancer != balancer {\n\t\tt.Errorf(\"Balancer not set correctly\")\n\t}\n}\n\nfunc testWriterRoundRobin1(t *testing.T) {\n\tconst topic = \"test-writer-1\"\n\tcreateTopic(t, topic, 1)\n\tdefer deleteTopic(t, topic)\n\n\toffset, err := readOffset(topic, 0)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tw := newTestWriter(WriterConfig{\n\t\tTopic: topic,\n\t\tBalancer: &RoundRobin{},\n\t})\n\tdefer w.Close()\n\n\tif err := w.WriteMessages(context.Background(), Message{\n\t\tValue: []byte(\"Hello World!\"),\n\t}); err != nil {\n\t\tt.Error(err)\n\t\treturn\n\t}\n\n\tmsgs, err := readPartition(topic, 0, offset)\n\tif err != nil {\n\t\tt.Error(\"error reading partition\", err)\n\t\treturn\n\t}\n\n\tif len(msgs) != 1 {\n\t\tt.Error(\"bad messages in partition\", msgs)\n\t\treturn\n\t}\n\n\tfor _, m := range msgs {\n\t\tif string(m.Value) != \"Hello World!\" {\n\t\t\tt.Error(\"bad messages in partition\", msgs)\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nfunc TestValidateWriter(t *testing.T) {\n\ttests := []struct {\n\t\tconfig WriterConfig\n\t\terrorOccured bool\n\t}{\n\t\t{config: WriterConfig{}, errorOccured: true},\n\t\t{config: WriterConfig{Brokers: []string{\"broker1\", \"broker2\"}}, errorOccured: false},\n\t\t{config: WriterConfig{Brokers: []string{\"broker1\"}, Topic: \"topic1\"}, errorOccured: false},\n\t}\n\tfor _, test := range tests {\n\t\terr := test.config.Validate()\n\t\tif test.errorOccured && err == nil {\n\t\t\tt.Fail()\n\t\t}\n\t\tif !test.errorOccured && err != nil {\n\t\t\tt.Fail()\n\t\t}\n\t}\n}\n\nfunc testWriterMaxAttemptsErr(t *testing.T) {\n\ttopic := makeTopic()\n\tcreateTopic(t, topic, 1)\n\tdefer deleteTopic(t, topic)\n\n\tw := newTestWriter(WriterConfig{\n\t\tBrokers: []string{\"localhost:9999\"}, // nothing is listening here\n\t\tTopic: topic,\n\t\tMaxAttempts: 3,\n\t\tBalancer: &RoundRobin{},\n\t})\n\tdefer w.Close()\n\n\tif err := w.WriteMessages(context.Background(), Message{\n\t\tValue: []byte(\"Hello World!\"),\n\t}); err == nil {\n\t\tt.Error(\"expected error\")\n\t\treturn\n\t}\n}\n\nfunc testWriterMaxBytes(t *testing.T) {\n\ttopic := makeTopic()\n\tcreateTopic(t, topic, 1)\n\tdefer deleteTopic(t, topic)\n\n\tw := newTestWriter(WriterConfig{\n\t\tTopic: topic,\n\t\tBatchBytes: 25,\n\t})\n\tdefer w.Close()\n\n\tif err := w.WriteMessages(context.Background(), Message{\n\t\tValue: []byte(\"Hi\"),\n\t}); err != nil {\n\t\tt.Error(err)\n\t\treturn\n\t}\n\n\tfirstMsg := []byte(\"Hello World!\")\n\tsecondMsg := []byte(\"LeftOver!\")\n\tmsgs := []Message{\n\t\t{\n\t\t\tValue: firstMsg,\n\t\t},\n\t\t{\n\t\t\tValue: secondMsg,\n\t\t},\n\t}\n\tif err := w.WriteMessages(context.Background(), msgs...); err == nil {\n\t\tt.Error(\"expected error\")\n\t\treturn\n\t} else if err != nil {\n\t\tvar e MessageTooLargeError\n\t\tswitch {\n\t\tcase errors.As(err, &e):\n\t\t\tif string(e.Message.Value) != string(firstMsg) {\n\t\t\t\tt.Errorf(\"unxpected returned message. Expected: %s, Got %s\", firstMsg, e.Message.Value)\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif len(e.Remaining) != 1 {\n\t\t\t\tt.Error(\"expected remaining errors; found none\")\n\t\t\t\treturn\n\t\t\t}\n\t\t\tif string(e.Remaining[0].Value) != string(secondMsg) {\n\t\t\t\tt.Errorf(\"unxpected returned message. Expected: %s, Got %s\", secondMsg, e.Message.Value)\n\t\t\t\treturn\n\t\t\t}\n\n\t\tdefault:\n\t\t\tt.Errorf(\"unexpected error: %s\", err)\n\t\t\treturn\n\t\t}\n\t}\n}\n\n// readOffset gets the latest offset for the given topic/partition.\nfunc readOffset(topic string, partition int) (offset int64, err error) {\n\tvar conn *Conn\n\n\tctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)\n\tdefer cancel()\n\n\tif conn, err = DialLeader(ctx, \"tcp\", \"localhost:9092\", topic, partition); err != nil {\n\t\terr = fmt.Errorf(\"readOffset, DialLeader: %w\", err)\n\t\treturn\n\t}\n\tdefer conn.Close()\n\n\toffset, err = conn.ReadLastOffset()\n\tif err != nil {\n\t\terr = fmt.Errorf(\"readOffset, conn.ReadLastOffset: %w\", err)\n\t}\n\treturn\n}\n\nfunc readPartition(topic string, partition int, offset int64) (msgs []Message, err error) {\n\tvar conn *Conn\n\n\tif conn, err = DialLeader(context.Background(), \"tcp\", \"localhost:9092\", topic, partition); err != nil {\n\t\treturn\n\t}\n\tdefer conn.Close()\n\n\tconn.Seek(offset, SeekAbsolute)\n\tconn.SetReadDeadline(time.Now().Add(10 * time.Second))\n\tbatch := conn.ReadBatch(0, 1000000000)\n\tdefer batch.Close()\n\n\tfor {\n\t\tvar msg Message\n\n\t\tif msg, err = batch.ReadMessage(); err != nil {\n\t\t\tif errors.Is(err, io.EOF) {\n\t\t\t\terr = nil\n\t\t\t}\n\t\t\treturn\n\t\t}\n\n\t\tmsgs = append(msgs, msg)\n\t}\n}\n\nfunc testWriterBatchBytes(t *testing.T) {\n\ttopic := makeTopic()\n\tcreateTopic(t, topic, 1)\n\tdefer deleteTopic(t, topic)\n\n\toffset, err := readOffset(topic, 0)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tw := newTestWriter(WriterConfig{\n\t\tTopic: topic,\n\t\tBatchBytes: 50,\n\t\tBatchTimeout: math.MaxInt32 * time.Second,\n\t\tBalancer: &RoundRobin{},\n\t})\n\tdefer w.Close()\n\n\tctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)\n\tdefer cancel()\n\tif err := w.WriteMessages(ctx, []Message{\n\t\t{Value: []byte(\"M0\")}, // 25 Bytes\n\t\t{Value: []byte(\"M1\")}, // 25 Bytes\n\t\t{Value: []byte(\"M2\")}, // 25 Bytes\n\t\t{Value: []byte(\"M3\")}, // 25 Bytes\n\t}...); err != nil {\n\t\tt.Error(err)\n\t\treturn\n\t}\n\n\tif w.Stats().Writes != 2 {\n\t\tt.Error(\"didn't create expected batches\")\n\t\treturn\n\t}\n\tmsgs, err := readPartition(topic, 0, offset)\n\tif err != nil {\n\t\tt.Error(\"error reading partition\", err)\n\t\treturn\n\t}\n\n\tif len(msgs) != 4 {\n\t\tt.Error(\"bad messages in partition\", msgs)\n\t\treturn\n\t}\n\n\tfor i, m := range msgs {\n\t\tif string(m.Value) == \"M\"+strconv.Itoa(i) {\n\t\t\tcontinue\n\t\t}\n\t\tt.Error(\"bad messages in partition\", string(m.Value))\n\t}\n}\n\nfunc testWriterBatchSize(t *testing.T) {\n\ttopic := makeTopic()\n\tcreateTopic(t, topic, 1)\n\tdefer deleteTopic(t, topic)\n\n\toffset, err := readOffset(topic, 0)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tw := newTestWriter(WriterConfig{\n\t\tTopic: topic,\n\t\tBatchSize: 2,\n\t\tBatchTimeout: math.MaxInt32 * time.Second,\n\t\tBalancer: &RoundRobin{},\n\t})\n\tdefer w.Close()\n\n\tctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)\n\tdefer cancel()\n\tif err := w.WriteMessages(ctx, []Message{\n\t\t{Value: []byte(\"Hi\")}, // 24 Bytes\n\t\t{Value: []byte(\"By\")}, // 24 Bytes\n\t}...); err != nil {\n\t\tt.Error(err)\n\t\treturn\n\t}\n\n\tif w.Stats().Writes > 1 {\n\t\tt.Error(\"didn't batch messages\")\n\t\treturn\n\t}\n\tmsgs, err := readPartition(topic, 0, offset)\n\tif err != nil {\n\t\tt.Error(\"error reading partition\", err)\n\t\treturn\n\t}\n\n\tif len(msgs) != 2 {\n\t\tt.Error(\"bad messages in partition\", msgs)\n\t\treturn\n\t}\n\n\tfor _, m := range msgs {\n\t\tif string(m.Value) == \"Hi\" || string(m.Value) == \"By\" {\n\t\t\tcontinue\n\t\t}\n\t\tt.Error(\"bad messages in partition\", msgs)\n\t}\n}\n\nfunc testWriterSmallBatchBytes(t *testing.T) {\n\ttopic := makeTopic()\n\tcreateTopic(t, topic, 1)\n\tdefer deleteTopic(t, topic)\n\n\toffset, err := readOffset(topic, 0)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tw := newTestWriter(WriterConfig{\n\t\tTopic: topic,\n\t\tBatchBytes: 25,\n\t\tBatchTimeout: 50 * time.Millisecond,\n\t\tBalancer: &RoundRobin{},\n\t})\n\tdefer w.Close()\n\n\tctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)\n\tdefer cancel()\n\tif err := w.WriteMessages(ctx, []Message{\n\t\t{Value: []byte(\"Hi\")}, // 24 Bytes\n\t\t{Value: []byte(\"By\")}, // 24 Bytes\n\t}...); err != nil {\n\t\tt.Error(err)\n\t\treturn\n\t}\n\tws := w.Stats()\n\tif ws.Writes != 2 {\n\t\tt.Error(\"didn't batch messages; Writes: \", ws.Writes)\n\t\treturn\n\t}\n\tmsgs, err := readPartition(topic, 0, offset)\n\tif err != nil {\n\t\tt.Error(\"error reading partition\", err)\n\t\treturn\n\t}\n\n\tif len(msgs) != 2 {\n\t\tt.Error(\"bad messages in partition\", msgs)\n\t\treturn\n\t}\n\n\tfor _, m := range msgs {\n\t\tif string(m.Value) == \"Hi\" || string(m.Value) == \"By\" {\n\t\t\tcontinue\n\t\t}\n\t\tt.Error(\"bad messages in partition\", msgs)\n\t}\n}\n\nfunc testWriterBatchBytesHeaders(t *testing.T) {\n\ttopic := makeTopic()\n\tcreateTopic(t, topic, 1)\n\tdefer deleteTopic(t, topic)\n\n\toffset, err := readOffset(topic, 0)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tw := newTestWriter(WriterConfig{\n\t\tTopic: topic,\n\t\tBatchBytes: 100,\n\t\tBatchTimeout: 50 * time.Millisecond,\n\t\tBalancer: &RoundRobin{},\n\t})\n\tdefer w.Close()\n\n\tctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)\n\tdefer cancel()\n\tif err := w.WriteMessages(ctx, []Message{\n\t\t{\n\t\t\tValue: []byte(\"Hello World 1\"),\n\t\t\tHeaders: []Header{\n\t\t\t\t{Key: \"User-Agent\", Value: []byte(\"abc/xyz\")},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tValue: []byte(\"Hello World 2\"),\n\t\t\tHeaders: []Header{\n\t\t\t\t{Key: \"User-Agent\", Value: []byte(\"abc/xyz\")},\n\t\t\t},\n\t\t},\n\t}...); err != nil {\n\t\tt.Error(err)\n\t\treturn\n\t}\n\tws := w.Stats()\n\tif ws.Writes != 2 {\n\t\tt.Error(\"didn't batch messages; Writes: \", ws.Writes)\n\t\treturn\n\t}\n\tmsgs, err := readPartition(topic, 0, offset)\n\tif err != nil {\n\t\tt.Error(\"error reading partition\", err)\n\t\treturn\n\t}\n\n\tif len(msgs) != 2 {\n\t\tt.Error(\"bad messages in partition\", msgs)\n\t\treturn\n\t}\n\n\tfor _, m := range msgs {\n\t\tif strings.HasPrefix(string(m.Value), \"Hello World\") {\n\t\t\tcontinue\n\t\t}\n\t\tt.Error(\"bad messages in partition\", msgs)\n\t}\n}\n\nfunc testWriterMultipleTopics(t *testing.T) {\n\ttopic1 := makeTopic()\n\tcreateTopic(t, topic1, 1)\n\tdefer deleteTopic(t, topic1)\n\n\toffset1, err := readOffset(topic1, 0)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\ttopic2 := makeTopic()\n\tcreateTopic(t, topic2, 1)\n\tdefer deleteTopic(t, topic2)\n\n\toffset2, err := readOffset(topic2, 0)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tw := newTestWriter(WriterConfig{\n\t\tBalancer: &RoundRobin{},\n\t})\n\tdefer w.Close()\n\n\tmsg1 := Message{Topic: topic1, Value: []byte(\"Hello\")}\n\tmsg2 := Message{Topic: topic2, Value: []byte(\"World\")}\n\n\tctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)\n\tdefer cancel()\n\tif err := w.WriteMessages(ctx, msg1, msg2); err != nil {\n\t\tt.Error(err)\n\t\treturn\n\t}\n\tws := w.Stats()\n\tif ws.Writes != 2 {\n\t\tt.Error(\"didn't batch messages; Writes: \", ws.Writes)\n\t\treturn\n\t}\n\n\tmsgs1, err := readPartition(topic1, 0, offset1)\n\tif err != nil {\n\t\tt.Error(\"error reading partition\", err)\n\t\treturn\n\t}\n\tif len(msgs1) != 1 {\n\t\tt.Error(\"bad messages in partition\", msgs1)\n\t\treturn\n\t}\n\tif string(msgs1[0].Value) != \"Hello\" {\n\t\tt.Error(\"bad message in partition\", msgs1)\n\t}\n\n\tmsgs2, err := readPartition(topic2, 0, offset2)\n\tif err != nil {\n\t\tt.Error(\"error reading partition\", err)\n\t\treturn\n\t}\n\tif len(msgs2) != 1 {\n\t\tt.Error(\"bad messages in partition\", msgs2)\n\t\treturn\n\t}\n\tif string(msgs2[0].Value) != \"World\" {\n\t\tt.Error(\"bad message in partition\", msgs2)\n\t}\n}\n\nfunc testWriterMissingTopic(t *testing.T) {\n\tctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)\n\tdefer cancel()\n\n\tw := newTestWriter(WriterConfig{\n\t\t// no topic\n\t\tBalancer: &RoundRobin{},\n\t})\n\tdefer w.Close()\n\n\tmsg := Message{Value: []byte(\"Hello World\")} // no topic\n\n\tif err := w.WriteMessages(ctx, msg); err == nil {\n\t\tt.Error(\"expected error\")\n\t\treturn\n\t}\n}\n\nfunc testWriterInvalidPartition(t *testing.T) {\n\tctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)\n\tdefer cancel()\n\n\ttopic := makeTopic()\n\tcreateTopic(t, topic, 1)\n\tdefer deleteTopic(t, topic)\n\n\tw := newTestWriter(WriterConfig{\n\t\tTopic: topic,\n\t\tMaxAttempts: 1, // only try once to get the error back immediately\n\t\tBalancer: &staticBalancer{partition: -1}, // intentionally invalid partition\n\t})\n\tdefer w.Close()\n\n\tmsg := Message{\n\t\tValue: []byte(\"Hello World!\"),\n\t}\n\n\t// this call should return an error and not panic (see issue #517)\n\tif err := w.WriteMessages(ctx, msg); err == nil {\n\t\tt.Fatal(\"expected error attempting to write message\")\n\t}\n}\n\nfunc testWriterUnexpectedMessageTopic(t *testing.T) {\n\tctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)\n\tdefer cancel()\n\n\ttopic := makeTopic()\n\tcreateTopic(t, topic, 1)\n\tdefer deleteTopic(t, topic)\n\n\tw := newTestWriter(WriterConfig{\n\t\tTopic: topic,\n\t\tBalancer: &RoundRobin{},\n\t})\n\tdefer w.Close()\n\n\tmsg := Message{Topic: \"should-fail\", Value: []byte(\"Hello World\")}\n\n\tif err := w.WriteMessages(ctx, msg); err == nil {\n\t\tt.Error(\"expected error\")\n\t\treturn\n\t}\n}\n\nfunc testWriteMessageWithWriterData(t *testing.T) {\n\ttopic := makeTopic()\n\tcreateTopic(t, topic, 1)\n\tdefer deleteTopic(t, topic)\n\tw := newTestWriter(WriterConfig{\n\t\tTopic: topic,\n\t\tBalancer: &RoundRobin{},\n\t})\n\tdefer w.Close()\n\n\tindex := 0\n\tw.Completion = func(messages []Message, err error) {\n\t\tif err != nil {\n\t\t\tt.Errorf(\"unexpected error %v\", err)\n\t\t}\n\n\t\tfor _, msg := range messages {\n\t\t\tmeta := msg.WriterData.(int)\n\t\t\tif index != meta {\n\t\t\t\tt.Errorf(\"metadata is not correct, index = %d, writerData = %d\", index, meta)\n\t\t\t}\n\t\t\tindex += 1\n\t\t}\n\t}\n\n\tmsg := Message{Key: []byte(\"key\"), Value: []byte(\"Hello World\")}\n\tfor i := 0; i < 5; i++ {\n\t\tctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)\n\t\tdefer cancel()\n\n\t\tmsg.WriterData = i\n\t\terr := w.WriteMessages(ctx, msg)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"unexpected error %v\", err)\n\t\t}\n\t}\n\n}\n\nfunc testWriterAutoCreateTopic(t *testing.T) {\n\ttopic := makeTopic()\n\t// Assume it's going to get created.\n\tdefer deleteTopic(t, topic)\n\n\tw := newTestWriter(WriterConfig{\n\t\tTopic: topic,\n\t\tBalancer: &RoundRobin{},\n\t})\n\tw.AllowAutoTopicCreation = true\n\tdefer w.Close()\n\n\tmsg := Message{Key: []byte(\"key\"), Value: []byte(\"Hello World\")}\n\n\tvar err error\n\tconst retries = 5\n\tfor i := 0; i < retries; i++ {\n\t\tctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)\n\t\tdefer cancel()\n\t\terr = w.WriteMessages(ctx, msg)\n\t\tif errors.Is(err, LeaderNotAvailable) || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, UnknownTopicOrPartition) {\n\t\t\ttime.Sleep(time.Millisecond * 250)\n\t\t\tcontinue\n\t\t}\n\n\t\tif err != nil {\n\t\t\tt.Errorf(\"unexpected error %v\", err)\n\t\t\treturn\n\t\t}\n\t}\n\tif err != nil {\n\t\tt.Errorf(\"unable to create topic %v\", err)\n\t}\n}\n\nfunc testWriterTerminateMissingTopic(t *testing.T) {\n\ttopic := makeTopic()\n\n\ttransport := &Transport{}\n\tdefer transport.CloseIdleConnections()\n\n\twriter := &Writer{\n\t\tAddr: TCP(\"localhost:9092\"),\n\t\tTopic: topic,\n\t\tBalancer: &RoundRobin{},\n\t\tRequiredAcks: RequireNone,\n\t\tAllowAutoTopicCreation: false,\n\t\tTransport: transport,\n\t}\n\tdefer writer.Close()\n\n\tmsg := Message{Value: []byte(\"FooBar\")}\n\n\tif err := writer.WriteMessages(context.Background(), msg); err == nil {\n\t\tt.Fatal(\"Kafka error [3] UNKNOWN_TOPIC_OR_PARTITION is expected\")\n\t\treturn\n\t}\n}\n\nfunc testWriterSasl(t *testing.T) {\n\ttopic := makeTopic()\n\tdefer deleteTopic(t, topic)\n\tdialer := &Dialer{\n\t\tTimeout: 10 * time.Second,\n\t\tSASLMechanism: plain.Mechanism{\n\t\t\tUsername: \"adminplain\",\n\t\t\tPassword: \"admin-secret\",\n\t\t},\n\t}\n\n\tw := newTestWriter(WriterConfig{\n\t\tDialer: dialer,\n\t\tTopic: topic,\n\t\tBrokers: []string{\"localhost:9093\"},\n\t})\n\n\tw.AllowAutoTopicCreation = true\n\n\tdefer w.Close()\n\n\tmsg := Message{Key: []byte(\"key\"), Value: []byte(\"Hello World\")}\n\n\tvar err error\n\tconst retries = 5\n\tfor i := 0; i < retries; i++ {\n\t\tctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)\n\t\tdefer cancel()\n\t\terr = w.WriteMessages(ctx, msg)\n\t\tif errors.Is(err, LeaderNotAvailable) || errors.Is(err, context.DeadlineExceeded) || errors.Is(err, UnknownTopicOrPartition) {\n\t\t\ttime.Sleep(time.Millisecond * 250)\n\t\t\tcontinue\n\t\t}\n\n\t\tif err != nil {\n\t\t\tt.Errorf(\"unexpected error %v\", err)\n\t\t\treturn\n\t\t}\n\t}\n\tif err != nil {\n\t\tt.Errorf(\"unable to create topic %v\", err)\n\t}\n}\n\nfunc testWriterDefaults(t *testing.T) {\n\tw := &Writer{}\n\tdefer w.Close()\n\n\tif w.writeBackoffMin() != 100*time.Millisecond {\n\t\tt.Error(\"Incorrect default min write backoff delay\")\n\t}\n\n\tif w.writeBackoffMax() != 1*time.Second {\n\t\tt.Error(\"Incorrect default max write backoff delay\")\n\t}\n}\n\nfunc testWriterDefaultStats(t *testing.T) {\n\tw := &Writer{}\n\tdefer w.Close()\n\n\tstats := w.Stats()\n\n\tif stats.MaxAttempts == 0 {\n\t\tt.Error(\"Incorrect default MaxAttempts value\")\n\t}\n\n\tif stats.WriteBackoffMin == 0 {\n\t\tt.Error(\"Incorrect default WriteBackoffMin value\")\n\t}\n\n\tif stats.WriteBackoffMax == 0 {\n\t\tt.Error(\"Incorrect default WriteBackoffMax value\")\n\t}\n\n\tif stats.MaxBatchSize == 0 {\n\t\tt.Error(\"Incorrect default MaxBatchSize value\")\n\t}\n\n\tif stats.BatchTimeout == 0 {\n\t\tt.Error(\"Incorrect default BatchTimeout value\")\n\t}\n\n\tif stats.ReadTimeout == 0 {\n\t\tt.Error(\"Incorrect default ReadTimeout value\")\n\t}\n\n\tif stats.WriteTimeout == 0 {\n\t\tt.Error(\"Incorrect default WriteTimeout value\")\n\t}\n}\n\nfunc testWriterOverrideConfigStats(t *testing.T) {\n\tw := &Writer{\n\t\tMaxAttempts: 6,\n\t\tWriteBackoffMin: 2,\n\t\tWriteBackoffMax: 4,\n\t\tBatchSize: 1024,\n\t\tBatchTimeout: 16,\n\t\tReadTimeout: 24,\n\t\tWriteTimeout: 32,\n\t}\n\tdefer w.Close()\n\n\tstats := w.Stats()\n\n\tif stats.MaxAttempts != 6 {\n\t\tt.Error(\"Incorrect MaxAttempts value\")\n\t}\n\n\tif stats.WriteBackoffMin != 2 {\n\t\tt.Error(\"Incorrect WriteBackoffMin value\")\n\t}\n\n\tif stats.WriteBackoffMax != 4 {\n\t\tt.Error(\"Incorrect WriteBackoffMax value\")\n\t}\n\n\tif stats.MaxBatchSize != 1024 {\n\t\tt.Error(\"Incorrect MaxBatchSize value\")\n\t}\n\n\tif stats.BatchTimeout != 16 {\n\t\tt.Error(\"Incorrect BatchTimeout value\")\n\t}\n\n\tif stats.ReadTimeout != 24 {\n\t\tt.Error(\"Incorrect ReadTimeout value\")\n\t}\n\n\tif stats.WriteTimeout != 32 {\n\t\tt.Error(\"Incorrect WriteTimeout value\")\n\t}\n}\n\ntype staticBalancer struct {\n\tpartition int\n}\n\nfunc (b *staticBalancer) Balance(_ Message, partitions ...int) int {\n\treturn b.partition\n}\n" }, { "path": "zstd/zstd.go", "content": "// Package zstd does nothing, it's kept for backward compatibility to avoid\n// breaking the majority of programs that imported it to install the compression\n// codec, which is now always included.\npackage zstd\n\nimport \"github.com/segmentio/kafka-go/compress/zstd\"\n\nconst (\n\tCode = 4\n\tDefaultCompressionLevel = 3\n)\n\ntype CompressionCodec = zstd.Codec\n\nfunc NewCompressionCodec() *CompressionCodec {\n\treturn NewCompressionCodecWith(DefaultCompressionLevel)\n}\n\nfunc NewCompressionCodecWith(level int) *CompressionCodec {\n\treturn &CompressionCodec{Level: level}\n}\n" } ]