Showing preview only (1,434K chars total). Download the full file or copy to clipboard to get everything.
Repository: segmentio/kafka-go
Branch: main
Commit: 3ce29796ef96
Files: 366
Total size: 1.3 MB
Directory structure:
gitextract_308eihdc/
├── .circleci/
│ └── config.yml
├── .gitattributes
├── .github/
│ └── ISSUE_TEMPLATE/
│ ├── bug_report.md
│ └── feature_request.md
├── .gitignore
├── .golangci.yml
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── LICENSE
├── Makefile
├── README.md
├── addoffsetstotxn.go
├── addoffsetstotxn_test.go
├── addpartitionstotxn.go
├── addpartitionstotxn_test.go
├── address.go
├── address_test.go
├── alterclientquotas.go
├── alterclientquotas_test.go
├── alterconfigs.go
├── alterconfigs_test.go
├── alterpartitionreassignments.go
├── alterpartitionreassignments_test.go
├── alteruserscramcredentials.go
├── alteruserscramcredentials_test.go
├── apiversions.go
├── apiversions_test.go
├── balancer.go
├── balancer_test.go
├── batch.go
├── batch_test.go
├── buffer.go
├── builder_test.go
├── client.go
├── client_test.go
├── commit.go
├── commit_test.go
├── compress/
│ ├── compress.go
│ ├── compress_test.go
│ ├── gzip/
│ │ └── gzip.go
│ ├── lz4/
│ │ └── lz4.go
│ ├── snappy/
│ │ ├── go-xerial-snappy/
│ │ │ ├── LICENSE
│ │ │ ├── README.md
│ │ │ ├── corpus/
│ │ │ │ ├── 020dfb19a68cbcf99dc93dc1030068d4c9968ad0-2
│ │ │ │ ├── 05979b224be0294bf350310d4ba5257c9bb815db-3
│ │ │ │ ├── 0e64ca2823923c5efa03ff2bd6e0aa1018eeca3b-9
│ │ │ │ ├── 1
│ │ │ │ ├── 361a1c6d2a8f80780826c3d83ad391d0475c922f-4
│ │ │ │ ├── 4117af68228fa64339d362cf980c68ffadff96c8-12
│ │ │ │ ├── 4142249be82c8a617cf838eef05394ece39becd3-9
│ │ │ │ ├── 41ea8c7d904f1cd913b52e9ead4a96c639d76802-10
│ │ │ │ ├── 44083e1447694980c0ee682576e32358c9ee883f-2
│ │ │ │ ├── 4d6b359bd538feaa7d36c89235d07d0a443797ac-1
│ │ │ │ ├── 521e7e67b6063a75e0eeb24b0d1dd20731d34ad8-4
│ │ │ │ ├── 526e6f85d1b8777f0d9f70634c9f8b77fbdccdff-7
│ │ │ │ ├── 581b8fe7088f921567811fdf30e1f527c9f48e5e
│ │ │ │ ├── 60cd10738158020f5843b43960158c3d116b3a71-11
│ │ │ │ ├── 652b031b4b9d601235f86ef62523e63d733b8623-3
│ │ │ │ ├── 684a011f6fdfc7ae9863e12381165e82d2a2e356-9
│ │ │ │ ├── 72e42fc8e5eaed6a8a077f420fc3bd1f9a7c0919-1
│ │ │ │ ├── 80881d1b911b95e0203b3b0e7dc6360c35f7620f-7
│ │ │ │ ├── 8484b3082d522e0a1f315db1fa1b2a5118be7cc3-8
│ │ │ │ ├── 9635bb09260f100bc4a2ee4e3b980fecc5b874ce-1
│ │ │ │ ├── 99d36b0b5b1be7151a508dd440ec725a2576c41c-1
│ │ │ │ ├── 9d339eddb4e2714ea319c3fb571311cb95fdb067-6
│ │ │ │ ├── b2419fcb7a9aef359de67cb6bd2b8a8c1f5c100f-4
│ │ │ │ ├── c1951b29109ec1017f63535ce3699630f46f54e1-5
│ │ │ │ ├── cb806bc4f67316af02d6ae677332a3b6005a18da-5
│ │ │ │ ├── cd7dd228703739e9252c7ea76f1c5f82ab44686a-10
│ │ │ │ ├── ce3671e91907349cea04fc3f2a4b91c65b99461d-3
│ │ │ │ ├── ce3c6f4c31f74d72fbf74c17d14a8d29aa62059e-6
│ │ │ │ ├── da39a3ee5e6b4b0d3255bfef95601890afd80709-1
│ │ │ │ ├── e2230aa0ecaebb9b890440effa13f501a89247b2-1
│ │ │ │ ├── efa11d676fb2a77afb8eac3d7ed30e330a7c2efe-11
│ │ │ │ ├── f0445ac39e03978bbc8011316ac8468015ddb72c-1
│ │ │ │ └── f241da53c6bc1fe3368c55bf28db86ce15a2c784-2
│ │ │ ├── fuzz.go
│ │ │ ├── snappy.go
│ │ │ └── snappy_test.go
│ │ ├── snappy.go
│ │ ├── xerial.go
│ │ └── xerial_test.go
│ └── zstd/
│ └── zstd.go
├── compression.go
├── conn.go
├── conn_test.go
├── consumergroup.go
├── consumergroup_test.go
├── crc32.go
├── crc32_test.go
├── createacls.go
├── createacls_test.go
├── createpartitions.go
├── createpartitions_test.go
├── createtopics.go
├── createtopics_test.go
├── deleteacls.go
├── deleteacls_test.go
├── deletegroups.go
├── deletegroups_test.go
├── deletetopics.go
├── deletetopics_test.go
├── describeacls.go
├── describeacls_test.go
├── describeclientquotas.go
├── describeconfigs.go
├── describeconfigs_test.go
├── describegroups.go
├── describegroups_test.go
├── describeuserscramcredentials.go
├── describeuserscramcredentials_test.go
├── dialer.go
├── dialer_test.go
├── discard.go
├── discard_test.go
├── docker-compose.yml
├── docker_compose_versions/
│ ├── README.md
│ ├── docker-compose-270.yml
│ ├── docker-compose-370.yml
│ └── docker-compose-400.yml
├── electleaders.go
├── electleaders_test.go
├── endtxn.go
├── error.go
├── error_test.go
├── example_consumergroup_test.go
├── example_groupbalancer_test.go
├── example_writer_test.go
├── examples/
│ ├── .gitignore
│ ├── consumer-logger/
│ │ ├── Dockerfile
│ │ ├── go.mod
│ │ ├── go.sum
│ │ └── main.go
│ ├── consumer-mongo-db/
│ │ ├── Dockerfile
│ │ ├── go.mod
│ │ ├── go.sum
│ │ └── main.go
│ ├── docker-compose.yaml
│ ├── kafka/
│ │ └── kafka-variables.env
│ ├── producer-api/
│ │ ├── Dockerfile
│ │ ├── go.mod
│ │ ├── go.sum
│ │ ├── main.go
│ │ └── test.http
│ └── producer-random/
│ ├── Dockerfile
│ ├── go.mod
│ ├── go.sum
│ └── main.go
├── fetch.go
├── fetch_test.go
├── findcoordinator.go
├── findcoordinator_test.go
├── fixtures/
│ ├── v1-v1.hex
│ ├── v1-v1.pcapng
│ ├── v1-v1c-v2-v2c-v2b-v2b-v2b-v2bc-v1b-v1bc.hex
│ ├── v1-v1c-v2-v2c-v2b-v2b-v2b-v2bc-v1b-v1bc.pcapng
│ ├── v1c-v1-v1c.hex
│ ├── v1c-v1-v1c.pcapng
│ ├── v1c-v1c.hex
│ ├── v1c-v1c.pcapng
│ ├── v2-v2.hex
│ ├── v2-v2.pcapng
│ ├── v2b-v1.hex
│ ├── v2b-v1.pcapng
│ ├── v2bc-v1-v1c.hex
│ ├── v2bc-v1-v1c.pcapng
│ ├── v2bc-v1.hex
│ ├── v2bc-v1.pcapng
│ ├── v2bc-v1c.hex
│ ├── v2bc-v1c.pcapng
│ ├── v2c-v2-v2c.hex
│ ├── v2c-v2-v2c.pcapng
│ ├── v2c-v2c.hex
│ └── v2c-v2c.pcapng
├── go.mod
├── go.sum
├── groupbalancer.go
├── groupbalancer_test.go
├── gzip/
│ └── gzip.go
├── heartbeat.go
├── heartbeat_test.go
├── incrementalalterconfigs.go
├── incrementalalterconfigs_test.go
├── initproducerid.go
├── initproducerid_test.go
├── joingroup.go
├── joingroup_test.go
├── kafka.go
├── kafka_test.go
├── leavegroup.go
├── leavegroup_test.go
├── listgroups.go
├── listgroups_test.go
├── listoffset.go
├── listoffset_test.go
├── listpartitionreassignments.go
├── listpartitionreassignments_test.go
├── logger.go
├── lz4/
│ └── lz4.go
├── message.go
├── message_reader.go
├── message_test.go
├── metadata.go
├── metadata_test.go
├── offsetcommit.go
├── offsetcommit_test.go
├── offsetdelete.go
├── offsetdelete_test.go
├── offsetfetch.go
├── offsetfetch_test.go
├── produce.go
├── produce_test.go
├── protocol/
│ ├── addoffsetstotxn/
│ │ ├── addoffsetstotxn.go
│ │ └── addoffsetstotxn_test.go
│ ├── addpartitionstotxn/
│ │ ├── addpartitionstotxn.go
│ │ └── addpartitionstotxn_test.go
│ ├── alterclientquotas/
│ │ ├── alterclientquotas.go
│ │ └── alterclientquotas_test.go
│ ├── alterconfigs/
│ │ ├── alterconfigs.go
│ │ └── alterconfigs_test.go
│ ├── alterpartitionreassignments/
│ │ ├── alterpartitionreassignments.go
│ │ └── alterpartitionreassignments_test.go
│ ├── alteruserscramcredentials/
│ │ ├── alteruserscramcredentials.go
│ │ └── alteruserscramcredentials_test.go
│ ├── apiversions/
│ │ ├── apiversions.go
│ │ └── apiversions_test.go
│ ├── buffer.go
│ ├── buffer_test.go
│ ├── cluster.go
│ ├── conn.go
│ ├── consumer/
│ │ ├── consumer.go
│ │ └── consumer_test.go
│ ├── createacls/
│ │ ├── createacls.go
│ │ └── createacls_test.go
│ ├── createpartitions/
│ │ ├── createpartitions.go
│ │ └── createpartitions_test.go
│ ├── createtopics/
│ │ └── createtopics.go
│ ├── decode.go
│ ├── deleteacls/
│ │ ├── deleteacls.go
│ │ └── deleteacls_test.go
│ ├── deletegroups/
│ │ ├── deletegroups.go
│ │ └── deletegroups_test.go
│ ├── deletetopics/
│ │ ├── deletetopics.go
│ │ └── deletetopics_test.go
│ ├── describeacls/
│ │ ├── describeacls.go
│ │ └── describeacls_test.go
│ ├── describeclientquotas/
│ │ ├── describeclientquotas.go
│ │ └── describeclientquotas_test.go
│ ├── describeconfigs/
│ │ ├── describeconfigs.go
│ │ └── describeconfigs_test.go
│ ├── describegroups/
│ │ ├── describegroups.go
│ │ └── describegroups_test.go
│ ├── describeuserscramcredentials/
│ │ ├── describeuserscramcredentials.go
│ │ └── describeuserscramcredentials_test.go
│ ├── electleaders/
│ │ ├── electleaders.go
│ │ └── electleaders_test.go
│ ├── encode.go
│ ├── endtxn/
│ │ ├── endtxn.go
│ │ └── endtxn_test.go
│ ├── error.go
│ ├── fetch/
│ │ ├── fetch.go
│ │ └── fetch_test.go
│ ├── findcoordinator/
│ │ └── findcoordinator.go
│ ├── heartbeat/
│ │ ├── heartbeat.go
│ │ └── heartbeat_test.go
│ ├── incrementalalterconfigs/
│ │ ├── incrementalalterconfigs.go
│ │ └── incrementalalterconfigs_test.go
│ ├── initproducerid/
│ │ ├── initproducerid.go
│ │ └── initproducerid_test.go
│ ├── joingroup/
│ │ ├── joingroup.go
│ │ └── joingroup_test.go
│ ├── leavegroup/
│ │ ├── leavegroup.go
│ │ └── leavegroup_test.go
│ ├── listgroups/
│ │ └── listgroups.go
│ ├── listoffsets/
│ │ ├── listoffsets.go
│ │ └── listoffsets_test.go
│ ├── listpartitionreassignments/
│ │ ├── listpartitionreassignments.go
│ │ └── listpartitionreassignments_test.go
│ ├── metadata/
│ │ ├── metadata.go
│ │ └── metadata_test.go
│ ├── offsetcommit/
│ │ ├── offsetcommit.go
│ │ └── offsetcommit_test.go
│ ├── offsetdelete/
│ │ ├── offsetdelete.go
│ │ └── offsetdelete_test.go
│ ├── offsetfetch/
│ │ └── offsetfetch.go
│ ├── produce/
│ │ ├── produce.go
│ │ └── produce_test.go
│ ├── protocol.go
│ ├── protocol_test.go
│ ├── prototest/
│ │ ├── bytes.go
│ │ ├── prototest.go
│ │ ├── reflect.go
│ │ ├── request.go
│ │ └── response.go
│ ├── rawproduce/
│ │ ├── rawproduce.go
│ │ └── rawproduce_test.go
│ ├── record.go
│ ├── record_batch.go
│ ├── record_batch_test.go
│ ├── record_v1.go
│ ├── record_v2.go
│ ├── reflect.go
│ ├── reflect_unsafe.go
│ ├── request.go
│ ├── response.go
│ ├── response_test.go
│ ├── roundtrip.go
│ ├── saslauthenticate/
│ │ └── saslauthenticate.go
│ ├── saslhandshake/
│ │ └── saslhandshake.go
│ ├── size.go
│ ├── syncgroup/
│ │ ├── syncgroup.go
│ │ └── syncgroup_test.go
│ └── txnoffsetcommit/
│ ├── txnoffsetcommit.go
│ └── txnoffsetcommit_test.go
├── protocol.go
├── protocol_test.go
├── rawproduce.go
├── rawproduce_test.go
├── read.go
├── read_test.go
├── reader.go
├── reader_test.go
├── record.go
├── recordbatch.go
├── resolver.go
├── resource.go
├── resource_test.go
├── sasl/
│ ├── aws_msk_iam/
│ │ ├── go.mod
│ │ ├── go.sum
│ │ ├── msk_iam.go
│ │ └── msk_iam_test.go
│ ├── aws_msk_iam_v2/
│ │ ├── README.md
│ │ ├── example_test.go
│ │ ├── go.mod
│ │ ├── go.sum
│ │ ├── msk_iam.go
│ │ └── msk_iam_test.go
│ ├── plain/
│ │ └── plain.go
│ ├── sasl.go
│ ├── sasl_test.go
│ └── scram/
│ └── scram.go
├── saslauthenticate.go
├── saslauthenticate_test.go
├── saslhandshake.go
├── saslhandshake_test.go
├── scripts/
│ └── wait-for-kafka.sh
├── sizeof.go
├── snappy/
│ └── snappy.go
├── stats.go
├── syncgroup.go
├── syncgroup_test.go
├── testing/
│ ├── conn.go
│ ├── version.go
│ └── version_test.go
├── time.go
├── topics/
│ ├── list_topics.go
│ └── list_topics_test.go
├── transport.go
├── transport_test.go
├── txnoffsetcommit.go
├── txnoffsetcommit_test.go
├── write.go
├── write_test.go
├── writer.go
├── writer_test.go
└── zstd/
└── zstd.go
================================================
FILE CONTENTS
================================================
================================================
FILE: .circleci/config.yml
================================================
version: 2
jobs:
lint:
docker:
- image: golangci/golangci-lint:v1.45-alpine
steps:
- checkout
- run: golangci-lint run
kafka-270:
working_directory: &working_directory /go/src/github.com/segmentio/kafka-go
environment:
KAFKA_VERSION: "2.7.0"
# Need to skip nettest to avoid these kinds of errors:
# --- FAIL: TestConn/nettest (17.56s)
# --- FAIL: TestConn/nettest/PingPong (7.40s)
# conntest.go:112: unexpected Read error: [7] Request Timed Out: the request exceeded the user-specified time limit in the request
# conntest.go:118: mismatching value: got 77, want 78
# conntest.go:118: mismatching value: got 78, want 79
# ...
#
# TODO: Figure out why these are happening and fix them (they don't appear to be new).
KAFKA_SKIP_NETTEST: "1"
docker:
- image: circleci/golang
- image: bitnamilegacy/zookeeper:latest
ports:
- 2181:2181
environment:
ALLOW_ANONYMOUS_LOGIN: yes
- image: bitnamilegacy/kafka:2.7.0
ports:
- 9092:9092
- 9093:9093
environment: &environment
KAFKA_CFG_BROKER_ID: 1
KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
KAFKA_CFG_ADVERTISED_HOST_NAME: 'localhost'
KAFKA_CFG_ADVERTISED_PORT: '9092'
KAFKA_CFG_ZOOKEEPER_CONNECT: localhost:2181
KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'true'
KAFKA_CFG_MESSAGE_MAX_BYTES: '200000000'
KAFKA_CFG_LISTENERS: 'PLAINTEXT://:9092,SASL_PLAINTEXT://:9093'
KAFKA_CFG_ADVERTISED_LISTENERS: 'PLAINTEXT://localhost:9092,SASL_PLAINTEXT://localhost:9093'
KAFKA_CFG_SASL_ENABLED_MECHANISMS: 'PLAIN,SCRAM-SHA-256,SCRAM-SHA-512'
KAFKA_CFG_AUTHORIZER_CLASS_NAME: 'kafka.security.auth.SimpleAclAuthorizer'
KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true'
KAFKA_OPTS: "-Djava.security.auth.login.config=/opt/bitnami/kafka/config/kafka_jaas.conf"
ALLOW_PLAINTEXT_LISTENER: yes
entrypoint: &entrypoint
- "/bin/bash"
- "-c"
- echo -e 'KafkaServer {\norg.apache.kafka.common.security.scram.ScramLoginModule required\n username="adminscram"\n password="admin-secret";\n org.apache.kafka.common.security.plain.PlainLoginModule required\n username="adminplain"\n password="admin-secret"\n user_adminplain="admin-secret";\n };' > /opt/bitnami/kafka/config/kafka_jaas.conf; /opt/bitnami/kafka/bin/kafka-configs.sh --zookeeper localhost:2181 --alter --add-config "SCRAM-SHA-256=[password=admin-secret-256],SCRAM-SHA-512=[password=admin-secret-512]" --entity-type users --entity-name adminscram; exec /entrypoint.sh /run.sh
steps: &steps
- checkout
- restore_cache:
key: kafka-go-mod-{{ checksum "go.sum" }}-1
- run:
name: Download dependencies
command: go mod download
- save_cache:
key: kafka-go-mod-{{ checksum "go.sum" }}-1
paths:
- /go/pkg/mod
- run:
name: Wait for kafka
command: ./scripts/wait-for-kafka.sh
- run:
name: Test kafka-go
command: go test -race -cover ./...
- run:
name: Test kafka-go unsafe
command: go test -tags=unsafe -race -cover ./...
- run:
name: Test kafka-go/sasl/aws_msk_iam
working_directory: ./sasl/aws_msk_iam
command: go test -race -cover ./...
kafka-281:
working_directory: *working_directory
environment:
KAFKA_VERSION: "2.8.1"
# Need to skip nettest to avoid these kinds of errors:
# --- FAIL: TestConn/nettest (17.56s)
# --- FAIL: TestConn/nettest/PingPong (7.40s)
# conntest.go:112: unexpected Read error: [7] Request Timed Out: the request exceeded the user-specified time limit in the request
# conntest.go:118: mismatching value: got 77, want 78
# conntest.go:118: mismatching value: got 78, want 79
# ...
#
# TODO: Figure out why these are happening and fix them (they don't appear to be new).
KAFKA_SKIP_NETTEST: "1"
docker:
- image: circleci/golang
- image: bitnamilegacy/zookeeper:latest
ports:
- 2181:2181
environment:
ALLOW_ANONYMOUS_LOGIN: yes
- image: bitnamilegacy/kafka:2.8.1
ports:
- 9092:9092
- 9093:9093
environment: *environment
entrypoint: *entrypoint
steps: *steps
kafka-370:
working_directory: *working_directory
environment:
KAFKA_VERSION: "3.7.0"
# Need to skip nettest to avoid these kinds of errors:
# --- FAIL: TestConn/nettest (17.56s)
# --- FAIL: TestConn/nettest/PingPong (7.40s)
# conntest.go:112: unexpected Read error: [7] Request Timed Out: the request exceeded the user-specified time limit in the request
# conntest.go:118: mismatching value: got 77, want 78
# conntest.go:118: mismatching value: got 78, want 79
# ...
#
# TODO: Figure out why these are happening and fix them (they don't appear to be new).
KAFKA_SKIP_NETTEST: "1"
docker:
- image: circleci/golang
- image: bitnamilegacy/zookeeper:latest
ports:
- 2181:2181
environment:
ALLOW_ANONYMOUS_LOGIN: yes
- image: bitnamilegacy/kafka:3.7.0
ports:
- 9092:9092
- 9093:9093
environment:
<<: *environment
KAFKA_CFG_AUTHORIZER_CLASS_NAME: 'kafka.security.authorizer.AclAuthorizer'
entrypoint: *entrypoint
steps: *steps
# NOTE: this fails quite often due to Java heap errors from Kafka.
# Once we figure out how to fix that, we can re-enable this.
# https://github.com/segmentio/kafka-go/issues/1360#issuecomment-2858935900
# kafka-400:
# working_directory: *working_directory
# environment:
# KAFKA_VERSION: "4.0.0"
# # Need to skip nettest to avoid these kinds of errors:
# # --- FAIL: TestConn/nettest (17.56s)
# # --- FAIL: TestConn/nettest/PingPong (7.40s)
# # conntest.go:112: unexpected Read error: [7] Request Timed Out: the request exceeded the user-specified time limit in the request
# # conntest.go:118: mismatching value: got 77, want 78
# # conntest.go:118: mismatching value: got 78, want 79
# # ...
# #
# # TODO: Figure out why these are happening and fix them (they don't appear to be new).
# KAFKA_SKIP_NETTEST: "1"
# docker:
# - image: circleci/golang
# - image: bitnamilegacy/kafka:4.0.0
# ports:
# - 9092:9092
# - 9093:9093
# environment:
# KAFKA_CFG_NODE_ID: 1
# KAFKA_CFG_BROKER_ID: 1
# KAFKA_CFG_PROCESS_ROLES: broker,controller
# KAFKA_CFG_ADVERTISED_HOST_NAME: 'localhost'
# KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
# KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAIN:PLAINTEXT,SASL:SASL_PLAINTEXT
# KAFKA_CFG_LISTENERS: CONTROLLER://:9094,PLAIN://:9092,SASL://:9093
# KAFKA_CFG_ADVERTISED_LISTENERS: PLAIN://localhost:9092,SASL://localhost:9093
# KAFKA_CFG_INTER_BROKER_LISTENER_NAME: PLAIN
# KAFKA_CFG_SASL_ENABLED_MECHANISMS: 'PLAIN,SCRAM-SHA-256,SCRAM-SHA-512'
# KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: 1@localhost:9094
# ALLOW_PLAINTEXT_LISTENER: yes
# KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true'
# KAFKA_OPTS: "-Djava.security.auth.login.config=/opt/bitnami/kafka/config/kafka_jaas.conf"
# KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'true'
# KAFKA_CFG_DELETE_TOPIC_ENABLE: 'true'
# KAFKA_CFG_MESSAGE_MAX_BYTES: '200000000'
# KAFKA_CFG_AUTHORIZER_CLASS_NAME: 'org.apache.kafka.metadata.authorizer.StandardAuthorizer'
# KAFKA_CFG_SUPER_USERS: User:adminscram256;User:adminscram512;User:adminplain
# KAFKA_CLIENT_USERS: adminscram256,adminscram512,adminplain
# KAFKA_CLIENT_PASSWORDS: admin-secret-256,admin-secret-512,admin-secret
# KAFKA_CLIENT_SASL_MECHANISMS: SCRAM-SHA-256,SCRAM-SHA-512,PLAIN
# KAFKA_INTER_BROKER_USER: adminscram512
# KAFKA_INTER_BROKER_PASSWORD: admin-secret-512
# KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL: SCRAM-SHA-512
# steps: *steps
workflows:
version: 2
run:
jobs:
- lint
- kafka-270
- kafka-281
- kafka-370
#- kafka-400
================================================
FILE: .gitattributes
================================================
fixtures/*.hex binary
================================================
FILE: .github/ISSUE_TEMPLATE/bug_report.md
================================================
---
name: Bug report
about: Create a report to help us improve
title: ''
labels: bug
assignees: ''
---
**Describe the bug**
> A clear and concise description of what the bug is.
**Kafka Version**
> * What version(s) of Kafka are you testing against?
> * What version of kafka-go are you using?
**To Reproduce**
> Resources to reproduce the behavior:
```yaml
---
# docker-compose.yaml
#
# Adding a docker-compose file will help the maintainers setup the environment
# to reproduce the issue.
#
# If one the docker-compose files available in the repository may be used,
# mentioning it is also a useful alternative.
...
```
```go
package main
import (
"github.com/segmentio/kafka-go"
)
func main() {
// Adding a fully reproducible example will help maintainers provide
// assistance to debug the issues.
...
}
```
**Expected Behavior**
> A clear and concise description of what you expected to happen.
**Observed Behavior**
> A clear and concise description of the behavior you observed.
```
Often times, pasting the logging output from a kafka.Reader or kafka.Writer will
provide useful details to help maintainers investigate the issue and provide a
fix. If possible, providing stack traces or CPU/memory profiles may also contain
valuable information to understand the conditions that triggered the issue.
```
**Additional Context**
> Add any other context about the problem here.
================================================
FILE: .github/ISSUE_TEMPLATE/feature_request.md
================================================
---
name: Feature request
about: Suggest an idea for this project
title: ''
labels: enhancement
assignees: ''
---
**Describe the solution you would like**
> A clear and concise description of what you want to happen.
**Supporting documentation**
> Please provides links to relevant Kafka protocol docs and/or KIPs.
================================================
FILE: .gitignore
================================================
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof
/kafkacli
# Emacs
*~
# VIM
*.swp
# Goland
.idea
#IntelliJ
*.iml
# govendor
/vendor/*/
================================================
FILE: .golangci.yml
================================================
linters:
enable:
- bodyclose
- errorlint
- goconst
- godot
- gofmt
- goimports
- prealloc
disable:
# Temporarily disabling so it can be addressed in a dedicated PR.
- errcheck
- goerr113
linters-settings:
goconst:
ignore-tests: true
================================================
FILE: CODE_OF_CONDUCT.md
================================================
# Contributor Covenant Code of Conduct
## Our Pledge
In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to making participation in our project and
our community a harassment-free experience for everyone, regardless of age, body
size, disability, ethnicity, sex characteristics, gender identity and expression,
level of experience, education, socio-economic status, nationality, personal
appearance, race, religion, or sexual identity and orientation.
## Our Standards
Examples of behavior that contributes to creating a positive environment
include:
- Using welcoming and inclusive language
- Being respectful of differing viewpoints and experiences
- Gracefully accepting constructive criticism
- Focusing on what is best for the community
- Showing empathy towards other community members
Examples of unacceptable behavior by participants include:
- The use of sexualized language or imagery and unwelcome sexual attention or
advances
- Trolling, insulting/derogatory comments, and personal or political attacks
- Public or private harassment
- Publishing others' private information, such as a physical or electronic
address, without explicit permission
- Other conduct which could reasonably be considered inappropriate in a
professional setting
## Our Responsibilities
Project maintainers are responsible for clarifying the standards of acceptable
behavior and are expected to take appropriate and fair corrective action in
response to any instances of unacceptable behavior.
Project maintainers have the right and responsibility to remove, edit, or
reject comments, commits, code, wiki edits, issues, and other contributions
that are not aligned to this Code of Conduct, or to ban temporarily or
permanently any contributor for other behaviors that they deem inappropriate,
threatening, offensive, or harmful.
Project maintainers are available at [#kafka-go](https://gophers.slack.com/archives/CG4H0N9PX) channel inside the [Gophers Slack](https://gophers.slack.com)
## Scope
This Code of Conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community. Examples of
representing a project or community include using an official project e-mail
address, posting via an official social media account, or acting as an appointed
representative at an online or offline event. Representation of a project may be
further defined and clarified by project maintainers.
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting the project team at open-source@twilio.com. All
complaints will be reviewed and investigated and will result in a response that
is deemed necessary and appropriate to the circumstances. The project team is
obligated to maintain confidentiality with regard to the reporter of an incident.
Further details of specific enforcement policies may be posted separately.
Project maintainers who do not follow or enforce the Code of Conduct in good
faith may face temporary or permanent repercussions as determined by other
members of the project's leadership.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
[homepage]: https://www.contributor-covenant.org
================================================
FILE: CONTRIBUTING.md
================================================
# Contributing to kafka-go
kafka-go is an open source project. We welcome contributions to kafka-go of any kind including documentation,
organization, tutorials, bug reports, issues, feature requests, feature implementations, pull requests, etc.
## Table of Contents
* [Reporting Issues](#reporting-issues)
* [Submitting Patches](#submitting-patches)
* [Code Contribution Guidelines](#code-contribution-guidelines)
* [Git Commit Message Guidelines](#git-commit-message-guidelines)
* [Fetching the Source From GitHub](#fetching-the-sources-from-github)
* [Building kafka-go with Your Changes](#building-kakfa-go-with-your-changes)
## Reporting Issues
If you believe you have found a defect in kafka-go, use the GitHub issue tracker to report
the problem to the maintainers.
When reporting the issue, please provide the version of kafka-go, what version(s) of Kafka
are you testing against, and your operating system.
- [kafka-go Issues segmentio/kafka-go](https://github.com/segmentio/kafka-go/issues)
## Submitting Patches
kafka-go project welcomes all contributors and contributions regardless of skill or experience levels. If you are
interested in helping with the project, we will help you with your contribution.
### Code Contribution
To make contributions as seamless as possible, we ask the following:
* Go ahead and fork the project and make your changes. We encourage pull requests to allow for review and discussion of code changes.
* When you’re ready to create a pull request, be sure to:
* Have test cases for the new code. If you have questions about how to do this, please ask in your pull request.
* Run `go fmt`.
* Squash your commits into a single commit. `git rebase -i`. It’s okay to force update your pull request with `git push -f`.
* Follow the **Git Commit Message Guidelines** below.
### Git Commit Message Guidelines
This [blog article](http://chris.beams.io/posts/git-commit/) is a good resource for learning how to write good commit messages,
the most important part being that each commit message should have a title/subject in imperative mood starting with a capital letter and no trailing period:
*"Return error on wrong use of the Reader"*, **NOT** *"returning some error."*
Also, if your commit references one or more GitHub issues, always end your commit message body with *See #1234* or *Fixes #1234*.
Replace *1234* with the GitHub issue ID. The last example will close the issue when the commit is merged into *master*.
Please use a short and descriptive branch name, e.g. NOT "patch-1". It's very common but creates a naming conflict each
time when a submission is pulled for a review.
An example:
```text
Add Code of Conduct and Code Contribution Guidelines
Add a full Code of Conduct and Code Contribution Guidelines document.
Provide description on how best to retrieve code, fork, checkout, and commit changes.
Fixes #688
```
### Fetching the Sources From GitHub
We use Go Modules support built into Go 1.11 to build. The easiest way is to clone kafka-go into a directory outside of
`GOPATH`, as in the following example:
```bash
mkdir $HOME/src
cd $HOME/src
git clone https://github.com/segmentio/kafka-go.git
cd kafka-go
go build ./...
```
To make changes to kafka-go's source:
1. Create a new branch for your changes (the branch name is arbitrary):
```bash
git checkout -b branch1234
```
1. After making your changes, commit them to your new branch:
```bash
git commit -a -v
```
1. Fork kafka-go in GitHub
1. Add your fork as a new remote (the remote name, "upstream" in this example, is arbitrary):
```bash
git remote add upstream git@github.com:USERNAME/kafka-go.git
```
1. Push your branch (the remote name, "upstream" in this example, is arbitrary):
```bash
git push upstream
```
1. You are now ready to submit a PR based upon the new branch in your forked repository.
### Using the forked library
To replace the original version of kafka-go library with a forked version is accomplished this way.
1. Make sure your application already has a go.mod entry depending on kafka-go
```bash
module github.com/myusername/myapp
require (
...
github.com/segmentio/kafka-go v1.2.3
...
)
```
1. Add the following entry to the beginning of the modules file.
```bash
module github.com/myusername/myapp
replace github.com/segmentio/kafka-go v1.2.3 => ../local/directory
require (
...
github.com/segmentio/kafka-go v1.2.3
...
)
```
1. Depending on if you are using `vendor`ing or not you might need to run the following command to pull in the new bits.
```bash
> go mod vendor
```
================================================
FILE: LICENSE
================================================
MIT License
Copyright (c) 2017 Segment
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
================================================
FILE: Makefile
================================================
test:
KAFKA_SKIP_NETTEST=1 \
KAFKA_VERSION=2.3.1 \
go test -race -cover ./...
docker:
docker compose up -d
================================================
FILE: README.md
================================================
# kafka-go [](https://circleci.com/gh/segmentio/kafka-go) [](https://goreportcard.com/report/github.com/segmentio/kafka-go) [](https://godoc.org/github.com/segmentio/kafka-go)
## Motivations
We rely on both Go and Kafka a lot at Segment. Unfortunately, the state of the Go
client libraries for Kafka at the time of this writing was not ideal. The available
options were:
- [sarama](https://github.com/Shopify/sarama), which is by far the most popular
but is quite difficult to work with. It is poorly documented, the API exposes
low level concepts of the Kafka protocol, and it doesn't support recent Go features
like [contexts](https://golang.org/pkg/context/). It also passes all values as
pointers which causes large numbers of dynamic memory allocations, more frequent
garbage collections, and higher memory usage.
- [confluent-kafka-go](https://github.com/confluentinc/confluent-kafka-go) is a
cgo based wrapper around [librdkafka](https://github.com/edenhill/librdkafka),
which means it introduces a dependency to a C library on all Go code that uses
the package. It has much better documentation than sarama but still lacks support
for Go contexts.
- [goka](https://github.com/lovoo/goka) is a more recent Kafka client for Go
which focuses on a specific usage pattern. It provides abstractions for using Kafka
as a message passing bus between services rather than an ordered log of events, but
this is not the typical use case of Kafka for us at Segment. The package also
depends on sarama for all interactions with Kafka.
This is where `kafka-go` comes into play. It provides both low and high level
APIs for interacting with Kafka, mirroring concepts and implementing interfaces of
the Go standard library to make it easy to use and integrate with existing
software.
#### Note:
In order to better align with our newly adopted Code of Conduct, the kafka-go
project has renamed our default branch to `main`. For the full details of our
Code Of Conduct see [this](./CODE_OF_CONDUCT.md) document.
## Kafka versions
`kafka-go` is currently tested with Kafka versions 0.10.1.0 to 2.7.1.
While it should also be compatible with later versions, newer features available
in the Kafka API may not yet be implemented in the client.
## Go versions
`kafka-go` requires Go version 1.15 or later.
## Connection [](https://godoc.org/github.com/segmentio/kafka-go#Conn)
The `Conn` type is the core of the `kafka-go` package. It wraps around a raw
network connection to expose a low-level API to a Kafka server.
Here are some examples showing typical use of a connection object:
```go
// to produce messages
topic := "my-topic"
partition := 0
conn, err := kafka.DialLeader(context.Background(), "tcp", "localhost:9092", topic, partition)
if err != nil {
log.Fatal("failed to dial leader:", err)
}
conn.SetWriteDeadline(time.Now().Add(10*time.Second))
_, err = conn.WriteMessages(
kafka.Message{Value: []byte("one!")},
kafka.Message{Value: []byte("two!")},
kafka.Message{Value: []byte("three!")},
)
if err != nil {
log.Fatal("failed to write messages:", err)
}
if err := conn.Close(); err != nil {
log.Fatal("failed to close writer:", err)
}
```
```go
// to consume messages
topic := "my-topic"
partition := 0
conn, err := kafka.DialLeader(context.Background(), "tcp", "localhost:9092", topic, partition)
if err != nil {
log.Fatal("failed to dial leader:", err)
}
conn.SetReadDeadline(time.Now().Add(10*time.Second))
batch := conn.ReadBatch(10e3, 1e6) // fetch 10KB min, 1MB max
b := make([]byte, 10e3) // 10KB max per message
for {
n, err := batch.Read(b)
if err != nil {
break
}
fmt.Println(string(b[:n]))
}
if err := batch.Close(); err != nil {
log.Fatal("failed to close batch:", err)
}
if err := conn.Close(); err != nil {
log.Fatal("failed to close connection:", err)
}
```
### To Create Topics
By default kafka has the `auto.create.topics.enable='true'` (`KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE='true'` in the bitnami/kafka kafka docker image). If this value is set to `'true'` then topics will be created as a side effect of `kafka.DialLeader` like so:
```go
// to create topics when auto.create.topics.enable='true'
conn, err := kafka.DialLeader(context.Background(), "tcp", "localhost:9092", "my-topic", 0)
if err != nil {
panic(err.Error())
}
```
If `auto.create.topics.enable='false'` then you will need to create topics explicitly like so:
```go
// to create topics when auto.create.topics.enable='false'
topic := "my-topic"
conn, err := kafka.Dial("tcp", "localhost:9092")
if err != nil {
panic(err.Error())
}
defer conn.Close()
controller, err := conn.Controller()
if err != nil {
panic(err.Error())
}
var controllerConn *kafka.Conn
controllerConn, err = kafka.Dial("tcp", net.JoinHostPort(controller.Host, strconv.Itoa(controller.Port)))
if err != nil {
panic(err.Error())
}
defer controllerConn.Close()
topicConfigs := []kafka.TopicConfig{
{
Topic: topic,
NumPartitions: 1,
ReplicationFactor: 1,
},
}
err = controllerConn.CreateTopics(topicConfigs...)
if err != nil {
panic(err.Error())
}
```
### To Connect To Leader Via a Non-leader Connection
```go
// to connect to the kafka leader via an existing non-leader connection rather than using DialLeader
conn, err := kafka.Dial("tcp", "localhost:9092")
if err != nil {
panic(err.Error())
}
defer conn.Close()
controller, err := conn.Controller()
if err != nil {
panic(err.Error())
}
var connLeader *kafka.Conn
connLeader, err = kafka.Dial("tcp", net.JoinHostPort(controller.Host, strconv.Itoa(controller.Port)))
if err != nil {
panic(err.Error())
}
defer connLeader.Close()
```
### To list topics
```go
conn, err := kafka.Dial("tcp", "localhost:9092")
if err != nil {
panic(err.Error())
}
defer conn.Close()
partitions, err := conn.ReadPartitions()
if err != nil {
panic(err.Error())
}
m := map[string]struct{}{}
for _, p := range partitions {
m[p.Topic] = struct{}{}
}
for k := range m {
fmt.Println(k)
}
```
Because it is low level, the `Conn` type turns out to be a great building block
for higher level abstractions, like the `Reader` for example.
## Reader [](https://godoc.org/github.com/segmentio/kafka-go#Reader)
A `Reader` is another concept exposed by the `kafka-go` package, which intends
to make it simpler to implement the typical use case of consuming from a single
topic-partition pair.
A `Reader` also automatically handles reconnections and offset management, and
exposes an API that supports asynchronous cancellations and timeouts using Go
contexts.
Note that it is important to call `Close()` on a `Reader` when a process exits.
The kafka server needs a graceful disconnect to stop it from continuing to
attempt to send messages to the connected clients. The given example will not
call `Close()` if the process is terminated with SIGINT (ctrl-c at the shell) or
SIGTERM (as docker stop or a kubernetes restart does). This can result in a
delay when a new reader on the same topic connects (e.g. new process started
or new container running). Use a `signal.Notify` handler to close the reader on
process shutdown.
```go
// make a new reader that consumes from topic-A, partition 0, at offset 42
r := kafka.NewReader(kafka.ReaderConfig{
Brokers: []string{"localhost:9092","localhost:9093", "localhost:9094"},
Topic: "topic-A",
Partition: 0,
MaxBytes: 10e6, // 10MB
})
r.SetOffset(42)
for {
m, err := r.ReadMessage(context.Background())
if err != nil {
break
}
fmt.Printf("message at offset %d: %s = %s\n", m.Offset, string(m.Key), string(m.Value))
}
if err := r.Close(); err != nil {
log.Fatal("failed to close reader:", err)
}
```
### Consumer Groups
```kafka-go``` also supports Kafka consumer groups including broker managed offsets.
To enable consumer groups, simply specify the GroupID in the ReaderConfig.
ReadMessage automatically commits offsets when using consumer groups.
```go
// make a new reader that consumes from topic-A
r := kafka.NewReader(kafka.ReaderConfig{
Brokers: []string{"localhost:9092", "localhost:9093", "localhost:9094"},
GroupID: "consumer-group-id",
Topic: "topic-A",
MaxBytes: 10e6, // 10MB
})
for {
m, err := r.ReadMessage(context.Background())
if err != nil {
break
}
fmt.Printf("message at topic/partition/offset %v/%v/%v: %s = %s\n", m.Topic, m.Partition, m.Offset, string(m.Key), string(m.Value))
}
if err := r.Close(); err != nil {
log.Fatal("failed to close reader:", err)
}
```
There are a number of limitations when using consumer groups:
* ```(*Reader).SetOffset``` will return an error when GroupID is set
* ```(*Reader).Offset``` will always return ```-1``` when GroupID is set
* ```(*Reader).Lag``` will always return ```-1``` when GroupID is set
* ```(*Reader).ReadLag``` will return an error when GroupID is set
* ```(*Reader).Stats``` will return a partition of ```-1``` when GroupID is set
### Explicit Commits
```kafka-go``` also supports explicit commits. Instead of calling ```ReadMessage```,
call ```FetchMessage``` followed by ```CommitMessages```.
```go
ctx := context.Background()
for {
m, err := r.FetchMessage(ctx)
if err != nil {
break
}
fmt.Printf("message at topic/partition/offset %v/%v/%v: %s = %s\n", m.Topic, m.Partition, m.Offset, string(m.Key), string(m.Value))
if err := r.CommitMessages(ctx, m); err != nil {
log.Fatal("failed to commit messages:", err)
}
}
```
When committing messages in consumer groups, the message with the highest offset
for a given topic/partition determines the value of the committed offset for
that partition. For example, if messages at offset 1, 2, and 3 of a single
partition were retrieved by call to `FetchMessage`, calling `CommitMessages`
with message offset 3 will also result in committing the messages at offsets 1
and 2 for that partition.
### Managing Commits
By default, CommitMessages will synchronously commit offsets to Kafka. For
improved performance, you can instead periodically commit offsets to Kafka
by setting CommitInterval on the ReaderConfig.
```go
// make a new reader that consumes from topic-A
r := kafka.NewReader(kafka.ReaderConfig{
Brokers: []string{"localhost:9092", "localhost:9093", "localhost:9094"},
GroupID: "consumer-group-id",
Topic: "topic-A",
MaxBytes: 10e6, // 10MB
CommitInterval: time.Second, // flushes commits to Kafka every second
})
```
## Writer [](https://godoc.org/github.com/segmentio/kafka-go#Writer)
To produce messages to Kafka, a program may use the low-level `Conn` API, but
the package also provides a higher level `Writer` type which is more appropriate
to use in most cases as it provides additional features:
- Automatic retries and reconnections on errors.
- Configurable distribution of messages across available partitions.
- Synchronous or asynchronous writes of messages to Kafka.
- Asynchronous cancellation using contexts.
- Flushing of pending messages on close to support graceful shutdowns.
- Creation of a missing topic before publishing a message. *Note!* it was the default behaviour up to the version `v0.4.30`.
```go
// make a writer that produces to topic-A, using the least-bytes distribution
w := &kafka.Writer{
Addr: kafka.TCP("localhost:9092", "localhost:9093", "localhost:9094"),
Topic: "topic-A",
Balancer: &kafka.LeastBytes{},
}
err := w.WriteMessages(context.Background(),
kafka.Message{
Key: []byte("Key-A"),
Value: []byte("Hello World!"),
},
kafka.Message{
Key: []byte("Key-B"),
Value: []byte("One!"),
},
kafka.Message{
Key: []byte("Key-C"),
Value: []byte("Two!"),
},
)
if err != nil {
log.Fatal("failed to write messages:", err)
}
if err := w.Close(); err != nil {
log.Fatal("failed to close writer:", err)
}
```
### Missing topic creation before publication
```go
// Make a writer that publishes messages to topic-A.
// The topic will be created if it is missing.
w := &Writer{
Addr: kafka.TCP("localhost:9092", "localhost:9093", "localhost:9094"),
Topic: "topic-A",
AllowAutoTopicCreation: true,
}
messages := []kafka.Message{
{
Key: []byte("Key-A"),
Value: []byte("Hello World!"),
},
{
Key: []byte("Key-B"),
Value: []byte("One!"),
},
{
Key: []byte("Key-C"),
Value: []byte("Two!"),
},
}
var err error
const retries = 3
for i := 0; i < retries; i++ {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
// attempt to create topic prior to publishing the message
err = w.WriteMessages(ctx, messages...)
if errors.Is(err, kafka.LeaderNotAvailable) || errors.Is(err, context.DeadlineExceeded) {
time.Sleep(time.Millisecond * 250)
continue
}
if err != nil {
log.Fatalf("unexpected error %v", err)
}
break
}
if err := w.Close(); err != nil {
log.Fatal("failed to close writer:", err)
}
```
### Writing to multiple topics
Normally, the `WriterConfig.Topic` is used to initialize a single-topic writer.
By excluding that particular configuration, you are given the ability to define
the topic on a per-message basis by setting `Message.Topic`.
```go
w := &kafka.Writer{
Addr: kafka.TCP("localhost:9092", "localhost:9093", "localhost:9094"),
// NOTE: When Topic is not defined here, each Message must define it instead.
Balancer: &kafka.LeastBytes{},
}
err := w.WriteMessages(context.Background(),
// NOTE: Each Message has Topic defined, otherwise an error is returned.
kafka.Message{
Topic: "topic-A",
Key: []byte("Key-A"),
Value: []byte("Hello World!"),
},
kafka.Message{
Topic: "topic-B",
Key: []byte("Key-B"),
Value: []byte("One!"),
},
kafka.Message{
Topic: "topic-C",
Key: []byte("Key-C"),
Value: []byte("Two!"),
},
)
if err != nil {
log.Fatal("failed to write messages:", err)
}
if err := w.Close(); err != nil {
log.Fatal("failed to close writer:", err)
}
```
**NOTE:** These 2 patterns are mutually exclusive, if you set `Writer.Topic`,
you must not also explicitly define `Message.Topic` on the messages you are
writing. The opposite applies when you do not define a topic for the writer.
The `Writer` will return an error if it detects this ambiguity.
### Compatibility with other clients
#### Sarama
If you're switching from Sarama and need/want to use the same algorithm for message partitioning, you can either use
the `kafka.Hash` balancer or the `kafka.ReferenceHash` balancer:
* `kafka.Hash` = `sarama.NewHashPartitioner`
* `kafka.ReferenceHash` = `sarama.NewReferenceHashPartitioner`
The `kafka.Hash` and `kafka.ReferenceHash` balancers would route messages to the same partitions that the two
aforementioned Sarama partitioners would route them to.
```go
w := &kafka.Writer{
Addr: kafka.TCP("localhost:9092", "localhost:9093", "localhost:9094"),
Topic: "topic-A",
Balancer: &kafka.Hash{},
}
```
#### librdkafka and confluent-kafka-go
Use the ```kafka.CRC32Balancer``` balancer to get the same behaviour as librdkafka's
default ```consistent_random``` partition strategy.
```go
w := &kafka.Writer{
Addr: kafka.TCP("localhost:9092", "localhost:9093", "localhost:9094"),
Topic: "topic-A",
Balancer: kafka.CRC32Balancer{},
}
```
#### Java
Use the ```kafka.Murmur2Balancer``` balancer to get the same behaviour as the canonical
Java client's default partitioner. Note: the Java class allows you to directly specify
the partition which is not permitted.
```go
w := &kafka.Writer{
Addr: kafka.TCP("localhost:9092", "localhost:9093", "localhost:9094"),
Topic: "topic-A",
Balancer: kafka.Murmur2Balancer{},
}
```
### Compression
Compression can be enabled on the `Writer` by setting the `Compression` field:
```go
w := &kafka.Writer{
Addr: kafka.TCP("localhost:9092", "localhost:9093", "localhost:9094"),
Topic: "topic-A",
Compression: kafka.Snappy,
}
```
The `Reader` will by determine if the consumed messages are compressed by
examining the message attributes. However, the package(s) for all expected
codecs must be imported so that they get loaded correctly.
_Note: in versions prior to 0.4 programs had to import compression packages to
install codecs and support reading compressed messages from kafka. This is no
longer the case and import of the compression packages are now no-ops._
## TLS Support
For a bare bones Conn type or in the Reader/Writer configs you can specify a dialer option for TLS support. If the TLS field is nil, it will not connect with TLS.
*Note:* Connecting to a Kafka cluster with TLS enabled without configuring TLS on the Conn/Reader/Writer can manifest in opaque io.ErrUnexpectedEOF errors.
### Connection
```go
dialer := &kafka.Dialer{
Timeout: 10 * time.Second,
DualStack: true,
TLS: &tls.Config{...tls config...},
}
conn, err := dialer.DialContext(ctx, "tcp", "localhost:9093")
```
### Reader
```go
dialer := &kafka.Dialer{
Timeout: 10 * time.Second,
DualStack: true,
TLS: &tls.Config{...tls config...},
}
r := kafka.NewReader(kafka.ReaderConfig{
Brokers: []string{"localhost:9092", "localhost:9093", "localhost:9094"},
GroupID: "consumer-group-id",
Topic: "topic-A",
Dialer: dialer,
})
```
### Writer
Direct Writer creation
```go
w := kafka.Writer{
Addr: kafka.TCP("localhost:9092", "localhost:9093", "localhost:9094"),
Topic: "topic-A",
Balancer: &kafka.Hash{},
Transport: &kafka.Transport{
TLS: &tls.Config{},
},
}
```
Using `kafka.NewWriter`
```go
dialer := &kafka.Dialer{
Timeout: 10 * time.Second,
DualStack: true,
TLS: &tls.Config{...tls config...},
}
w := kafka.NewWriter(kafka.WriterConfig{
Brokers: []string{"localhost:9092", "localhost:9093", "localhost:9094"},
Topic: "topic-A",
Balancer: &kafka.Hash{},
Dialer: dialer,
})
```
Note that `kafka.NewWriter` and `kafka.WriterConfig` are deprecated and will be removed in a future release.
## SASL Support
You can specify an option on the `Dialer` to use SASL authentication. The `Dialer` can be used directly to open a `Conn` or it can be passed to a `Reader` or `Writer` via their respective configs. If the `SASLMechanism` field is `nil`, it will not authenticate with SASL.
### SASL Authentication Types
#### [Plain](https://godoc.org/github.com/segmentio/kafka-go/sasl/plain#Mechanism)
```go
mechanism := plain.Mechanism{
Username: "username",
Password: "password",
}
```
#### [SCRAM](https://godoc.org/github.com/segmentio/kafka-go/sasl/scram#Mechanism)
```go
mechanism, err := scram.Mechanism(scram.SHA512, "username", "password")
if err != nil {
panic(err)
}
```
### Connection
```go
mechanism, err := scram.Mechanism(scram.SHA512, "username", "password")
if err != nil {
panic(err)
}
dialer := &kafka.Dialer{
Timeout: 10 * time.Second,
DualStack: true,
SASLMechanism: mechanism,
}
conn, err := dialer.DialContext(ctx, "tcp", "localhost:9093")
```
### Reader
```go
mechanism, err := scram.Mechanism(scram.SHA512, "username", "password")
if err != nil {
panic(err)
}
dialer := &kafka.Dialer{
Timeout: 10 * time.Second,
DualStack: true,
SASLMechanism: mechanism,
}
r := kafka.NewReader(kafka.ReaderConfig{
Brokers: []string{"localhost:9092","localhost:9093", "localhost:9094"},
GroupID: "consumer-group-id",
Topic: "topic-A",
Dialer: dialer,
})
```
### Writer
```go
mechanism, err := scram.Mechanism(scram.SHA512, "username", "password")
if err != nil {
panic(err)
}
// Transports are responsible for managing connection pools and other resources,
// it's generally best to create a few of these and share them across your
// application.
sharedTransport := &kafka.Transport{
SASL: mechanism,
}
w := kafka.Writer{
Addr: kafka.TCP("localhost:9092", "localhost:9093", "localhost:9094"),
Topic: "topic-A",
Balancer: &kafka.Hash{},
Transport: sharedTransport,
}
```
### Client
```go
mechanism, err := scram.Mechanism(scram.SHA512, "username", "password")
if err != nil {
panic(err)
}
// Transports are responsible for managing connection pools and other resources,
// it's generally best to create a few of these and share them across your
// application.
sharedTransport := &kafka.Transport{
SASL: mechanism,
}
client := &kafka.Client{
Addr: kafka.TCP("localhost:9092", "localhost:9093", "localhost:9094"),
Timeout: 10 * time.Second,
Transport: sharedTransport,
}
```
#### Reading all messages within a time range
```go
startTime := time.Now().Add(-time.Hour)
endTime := time.Now()
batchSize := int(10e6) // 10MB
r := kafka.NewReader(kafka.ReaderConfig{
Brokers: []string{"localhost:9092", "localhost:9093", "localhost:9094"},
Topic: "my-topic1",
Partition: 0,
MaxBytes: batchSize,
})
r.SetOffsetAt(context.Background(), startTime)
for {
m, err := r.ReadMessage(context.Background())
if err != nil {
break
}
if m.Time.After(endTime) {
break
}
// TODO: process message
fmt.Printf("message at offset %d: %s = %s\n", m.Offset, string(m.Key), string(m.Value))
}
if err := r.Close(); err != nil {
log.Fatal("failed to close reader:", err)
}
```
## Logging
For visiblity into the operations of the Reader/Writer types, configure a logger on creation.
### Reader
```go
func logf(msg string, a ...interface{}) {
fmt.Printf(msg, a...)
fmt.Println()
}
r := kafka.NewReader(kafka.ReaderConfig{
Brokers: []string{"localhost:9092", "localhost:9093", "localhost:9094"},
Topic: "my-topic1",
Partition: 0,
Logger: kafka.LoggerFunc(logf),
ErrorLogger: kafka.LoggerFunc(logf),
})
```
### Writer
```go
func logf(msg string, a ...interface{}) {
fmt.Printf(msg, a...)
fmt.Println()
}
w := &kafka.Writer{
Addr: kafka.TCP("localhost:9092"),
Topic: "topic",
Logger: kafka.LoggerFunc(logf),
ErrorLogger: kafka.LoggerFunc(logf),
}
```
## Testing
Subtle behavior changes in later Kafka versions have caused some historical tests to break, if you are running against Kafka 2.3.1 or later, exporting the `KAFKA_SKIP_NETTEST=1` environment variables will skip those tests.
Run Kafka locally in docker
```bash
docker-compose up -d
```
Run tests
```bash
KAFKA_VERSION=2.3.1 \
KAFKA_SKIP_NETTEST=1 \
go test -race ./...
```
(or) to clean up the cached test results and run tests:
```
go clean -cache && make test
```
================================================
FILE: addoffsetstotxn.go
================================================
package kafka
import (
"context"
"fmt"
"net"
"time"
"github.com/segmentio/kafka-go/protocol/addoffsetstotxn"
)
// AddOffsetsToTxnRequest is the request structure for the AddOffsetsToTxn function.
type AddOffsetsToTxnRequest struct {
// Address of the kafka broker to send the request to.
Addr net.Addr
// The transactional id key
TransactionalID string
// The Producer ID (PID) for the current producer session;
// received from an InitProducerID request.
ProducerID int
// The epoch associated with the current producer session for the given PID
ProducerEpoch int
// The unique group identifier.
GroupID string
}
// AddOffsetsToTxnResponse is the response structure for the AddOffsetsToTxn function.
type AddOffsetsToTxnResponse struct {
// The amount of time that the broker throttled the request.
Throttle time.Duration
// An error that may have occurred when attempting to add the offsets
// to a transaction.
//
// The errors contain the kafka error code. Programs may use the standard
// errors.Is function to test the error against kafka error codes.
Error error
}
// AddOffsetsToTnx sends an add offsets to txn request to a kafka broker and returns the response.
func (c *Client) AddOffsetsToTxn(
ctx context.Context,
req *AddOffsetsToTxnRequest,
) (*AddOffsetsToTxnResponse, error) {
m, err := c.roundTrip(ctx, req.Addr, &addoffsetstotxn.Request{
TransactionalID: req.TransactionalID,
ProducerID: int64(req.ProducerID),
ProducerEpoch: int16(req.ProducerEpoch),
GroupID: req.GroupID,
})
if err != nil {
return nil, fmt.Errorf("kafka.(*Client).AddOffsetsToTxn: %w", err)
}
r := m.(*addoffsetstotxn.Response)
res := &AddOffsetsToTxnResponse{
Throttle: makeDuration(r.ThrottleTimeMs),
Error: makeError(r.ErrorCode, ""),
}
return res, nil
}
================================================
FILE: addoffsetstotxn_test.go
================================================
package kafka
import (
"context"
"log"
"os"
"testing"
"time"
ktesting "github.com/segmentio/kafka-go/testing"
)
func TestClientAddOffsetsToTxn(t *testing.T) {
if !ktesting.KafkaIsAtLeast("0.11.0") {
t.Skip("Skipping test because kafka version is not high enough.")
}
// TODO: look into why this test fails on Kafka 3.0.0 and higher when transactional support
// work is revisited.
if ktesting.KafkaIsAtLeast("3.0.0") {
t.Skip("Skipping test because it fails on Kafka version 3.0.0 or higher.")
}
topic := makeTopic()
transactionalID := makeTransactionalID()
client, shutdown := newLocalClient()
defer shutdown()
err := clientCreateTopic(client, topic, 3)
defer deleteTopic(t, topic)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
waitForTopic(ctx, t, topic)
defer cancel()
respc, err := waitForCoordinatorIndefinitely(ctx, client, &FindCoordinatorRequest{
Addr: client.Addr,
Key: transactionalID,
KeyType: CoordinatorKeyTypeConsumer,
})
if err != nil {
t.Fatal(err)
}
if respc.Error != nil {
t.Fatal(err)
}
groupID := makeGroupID()
group, err := NewConsumerGroup(ConsumerGroupConfig{
ID: groupID,
Topics: []string{topic},
Brokers: []string{"localhost:9092"},
HeartbeatInterval: 2 * time.Second,
RebalanceTimeout: 2 * time.Second,
RetentionTime: time.Hour,
Logger: log.New(os.Stdout, "cg-test: ", 0),
})
if err != nil {
t.Fatal(err)
}
defer group.Close()
ctx, cancel = context.WithTimeout(context.Background(), time.Second*30)
defer cancel()
_, err = group.Next(ctx)
if err != nil {
t.Fatal(err)
}
ctx, cancel = context.WithTimeout(context.Background(), time.Second*30)
defer cancel()
respc, err = waitForCoordinatorIndefinitely(ctx, client, &FindCoordinatorRequest{
Addr: client.Addr,
Key: transactionalID,
KeyType: CoordinatorKeyTypeTransaction,
})
if err != nil {
t.Fatal(err)
}
if respc.Error != nil {
t.Fatal(err)
}
ipResp, err := client.InitProducerID(ctx, &InitProducerIDRequest{
TransactionalID: transactionalID,
TransactionTimeoutMs: 10000,
})
if err != nil {
t.Fatal(err)
}
if ipResp.Error != nil {
t.Fatal(ipResp.Error)
}
defer func() {
err := clientEndTxn(client, &EndTxnRequest{
TransactionalID: transactionalID,
ProducerID: ipResp.Producer.ProducerID,
ProducerEpoch: ipResp.Producer.ProducerEpoch,
Committed: false,
})
if err != nil {
t.Fatal(err)
}
}()
ctx, cancel = context.WithTimeout(context.Background(), time.Second*30)
defer cancel()
resp, err := client.AddOffsetsToTxn(ctx, &AddOffsetsToTxnRequest{
TransactionalID: transactionalID,
ProducerID: ipResp.Producer.ProducerID,
ProducerEpoch: ipResp.Producer.ProducerEpoch,
GroupID: groupID,
})
if err != nil {
t.Fatal(err)
}
if resp.Error != nil {
t.Fatal(err)
}
}
================================================
FILE: addpartitionstotxn.go
================================================
package kafka
import (
"context"
"fmt"
"net"
"time"
"github.com/segmentio/kafka-go/protocol/addpartitionstotxn"
)
// AddPartitionToTxn represents a partition to be added
// to a transaction.
type AddPartitionToTxn struct {
// Partition is the ID of a partition to add to the transaction.
Partition int
}
// AddPartitionsToTxnRequest is the request structure fo the AddPartitionsToTxn function.
type AddPartitionsToTxnRequest struct {
// Address of the kafka broker to send the request to.
Addr net.Addr
// The transactional id key
TransactionalID string
// The Producer ID (PID) for the current producer session;
// received from an InitProducerID request.
ProducerID int
// The epoch associated with the current producer session for the given PID
ProducerEpoch int
// Mappings of topic names to lists of partitions.
Topics map[string][]AddPartitionToTxn
}
// AddPartitionsToTxnResponse is the response structure for the AddPartitionsToTxn function.
type AddPartitionsToTxnResponse struct {
// The amount of time that the broker throttled the request.
Throttle time.Duration
// Mappings of topic names to partitions being added to a transactions.
Topics map[string][]AddPartitionToTxnPartition
}
// AddPartitionToTxnPartition represents the state of a single partition
// in response to adding to a transaction.
type AddPartitionToTxnPartition struct {
// The ID of the partition.
Partition int
// An error that may have occurred when attempting to add the partition
// to a transaction.
//
// The errors contain the kafka error code. Programs may use the standard
// errors.Is function to test the error against kafka error codes.
Error error
}
// AddPartitionsToTnx sends an add partitions to txn request to a kafka broker and returns the response.
func (c *Client) AddPartitionsToTxn(
ctx context.Context,
req *AddPartitionsToTxnRequest,
) (*AddPartitionsToTxnResponse, error) {
protoReq := &addpartitionstotxn.Request{
TransactionalID: req.TransactionalID,
ProducerID: int64(req.ProducerID),
ProducerEpoch: int16(req.ProducerEpoch),
}
protoReq.Topics = make([]addpartitionstotxn.RequestTopic, 0, len(req.Topics))
for topic, partitions := range req.Topics {
reqTopic := addpartitionstotxn.RequestTopic{
Name: topic,
Partitions: make([]int32, len(partitions)),
}
for i, partition := range partitions {
reqTopic.Partitions[i] = int32(partition.Partition)
}
protoReq.Topics = append(protoReq.Topics, reqTopic)
}
m, err := c.roundTrip(ctx, req.Addr, protoReq)
if err != nil {
return nil, fmt.Errorf("kafka.(*Client).AddPartitionsToTxn: %w", err)
}
r := m.(*addpartitionstotxn.Response)
res := &AddPartitionsToTxnResponse{
Throttle: makeDuration(r.ThrottleTimeMs),
Topics: make(map[string][]AddPartitionToTxnPartition, len(r.Results)),
}
for _, result := range r.Results {
partitions := make([]AddPartitionToTxnPartition, 0, len(result.Results))
for _, rp := range result.Results {
partitions = append(partitions, AddPartitionToTxnPartition{
Partition: int(rp.PartitionIndex),
Error: makeError(rp.ErrorCode, ""),
})
}
res.Topics[result.Name] = partitions
}
return res, nil
}
================================================
FILE: addpartitionstotxn_test.go
================================================
package kafka
import (
"context"
"net"
"strconv"
"testing"
"time"
ktesting "github.com/segmentio/kafka-go/testing"
)
func TestClientAddPartitionsToTxn(t *testing.T) {
if !ktesting.KafkaIsAtLeast("0.11.0") {
t.Skip("Skipping test because kafka version is not high enough.")
}
// TODO: look into why this test fails on Kafka 3.0.0 and higher when transactional support
// work is revisited.
if ktesting.KafkaIsAtLeast("3.0.0") {
t.Skip("Skipping test because it fails on Kafka version 3.0.0 or higher.")
}
topic1 := makeTopic()
topic2 := makeTopic()
client, shutdown := newLocalClient()
defer shutdown()
err := clientCreateTopic(client, topic1, 3)
if err != nil {
t.Fatal(err)
}
err = clientCreateTopic(client, topic2, 3)
if err != nil {
t.Fatal(err)
}
transactionalID := makeTransactionalID()
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
defer cancel()
respc, err := waitForCoordinatorIndefinitely(ctx, client, &FindCoordinatorRequest{
Addr: client.Addr,
Key: transactionalID,
KeyType: CoordinatorKeyTypeTransaction,
})
if err != nil {
t.Fatal(err)
}
transactionCoordinator := TCP(net.JoinHostPort(respc.Coordinator.Host, strconv.Itoa(int(respc.Coordinator.Port))))
client, shutdown = newClient(transactionCoordinator)
defer shutdown()
ipResp, err := client.InitProducerID(ctx, &InitProducerIDRequest{
TransactionalID: transactionalID,
TransactionTimeoutMs: 10000,
})
if err != nil {
t.Fatal(err)
}
if ipResp.Error != nil {
t.Fatal(ipResp.Error)
}
defer func() {
err := clientEndTxn(client, &EndTxnRequest{
TransactionalID: transactionalID,
ProducerID: ipResp.Producer.ProducerID,
ProducerEpoch: ipResp.Producer.ProducerEpoch,
Committed: false,
})
if err != nil {
t.Fatal(err)
}
}()
ctx, cancel = context.WithTimeout(context.Background(), time.Second*30)
defer cancel()
resp, err := client.AddPartitionsToTxn(ctx, &AddPartitionsToTxnRequest{
TransactionalID: transactionalID,
ProducerID: ipResp.Producer.ProducerID,
ProducerEpoch: ipResp.Producer.ProducerEpoch,
Topics: map[string][]AddPartitionToTxn{
topic1: {
{
Partition: 0,
},
{
Partition: 1,
},
{
Partition: 2,
},
},
topic2: {
{
Partition: 0,
},
{
Partition: 2,
},
},
},
})
if err != nil {
t.Fatal(err)
}
if len(resp.Topics) != 2 {
t.Errorf("expected responses for 2 topics; got: %d", len(resp.Topics))
}
for topic, partitions := range resp.Topics {
if topic == topic1 {
if len(partitions) != 3 {
t.Errorf("expected 3 partitions in response for topic %s; got: %d", topic, len(partitions))
}
}
if topic == topic2 {
if len(partitions) != 2 {
t.Errorf("expected 2 partitions in response for topic %s; got: %d", topic, len(partitions))
}
}
for _, partition := range partitions {
if partition.Error != nil {
t.Error(partition.Error)
}
}
}
}
================================================
FILE: address.go
================================================
package kafka
import (
"net"
"strings"
)
// TCP constructs an address with the network set to "tcp".
func TCP(address ...string) net.Addr { return makeNetAddr("tcp", address) }
func makeNetAddr(network string, addresses []string) net.Addr {
switch len(addresses) {
case 0:
return nil // maybe panic instead?
case 1:
return makeAddr(network, addresses[0])
default:
return makeMultiAddr(network, addresses)
}
}
func makeAddr(network, address string) net.Addr {
return &networkAddress{
network: network,
address: canonicalAddress(address),
}
}
func makeMultiAddr(network string, addresses []string) net.Addr {
multi := make(multiAddr, len(addresses))
for i, address := range addresses {
multi[i] = makeAddr(network, address)
}
return multi
}
type networkAddress struct {
network string
address string
}
func (a *networkAddress) Network() string { return a.network }
func (a *networkAddress) String() string { return a.address }
type multiAddr []net.Addr
func (m multiAddr) Network() string { return m.join(net.Addr.Network) }
func (m multiAddr) String() string { return m.join(net.Addr.String) }
func (m multiAddr) join(f func(net.Addr) string) string {
switch len(m) {
case 0:
return ""
case 1:
return f(m[0])
}
s := make([]string, len(m))
for i, a := range m {
s[i] = f(a)
}
return strings.Join(s, ",")
}
================================================
FILE: address_test.go
================================================
package kafka
import (
"net"
"testing"
)
func TestNetworkAddress(t *testing.T) {
tests := []struct {
addr net.Addr
network string
address string
}{
{
addr: TCP("127.0.0.1"),
network: "tcp",
address: "127.0.0.1:9092",
},
{
addr: TCP("::1"),
network: "tcp",
address: "[::1]:9092",
},
{
addr: TCP("localhost"),
network: "tcp",
address: "localhost:9092",
},
{
addr: TCP("localhost:9092"),
network: "tcp",
address: "localhost:9092",
},
{
addr: TCP("localhost", "localhost:9093", "localhost:9094"),
network: "tcp,tcp,tcp",
address: "localhost:9092,localhost:9093,localhost:9094",
},
}
for _, test := range tests {
t.Run(test.network+"+"+test.address, func(t *testing.T) {
if s := test.addr.Network(); s != test.network {
t.Errorf("network mismatch: want %q but got %q", test.network, s)
}
if s := test.addr.String(); s != test.address {
t.Errorf("network mismatch: want %q but got %q", test.address, s)
}
})
}
}
================================================
FILE: alterclientquotas.go
================================================
package kafka
import (
"context"
"fmt"
"net"
"time"
"github.com/segmentio/kafka-go/protocol/alterclientquotas"
)
// AlterClientQuotasRequest represents a request sent to a kafka broker to
// alter client quotas.
type AlterClientQuotasRequest struct {
// Address of the kafka broker to send the request to.
Addr net.Addr
// List of client quotas entries to alter.
Entries []AlterClientQuotaEntry
// Whether the alteration should be validated, but not performed.
ValidateOnly bool
}
type AlterClientQuotaEntry struct {
// The quota entities to alter.
Entities []AlterClientQuotaEntity
// An individual quota configuration entry to alter.
Ops []AlterClientQuotaOps
}
type AlterClientQuotaEntity struct {
// The quota entity type.
EntityType string
// The name of the quota entity, or null if the default.
EntityName string
}
type AlterClientQuotaOps struct {
// The quota configuration key.
Key string
// The quota configuration value to set, otherwise ignored if the value is to be removed.
Value float64
// Whether the quota configuration value should be removed, otherwise set.
Remove bool
}
type AlterClientQuotaResponseQuotas struct {
// Error is set to a non-nil value including the code and message if a top-level
// error was encountered when doing the update.
Error error
// The altered quota entities.
Entities []AlterClientQuotaEntity
}
// AlterClientQuotasResponse represents a response from a kafka broker to an alter client
// quotas request.
type AlterClientQuotasResponse struct {
// The amount of time that the broker throttled the request.
Throttle time.Duration
// List of altered client quotas responses.
Entries []AlterClientQuotaResponseQuotas
}
// AlterClientQuotas sends client quotas alteration request to a kafka broker and returns
// the response.
func (c *Client) AlterClientQuotas(ctx context.Context, req *AlterClientQuotasRequest) (*AlterClientQuotasResponse, error) {
entries := make([]alterclientquotas.Entry, len(req.Entries))
for entryIdx, entry := range req.Entries {
entities := make([]alterclientquotas.Entity, len(entry.Entities))
for entityIdx, entity := range entry.Entities {
entities[entityIdx] = alterclientquotas.Entity{
EntityType: entity.EntityType,
EntityName: entity.EntityName,
}
}
ops := make([]alterclientquotas.Ops, len(entry.Ops))
for opsIdx, op := range entry.Ops {
ops[opsIdx] = alterclientquotas.Ops{
Key: op.Key,
Value: op.Value,
Remove: op.Remove,
}
}
entries[entryIdx] = alterclientquotas.Entry{
Entities: entities,
Ops: ops,
}
}
m, err := c.roundTrip(ctx, req.Addr, &alterclientquotas.Request{
Entries: entries,
ValidateOnly: req.ValidateOnly,
})
if err != nil {
return nil, fmt.Errorf("kafka.(*Client).AlterClientQuotas: %w", err)
}
res := m.(*alterclientquotas.Response)
responseEntries := make([]AlterClientQuotaResponseQuotas, len(res.Results))
for responseEntryIdx, responseEntry := range res.Results {
responseEntities := make([]AlterClientQuotaEntity, len(responseEntry.Entities))
for responseEntityIdx, responseEntity := range responseEntry.Entities {
responseEntities[responseEntityIdx] = AlterClientQuotaEntity{
EntityType: responseEntity.EntityType,
EntityName: responseEntity.EntityName,
}
}
responseEntries[responseEntryIdx] = AlterClientQuotaResponseQuotas{
Error: makeError(responseEntry.ErrorCode, responseEntry.ErrorMessage),
Entities: responseEntities,
}
}
ret := &AlterClientQuotasResponse{
Throttle: makeDuration(res.ThrottleTimeMs),
Entries: responseEntries,
}
return ret, nil
}
================================================
FILE: alterclientquotas_test.go
================================================
package kafka
import (
"context"
"testing"
"time"
ktesting "github.com/segmentio/kafka-go/testing"
"github.com/stretchr/testify/assert"
)
func TestClientAlterClientQuotas(t *testing.T) {
// Added in Version 2.6.0 https://issues.apache.org/jira/browse/KAFKA-7740
if !ktesting.KafkaIsAtLeast("2.6.0") {
return
}
const (
entityType = "client-id"
entityName = "my-client-id"
key = "producer_byte_rate"
value = 500000.0
)
client, shutdown := newLocalClient()
defer shutdown()
alterResp, err := client.AlterClientQuotas(context.Background(), &AlterClientQuotasRequest{
Entries: []AlterClientQuotaEntry{
{
Entities: []AlterClientQuotaEntity{
{
EntityType: entityType,
EntityName: entityName,
},
},
Ops: []AlterClientQuotaOps{
{
Key: key,
Value: value,
Remove: false,
},
},
},
},
})
if err != nil {
t.Fatal(err)
}
expectedAlterResp := AlterClientQuotasResponse{
Throttle: 0,
Entries: []AlterClientQuotaResponseQuotas{
{
Error: makeError(0, ""),
Entities: []AlterClientQuotaEntity{
{
EntityName: entityName,
EntityType: entityType,
},
},
},
},
}
assert.Equal(t, expectedAlterResp, *alterResp)
// kraft mode is slow
if ktesting.KafkaIsAtLeast("3.7.0") {
time.Sleep(3 * time.Second)
}
describeResp, err := client.DescribeClientQuotas(context.Background(), &DescribeClientQuotasRequest{
Components: []DescribeClientQuotasRequestComponent{
{
EntityType: entityType,
MatchType: 0,
Match: entityName,
},
},
})
if err != nil {
t.Fatal(err)
}
expectedDescribeResp := DescribeClientQuotasResponse{
Throttle: 0,
Error: makeError(0, ""),
Entries: []DescribeClientQuotasResponseQuotas{
{
Entities: []DescribeClientQuotasEntity{
{
EntityType: entityType,
EntityName: entityName,
},
},
Values: []DescribeClientQuotasValue{
{
Key: key,
Value: value,
},
},
},
},
}
assert.Equal(t, expectedDescribeResp, *describeResp)
}
================================================
FILE: alterconfigs.go
================================================
package kafka
import (
"context"
"fmt"
"net"
"time"
"github.com/segmentio/kafka-go/protocol/alterconfigs"
)
// AlterConfigsRequest represents a request sent to a kafka broker to alter configs.
type AlterConfigsRequest struct {
// Address of the kafka broker to send the request to.
Addr net.Addr
// List of resources to update.
Resources []AlterConfigRequestResource
// When set to true, topics are not created but the configuration is
// validated as if they were.
ValidateOnly bool
}
type AlterConfigRequestResource struct {
// Resource Type
ResourceType ResourceType
// Resource Name
ResourceName string
// Configs is a list of configuration updates.
Configs []AlterConfigRequestConfig
}
type AlterConfigRequestConfig struct {
// Configuration key name
Name string
// The value to set for the configuration key.
Value string
}
// AlterConfigsResponse represents a response from a kafka broker to an alter config request.
type AlterConfigsResponse struct {
// Duration for which the request was throttled due to a quota violation.
Throttle time.Duration
// Mapping of topic names to errors that occurred while attempting to create
// the topics.
//
// The errors contain the kafka error code. Programs may use the standard
// errors.Is function to test the error against kafka error codes.
Errors map[AlterConfigsResponseResource]error
}
// AlterConfigsResponseResource helps map errors to specific resources in an
// alter config response.
type AlterConfigsResponseResource struct {
Type int8
Name string
}
// AlterConfigs sends a config altering request to a kafka broker and returns the
// response.
func (c *Client) AlterConfigs(ctx context.Context, req *AlterConfigsRequest) (*AlterConfigsResponse, error) {
resources := make([]alterconfigs.RequestResources, len(req.Resources))
for i, t := range req.Resources {
configs := make([]alterconfigs.RequestConfig, len(t.Configs))
for j, v := range t.Configs {
configs[j] = alterconfigs.RequestConfig{
Name: v.Name,
Value: v.Value,
}
}
resources[i] = alterconfigs.RequestResources{
ResourceType: int8(t.ResourceType),
ResourceName: t.ResourceName,
Configs: configs,
}
}
m, err := c.roundTrip(ctx, req.Addr, &alterconfigs.Request{
Resources: resources,
ValidateOnly: req.ValidateOnly,
})
if err != nil {
return nil, fmt.Errorf("kafka.(*Client).AlterConfigs: %w", err)
}
res := m.(*alterconfigs.Response)
ret := &AlterConfigsResponse{
Throttle: makeDuration(res.ThrottleTimeMs),
Errors: make(map[AlterConfigsResponseResource]error, len(res.Responses)),
}
for _, t := range res.Responses {
ret.Errors[AlterConfigsResponseResource{
Type: t.ResourceType,
Name: t.ResourceName,
}] = makeError(t.ErrorCode, t.ErrorMessage)
}
return ret, nil
}
================================================
FILE: alterconfigs_test.go
================================================
package kafka
import (
"context"
"testing"
ktesting "github.com/segmentio/kafka-go/testing"
"github.com/stretchr/testify/assert"
)
func TestClientAlterConfigs(t *testing.T) {
if !ktesting.KafkaIsAtLeast("0.11.0") {
return
}
const (
MaxMessageBytes = "max.message.bytes"
MaxMessageBytesValue = "200000"
)
client, shutdown := newLocalClient()
defer shutdown()
topic := makeTopic()
createTopic(t, topic, 1)
defer deleteTopic(t, topic)
_, err := client.AlterConfigs(context.Background(), &AlterConfigsRequest{
Resources: []AlterConfigRequestResource{{
ResourceType: ResourceTypeTopic,
ResourceName: topic,
Configs: []AlterConfigRequestConfig{{
Name: MaxMessageBytes,
Value: MaxMessageBytesValue,
},
},
}},
})
if err != nil {
t.Fatal(err)
}
describeResp, err := client.DescribeConfigs(context.Background(), &DescribeConfigsRequest{
Resources: []DescribeConfigRequestResource{{
ResourceType: ResourceTypeTopic,
ResourceName: topic,
ConfigNames: []string{MaxMessageBytes},
}},
})
if err != nil {
t.Fatal(err)
}
maxMessageBytesValue := "0"
for _, resource := range describeResp.Resources {
if resource.ResourceType == int8(ResourceTypeTopic) && resource.ResourceName == topic {
for _, entry := range resource.ConfigEntries {
if entry.ConfigName == MaxMessageBytes {
maxMessageBytesValue = entry.ConfigValue
}
}
}
}
assert.Equal(t, maxMessageBytesValue, MaxMessageBytesValue)
}
================================================
FILE: alterpartitionreassignments.go
================================================
package kafka
import (
"context"
"net"
"time"
"github.com/segmentio/kafka-go/protocol/alterpartitionreassignments"
)
// AlterPartitionReassignmentsRequest is a request to the AlterPartitionReassignments API.
type AlterPartitionReassignmentsRequest struct {
// Address of the kafka broker to send the request to.
Addr net.Addr
// Topic is the name of the topic to alter partitions in. Keep this field empty and use Topic in AlterPartitionReassignmentsRequestAssignment to
// reassign to multiple topics.
Topic string
// Assignments is the list of partition reassignments to submit to the API.
Assignments []AlterPartitionReassignmentsRequestAssignment
// Timeout is the amount of time to wait for the request to complete.
Timeout time.Duration
}
// AlterPartitionReassignmentsRequestAssignment contains the requested reassignments for a single
// partition.
type AlterPartitionReassignmentsRequestAssignment struct {
// Topic is the name of the topic to alter partitions in. If empty, the value of Topic in AlterPartitionReassignmentsRequest is used.
Topic string
// PartitionID is the ID of the partition to make the reassignments in.
PartitionID int
// BrokerIDs is a slice of brokers to set the partition replicas to, or null to cancel a pending reassignment for this partition.
BrokerIDs []int
}
// AlterPartitionReassignmentsResponse is a response from the AlterPartitionReassignments API.
type AlterPartitionReassignmentsResponse struct {
// Error is set to a non-nil value including the code and message if a top-level
// error was encountered when doing the update.
Error error
// PartitionResults contains the specific results for each partition.
PartitionResults []AlterPartitionReassignmentsResponsePartitionResult
}
// AlterPartitionReassignmentsResponsePartitionResult contains the detailed result of
// doing reassignments for a single partition.
type AlterPartitionReassignmentsResponsePartitionResult struct {
// Topic is the topic name.
Topic string
// PartitionID is the ID of the partition that was altered.
PartitionID int
// Error is set to a non-nil value including the code and message if an error was encountered
// during the update for this partition.
Error error
}
func (c *Client) AlterPartitionReassignments(
ctx context.Context,
req *AlterPartitionReassignmentsRequest,
) (*AlterPartitionReassignmentsResponse, error) {
apiTopicMap := make(map[string]*alterpartitionreassignments.RequestTopic)
for _, assignment := range req.Assignments {
topic := assignment.Topic
if topic == "" {
topic = req.Topic
}
apiTopic := apiTopicMap[topic]
if apiTopic == nil {
apiTopic = &alterpartitionreassignments.RequestTopic{
Name: topic,
}
apiTopicMap[topic] = apiTopic
}
replicas := []int32{}
for _, brokerID := range assignment.BrokerIDs {
replicas = append(replicas, int32(brokerID))
}
apiTopic.Partitions = append(
apiTopic.Partitions,
alterpartitionreassignments.RequestPartition{
PartitionIndex: int32(assignment.PartitionID),
Replicas: replicas,
},
)
}
apiReq := &alterpartitionreassignments.Request{
TimeoutMs: int32(req.Timeout.Milliseconds()),
}
for _, apiTopic := range apiTopicMap {
apiReq.Topics = append(apiReq.Topics, *apiTopic)
}
protoResp, err := c.roundTrip(
ctx,
req.Addr,
apiReq,
)
if err != nil {
return nil, err
}
apiResp := protoResp.(*alterpartitionreassignments.Response)
resp := &AlterPartitionReassignmentsResponse{
Error: makeError(apiResp.ErrorCode, apiResp.ErrorMessage),
}
for _, topicResult := range apiResp.Results {
for _, partitionResult := range topicResult.Partitions {
resp.PartitionResults = append(
resp.PartitionResults,
AlterPartitionReassignmentsResponsePartitionResult{
Topic: topicResult.Name,
PartitionID: int(partitionResult.PartitionIndex),
Error: makeError(partitionResult.ErrorCode, partitionResult.ErrorMessage),
},
)
}
}
return resp, nil
}
================================================
FILE: alterpartitionreassignments_test.go
================================================
package kafka
import (
"context"
"testing"
"time"
ktesting "github.com/segmentio/kafka-go/testing"
)
func TestClientAlterPartitionReassignments(t *testing.T) {
if !ktesting.KafkaIsAtLeast("2.4.0") {
return
}
ctx := context.Background()
client, shutdown := newLocalClient()
defer shutdown()
topic := makeTopic()
createTopic(t, topic, 2)
defer deleteTopic(t, topic)
// Local kafka only has 1 broker, so any partition reassignments are really no-ops.
resp, err := client.AlterPartitionReassignments(
ctx,
&AlterPartitionReassignmentsRequest{
Topic: topic,
Assignments: []AlterPartitionReassignmentsRequestAssignment{
{
PartitionID: 0,
BrokerIDs: []int{1},
},
{
PartitionID: 1,
BrokerIDs: []int{1},
},
},
Timeout: 5 * time.Second,
},
)
if err != nil {
t.Fatal(err)
}
if resp.Error != nil {
t.Error(
"Unexpected error in response",
"expected", nil,
"got", resp.Error,
)
}
if len(resp.PartitionResults) != 2 {
t.Error(
"Unexpected length of partition results",
"expected", 2,
"got", len(resp.PartitionResults),
)
}
}
func TestClientAlterPartitionReassignmentsMultiTopics(t *testing.T) {
if !ktesting.KafkaIsAtLeast("2.4.0") {
return
}
ctx := context.Background()
client, shutdown := newLocalClient()
defer shutdown()
topic1 := makeTopic()
topic2 := makeTopic()
createTopic(t, topic1, 2)
createTopic(t, topic2, 2)
defer func() {
deleteTopic(t, topic1)
deleteTopic(t, topic2)
}()
// Local kafka only has 1 broker, so any partition reassignments are really no-ops.
resp, err := client.AlterPartitionReassignments(
ctx,
&AlterPartitionReassignmentsRequest{
Assignments: []AlterPartitionReassignmentsRequestAssignment{
{
Topic: topic1,
PartitionID: 0,
BrokerIDs: []int{1},
},
{
Topic: topic1,
PartitionID: 1,
BrokerIDs: []int{1},
},
{
Topic: topic2,
PartitionID: 0,
BrokerIDs: []int{1},
},
},
Timeout: 5 * time.Second,
},
)
if err != nil {
t.Fatal(err)
}
if resp.Error != nil {
t.Error(
"Unexpected error in response",
"expected", nil,
"got", resp.Error,
)
}
if len(resp.PartitionResults) != 3 {
t.Error(
"Unexpected length of partition results",
"expected", 3,
"got", len(resp.PartitionResults),
)
}
}
================================================
FILE: alteruserscramcredentials.go
================================================
package kafka
import (
"context"
"fmt"
"net"
"time"
"github.com/segmentio/kafka-go/protocol/alteruserscramcredentials"
)
// AlterUserScramCredentialsRequest represents a request sent to a kafka broker to
// alter user scram credentials.
type AlterUserScramCredentialsRequest struct {
// Address of the kafka broker to send the request to.
Addr net.Addr
// List of credentials to delete.
Deletions []UserScramCredentialsDeletion
// List of credentials to upsert.
Upsertions []UserScramCredentialsUpsertion
}
type ScramMechanism int8
const (
ScramMechanismUnknown ScramMechanism = iota // 0
ScramMechanismSha256 // 1
ScramMechanismSha512 // 2
)
type UserScramCredentialsDeletion struct {
Name string
Mechanism ScramMechanism
}
type UserScramCredentialsUpsertion struct {
Name string
Mechanism ScramMechanism
Iterations int
Salt []byte
SaltedPassword []byte
}
// AlterUserScramCredentialsResponse represents a response from a kafka broker to an alter user
// credentials request.
type AlterUserScramCredentialsResponse struct {
// The amount of time that the broker throttled the request.
Throttle time.Duration
// List of altered user scram credentials.
Results []AlterUserScramCredentialsResponseUser
}
type AlterUserScramCredentialsResponseUser struct {
User string
Error error
}
// AlterUserScramCredentials sends user scram credentials alteration request to a kafka broker and returns
// the response.
func (c *Client) AlterUserScramCredentials(ctx context.Context, req *AlterUserScramCredentialsRequest) (*AlterUserScramCredentialsResponse, error) {
deletions := make([]alteruserscramcredentials.RequestUserScramCredentialsDeletion, len(req.Deletions))
upsertions := make([]alteruserscramcredentials.RequestUserScramCredentialsUpsertion, len(req.Upsertions))
for deletionIdx, deletion := range req.Deletions {
deletions[deletionIdx] = alteruserscramcredentials.RequestUserScramCredentialsDeletion{
Name: deletion.Name,
Mechanism: int8(deletion.Mechanism),
}
}
for upsertionIdx, upsertion := range req.Upsertions {
upsertions[upsertionIdx] = alteruserscramcredentials.RequestUserScramCredentialsUpsertion{
Name: upsertion.Name,
Mechanism: int8(upsertion.Mechanism),
Iterations: int32(upsertion.Iterations),
Salt: upsertion.Salt,
SaltedPassword: upsertion.SaltedPassword,
}
}
m, err := c.roundTrip(ctx, req.Addr, &alteruserscramcredentials.Request{
Deletions: deletions,
Upsertions: upsertions,
})
if err != nil {
return nil, fmt.Errorf("kafka.(*Client).AlterUserScramCredentials: %w", err)
}
res := m.(*alteruserscramcredentials.Response)
responseEntries := make([]AlterUserScramCredentialsResponseUser, len(res.Results))
for responseIdx, responseResult := range res.Results {
responseEntries[responseIdx] = AlterUserScramCredentialsResponseUser{
User: responseResult.User,
Error: makeError(responseResult.ErrorCode, responseResult.ErrorMessage),
}
}
ret := &AlterUserScramCredentialsResponse{
Throttle: makeDuration(res.ThrottleTimeMs),
Results: responseEntries,
}
return ret, nil
}
================================================
FILE: alteruserscramcredentials_test.go
================================================
package kafka
import (
"context"
"testing"
ktesting "github.com/segmentio/kafka-go/testing"
)
func TestAlterUserScramCredentials(t *testing.T) {
// https://issues.apache.org/jira/browse/KAFKA-10259
if !ktesting.KafkaIsAtLeast("2.7.0") {
return
}
client, shutdown := newLocalClient()
defer shutdown()
name := makeTopic()
createRes, err := client.AlterUserScramCredentials(context.Background(), &AlterUserScramCredentialsRequest{
Upsertions: []UserScramCredentialsUpsertion{
{
Name: name,
Mechanism: ScramMechanismSha512,
Iterations: 15000,
Salt: []byte("my-salt"),
SaltedPassword: []byte("my-salted-password"),
},
},
})
if err != nil {
t.Fatal(err)
}
if len(createRes.Results) != 1 {
t.Fatalf("expected 1 createResult; got %d", len(createRes.Results))
}
if createRes.Results[0].User != name {
t.Fatalf("expected createResult with user: %s, got %s", name, createRes.Results[0].User)
}
if createRes.Results[0].Error != nil {
t.Fatalf("didn't expect an error in createResult, got %v", createRes.Results[0].Error)
}
deleteRes, err := client.AlterUserScramCredentials(context.Background(), &AlterUserScramCredentialsRequest{
Deletions: []UserScramCredentialsDeletion{
{
Name: name,
Mechanism: ScramMechanismSha512,
},
},
})
if err != nil {
t.Fatal(err)
}
if len(deleteRes.Results) != 1 {
t.Fatalf("expected 1 deleteResult; got %d", len(deleteRes.Results))
}
if deleteRes.Results[0].User != name {
t.Fatalf("expected deleteResult with user: %s, got %s", name, deleteRes.Results[0].User)
}
if deleteRes.Results[0].Error != nil {
t.Fatalf("didn't expect an error in deleteResult, got %v", deleteRes.Results[0].Error)
}
}
================================================
FILE: apiversions.go
================================================
package kafka
import (
"context"
"net"
"github.com/segmentio/kafka-go/protocol"
"github.com/segmentio/kafka-go/protocol/apiversions"
)
// ApiVersionsRequest is a request to the ApiVersions API.
type ApiVersionsRequest struct {
// Address of the kafka broker to send the request to.
Addr net.Addr
}
// ApiVersionsResponse is a response from the ApiVersions API.
type ApiVersionsResponse struct {
// Error is set to a non-nil value if an error was encountered.
Error error
// ApiKeys contains the specific details of each supported API.
ApiKeys []ApiVersionsResponseApiKey
}
// ApiVersionsResponseApiKey includes the details of which versions are supported for a single API.
type ApiVersionsResponseApiKey struct {
// ApiKey is the ID of the API.
ApiKey int
// ApiName is a human-friendly description of the API.
ApiName string
// MinVersion is the minimum API version supported by the broker.
MinVersion int
// MaxVersion is the maximum API version supported by the broker.
MaxVersion int
}
func (c *Client) ApiVersions(
ctx context.Context,
req *ApiVersionsRequest,
) (*ApiVersionsResponse, error) {
apiReq := &apiversions.Request{}
protoResp, err := c.roundTrip(
ctx,
req.Addr,
apiReq,
)
if err != nil {
return nil, err
}
apiResp := protoResp.(*apiversions.Response)
resp := &ApiVersionsResponse{
Error: makeError(apiResp.ErrorCode, ""),
}
for _, apiKey := range apiResp.ApiKeys {
resp.ApiKeys = append(
resp.ApiKeys,
ApiVersionsResponseApiKey{
ApiKey: int(apiKey.ApiKey),
ApiName: protocol.ApiKey(apiKey.ApiKey).String(),
MinVersion: int(apiKey.MinVersion),
MaxVersion: int(apiKey.MaxVersion),
},
)
}
return resp, err
}
================================================
FILE: apiversions_test.go
================================================
package kafka
import (
"context"
"testing"
)
func TestClientApiVersions(t *testing.T) {
ctx := context.Background()
client, shutdown := newLocalClient()
defer shutdown()
resp, err := client.ApiVersions(ctx, &ApiVersionsRequest{})
if err != nil {
t.Fatal(err)
}
if resp.Error != nil {
t.Error(
"Unexpected error in response",
"expected", nil,
"got", resp.Error,
)
}
if len(resp.ApiKeys) == 0 {
t.Error(
"Unexpected apiKeys length",
"expected greater than", 0,
"got", 0,
)
}
}
================================================
FILE: balancer.go
================================================
package kafka
import (
"hash"
"hash/crc32"
"hash/fnv"
"math/rand"
"sort"
"sync"
)
// The Balancer interface provides an abstraction of the message distribution
// logic used by Writer instances to route messages to the partitions available
// on a kafka cluster.
//
// Balancers must be safe to use concurrently from multiple goroutines.
type Balancer interface {
// Balance receives a message and a set of available partitions and
// returns the partition number that the message should be routed to.
//
// An application should refrain from using a balancer to manage multiple
// sets of partitions (from different topics for examples), use one balancer
// instance for each partition set, so the balancer can detect when the
// partitions change and assume that the kafka topic has been rebalanced.
Balance(msg Message, partitions ...int) (partition int)
}
// BalancerFunc is an implementation of the Balancer interface that makes it
// possible to use regular functions to distribute messages across partitions.
type BalancerFunc func(Message, ...int) int
// Balance calls f, satisfies the Balancer interface.
func (f BalancerFunc) Balance(msg Message, partitions ...int) int {
return f(msg, partitions...)
}
// RoundRobin is an Balancer implementation that equally distributes messages
// across all available partitions. It can take an optional chunk size to send
// ChunkSize messages to the same partition before moving to the next partition.
// This can be used to improve batch sizes.
type RoundRobin struct {
ChunkSize int
// Use a 32 bits integer so RoundRobin values don't need to be aligned to
// apply increments.
counter uint32
mutex sync.Mutex
}
// Balance satisfies the Balancer interface.
func (rr *RoundRobin) Balance(msg Message, partitions ...int) int {
return rr.balance(partitions)
}
func (rr *RoundRobin) balance(partitions []int) int {
rr.mutex.Lock()
defer rr.mutex.Unlock()
if rr.ChunkSize < 1 {
rr.ChunkSize = 1
}
length := len(partitions)
counterNow := rr.counter
offset := int(counterNow / uint32(rr.ChunkSize))
rr.counter++
return partitions[offset%length]
}
// LeastBytes is a Balancer implementation that routes messages to the partition
// that has received the least amount of data.
//
// Note that no coordination is done between multiple producers, having good
// balancing relies on the fact that each producer using a LeastBytes balancer
// should produce well balanced messages.
type LeastBytes struct {
mutex sync.Mutex
counters []leastBytesCounter
}
type leastBytesCounter struct {
partition int
bytes uint64
}
// Balance satisfies the Balancer interface.
func (lb *LeastBytes) Balance(msg Message, partitions ...int) int {
lb.mutex.Lock()
defer lb.mutex.Unlock()
// partitions change
if len(partitions) != len(lb.counters) {
lb.counters = lb.makeCounters(partitions...)
}
minBytes := lb.counters[0].bytes
minIndex := 0
for i, c := range lb.counters[1:] {
if c.bytes < minBytes {
minIndex = i + 1
minBytes = c.bytes
}
}
c := &lb.counters[minIndex]
c.bytes += uint64(len(msg.Key)) + uint64(len(msg.Value))
return c.partition
}
func (lb *LeastBytes) makeCounters(partitions ...int) (counters []leastBytesCounter) {
counters = make([]leastBytesCounter, len(partitions))
for i, p := range partitions {
counters[i].partition = p
}
sort.Slice(counters, func(i int, j int) bool {
return counters[i].partition < counters[j].partition
})
return
}
var (
fnv1aPool = &sync.Pool{
New: func() interface{} {
return fnv.New32a()
},
}
)
// Hash is a Balancer that uses the provided hash function to determine which
// partition to route messages to. This ensures that messages with the same key
// are routed to the same partition.
//
// The logic to calculate the partition is:
//
// hasher.Sum32() % len(partitions) => partition
//
// By default, Hash uses the FNV-1a algorithm. This is the same algorithm used
// by the Sarama Producer and ensures that messages produced by kafka-go will
// be delivered to the same topics that the Sarama producer would be delivered to.
type Hash struct {
rr RoundRobin
Hasher hash.Hash32
// lock protects Hasher while calculating the hash code. It is assumed that
// the Hasher field is read-only once the Balancer is created, so as a
// performance optimization, reads of the field are not protected.
lock sync.Mutex
}
func (h *Hash) Balance(msg Message, partitions ...int) int {
if msg.Key == nil {
return h.rr.Balance(msg, partitions...)
}
hasher := h.Hasher
if hasher != nil {
h.lock.Lock()
defer h.lock.Unlock()
} else {
hasher = fnv1aPool.Get().(hash.Hash32)
defer fnv1aPool.Put(hasher)
}
hasher.Reset()
if _, err := hasher.Write(msg.Key); err != nil {
panic(err)
}
// uses same algorithm that Sarama's hashPartitioner uses
// note the type conversions here. if the uint32 hash code is not cast to
// an int32, we do not get the same result as sarama.
partition := int32(hasher.Sum32()) % int32(len(partitions))
if partition < 0 {
partition = -partition
}
return int(partition)
}
// ReferenceHash is a Balancer that uses the provided hash function to determine which
// partition to route messages to. This ensures that messages with the same key
// are routed to the same partition.
//
// The logic to calculate the partition is:
//
// (int32(hasher.Sum32()) & 0x7fffffff) % len(partitions) => partition
//
// By default, ReferenceHash uses the FNV-1a algorithm. This is the same algorithm as
// the Sarama NewReferenceHashPartitioner and ensures that messages produced by kafka-go will
// be delivered to the same topics that the Sarama producer would be delivered to.
type ReferenceHash struct {
rr randomBalancer
Hasher hash.Hash32
// lock protects Hasher while calculating the hash code. It is assumed that
// the Hasher field is read-only once the Balancer is created, so as a
// performance optimization, reads of the field are not protected.
lock sync.Mutex
}
func (h *ReferenceHash) Balance(msg Message, partitions ...int) int {
if msg.Key == nil {
return h.rr.Balance(msg, partitions...)
}
hasher := h.Hasher
if hasher != nil {
h.lock.Lock()
defer h.lock.Unlock()
} else {
hasher = fnv1aPool.Get().(hash.Hash32)
defer fnv1aPool.Put(hasher)
}
hasher.Reset()
if _, err := hasher.Write(msg.Key); err != nil {
panic(err)
}
// uses the same algorithm as the Sarama's referenceHashPartitioner.
// note the type conversions here. if the uint32 hash code is not cast to
// an int32, we do not get the same result as sarama.
partition := (int32(hasher.Sum32()) & 0x7fffffff) % int32(len(partitions))
return int(partition)
}
type randomBalancer struct {
mock int // mocked return value, used for testing
}
func (b randomBalancer) Balance(msg Message, partitions ...int) (partition int) {
if b.mock != 0 {
return b.mock
}
return partitions[rand.Int()%len(partitions)]
}
// CRC32Balancer is a Balancer that uses the CRC32 hash function to determine
// which partition to route messages to. This ensures that messages with the
// same key are routed to the same partition. This balancer is compatible with
// the built-in hash partitioners in librdkafka and the language bindings that
// are built on top of it, including the
// github.com/confluentinc/confluent-kafka-go Go package.
//
// With the Consistent field false (default), this partitioner is equivalent to
// the "consistent_random" setting in librdkafka. When Consistent is true, this
// partitioner is equivalent to the "consistent" setting. The latter will hash
// empty or nil keys into the same partition.
//
// Unless you are absolutely certain that all your messages will have keys, it's
// best to leave the Consistent flag off. Otherwise, you run the risk of
// creating a very hot partition.
type CRC32Balancer struct {
Consistent bool
random randomBalancer
}
func (b CRC32Balancer) Balance(msg Message, partitions ...int) (partition int) {
// NOTE: the crc32 balancers in librdkafka don't differentiate between nil
// and empty keys. both cases are treated as unset.
if len(msg.Key) == 0 && !b.Consistent {
return b.random.Balance(msg, partitions...)
}
idx := crc32.ChecksumIEEE(msg.Key) % uint32(len(partitions))
return partitions[idx]
}
// Murmur2Balancer is a Balancer that uses the Murmur2 hash function to
// determine which partition to route messages to. This ensures that messages
// with the same key are routed to the same partition. This balancer is
// compatible with the partitioner used by the Java library and by librdkafka's
// "murmur2" and "murmur2_random" partitioners.
//
// With the Consistent field false (default), this partitioner is equivalent to
// the "murmur2_random" setting in librdkafka. When Consistent is true, this
// partitioner is equivalent to the "murmur2" setting. The latter will hash
// nil keys into the same partition. Empty, non-nil keys are always hashed to
// the same partition regardless of configuration.
//
// Unless you are absolutely certain that all your messages will have keys, it's
// best to leave the Consistent flag off. Otherwise, you run the risk of
// creating a very hot partition.
//
// Note that the librdkafka documentation states that the "murmur2_random" is
// functionally equivalent to the default Java partitioner. That's because the
// Java partitioner will use a round robin balancer instead of random on nil
// keys. We choose librdkafka's implementation because it arguably has a larger
// install base.
type Murmur2Balancer struct {
Consistent bool
random randomBalancer
}
func (b Murmur2Balancer) Balance(msg Message, partitions ...int) (partition int) {
// NOTE: the murmur2 balancers in java and librdkafka treat a nil key as
// non-existent while treating an empty slice as a defined value.
if msg.Key == nil && !b.Consistent {
return b.random.Balance(msg, partitions...)
}
idx := (murmur2(msg.Key) & 0x7fffffff) % uint32(len(partitions))
return partitions[idx]
}
// Go port of the Java library's murmur2 function.
// https://github.com/apache/kafka/blob/1.0/clients/src/main/java/org/apache/kafka/common/utils/Utils.java#L353
func murmur2(data []byte) uint32 {
length := len(data)
const (
seed uint32 = 0x9747b28c
// 'm' and 'r' are mixing constants generated offline.
// They're not really 'magic', they just happen to work well.
m = 0x5bd1e995
r = 24
)
// Initialize the hash to a random value
h := seed ^ uint32(length)
length4 := length / 4
for i := 0; i < length4; i++ {
i4 := i * 4
k := (uint32(data[i4+0]) & 0xff) + ((uint32(data[i4+1]) & 0xff) << 8) + ((uint32(data[i4+2]) & 0xff) << 16) + ((uint32(data[i4+3]) & 0xff) << 24)
k *= m
k ^= k >> r
k *= m
h *= m
h ^= k
}
// Handle the last few bytes of the input array
extra := length % 4
if extra >= 3 {
h ^= (uint32(data[(length & ^3)+2]) & 0xff) << 16
}
if extra >= 2 {
h ^= (uint32(data[(length & ^3)+1]) & 0xff) << 8
}
if extra >= 1 {
h ^= uint32(data[length & ^3]) & 0xff
h *= m
}
h ^= h >> 13
h *= m
h ^= h >> 15
return h
}
================================================
FILE: balancer_test.go
================================================
package kafka
import (
"fmt"
"hash"
"hash/crc32"
"testing"
)
func TestHashBalancer(t *testing.T) {
testCases := map[string]struct {
Key []byte
Hasher hash.Hash32
Partitions []int
Partition int
}{
"nil": {
Key: nil,
Partitions: []int{0, 1, 2},
Partition: 0,
},
"partition-0": {
Key: []byte("blah"),
Partitions: []int{0, 1},
Partition: 0,
},
"partition-1": {
Key: []byte("blah"),
Partitions: []int{0, 1, 2},
Partition: 1,
},
"partition-2": {
Key: []byte("boop"),
Partitions: []int{0, 1, 2},
Partition: 2,
},
"custom hash": {
Key: []byte("boop"),
Hasher: crc32.NewIEEE(),
Partitions: []int{0, 1, 2},
Partition: 1,
},
// in a previous version, this test would select a different partition
// than sarama's hash partitioner.
"hash code with MSB set": {
Key: []byte("20"),
Partitions: []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
Partition: 1,
},
}
for label, test := range testCases {
t.Run(label, func(t *testing.T) {
msg := Message{Key: test.Key}
h := Hash{
Hasher: test.Hasher,
}
partition := h.Balance(msg, test.Partitions...)
if partition != test.Partition {
t.Errorf("expected %v; got %v", test.Partition, partition)
}
})
}
}
func TestReferenceHashBalancer(t *testing.T) {
testCases := map[string]struct {
Key []byte
Hasher hash.Hash32
Partitions []int
Partition int
RndBalancerResult int
}{
"nil": {
Key: nil, // nil key means random partition
Partitions: []int{0, 1, 2},
Partition: 123,
RndBalancerResult: 123,
},
"partition-0": {
Key: []byte("blah"),
Partitions: []int{0, 1},
Partition: 0,
},
"partition-1": {
Key: []byte("blah"),
Partitions: []int{0, 1, 2},
Partition: 1,
},
"partition-2": {
Key: []byte("castle"),
Partitions: []int{0, 1, 2},
Partition: 2,
},
"custom hash": {
Key: []byte("boop"),
Hasher: crc32.NewIEEE(),
Partitions: []int{0, 1, 2},
Partition: 1,
},
"hash code with MSB set": {
Key: []byte("20"),
Partitions: []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15},
Partition: 15,
},
}
for label, test := range testCases {
t.Run(label, func(t *testing.T) {
var rr randomBalancer
if test.Key == nil {
rr.mock = test.RndBalancerResult
}
msg := Message{Key: test.Key}
h := ReferenceHash{Hasher: test.Hasher, rr: rr}
partition := h.Balance(msg, test.Partitions...)
if partition != test.Partition {
t.Errorf("expected %v; got %v", test.Partition, partition)
}
})
}
}
func TestCRC32Balancer(t *testing.T) {
// These tests are taken from the default "consistent_random" partitioner from
// https://github.com/edenhill/librdkafka/blob/master/tests/0048-partitioner.c
partitionCount := 17
var partitions []int
for i := 0; i < partitionCount; i++ {
partitions = append(partitions, i*i)
}
testCases := map[string]struct {
Key []byte
Partitions []int
Partition int
}{
"nil": {
Key: nil,
Partitions: partitions,
Partition: -1,
},
"empty": {
Key: []byte{},
Partitions: partitions,
Partition: -1,
},
"unaligned": {
Key: []byte("23456"),
Partitions: partitions,
Partition: partitions[0xb1b451d7%partitionCount],
},
"long key": {
Key: []byte("this is another string with more length to it perhaps"),
Partitions: partitions,
Partition: partitions[0xb0150df7%partitionCount],
},
"short key": {
Key: []byte("hejsan"),
Partitions: partitions,
Partition: partitions[0xd077037e%partitionCount],
},
}
t.Run("default", func(t *testing.T) {
for label, test := range testCases {
t.Run(label, func(t *testing.T) {
b := CRC32Balancer{}
b.random.mock = -1
msg := Message{Key: test.Key}
partition := b.Balance(msg, test.Partitions...)
if partition != test.Partition {
t.Errorf("expected %v; got %v", test.Partition, partition)
}
})
}
})
t.Run("consistent", func(t *testing.T) {
b := CRC32Balancer{Consistent: true}
b.random.mock = -1
p := b.Balance(Message{}, partitions...)
if p < 0 {
t.Fatal("should not have gotten a random partition")
}
for i := 0; i < 10; i++ {
if p != b.Balance(Message{}, partitions...) {
t.Fatal("nil key should always hash consistently")
}
if p != b.Balance(Message{Key: []byte{}}, partitions...) {
t.Fatal("empty key should always hash consistently and have same result as nil key")
}
}
})
}
func TestMurmur2(t *testing.T) {
// These tests are taken from the "murmur2" implementation from
// https://github.com/edenhill/librdkafka/blob/master/src/rdmurmur2.c
testCases := []struct {
Key []byte
JavaMurmur2Result uint32
}{
{Key: []byte("kafka"), JavaMurmur2Result: 0xd067cf64},
{Key: []byte("giberish123456789"), JavaMurmur2Result: 0x8f552b0c},
{Key: []byte("1234"), JavaMurmur2Result: 0x9fc97b14},
{Key: []byte("234"), JavaMurmur2Result: 0xe7c009ca},
{Key: []byte("34"), JavaMurmur2Result: 0x873930da},
{Key: []byte("4"), JavaMurmur2Result: 0x5a4b5ca1},
{Key: []byte("PreAmbleWillBeRemoved,ThePrePartThatIs"), JavaMurmur2Result: 0x78424f1c},
{Key: []byte("reAmbleWillBeRemoved,ThePrePartThatIs"), JavaMurmur2Result: 0x4a62b377},
{Key: []byte("eAmbleWillBeRemoved,ThePrePartThatIs"), JavaMurmur2Result: 0xe0e4e09e},
{Key: []byte("AmbleWillBeRemoved,ThePrePartThatIs"), JavaMurmur2Result: 0x62b8b43f},
{Key: []byte(""), JavaMurmur2Result: 0x106e08d9},
{Key: nil, JavaMurmur2Result: 0x106e08d9},
}
for _, test := range testCases {
t.Run(fmt.Sprintf("key:%s", test.Key), func(t *testing.T) {
got := murmur2(test.Key)
if got != test.JavaMurmur2Result {
t.Errorf("expected %v; got %v", test.JavaMurmur2Result, got)
}
})
}
}
func TestMurmur2Balancer(t *testing.T) {
// These tests are taken from the "murmur2_random" partitioner from
// https://github.com/edenhill/librdkafka/blob/master/tests/0048-partitioner.c
partitionCount := 17
librdkafkaPartitions := make([]int, partitionCount)
for i := 0; i < partitionCount; i++ {
librdkafkaPartitions[i] = i * i
}
// These tests are taken from the Murmur2Partitioner Python class from
// https://github.com/dpkp/kafka-python/blob/master/test/test_partitioner.py
pythonPartitions := make([]int, 1000)
for i := 0; i < 1000; i++ {
pythonPartitions[i] = i
}
testCases := map[string]struct {
Key []byte
Partitions []int
Partition int
}{
"librdkafka-nil": {
Key: nil,
Partitions: librdkafkaPartitions,
Partition: 123,
},
"librdkafka-empty": {
Key: []byte{},
Partitions: librdkafkaPartitions,
Partition: librdkafkaPartitions[0x106e08d9%partitionCount],
},
"librdkafka-unaligned": {
Key: []byte("23456"),
Partitions: librdkafkaPartitions,
Partition: librdkafkaPartitions[0x058d780f%partitionCount],
},
"librdkafka-long key": {
Key: []byte("this is another string with more length to it perhaps"),
Partitions: librdkafkaPartitions,
Partition: librdkafkaPartitions[0x4f7703da%partitionCount],
},
"librdkafka-short key": {
Key: []byte("hejsan"),
Partitions: librdkafkaPartitions,
Partition: librdkafkaPartitions[0x5ec19395%partitionCount],
},
"python-empty": {
Key: []byte(""),
Partitions: pythonPartitions,
Partition: 681,
},
"python-a": {
Key: []byte("a"),
Partitions: pythonPartitions,
Partition: 524,
},
"python-ab": {
Key: []byte("ab"),
Partitions: pythonPartitions,
Partition: 434,
},
"python-abc": {
Key: []byte("abc"),
Partitions: pythonPartitions,
Partition: 107,
},
"python-123456789": {
Key: []byte("123456789"),
Partitions: pythonPartitions,
Partition: 566,
},
"python-\x00 ": {
Key: []byte{0, 32},
Partitions: pythonPartitions,
Partition: 742,
},
}
t.Run("default", func(t *testing.T) {
for label, test := range testCases {
t.Run(label, func(t *testing.T) {
b := Murmur2Balancer{}
b.random.mock = 123
msg := Message{Key: test.Key}
partition := b.Balance(msg, test.Partitions...)
if partition != test.Partition {
t.Errorf("expected %v; got %v", test.Partition, partition)
}
})
}
})
t.Run("consistent", func(t *testing.T) {
b := Murmur2Balancer{Consistent: true}
b.random.mock = -1
p := b.Balance(Message{}, librdkafkaPartitions...)
if p < 0 {
t.Fatal("should not have gotten a random partition")
}
for i := 0; i < 10; i++ {
if p != b.Balance(Message{}, librdkafkaPartitions...) {
t.Fatal("nil key should always hash consistently")
}
}
})
}
func TestLeastBytes(t *testing.T) {
testCases := map[string]struct {
Keys [][]byte
Partitions [][]int
Partition int
}{
"single message": {
Keys: [][]byte{
[]byte("key"),
},
Partitions: [][]int{
{0, 1, 2},
},
Partition: 0,
},
"multiple messages, no partition change": {
Keys: [][]byte{
[]byte("a"),
[]byte("ab"),
[]byte("abc"),
[]byte("abcd"),
},
Partitions: [][]int{
{0, 1, 2},
{0, 1, 2},
{0, 1, 2},
{0, 1, 2},
},
Partition: 0,
},
"partition gained": {
Keys: [][]byte{
[]byte("hello world 1"),
[]byte("hello world 2"),
[]byte("hello world 3"),
},
Partitions: [][]int{
{0, 1},
{0, 1},
{0, 1, 2},
},
Partition: 0,
},
"partition lost": {
Keys: [][]byte{
[]byte("hello world 1"),
[]byte("hello world 2"),
[]byte("hello world 3"),
},
Partitions: [][]int{
{0, 1, 2},
{0, 1, 2},
{0, 1},
},
Partition: 0,
},
}
for label, test := range testCases {
t.Run(label, func(t *testing.T) {
lb := &LeastBytes{}
var partition int
for i, key := range test.Keys {
msg := Message{Key: key}
partition = lb.Balance(msg, test.Partitions[i]...)
}
if partition != test.Partition {
t.Errorf("expected %v; got %v", test.Partition, partition)
}
})
}
}
func TestRoundRobin(t *testing.T) {
testCases := map[string]struct {
Partitions []int
ChunkSize int
}{
"default - odd partition count": {
Partitions: []int{0, 1, 2, 3, 4, 5, 6},
},
"negative chunk size - odd partition count": {
Partitions: []int{0, 1, 2, 3, 4, 5, 6},
ChunkSize: -1,
},
"0 chunk size - odd partition count": {
Partitions: []int{0, 1, 2, 3, 4, 5, 6},
ChunkSize: 0,
},
"5 chunk size - odd partition count": {
Partitions: []int{0, 1, 2, 3, 4, 5, 6},
ChunkSize: 5,
},
"12 chunk size - odd partition count": {
Partitions: []int{0, 1, 2, 3, 4, 5, 6},
ChunkSize: 12,
},
"default - even partition count": {
Partitions: []int{0, 1, 2, 3, 4, 5, 6, 7},
},
"negative chunk size - even partition count": {
Partitions: []int{0, 1, 2, 3, 4, 5, 6, 7},
ChunkSize: -1,
},
"0 chunk size - even partition count": {
Partitions: []int{0, 1, 2, 3, 4, 5, 6, 7},
ChunkSize: 0,
},
"5 chunk size - even partition count": {
Partitions: []int{0, 1, 2, 3, 4, 5, 6, 7},
ChunkSize: 5,
},
"12 chunk size - even partition count": {
Partitions: []int{0, 1, 2, 3, 4, 5, 6, 7},
ChunkSize: 12,
},
}
for label, test := range testCases {
t.Run(label, func(t *testing.T) {
lb := &RoundRobin{ChunkSize: test.ChunkSize}
msg := Message{}
var partition int
var i int
expectedChunkSize := test.ChunkSize
if expectedChunkSize < 1 {
expectedChunkSize = 1
}
partitions := test.Partitions
for i = 0; i < 50; i++ {
partition = lb.Balance(msg, partitions...)
if partition != i/expectedChunkSize%len(partitions) {
t.Error("Returned partition", partition, "expecting", i/expectedChunkSize%len(partitions))
}
}
})
}
}
================================================
FILE: batch.go
================================================
package kafka
import (
"bufio"
"errors"
"io"
"sync"
"time"
)
// A Batch is an iterator over a sequence of messages fetched from a kafka
// server.
//
// Batches are created by calling (*Conn).ReadBatch. They hold a internal lock
// on the connection, which is released when the batch is closed. Failing to
// call a batch's Close method will likely result in a dead-lock when trying to
// use the connection.
//
// Batches are safe to use concurrently from multiple goroutines.
type Batch struct {
mutex sync.Mutex
conn *Conn
lock *sync.Mutex
msgs *messageSetReader
deadline time.Time
throttle time.Duration
topic string
partition int
offset int64
highWaterMark int64
err error
// The last offset in the batch.
//
// We use lastOffset to skip offsets that have been compacted away.
//
// We store lastOffset because we get lastOffset when we read a new message
// but only try to handle compaction when we receive an EOF. However, when
// we get an EOF we do not get the lastOffset. So there is a mismatch
// between when we receive it and need to use it.
lastOffset int64
}
// Throttle gives the throttling duration applied by the kafka server on the
// connection.
func (batch *Batch) Throttle() time.Duration {
return batch.throttle
}
// HighWaterMark returns the current highest watermark in a partition.
func (batch *Batch) HighWaterMark() int64 {
return batch.highWaterMark
}
// Partition returns the batch partition.
func (batch *Batch) Partition() int {
return batch.partition
}
// Offset returns the offset of the next message in the batch.
func (batch *Batch) Offset() int64 {
batch.mutex.Lock()
offset := batch.offset
batch.mutex.Unlock()
return offset
}
// Close closes the batch, releasing the connection lock and returning an error
// if reading the batch failed for any reason.
func (batch *Batch) Close() error {
batch.mutex.Lock()
err := batch.close()
batch.mutex.Unlock()
return err
}
func (batch *Batch) close() (err error) {
conn := batch.conn
lock := batch.lock
batch.conn = nil
batch.lock = nil
if batch.msgs != nil {
batch.msgs.discard()
}
if batch.msgs != nil && batch.msgs.decompressed != nil {
releaseBuffer(batch.msgs.decompressed)
batch.msgs.decompressed = nil
}
if err = batch.err; errors.Is(batch.err, io.EOF) {
err = nil
}
if conn != nil {
conn.rdeadline.unsetConnReadDeadline()
conn.mutex.Lock()
conn.offset = batch.offset
conn.mutex.Unlock()
if err != nil {
var kafkaError Error
if !errors.As(err, &kafkaError) && !errors.Is(err, io.ErrShortBuffer) {
conn.Close()
}
}
}
if lock != nil {
lock.Unlock()
}
return
}
// Err returns a non-nil error if the batch is broken. This is the same error
// that would be returned by Read, ReadMessage or Close (except in the case of
// io.EOF which is never returned by Close).
//
// This method is useful when building retry mechanisms for (*Conn).ReadBatch,
// the program can check whether the batch carried a error before attempting to
// read the first message.
//
// Note that checking errors on a batch is optional, calling Read or ReadMessage
// is always valid and can be used to either read a message or an error in cases
// where that's convenient.
func (batch *Batch) Err() error { return batch.err }
// Read reads the value of the next message from the batch into b, returning the
// number of bytes read, or an error if the next message couldn't be read.
//
// If an error is returned the batch cannot be used anymore and calling Read
// again will keep returning that error. All errors except io.EOF (indicating
// that the program consumed all messages from the batch) are also returned by
// Close.
//
// The method fails with io.ErrShortBuffer if the buffer passed as argument is
// too small to hold the message value.
func (batch *Batch) Read(b []byte) (int, error) {
n := 0
batch.mutex.Lock()
offset := batch.offset
_, _, _, err := batch.readMessage(
func(r *bufio.Reader, size int, nbytes int) (int, error) {
if nbytes < 0 {
return size, nil
}
return discardN(r, size, nbytes)
},
func(r *bufio.Reader, size int, nbytes int) (int, error) {
if nbytes < 0 {
return size, nil
}
// make sure there are enough bytes for the message value. return
// errShortRead if the message is truncated.
if nbytes > size {
return size, errShortRead
}
n = nbytes // return value
if nbytes > cap(b) {
nbytes = cap(b)
}
if nbytes > len(b) {
b = b[:nbytes]
}
nbytes, err := io.ReadFull(r, b[:nbytes])
if err != nil {
return size - nbytes, err
}
return discardN(r, size-nbytes, n-nbytes)
},
)
if err == nil && n > len(b) {
n, err = len(b), io.ErrShortBuffer
batch.err = io.ErrShortBuffer
batch.offset = offset // rollback
}
batch.mutex.Unlock()
return n, err
}
// ReadMessage reads and return the next message from the batch.
//
// Because this method allocate memory buffers for the message key and value
// it is less memory-efficient than Read, but has the advantage of never
// failing with io.ErrShortBuffer.
func (batch *Batch) ReadMessage() (Message, error) {
msg := Message{}
batch.mutex.Lock()
var offset, timestamp int64
var headers []Header
var err error
offset, timestamp, headers, err = batch.readMessage(
func(r *bufio.Reader, size int, nbytes int) (remain int, err error) {
msg.Key, remain, err = readNewBytes(r, size, nbytes)
return
},
func(r *bufio.Reader, size int, nbytes int) (remain int, err error) {
msg.Value, remain, err = readNewBytes(r, size, nbytes)
return
},
)
// A batch may start before the requested offset so skip messages
// until the requested offset is reached.
for batch.conn != nil && offset < batch.conn.offset {
if err != nil {
break
}
offset, timestamp, headers, err = batch.readMessage(
func(r *bufio.Reader, size int, nbytes int) (remain int, err error) {
msg.Key, remain, err = readNewBytes(r, size, nbytes)
return
},
func(r *bufio.Reader, size int, nbytes int) (remain int, err error) {
msg.Value, remain, err = readNewBytes(r, size, nbytes)
return
},
)
}
batch.mutex.Unlock()
msg.Topic = batch.topic
msg.Partition = batch.partition
msg.Offset = offset
msg.HighWaterMark = batch.highWaterMark
msg.Time = makeTime(timestamp)
msg.Headers = headers
return msg, err
}
func (batch *Batch) readMessage(
key func(*bufio.Reader, int, int) (int, error),
val func(*bufio.Reader, int, int) (int, error),
) (offset int64, timestamp int64, headers []Header, err error) {
if err = batch.err; err != nil {
return
}
var lastOffset int64
offset, lastOffset, timestamp, headers, err = batch.msgs.readMessage(batch.offset, key, val)
switch {
case err == nil:
batch.offset = offset + 1
batch.lastOffset = lastOffset
case errors.Is(err, errShortRead):
// As an "optimization" kafka truncates the returned response after
// producing MaxBytes, which could then cause the code to return
// errShortRead.
err = batch.msgs.discard()
switch {
case err != nil:
// Since io.EOF is used by the batch to indicate that there is are
// no more messages to consume, it is crucial that any io.EOF errors
// on the underlying connection are repackaged. Otherwise, the
// caller can't tell the difference between a batch that was fully
// consumed or a batch whose connection is in an error state.
batch.err = dontExpectEOF(err)
case batch.msgs.remaining() == 0:
// Because we use the adjusted deadline we could end up returning
// before the actual deadline occurred. This is necessary otherwise
// timing out the connection for real could end up leaving it in an
// unpredictable state, which would require closing it.
// This design decision was made to maximize the chances of keeping
// the connection open, the trade off being to lose precision on the
// read deadline management.
err = checkTimeoutErr(batch.deadline)
batch.err = err
// Checks the following:
// - `batch.err` for a "success" from the previous timeout check
// - `batch.msgs.lengthRemain` to ensure that this EOF is not due
// to MaxBytes truncation
// - `batch.lastOffset` to ensure that the message format contains
// `lastOffset`
if errors.Is(batch.err, io.EOF) && batch.msgs.lengthRemain == 0 && batch.lastOffset != -1 {
// Log compaction can create batches that end with compacted
// records so the normal strategy that increments the "next"
// offset as records are read doesn't work as the compacted
// records are "missing" and never get "read".
//
// In order to reliably reach the next non-compacted offset we
// jump past the saved lastOffset.
batch.offset = batch.lastOffset + 1
}
}
default:
// Since io.EOF is used by the batch to indicate that there is are
// no more messages to consume, it is crucial that any io.EOF errors
// on the underlying connection are repackaged. Otherwise, the
// caller can't tell the difference between a batch that was fully
// consumed or a batch whose connection is in an error state.
batch.err = dontExpectEOF(err)
}
return
}
func checkTimeoutErr(deadline time.Time) (err error) {
if !deadline.IsZero() && time.Now().After(deadline) {
err = RequestTimedOut
} else {
err = io.EOF
}
return
}
================================================
FILE: batch_test.go
================================================
package kafka
import (
"context"
"errors"
"io"
"net"
"strconv"
"testing"
)
func TestBatchDontExpectEOF(t *testing.T) {
topic := makeTopic()
broker, err := (&Dialer{
Resolver: &net.Resolver{},
}).LookupLeader(context.Background(), "tcp", "localhost:9092", topic, 0)
if err != nil {
t.Fatal("failed to open a new kafka connection:", err)
}
nc, err := net.Dial("tcp", net.JoinHostPort(broker.Host, strconv.Itoa(broker.Port)))
if err != nil {
t.Fatalf("cannot connect to partition leader at %s:%d: %s", broker.Host, broker.Port, err)
}
conn := NewConn(nc, topic, 0)
defer conn.Close()
nc.(*net.TCPConn).CloseRead()
batch := conn.ReadBatch(1024, 8192)
if _, err := batch.ReadMessage(); !errors.Is(err, io.ErrUnexpectedEOF) {
t.Error("bad error when reading message:", err)
}
if err := batch.Close(); !errors.Is(err, io.ErrUnexpectedEOF) {
t.Error("bad error when closing the batch:", err)
}
}
================================================
FILE: buffer.go
================================================
package kafka
import (
"bytes"
"sync"
)
var bufferPool = sync.Pool{
New: func() interface{} { return newBuffer() },
}
func newBuffer() *bytes.Buffer {
b := new(bytes.Buffer)
b.Grow(65536)
return b
}
func acquireBuffer() *bytes.Buffer {
return bufferPool.Get().(*bytes.Buffer)
}
func releaseBuffer(b *bytes.Buffer) {
if b != nil {
b.Reset()
bufferPool.Put(b)
}
}
================================================
FILE: builder_test.go
================================================
package kafka
import (
"bytes"
"fmt"
"io"
"time"
"github.com/segmentio/kafka-go/compress"
)
// This file defines builders to assist in creating kafka payloads for unit testing.
// fetchResponseBuilder builds v10 fetch responses. The version of the v10 fetch
// responses are not as important as the message sets contained within, as this
// type is ultimately used to unit test the message set reader that consumes the
// rest of the response once the header has been parsed.
type fetchResponseBuilder struct {
header fetchResponseHeader
msgSets []messageSetBuilder
rendered []byte
}
type fetchResponseHeader struct {
throttle int32
errorCode int16
sessionID int32
topic string
partition int32
partitionErrorCode int16
highWatermarkOffset int64
lastStableOffset int64
logStartOffset int64
}
func (b *fetchResponseBuilder) messages() (res []Message) {
for _, set := range b.msgSets {
res = append(res, set.messages()...)
}
return
}
func (b *fetchResponseBuilder) bytes() []byte {
if b.rendered == nil {
b.rendered = newWB().call(func(wb *kafkaWriteBuffer) {
wb.writeInt32(b.header.throttle)
wb.writeInt16(b.header.errorCode)
wb.writeInt32(b.header.sessionID)
wb.writeInt32(1) // num topics
wb.writeString(b.header.topic)
wb.writeInt32(1) // how many partitions
wb.writeInt32(b.header.partition)
wb.writeInt16(b.header.partitionErrorCode)
wb.writeInt64(b.header.highWatermarkOffset)
wb.writeInt64(b.header.lastStableOffset)
wb.writeInt64(b.header.logStartOffset)
wb.writeInt32(-1) // num aborted tx
wb.writeBytes(newWB().call(func(wb *kafkaWriteBuffer) {
for _, msgSet := range b.msgSets {
wb.Write(msgSet.bytes())
}
}))
})
}
return b.rendered
}
func (b *fetchResponseBuilder) Len() int {
return len(b.bytes())
}
type messageSetBuilder interface {
bytes() []byte
messages() []Message
}
type v0MessageSetBuilder struct {
msgs []Message
codec CompressionCodec
}
func (f v0MessageSetBuilder) messages() []Message {
return f.msgs
}
func (f v0MessageSetBuilder) bytes() []byte {
bs := newWB().call(func(wb *kafkaWriteBuffer) {
for _, msg := range f.msgs {
bs := newWB().call(func(wb *kafkaWriteBuffer) {
wb.writeInt64(msg.Offset) // offset
wb.writeBytes(newWB().call(func(wb *kafkaWriteBuffer) {
wb.writeInt32(-1) // crc, unused
wb.writeInt8(0) // magic
wb.writeInt8(0) // attributes -- zero, no compression for the inner message
wb.writeBytes(msg.Key)
wb.writeBytes(msg.Value)
}))
})
wb.Write(bs)
}
})
if f.codec != nil {
bs = newWB().call(func(wb *kafkaWriteBuffer) {
wb.writeInt64(f.msgs[0].Offset) // offset
wb.writeBytes(newWB().call(func(wb *kafkaWriteBuffer) {
compressed := mustCompress(bs, f.codec)
wb.writeInt32(-1) // crc, unused
wb.writeInt8(0) // magic
wb.writeInt8(f.codec.Code()) // attributes
wb.writeBytes(nil) // key is always nil for compressed
wb.writeBytes(compressed) // the value is the compressed message
}))
})
}
return bs
}
type v1MessageSetBuilder struct {
msgs []Message
codec CompressionCodec
}
func (f v1MessageSetBuilder) messages() []Message {
return f.msgs
}
func (f v1MessageSetBuilder) bytes() []byte {
bs := newWB().call(func(wb *kafkaWriteBuffer) {
for i, msg := range f.msgs {
bs := newWB().call(func(wb *kafkaWriteBuffer) {
if f.codec != nil {
wb.writeInt64(int64(i)) // compressed inner message offsets are relative
} else {
wb.writeInt64(msg.Offset) // offset
}
wb.writeBytes(newWB().call(func(wb *kafkaWriteBuffer) {
wb.writeInt32(-1) // crc, unused
wb.writeInt8(1) // magic
wb.writeInt8(0) // attributes -- zero, no compression for the inner message
wb.writeInt64(1000 * msg.Time.Unix()) // timestamp
wb.writeBytes(msg.Key)
wb.writeBytes(msg.Value)
}))
})
wb.Write(bs)
}
})
if f.codec != nil {
bs = newWB().call(func(wb *kafkaWriteBuffer) {
wb.writeInt64(f.msgs[len(f.msgs)-1].Offset) // offset of the wrapper message is the last offset of the inner messages
wb.writeBytes(newWB().call(func(wb *kafkaWriteBuffer) {
bs := mustCompress(bs, f.codec)
wb.writeInt32(-1) // crc, unused
wb.writeInt8(1) // magic
wb.writeInt8(f.codec.Code()) // attributes
wb.writeInt64(1000 * f.msgs[0].Time.Unix()) // timestamp
wb.writeBytes(nil) // key is always nil for compressed
wb.writeBytes(bs) // the value is the compressed message
}))
})
}
return bs
}
type v2MessageSetBuilder struct {
msgs []Message
codec CompressionCodec
}
func (f v2MessageSetBuilder) messages() []Message {
return f.msgs
}
func (f v2MessageSetBuilder) bytes() []byte {
attributes := int16(0)
if f.codec != nil {
attributes = int16(f.codec.Code()) // set codec code on attributes
}
return newWB().call(func(wb *kafkaWriteBuffer) {
wb.writeInt64(f.msgs[0].Offset)
wb.writeBytes(newWB().call(func(wb *kafkaWriteBuffer) {
wb.writeInt32(0) // leader epoch
wb.writeInt8(2) // magic = 2
wb.writeInt32(0) // crc, unused
wb.writeInt16(attributes) // record set attributes
wb.writeInt32(0) // record set last offset delta
wb.writeInt64(1000 * f.msgs[0].Time.Unix()) // record set first timestamp
wb.writeInt64(1000 * f.msgs[0].Time.Unix()) // record set last timestamp
wb.writeInt64(0) // record set producer id
wb.writeInt16(0) // record set producer epoch
wb.writeInt32(0) // record set base sequence
wb.writeInt32(int32(len(f.msgs))) // record set count
bs := newWB().call(func(wb *kafkaWriteBuffer) {
for i, msg := range f.msgs {
wb.Write(newWB().call(func(wb *kafkaWriteBuffer) {
bs := newWB().call(func(wb *kafkaWriteBuffer) {
wb.writeInt8(0) // record attributes, not used here
wb.writeVarInt(1000 * (time.Now().Unix() - msg.Time.Unix())) // timestamp
wb.writeVarInt(int64(i)) // offset delta
wb.writeVarInt(int64(len(msg.Key))) // key len
wb.Write(msg.Key) // key bytes
wb.writeVarInt(int64(len(msg.Value))) // value len
wb.Write(msg.Value) // value bytes
wb.writeVarInt(int64(len(msg.Headers))) // number of headers
for _, header := range msg.Headers {
wb.writeVarInt(int64(len(header.Key)))
wb.Write([]byte(header.Key))
wb.writeVarInt(int64(len(header.Value)))
wb.Write(header.Value)
}
})
wb.writeVarInt(int64(len(bs)))
wb.Write(bs)
}))
}
})
if f.codec != nil {
bs = mustCompress(bs, f.codec)
}
wb.Write(bs)
}))
})
}
// kafkaWriteBuffer is a write buffer that helps writing fetch responses.
type kafkaWriteBuffer struct {
writeBuffer
buf bytes.Buffer
}
func newWB() *kafkaWriteBuffer {
res := kafkaWriteBuffer{}
res.writeBuffer.w = &res.buf
return &res
}
func (f *kafkaWriteBuffer) Bytes() []byte {
return f.buf.Bytes()
}
// call is a convenience method that allows the kafkaWriteBuffer to be used
// in a functional manner. This is helpful when building
// nested structures, as the return value can be fed into
// other fwWB APIs.
func (f *kafkaWriteBuffer) call(cb func(wb *kafkaWriteBuffer)) []byte {
cb(f)
bs := f.Bytes()
if bs == nil {
bs = []byte{}
}
return bs
}
func mustCompress(bs []byte, codec compress.Codec) (res []byte) {
buf := bytes.Buffer{}
codecWriter := codec.NewWriter(&buf)
_, err := io.Copy(codecWriter, bytes.NewReader(bs))
if err != nil {
panic(fmt.Errorf("compress: %w", err))
}
err = codecWriter.Close()
if err != nil {
panic(fmt.Errorf("close codec writer: %w", err))
}
res = buf.Bytes()
return
}
================================================
FILE: client.go
================================================
package kafka
import (
"context"
"errors"
"fmt"
"net"
"time"
"github.com/segmentio/kafka-go/protocol"
)
const (
defaultCreateTopicsTimeout = 2 * time.Second
defaultDeleteTopicsTimeout = 2 * time.Second
defaultCreatePartitionsTimeout = 2 * time.Second
defaultProduceTimeout = 500 * time.Millisecond
defaultMaxWait = 500 * time.Millisecond
)
// Client is a high-level API to interract with kafka brokers.
//
// All methods of the Client type accept a context as first argument, which may
// be used to asynchronously cancel the requests.
//
// Clients are safe to use concurrently from multiple goroutines, as long as
// their configuration is not changed after first use.
type Client struct {
// Address of the kafka cluster (or specific broker) that the client will be
// sending requests to.
//
// This field is optional, the address may be provided in each request
// instead. The request address takes precedence if both were specified.
Addr net.Addr
// Time limit for requests sent by this client.
//
// If zero, no timeout is applied.
Timeout time.Duration
// A transport used to communicate with the kafka brokers.
//
// If nil, DefaultTransport is used.
Transport RoundTripper
}
// A ConsumerGroup and Topic as these are both strings we define a type for
// clarity when passing to the Client as a function argument
//
// N.B TopicAndGroup is currently experimental! Therefore, it is subject to
// change, including breaking changes between MINOR and PATCH releases.
//
// DEPRECATED: this type will be removed in version 1.0, programs should
// migrate to use kafka.(*Client).OffsetFetch instead.
type TopicAndGroup struct {
Topic string
GroupId string
}
// ConsumerOffsets returns a map[int]int64 of partition to committed offset for
// a consumer group id and topic.
//
// DEPRECATED: this method will be removed in version 1.0, programs should
// migrate to use kafka.(*Client).OffsetFetch instead.
func (c *Client) ConsumerOffsets(ctx context.Context, tg TopicAndGroup) (map[int]int64, error) {
metadata, err := c.Metadata(ctx, &MetadataRequest{
Topics: []string{tg.Topic},
})
if err != nil {
return nil, fmt.Errorf("failed to get topic metadata :%w", err)
}
topic := metadata.Topics[0]
partitions := make([]int, len(topic.Partitions))
for i := range topic.Partitions {
partitions[i] = topic.Partitions[i].ID
}
offsets, err := c.OffsetFetch(ctx, &OffsetFetchRequest{
GroupID: tg.GroupId,
Topics: map[string][]int{
tg.Topic: partitions,
},
})
if err != nil {
return nil, fmt.Errorf("failed to get offsets: %w", err)
}
topicOffsets := offsets.Topics[topic.Name]
partitionOffsets := make(map[int]int64, len(topicOffsets))
for _, off := range topicOffsets {
partitionOffsets[off.Partition] = off.CommittedOffset
}
return partitionOffsets, nil
}
func (c *Client) roundTrip(ctx context.Context, addr net.Addr, msg protocol.Message) (protocol.Message, error) {
if c.Timeout > 0 {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, c.Timeout)
defer cancel()
}
if addr == nil {
if addr = c.Addr; addr == nil {
return nil, errors.New("no address was given for the kafka cluster in the request or on the client")
}
}
return c.transport().RoundTrip(ctx, addr, msg)
}
func (c *Client) transport() RoundTripper {
if c.Transport != nil {
return c.Transport
}
return DefaultTransport
}
func (c *Client) timeout(ctx context.Context, defaultTimeout time.Duration) time.Duration {
timeout := c.Timeout
if deadline, ok := ctx.Deadline(); ok {
if remain := time.Until(deadline); remain < timeout {
timeout = remain
}
}
if timeout > 0 {
// Half the timeout because it is communicated to kafka in multiple
// requests (e.g. Fetch, Produce, etc...), this adds buffer to account
// for network latency when waiting for the response from kafka.
return timeout / 2
}
return defaultTimeout
}
func (c *Client) timeoutMs(ctx context.Context, defaultTimeout time.Duration) int32 {
return milliseconds(c.timeout(ctx, defaultTimeout))
}
================================================
FILE: client_test.go
================================================
package kafka
import (
"bytes"
"context"
"errors"
"io"
"math/rand"
"net"
"testing"
"time"
"github.com/segmentio/kafka-go/compress"
ktesting "github.com/segmentio/kafka-go/testing"
)
func newLocalClientAndTopic() (*Client, string, func()) {
topic := makeTopic()
client, shutdown := newLocalClientWithTopic(topic, 1)
return client, topic, shutdown
}
func newLocalClientWithTopic(topic string, partitions int) (*Client, func()) {
client, shutdown := newLocalClient()
if err := clientCreateTopic(client, topic, partitions); err != nil {
shutdown()
panic(err)
}
return client, func() {
client.DeleteTopics(context.Background(), &DeleteTopicsRequest{
Topics: []string{topic},
})
shutdown()
}
}
func clientCreateTopic(client *Client, topic string, partitions int) error {
_, err := client.CreateTopics(context.Background(), &CreateTopicsRequest{
Topics: []TopicConfig{{
Topic: topic,
NumPartitions: partitions,
ReplicationFactor: 1,
}},
})
if err != nil {
return err
}
// Topic creation seems to be asynchronous. Metadata for the topic partition
// layout in the cluster is available in the controller before being synced
// with the other brokers, which causes "Error:[3] Unknown Topic Or Partition"
// when sending requests to the partition leaders.
//
// This loop will wait up to 2 seconds polling the cluster until no errors
// are returned.
for i := 0; i < 20; i++ {
r, err := client.Fetch(context.Background(), &FetchRequest{
Topic: topic,
Partition: 0,
Offset: 0,
})
if err == nil && r.Error == nil {
break
}
time.Sleep(100 * time.Millisecond)
}
return nil
}
func clientEndTxn(client *Client, req *EndTxnRequest) error {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
defer cancel()
resp, err := client.EndTxn(ctx, req)
if err != nil {
return err
}
return resp.Error
}
func newLocalClient() (*Client, func()) {
return newClient(TCP("localhost"))
}
func newClient(addr net.Addr) (*Client, func()) {
conns := &ktesting.ConnWaitGroup{
DialFunc: (&net.Dialer{}).DialContext,
}
transport := &Transport{
Dial: conns.Dial,
Resolver: NewBrokerResolver(nil),
}
client := &Client{
Addr: addr,
Timeout: 5 * time.Second,
Transport: transport,
}
return client, func() { transport.CloseIdleConnections(); conns.Wait() }
}
func TestClient(t *testing.T) {
tests := []struct {
scenario string
function func(*testing.T, context.Context, *Client)
}{
{
scenario: "retrieve committed offsets for a consumer group and topic",
function: testConsumerGroupFetchOffsets,
},
}
for _, test := range tests {
testFunc := test.function
t.Run(test.scenario, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
client, shutdown := newLocalClient()
defer shutdown()
testFunc(t, ctx, client)
})
}
}
func testConsumerGroupFetchOffsets(t *testing.T, ctx context.Context, client *Client) {
const totalMessages = 144
const partitions = 12
const msgPerPartition = totalMessages / partitions
topic := makeTopic()
if err := clientCreateTopic(client, topic, partitions); err != nil {
t.Fatal(err)
}
groupId := makeGroupID()
brokers := []string{"localhost:9092"}
writer := &Writer{
Addr: TCP(brokers...),
Topic: topic,
Balancer: &RoundRobin{},
BatchSize: 1,
Transport: client.Transport,
}
if err := writer.WriteMessages(ctx, makeTestSequence(totalMessages)...); err != nil {
t.Fatalf("bad write messages: %v", err)
}
if err := writer.Close(); err != nil {
t.Fatalf("bad write err: %v", err)
}
r := NewReader(ReaderConfig{
Brokers: brokers,
Topic: topic,
GroupID: groupId,
MinBytes: 1,
MaxBytes: 10e6,
MaxWait: 100 * time.Millisecond,
})
defer r.Close()
for i := 0; i < totalMessages; i++ {
m, err := r.FetchMessage(ctx)
if err != nil {
t.Fatalf("error fetching message: %s", err)
}
if err := r.CommitMessages(context.Background(), m); err != nil {
t.Fatal(err)
}
}
offsets, err := client.ConsumerOffsets(ctx, TopicAndGroup{GroupId: groupId, Topic: topic})
if err != nil {
t.Fatal(err)
}
if len(offsets) != partitions {
t.Fatalf("expected %d partitions but only received offsets for %d", partitions, len(offsets))
}
for i := 0; i < partitions; i++ {
committedOffset := offsets[i]
if committedOffset != msgPerPartition {
t.Errorf("expected partition %d with committed offset of %d but received %d", i, msgPerPartition, committedOffset)
}
}
}
func TestClientProduceAndConsume(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// Tests a typical kafka use case, data is produced to a partition,
// then consumed back sequentially. We use snappy compression because
// kafka stream are often compressed, and verify that each record
// produced is exposed to the consumer, and order is preserved.
client, topic, shutdown := newLocalClientAndTopic()
defer shutdown()
epoch := time.Now()
seed := int64(0) // deterministic
prng := rand.New(rand.NewSource(seed))
offset := int64(0)
const numBatches = 100
const recordsPerBatch = 320
t.Logf("producing %d batches of %d records...", numBatches, recordsPerBatch)
for i := 0; i < numBatches; i++ { // produce 100 batches
records := make([]Record, recordsPerBatch)
for i := range records {
v := make([]byte, prng.Intn(999)+1)
io.ReadFull(prng, v)
records[i].Time = epoch
records[i].Value = NewBytes(v)
}
res, err := client.Produce(ctx, &ProduceRequest{
Topic: topic,
Partition: 0,
RequiredAcks: -1,
Records: NewRecordReader(records...),
Compression: compress.Snappy,
})
if err != nil {
t.Fatal(err)
}
if res.Error != nil {
t.Fatal(res.Error)
}
if res.BaseOffset != offset {
t.Fatalf("records were produced at an unexpected offset, want %d but got %d", offset, res.BaseOffset)
}
offset += int64(len(records))
}
prng.Seed(seed)
offset = 0 // reset
numFetches := 0
numRecords := 0
for numRecords < (numBatches * recordsPerBatch) {
res, err := client.Fetch(ctx, &FetchRequest{
Topic: topic,
Partition: 0,
Offset: offset,
MinBytes: 1,
MaxBytes: 256 * 1024,
MaxWait: 100 * time.Millisecond, // should only hit on the last fetch
})
if err != nil {
t.Fatal(err)
}
if res.Error != nil {
t.Fatal(err)
}
for {
r, err := res.Records.ReadRecord()
if err != nil {
if !errors.Is(err, io.EOF) {
t.Fatal(err)
}
break
}
if r.Key != nil {
r.Key.Close()
t.Error("unexpected non-null key on record at offset", r.Offset)
}
n := prng.Intn(999) + 1
a := make([]byte, n)
b := make([]byte, n)
io.ReadFull(prng, a)
_, err = io.ReadFull(r.Value, b)
r.Value.Close()
if err != nil {
t.Fatalf("reading record at offset %d: %v", r.Offset, err)
}
if !bytes.Equal(a, b) {
t.Fatalf("value of record at offset %d mismatches", r.Offset)
}
if r.Offset != offset {
t.Fatalf("record at offset %d was expected to have offset %d", r.Offset, offset)
}
offset = r.Offset + 1
numRecords++
}
numFetches++
}
t.Logf("%d records were read in %d fetches", numRecords, numFetches)
}
================================================
FILE: commit.go
================================================
package kafka
// A commit represents the instruction of publishing an update of the last
// offset read by a program for a topic and partition.
type commit struct {
topic string
partition int
offset int64
}
// makeCommit builds a commit value from a message, the resulting commit takes
// its topic, partition, and offset from the message.
func makeCommit(msg Message) commit {
return commit{
topic: msg.Topic,
partition: msg.Partition,
offset: msg.Offset + 1,
}
}
// makeCommits generates a slice of commits from a list of messages, it extracts
// the topic, partition, and offset of each message and builds the corresponding
// commit slice.
func makeCommits(msgs ...Message) []commit {
commits := make([]commit, len(msgs))
for i, m := range msgs {
commits[i] = makeCommit(m)
}
return commits
}
// commitRequest is the data type exchanged between the CommitMessages method
// and internals of the reader's implementation.
type commitRequest struct {
commits []commit
errch chan<- error
}
================================================
FILE: commit_test.go
================================================
package kafka
import "testing"
func TestMakeCommit(t *testing.T) {
msg := Message{
Topic: "blah",
Partition: 1,
Offset: 2,
}
commit := makeCommit(msg)
if commit.topic != msg.Topic {
t.Errorf("bad topic: expected %v; got %v", msg.Topic, commit.topic)
}
if commit.partition != msg.Partition {
t.Errorf("bad partition: expected %v; got %v", msg.Partition, commit.partition)
}
if commit.offset != msg.Offset+1 {
t.Errorf("expected committed offset to be 1 greater than msg offset")
}
}
================================================
FILE: compress/compress.go
================================================
package compress
import (
"encoding"
"fmt"
"io"
"strconv"
"strings"
"github.com/segmentio/kafka-go/compress/gzip"
"github.com/segmentio/kafka-go/compress/lz4"
"github.com/segmentio/kafka-go/compress/snappy"
"github.com/segmentio/kafka-go/compress/zstd"
)
// Compression represents the compression applied to a record set.
type Compression int8
const (
None Compression = 0
Gzip Compression = 1
Snappy Compression = 2
Lz4 Compression = 3
Zstd Compression = 4
)
func (c Compression) Codec() Codec {
if i := int(c); i >= 0 && i < len(Codecs) {
return Codecs[i]
}
return nil
}
func (c Compression) String() string {
if codec := c.Codec(); codec != nil {
return codec.Name()
}
return "uncompressed"
}
func (c Compression) MarshalText() ([]byte, error) {
return []byte(c.String()), nil
}
func (c *Compression) UnmarshalText(b []byte) error {
switch string(b) {
case "none", "uncompressed":
*c = None
return nil
}
for _, codec := range Codecs[None+1:] {
if codec.Name() == string(b) {
*c = Compression(codec.Code())
return nil
}
}
i, err := strconv.ParseInt(string(b), 10, 64)
if err == nil && i >= 0 && i < int64(len(Codecs)) {
*c = Compression(i)
return nil
}
s := &strings.Builder{}
s.WriteString("none, uncompressed")
for i, codec := range Codecs[None+1:] {
if i < (len(Codecs) - 1) {
s.WriteString(", ")
} else {
s.WriteString(", or ")
}
s.WriteString(codec.Name())
}
return fmt.Errorf("compression format must be one of %s, not %q", s, b)
}
var (
_ encoding.TextMarshaler = Compression(0)
_ encoding.TextUnmarshaler = (*Compression)(nil)
)
// Codec represents a compression codec to encode and decode the messages.
// See : https://cwiki.apache.org/confluence/display/KAFKA/Compression
//
// A Codec must be safe for concurrent access by multiple go routines.
type Codec interface {
// Code returns the compression codec code
Code() int8
// Human-readable name for the codec.
Name() string
// Constructs a new reader which decompresses data from r.
NewReader(r io.Reader) io.ReadCloser
// Constructs a new writer which writes compressed data to w.
NewWriter(w io.Writer) io.WriteCloser
}
var (
// The global gzip codec installed on the Codecs table.
GzipCodec gzip.Codec
// The global snappy codec installed on the Codecs table.
SnappyCodec snappy.Codec
// The global lz4 codec installed on the Codecs table.
Lz4Codec lz4.Codec
// The global zstd codec installed on the Codecs table.
ZstdCodec zstd.Codec
// The global table of compression codecs supported by the kafka protocol.
Codecs = [...]Codec{
None: nil,
Gzip: &GzipCodec,
Snappy: &SnappyCodec,
Lz4: &Lz4Codec,
Zstd: &ZstdCodec,
}
)
================================================
FILE: compress/compress_test.go
================================================
package compress_test
import (
"bytes"
"context"
"fmt"
"io"
"io/ioutil"
"math/rand"
"net"
"os"
"path/filepath"
"strconv"
"testing"
"text/tabwriter"
"time"
gz "github.com/klauspost/compress/gzip"
"github.com/segmentio/kafka-go"
pkg "github.com/segmentio/kafka-go/compress"
"github.com/segmentio/kafka-go/compress/gzip"
"github.com/segmentio/kafka-go/compress/lz4"
"github.com/segmentio/kafka-go/compress/snappy"
"github.com/segmentio/kafka-go/compress/zstd"
ktesting "github.com/segmentio/kafka-go/testing"
)
func init() {
// Seeding the random source is important to prevent multiple test runs from
// reusing the same topic names.
rand.Seed(time.Now().UnixNano())
}
func TestCodecs(t *testing.T) {
for i, c := range pkg.Codecs {
if c != nil {
if code := c.Code(); int8(code) != int8(i) {
t.Fatal("default compression codec table is misconfigured for", c.Name())
}
}
}
}
func TestCompression(t *testing.T) {
msg := kafka.Message{
Value: []byte("message"),
}
testEncodeDecode(t, msg, new(gzip.Codec))
testEncodeDecode(t, msg, new(snappy.Codec))
testEncodeDecode(t, msg, new(lz4.Codec))
if ktesting.KafkaIsAtLeast("2.1.0") {
testEncodeDecode(t, msg, new(zstd.Codec))
}
}
func compress(codec pkg.Codec, src []byte) ([]byte, error) {
b := new(bytes.Buffer)
r := bytes.NewReader(src)
w := codec.NewWriter(b)
if _, err := io.Copy(w, r); err != nil {
w.Close()
return nil, err
}
if err := w.Close(); err != nil {
return nil, err
}
return b.Bytes(), nil
}
func decompress(codec pkg.Codec, src []byte) ([]byte, error) {
b := new(bytes.Buffer)
r := codec.NewReader(bytes.NewReader(src))
if _, err := io.Copy(b, r); err != nil {
r.Close()
return nil, err
}
if err := r.Close(); err != nil {
return nil, err
}
return b.Bytes(), nil
}
func testEncodeDecode(t *testing.T, m kafka.Message, codec pkg.Codec) {
var r1, r2 []byte
var err error
t.Run("text format of "+codec.Name(), func(t *testing.T) {
c := pkg.Compression(codec.Code())
a := strconv.Itoa(int(c))
x := pkg.Compression(-1)
y := pkg.Compression(-1)
b, err := c.MarshalText()
if err != nil {
t.Fatal(err)
}
if err := x.UnmarshalText([]byte(a)); err != nil {
t.Fatal(err)
}
if err := y.UnmarshalText(b); err != nil {
t.Fatal(err)
}
if x != c {
t.Errorf("compression mismatch after marshal/unmarshal: want=%s got=%s", c, x)
}
if y != c {
t.Errorf("compression mismatch after marshal/unmarshal: want=%s got=%s", c, y)
}
})
t.Run("encode with "+codec.Name(), func(t *testing.T) {
r1, err = compress(codec, m.Value)
if err != nil {
t.Fatal(err)
}
})
t.Run("decode with "+codec.Name(), func(t *testing.T) {
if r1 == nil {
if r1, err = compress(codec, m.Value); err != nil {
t.Fatal(err)
}
}
r2, err = decompress(codec, r1)
if err != nil {
t.Fatal(err)
}
if string(r2) != "message" {
t.Error("bad message")
t.Logf("expected: %q", string(m.Value))
t.Logf("got: %q", string(r2))
}
})
}
func TestCompressedMessages(t *testing.T) {
testCompressedMessages(t, new(gzip.Codec))
testCompressedMessages(t, new(snappy.Codec))
testCompressedMessages(t, new(lz4.Codec))
if ktesting.KafkaIsAtLeast("2.1.0") {
testCompressedMessages(t, new(zstd.Codec))
}
}
func testCompressedMessages(t *testing.T, codec pkg.Codec) {
t.Run(codec.Name(), func(t *testing.T) {
client, topic, shutdown := newLocalClientAndTopic()
defer shutdown()
w := &kafka.Writer{
Addr: kafka.TCP("127.0.0.1:9092"),
Topic: topic,
Compression: kafka.Compression(codec.Code()),
BatchTimeout: 10 * time.Millisecond,
Transport: client.Transport,
}
defer w.Close()
offset := 0
var values []string
for i := 0; i < 10; i++ {
batch := make([]kafka.Message, i+1)
for j := range batch {
value := fmt.Sprintf("Hello World %d!", offset)
values = append(values, value)
batch[j] = kafka.Message{
Key: []byte(strconv.Itoa(offset)),
Value: []byte(value),
}
offset++
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
if err := w.WriteMessages(ctx, batch...); err != nil {
t.Errorf("error sending batch %d, reason: %+v", i+1, err)
}
cancel()
}
r := kafka.NewReader(kafka.ReaderConfig{
Brokers: []string{"127.0.0.1:9092"},
Topic: topic,
Partition: 0,
MaxWait: 10 * time.Millisecond,
MinBytes: 1,
MaxBytes: 1024,
})
defer r.Close()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// in order to ensure proper handling of decompressing message, read at
// offsets that we know to be in the middle of compressed message sets.
for base := range values {
r.SetOffset(int64(base))
for i := base; i < len(values); i++ {
msg, err := r.ReadMessage(ctx)
if err != nil {
t.Fatalf("error receiving message at loop %d, offset %d, reason: %+v", base, i, err)
}
if msg.Offset != int64(i) {
t.Fatalf("wrong offset at loop %d...expected %d but got %d", base, i, msg.Offset)
}
if strconv.Itoa(i) != string(msg.Key) {
t.Fatalf("wrong message key at loop %d...expected %d but got %s", base, i, string(msg.Key))
}
if values[i] != string(msg.Value) {
t.Fatalf("wrong message value at loop %d...expected %s but got %s", base, values[i], string(msg.Value))
}
}
}
})
}
func TestMixedCompressedMessages(t *testing.T) {
client, topic, shutdown := newLocalClientAndTopic()
defer shutdown()
offset := 0
var values []string
produce := func(n int, codec pkg.Codec) {
w := &kafka.Writer{
Addr: kafka.TCP("127.0.0.1:9092"),
Topic: topic,
Transport: client.Transport,
}
defer w.Close()
if codec != nil {
w.Compression = kafka.Compression(codec.Code())
}
msgs := make([]kafka.Message, n)
for i := range msgs {
value := fmt.Sprintf("Hello World %d!", offset)
values = append(values, value)
offset++
msgs[i] = kafka.Message{Value: []byte(value)}
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := w.WriteMessages(ctx, msgs...); err != nil {
t.Errorf("failed to produce messages: %+v", err)
}
}
// produce messages that interleave uncompressed messages and messages with
// different compression codecs. reader should be able to properly handle
// all of them.
produce(10, nil)
produce(20, new(gzip.Codec))
produce(5, nil)
produce(10, new(snappy.Codec))
produce(10, new(lz4.Codec))
produce(5, nil)
r := kafka.NewReader(kafka.ReaderConfig{
Brokers: []string{"127.0.0.1:9092"},
Topic: topic,
Partition: 0,
MaxWait: 10 * time.Millisecond,
MinBytes: 1,
MaxBytes: 1024,
})
defer r.Close()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
// in order to ensure proper handling of decompressing message, read at
// offsets that we know to be in the middle of compressed message sets.
for base := range values {
r.SetOffset(int64(base))
for i := base; i < len(values); i++ {
msg, err := r.ReadMessage(ctx)
if err != nil {
t.Errorf("error receiving message at loop %d, offset %d, reason: %+v", base, i, err)
}
if msg.Offset != int64(i) {
t.Errorf("wrong offset at loop %d...expected %d but got %d", base, i, msg.Offset)
}
if values[i] != string(msg.Value) {
t.Errorf("wrong message value at loop %d...expected %s but got %s", base, values[i], string(msg.Value))
}
}
}
}
type noopCodec struct{}
func (noopCodec) Code() int8 {
return 0
}
func (noopCodec) Name() string {
return "none"
}
func (noopCodec) NewReader(r io.Reader) io.ReadCloser {
return ioutil.NopCloser(r)
}
func (noopCodec) NewWriter(w io.Writer) io.WriteCloser {
return nopWriteCloser{w}
}
type nopWriteCloser struct{ io.Writer }
func (nopWriteCloser) Close() error { return nil }
func BenchmarkCompression(b *testing.B) {
benchmarks := []struct {
codec pkg.Codec
function func(*testing.B, pkg.Codec, *bytes.Buffer, []byte) float64
}{
{
codec: &noopCodec{},
function: benchmarkCompression,
},
{
codec: new(gzip.Codec),
function: benchmarkCompression,
},
{
codec: new(snappy.Codec),
function: benchmarkCompression,
},
{
codec: new(lz4.Codec),
function: benchmarkCompression,
},
{
codec: new(zstd.Codec),
function: benchmarkCompression,
},
}
f, err := os.Open(filepath.Join(os.Getenv("GOROOT"), "src/encoding/json/testdata/code.json.gz"))
if err != nil {
b.Fatal(err)
}
defer f.Close()
z, err := gz.NewReader(f)
if err != nil {
b.Fatal(err)
}
payload, err := ioutil.ReadAll(z)
if err != nil {
b.Fatal(err)
}
buffer := bytes.Buffer{}
buffer.Grow(len(payload))
ts := &bytes.Buffer{}
tw := tabwriter.NewWriter(ts, 0, 8, 0, '\t', 0)
defer func() {
tw.Flush()
fmt.Printf("input => %.2f MB\n", float64(len(payload))/(1024*1024))
fmt.Println(ts)
}()
for i := range benchmarks {
benchmark := &benchmarks[i]
ratio := 0.0
b.Run(benchmark.codec.Name(), func(b *testing.B) {
ratio = benchmark.function(b, benchmark.codec, &buffer, payload)
})
fmt.Fprintf(tw, " %s:\t%.2f%%\n", benchmark.codec.Name(), 100*ratio)
}
}
func benchmarkCompression(b *testing.B, codec pkg.Codec, buf *bytes.Buffer, payload []byte) float64 {
// In case only the decompression benchmark are run, we use this flags to
// detect whether we have to compress the payload before the decompression
// benchmarks.
compressed := false
b.Run("compress", func(b *testing.B) {
compressed = true
r := bytes.NewReader(payload)
b.ReportAllocs()
for i := 0; i < b.N; i++ {
buf.Reset()
r.Reset(payload)
w := codec.NewWriter(buf)
_, err := io.Copy(w, r)
if err != nil {
b.Fatal(err)
}
if err := w.Close(); err != nil {
b.Fatal(err)
}
}
b.SetBytes(int64(buf.Len()))
})
if !compressed {
r := bytes.NewReader(payload)
w := codec.NewWriter(buf)
_, err := io.Copy(w, r)
if err != nil {
b.Fatal(err)
}
if err := w.Close(); err != nil {
b.Fatal(err)
}
}
b.Run("decompress", func(b *testing.B) {
c := bytes.NewReader(buf.Bytes())
b.ReportAllocs()
for i := 0; i < b.N; i++ {
c.Reset(buf.Bytes())
r := codec.NewReader(c)
n, err := io.Copy(ioutil.Discard, r)
if err != nil {
b.Fatal(err)
}
if err := r.Close(); err != nil {
b.Fatal(err)
}
b.SetBytes(n)
}
})
return 1 - (float64(buf.Len()) / float64(len(payload)))
}
func init() {
rand.Seed(time.Now().UnixNano())
}
func makeTopic() string {
return fmt.Sprintf("kafka-go-%016x", rand.Int63())
}
func newLocalClientAndTopic() (*kafka.Client, string, func()) {
topic := makeTopic()
client, shutdown := newLocalClient()
_, err := client.CreateTopics(context.Background(), &kafka.CreateTopicsRequest{
Topics: []kafka.TopicConfig{{
Topic: topic,
NumPartitions: 1,
ReplicationFactor: 1,
}},
})
if err != nil {
shutdown()
panic(err)
}
// Topic creation seems to be asynchronous. Metadata for the topic partition
// layout in the cluster is available in the controller before being synced
// with the other brokers, which causes "Error:[3] Unknown Topic Or Partition"
// when sending requests to the partition leaders.
for i := 0; i < 20; i++ {
r, err := client.Fetch(context.Background(), &kafka.FetchRequest{
Topic: topic,
Partition: 0,
Offset: 0,
})
if err == nil && r.Error == nil {
break
}
time.Sleep(100 * time.Millisecond)
}
return client, topic, func() {
client.DeleteTopics(context.Background(), &kafka.DeleteTopicsRequest{
Topics: []string{topic},
})
shutdown()
}
}
func newLocalClient() (*kafka.Client, func()) {
return newClient(kafka.TCP("127.0.0.1:9092"))
}
func newClient(addr net.Addr) (*kafka.Client, func()) {
conns := &ktesting.ConnWaitGroup{
DialFunc: (&net.Dialer{}).DialContext,
}
transport := &kafka.Transport{
Dial: conns.Dial,
}
client := &kafka.Client{
Addr: addr,
Timeout: 5 * time.Second,
Transport: transport,
}
return client, func() { transport.CloseIdleConnections(); conns.Wait() }
}
================================================
FILE: compress/gzip/gzip.go
================================================
package gzip
import (
"io"
"sync"
"github.com/klauspost/compress/gzip"
)
var (
readerPool sync.Pool
)
// Codec is the implementation of a compress.Codec which supports creating
// readers and writers for kafka messages compressed with gzip.
type Codec struct {
// The compression level to configure on writers created by this codec.
// Acceptable values are defined in the standard gzip package.
//
// Default to gzip.DefaultCompressionLevel.
Level int
writerPool sync.Pool
}
// Code implements the compress.Codec interface.
func (c *Codec) Code() int8 { return 1 }
// Name implements the compress.Codec interface.
func (c *Codec) Name() string { return "gzip" }
// NewReader implements the compress.Codec interface.
func (c *Codec) NewReader(r io.Reader) io.ReadCloser {
var err error
z, _ := readerPool.Get().(*gzip.Reader)
if z != nil {
err = z.Reset(r)
} else {
z, err = gzip.NewReader(r)
}
if err != nil {
if z != nil {
readerPool.Put(z)
}
return &errorReader{err: err}
}
return &reader{Reader: z}
}
// NewWriter implements the compress.Codec interface.
func (c *Codec) NewWriter(w io.Writer) io.WriteCloser {
x := c.writerPool.Get()
z, _ := x.(*gzip.Writer)
if z == nil {
x, err := gzip.NewWriterLevel(w, c.level())
if err != nil {
return &errorWriter{err: err}
}
z = x
} else {
z.Reset(w)
}
return &writer{codec: c, Writer: z}
}
func (c *Codec) level() int {
if c.Level != 0 {
return c.Level
}
return gzip.DefaultCompression
}
type reader struct{ *gzip.Reader }
func (r *reader) Close() (err error) {
if z := r.Reader; z != nil {
r.Reader = nil
err = z.Close()
// Pass it an empty reader, which is a zero-size value implementing the
// flate.Reader interface to avoid the construction of a bufio.Reader in
// the call to Reset.
//
// Note: we could also not reset the reader at all, but that would cause
// the underlying reader to be retained until the gzip.Reader is freed,
// which may not be desirable.
z.Reset(emptyReader{})
readerPool.Put(z)
}
return
}
type writer struct {
codec *Codec
*gzip.Writer
}
func (w *writer) Close() (err error) {
if z := w.Writer; z != nil {
w.Writer = nil
err = z.Close()
z.Reset(nil)
w.codec.writerPool.Put(z)
}
return
}
type emptyReader struct{}
func (emptyReader) ReadByte() (byte, error) { return 0, io.EOF }
func (emptyReader) Read([]byte) (int, error) { return 0, io.EOF }
type errorReader struct{ err error }
func (r *errorReader) Close() error { return r.err }
func (r *errorReader) Read([]byte) (int, error) { return 0, r.err }
type errorWriter struct{ err error }
func (w *errorWriter) Close() error { return w.err }
func (w *errorWriter) Write([]byte) (int, error) { return 0, w.err }
================================================
FILE: compress/lz4/lz4.go
================================================
package lz4
import (
"io"
"sync"
"github.com/pierrec/lz4/v4"
)
var (
readerPool sync.Pool
writerPool sync.Pool
)
// Codec is the implementation of a compress.Codec which supports creating
// readers and writers for kafka messages compressed with lz4.
type Codec struct{}
// Code implements the compress.Codec interface.
func (c *Codec) Code() int8 { return 3 }
// Name implements the compress.Codec interface.
func (c *Codec) Name() string { return "lz4" }
// NewReader implements the compress.Codec interface.
func (c *Codec) NewReader(r io.Reader) io.ReadCloser {
z, _ := readerPool.Get().(*lz4.Reader)
if z != nil {
z.Reset(r)
} else {
z = lz4.NewReader(r)
}
return &reader{Reader: z}
}
// NewWriter implements the compress.Codec interface.
func (c *Codec) NewWriter(w io.Writer) io.WriteCloser {
z, _ := writerPool.Get().(*lz4.Writer)
if z != nil {
z.Reset(w)
} else {
z = lz4.NewWriter(w)
}
return &writer{Writer: z}
}
type reader struct{ *lz4.Reader }
func (r *reader) Close() (err error) {
if z := r.Reader; z != nil {
r.Reader = nil
z.Reset(nil)
readerPool.Put(z)
}
return
}
type writer struct{ *lz4.Writer }
func (w *writer) Close() (err error) {
if z := w.Writer; z != nil {
w.Writer = nil
err = z.Close()
z.Reset(nil)
writerPool.Put(z)
}
return
}
================================================
FILE: compress/snappy/go-xerial-snappy/LICENSE
================================================
The MIT License (MIT)
Copyright (c) 2016 Evan Huus
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
================================================
FILE: compress/snappy/go-xerial-snappy/README.md
================================================
# go-xerial-snappy
[](https://travis-ci.org/eapache/go-xerial-snappy)
Xerial-compatible Snappy framing support for golang.
Packages using Xerial for snappy encoding use a framing format incompatible with
basically everything else in existence. This package wraps Go's built-in snappy
package to support it.
Apps that use this format include Apache Kafka (see
https://github.com/dpkp/kafka-python/issues/126#issuecomment-35478921 for
details).
================================================
FILE: compress/snappy/go-xerial-snappy/corpus/05979b224be0294bf350310d4ba5257c9bb815db-3
================================================
���Y
================================================
FILE: compress/snappy/go-xerial-snappy/corpus/521e7e67b6063a75e0eeb24b0d1dd20731d34ad8-4
================================================
����Y
================================================
FILE: compress/snappy/go-xerial-snappy/corpus/581b8fe7088f921567811fdf30e1f527c9f48e5e
================================================
package
================================================
FILE: compress/snappy/go-xerial-snappy/corpus/80881d1b911b95e0203b3b0e7dc6360c35f7620f-7
================================================
墳￩��￩￩���꿽
================================================
FILE: compress/snappy/go-xerial-snappy/corpus/b2419fcb7a9aef359de67cb6bd2b8a8c1f5c100f-4
================================================
��Y
================================================
FILE: compress/snappy/go-xerial-snappy/corpus/cb806bc4f67316af02d6ae677332a3b6005a18da-5
================================================
墳�濽
================================================
FILE: compress/snappy/go-xerial-snappy/corpus/ce3c6f4c31f74d72fbf74c17d14a8d29aa62059e-6
================================================
墳￩���꿽
================================================
FILE: compress/snappy/go-xerial-snappy/corpus/da39a3ee5e6b4b0d3255bfef95601890afd80709-1
================================================
================================================
FILE: compress/snappy/go-xerial-snappy/fuzz.go
================================================
// +build gofuzz
package snappy
func Fuzz(data []byte) int {
decode, err := Decode(data)
if decode == nil && err == nil {
panic("nil error with nil result")
}
if err != nil {
return 0
}
return 1
}
================================================
FILE: compress/snappy/go-xerial-snappy/snappy.go
================================================
package snappy
import (
"bytes"
"encoding/binary"
"errors"
master "github.com/klauspost/compress/snappy"
)
const (
sizeOffset = 16
sizeBytes = 4
)
var (
xerialHeader = []byte{130, 83, 78, 65, 80, 80, 89, 0}
// This is xerial version 1 and minimally compatible with version 1.
xerialVersionInfo = []byte{0, 0, 0, 1, 0, 0, 0, 1}
// ErrMalformed is returned by the decoder when the xerial framing
// is malformed.
ErrMalformed = errors.New("malformed xerial framing")
)
func min(x, y int) int {
if x < y {
return x
}
return y
}
// Encode encodes data as snappy with no framing header.
func Encode(src []byte) []byte {
return master.Encode(nil, src)
}
// EncodeStream *appends* to the specified 'dst' the compressed
// 'src' in xerial framing format. If 'dst' does not have enough
// capacity, then a new slice will be allocated. If 'dst' has
// non-zero length, then if *must* have been built using this function.
func EncodeStream(dst, src []byte) []byte {
if len(dst) == 0 {
dst = append(dst, xerialHeader...)
dst = append(dst, xerialVersionInfo...)
}
// Snappy encode in blocks of maximum 32KB
var (
max = len(src)
blockSize = 32 * 1024
pos = 0
chunk []byte
)
for pos < max {
newPos := min(pos+blockSize, max)
chunk = master.Encode(chunk[:cap(chunk)], src[pos:newPos])
// First encode the compressed size (big-endian)
// Put* panics if the buffer is too small, so pad 4 bytes first
origLen := len(dst)
dst = append(dst, dst[0:4]...)
binary.BigEndian.PutUint32(dst[origLen:], uint32(len(chunk)))
// And now the compressed data
dst = append(dst, chunk...)
pos = newPos
}
return dst
}
// Decode decodes snappy data whether it is traditional unframed
// or includes the xerial framing format.
func Decode(src []byte) ([]byte, error) {
return DecodeInto(nil, src)
}
// DecodeInto decodes snappy data whether it is traditional unframed
// or includes the xerial framing format into the specified `dst`.
// It is assumed that the entirety of `dst` including all capacity is available
// for use by this function. If `dst` is nil *or* insufficiently large to hold
// the decoded `src`, new space will be allocated.
func DecodeInto(dst, src []byte) ([]byte, error) {
var max = len(src)
if max < len(xerialHeader) {
return nil, ErrMalformed
}
if !bytes.Equal(src[:8], xerialHeader) {
return master.Decode(dst[:cap(dst)], src)
}
if max < sizeOffset+sizeBytes {
return nil, ErrMalformed
}
if dst == nil {
dst = make([]byte, 0, len(src))
}
dst = dst[:0]
var (
pos = sizeOffset
chunk []byte
err error
)
for pos+sizeBytes <= max {
size := int(binary.BigEndian.Uint32(src[pos : pos+sizeBytes]))
pos += sizeBytes
nextPos := pos + size
// On architectures where int is 32-bytes wide size + pos could
// overflow so we need to check the low bound as well as the
// high
if nextPos < pos || nextPos > max {
return nil, ErrMalformed
}
chunk, err = master.Decode(chunk[:cap(chunk)], src[pos:nextPos])
if err != nil {
return nil, err
}
pos = nextPos
dst = append(dst, chunk...)
}
return dst, nil
}
================================================
FILE: compress/snappy/go-xerial-snappy/snappy_test.go
================================================
package snappy
import (
"bytes"
"errors"
"testing"
)
const largeString = `Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur? At vero eos et accusamus et iusto odio dignissimos ducimus qui blanditiis praesentium voluptatum deleniti atque corrupti quos dolores et quas molestias except`
var snappyTestCases = map[string][]byte{
"REPEATREPEATREPEATREPEATREPEATREPEAT": {36, 20, 82, 69, 80, 69, 65, 84, 118, 6, 0},
"REALLY SHORT": {12, 44, 82, 69, 65, 76, 76, 89, 32, 83, 72, 79, 82, 84},
"AXBXCXDXEXFX": {12, 44, 65, 88, 66, 88, 67, 88, 68, 88, 69, 88, 70, 88},
}
var snappyStreamTestCases = map[string][]byte{
"PLAINDATA": {130, 83, 78, 65, 80, 80, 89, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 11, 9, 32, 80, 76, 65, 73, 78, 68, 65, 84, 65},
`{"a":"UtaitILHMDAAAAfU","b":"日本"}`: {130, 83, 78, 65, 80, 80, 89, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 39, 37, 144, 123, 34, 97, 34, 58, 34, 85, 116, 97, 105, 116, 73, 76, 72, 77, 68, 65, 65, 65, 65, 102, 85, 34, 44, 34, 98, 34, 58, 34, 230, 151, 165, 230, 156, 172, 34, 125},
largeString: {130, 83, 78, 65, 80, 80, 89, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 3, 89, 128, 8, 240, 90, 83, 101, 100, 32, 117, 116, 32, 112, 101, 114, 115, 112, 105, 99, 105, 97, 116, 105, 115, 32, 117, 110, 100, 101, 32, 111, 109, 110, 105, 115, 32, 105, 115, 116, 101, 32, 110, 97, 116, 117, 115, 32, 101, 114, 114, 111, 114, 32, 115, 105, 116, 32, 118, 111, 108, 117, 112, 116, 97, 116, 101, 109, 32, 97, 99, 99, 117, 115, 97, 110, 116, 105, 117, 109, 32, 100, 111, 108, 111, 114, 101, 109, 113, 117, 101, 32, 108, 97, 117, 100, 97, 5, 22, 240, 60, 44, 32, 116, 111, 116, 97, 109, 32, 114, 101, 109, 32, 97, 112, 101, 114, 105, 97, 109, 44, 32, 101, 97, 113, 117, 101, 32, 105, 112, 115, 97, 32, 113, 117, 97, 101, 32, 97, 98, 32, 105, 108, 108, 111, 32, 105, 110, 118, 101, 110, 116, 111, 114, 101, 32, 118, 101, 114, 105, 116, 97, 1, 141, 4, 101, 116, 1, 36, 88, 115, 105, 32, 97, 114, 99, 104, 105, 116, 101, 99, 116, 111, 32, 98, 101, 97, 116, 97, 101, 32, 118, 105, 1, 6, 120, 100, 105, 99, 116, 97, 32, 115, 117, 110, 116, 32, 101, 120, 112, 108, 105, 99, 97, 98, 111, 46, 32, 78, 101, 109, 111, 32, 101, 110, 105, 109, 5, 103, 0, 109, 46, 180, 0, 12, 113, 117, 105, 97, 17, 16, 0, 115, 5, 209, 72, 97, 115, 112, 101, 114, 110, 97, 116, 117, 114, 32, 97, 117, 116, 32, 111, 100, 105, 116, 5, 9, 36, 102, 117, 103, 105, 116, 44, 32, 115, 101, 100, 9, 53, 32, 99, 111, 110, 115, 101, 113, 117, 117, 110, 1, 42, 20, 109, 97, 103, 110, 105, 32, 9, 245, 16, 115, 32, 101, 111, 115, 1, 36, 28, 32, 114, 97, 116, 105, 111, 110, 101, 17, 96, 33, 36, 1, 51, 36, 105, 32, 110, 101, 115, 99, 105, 117, 110, 116, 1, 155, 1, 254, 16, 112, 111, 114, 114, 111, 1, 51, 36, 115, 113, 117, 97, 109, 32, 101, 115, 116, 44, 1, 14, 13, 81, 5, 183, 4, 117, 109, 1, 18, 0, 97, 9, 19, 4, 32, 115, 1, 149, 12, 109, 101, 116, 44, 9, 135, 76, 99, 116, 101, 116, 117, 114, 44, 32, 97, 100, 105, 112, 105, 115, 99, 105, 32, 118, 101, 108, 50, 173, 0, 24, 110, 111, 110, 32, 110, 117, 109, 9, 94, 84, 105, 117, 115, 32, 109, 111, 100, 105, 32, 116, 101, 109, 112, 111, 114, 97, 32, 105, 110, 99, 105, 100, 33, 52, 20, 117, 116, 32, 108, 97, 98, 33, 116, 4, 101, 116, 9, 106, 0, 101, 5, 219, 20, 97, 109, 32, 97, 108, 105, 5, 62, 33, 164, 8, 114, 97, 116, 29, 212, 12, 46, 32, 85, 116, 41, 94, 52, 97, 100, 32, 109, 105, 110, 105, 109, 97, 32, 118, 101, 110, 105, 33, 221, 72, 113, 117, 105, 115, 32, 110, 111, 115, 116, 114, 117, 109, 32, 101, 120, 101, 114, 99, 105, 33, 202, 104, 111, 110, 101, 109, 32, 117, 108, 108, 97, 109, 32, 99, 111, 114, 112, 111, 114, 105, 115, 32, 115, 117, 115, 99, 105, 112, 105, 13, 130, 8, 105, 111, 115, 1, 64, 12, 110, 105, 115, 105, 1, 150, 5, 126, 44, 105, 100, 32, 101, 120, 32, 101, 97, 32, 99, 111, 109, 5, 192, 0, 99, 41, 131, 33, 172, 8, 63, 32, 81, 1, 107, 4, 97, 117, 33, 101, 96, 118, 101, 108, 32, 101, 117, 109, 32, 105, 117, 114, 101, 32, 114, 101, 112, 114, 101, 104, 101, 110, 100, 101, 114, 105, 65, 63, 12, 105, 32, 105, 110, 1, 69, 16, 118, 111, 108, 117, 112, 65, 185, 1, 47, 24, 105, 116, 32, 101, 115, 115, 101, 1, 222, 64, 109, 32, 110, 105, 104, 105, 108, 32, 109, 111, 108, 101, 115, 116, 105, 97, 101, 46, 103, 0, 0, 44, 1, 45, 16, 32, 105, 108, 108, 117, 37, 143, 45, 36, 0, 109, 5, 110, 65, 33, 20, 97, 116, 32, 113, 117, 111, 17, 92, 44, 115, 32, 110, 117, 108, 108, 97, 32, 112, 97, 114, 105, 9, 165, 24, 65, 116, 32, 118, 101, 114, 111, 69, 34, 44, 101, 116, 32, 97, 99, 99, 117, 115, 97, 109, 117, 115, 1, 13, 104, 105, 117, 115, 116, 111, 32, 111, 100, 105, 111, 32, 100, 105, 103, 110, 105, 115, 115, 105, 109, 111, 115, 32, 100, 117, 99, 105, 1, 34, 80, 113, 117, 105, 32, 98, 108, 97, 110, 100, 105, 116, 105, 105, 115, 32, 112, 114, 97, 101, 115, 101, 101, 87, 17, 111, 56, 116, 117, 109, 32, 100, 101, 108, 101, 110, 105, 116, 105, 32, 97, 116, 65, 89, 28, 99, 111, 114, 114, 117, 112, 116, 105, 1, 150, 0, 115, 13, 174, 5, 109, 8, 113, 117, 97, 65, 5, 52, 108, 101, 115, 116, 105, 97, 115, 32, 101, 120, 99, 101, 112, 116, 0, 0, 0, 1, 0},
}
func makeMassive(input string, numCopies int) string {
outBuff := make([]byte, len(input)*numCopies)
for i := 0; i < numCopies; i++ {
copy(outBuff[len(outBuff):], input)
}
return string(outBuff)
}
func TestSnappyEncode(t *testing.T) {
for src, exp := range snappyTestCases {
dst := Encode([]byte(src))
if !bytes.Equal(dst, exp) {
t.Errorf("Expected %s to generate %v, but was %v", src, exp, dst)
}
}
}
func TestSnappyEncodeStream(t *testing.T) {
for src := range snappyStreamTestCases {
dst := EncodeStream(nil, []byte(src))
// Block size can change the bytes generated, so let's just decode and make sure in matches out
dec, err := Decode(dst)
if err != nil {
t.Error(err)
}
if src != string(dec) {
t.Errorf("Expected decode to match encode orig = %s, decoded = %s", src, string(dec))
}
}
}
func TestSnappyLargeStringEncodeStream(t *testing.T) {
massiveString := makeMassive(largeString, 10000)
dst := EncodeStream(nil, []byte(massiveString))
dec, err := Decode(dst)
if err != nil {
t.Error(err)
}
if massiveString != string(dec) {
t.Errorf("Decoded string didn't match original input (not printing due to size)")
}
}
func TestSnappyDecode(t *testing.T) {
for exp, src := range snappyTestCases {
dst, err := Decode(src)
if err != nil {
t.Error("Encoding error: ", err)
} else if !bytes.Equal(dst, []byte(exp)) {
t.Errorf("Expected %s to be generated from %v, but was %s", exp, src, string(dst))
}
}
}
func TestSnappyDecodeStreams(t *testing.T) {
for exp, src := range snappyStreamTestCases {
dst, err := Decode(src)
if err != nil {
t.Error("Encoding error: ", err)
} else if !bytes.Equal(dst, []byte(exp)) {
t.Errorf("Expected %s to be generated from [%d]byte, but was %s", exp, len(src), string(dst))
}
}
}
func TestSnappyDecodeMalformedTruncatedHeader(t *testing.T) {
// Truncated headers should not cause a panic.
for i := 0; i < len(xerialHeader); i++ {
buf := make([]byte, i)
copy(buf, xerialHeader[:i])
if _, err := Decode(buf); !errors.Is(err, ErrMalformed) {
t.Errorf("expected ErrMalformed got %v", err)
}
}
}
func TestSnappyDecodeMalformedTruncatedSize(t *testing.T) {
// Inputs with valid Xerial header but truncated "size" field
sizes := []int{sizeOffset + 1, sizeOffset + 2, sizeOffset + 3}
for _, size := range sizes {
buf := make([]byte, size)
copy(buf, xerialHeader)
if _, err := Decode(buf); !errors.Is(err, ErrMalformed) {
t.Errorf("expected ErrMalformed got %v", err)
}
}
}
func TestSnappyDecodeMalformedBNoData(t *testing.T) {
// No data after the size field
buf := make([]byte, 20)
copy(buf, xerialHeader)
// indicate that there's one byte of data to be read
buf[len(buf)-1] = 1
if _, err := Decode(buf); !errors.Is(err, ErrMalformed) {
t.Errorf("expected ErrMalformed got %v", err)
}
}
func TestSnappyMasterDecodeFailed(t *testing.T) {
buf := make([]byte, 21)
copy(buf, xerialHeader)
// indicate that there's one byte of data to be read
buf[len(buf)-2] = 1
// A payload which will not decode
buf[len(buf)-1] = 1
if _, err := Decode(buf); errors.Is(err, ErrMalformed) || err == nil {
t.Errorf("unexpected err: %v", err)
}
}
func BenchmarkSnappyDecode(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
for n := 0; n < b.N; n++ {
bytes := 0
for _, test := range snappyTestCases {
dst, err := Decode(test)
if err != nil {
b.Error("Decoding error: ", err)
}
bytes += len(dst)
}
b.SetBytes(int64(bytes))
}
}
func BenchmarkSnappyDecodeInto(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
var (
dst []byte
err error
)
for n := 0; n < b.N; n++ {
bytes := 0
for _, test := range snappyTestCases {
dst, err = DecodeInto(dst, test)
if err != nil {
b.Error("Decoding error: ", err)
}
bytes += len(dst)
}
b.SetBytes(int64(bytes))
}
}
func BenchmarkSnappyStreamDecode(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
for n := 0; n < b.N; n++ {
bytes := 0
for _, test := range snappyStreamTestCases {
dst, err := Decode(test)
if err != nil {
b.Error("Decoding error: ", err)
}
bytes += len(dst)
}
b.SetBytes(int64(bytes))
}
}
func BenchmarkSnappyStreamDecodeInto(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
var (
dst = make([]byte, 1024)
err error
)
for n := 0; n < b.N; n++ {
bytes := 0
for _, test := range snappyStreamTestCases {
dst, err = DecodeInto(dst, test)
if err != nil {
b.Error("Decoding error: ", err)
}
bytes += len(dst)
}
b.SetBytes(int64(bytes))
}
}
func BenchmarkSnappyStreamDecodeMassive(b *testing.B) {
massiveString := makeMassive(largeString, 10000)
enc := EncodeStream(nil, []byte(massiveString))
b.ReportAllocs()
b.ResetTimer()
b.SetBytes(int64(len(massiveString)))
for n := 0; n < b.N; n++ {
_, err := Decode(enc)
if err != nil {
b.Error("Decoding error: ", err)
}
}
}
func BenchmarkSnappyStreamDecodeIntoMassive(b *testing.B) {
massiveString := makeMassive(largeString, 10000)
enc := EncodeStream(nil, []byte(massiveString))
b.ReportAllocs()
b.ResetTimer()
b.SetBytes(int64(len(massiveString)))
var (
dst = make([]byte, 1024)
err error
)
for n := 0; n < b.N; n++ {
dst, err = DecodeInto(dst, enc)
if err != nil {
b.Error("Decoding error: ", err)
}
}
}
================================================
FILE: compress/snappy/snappy.go
================================================
package snappy
import (
"io"
"sync"
"github.com/klauspost/compress/s2"
"github.com/klauspost/compress/snappy"
)
// Framing is an enumeration type used to enable or disable xerial framing of
// snappy messages.
type Framing int
const (
Framed Framing = iota
Unframed
)
// Compression level.
type Compression int
const (
DefaultCompression Compression = iota
FasterCompression
BetterCompression
BestCompression
)
var (
readerPool sync.Pool
writerPool sync.Pool
)
// Codec is the implementation of a compress.Codec which supports creating
// readers and writers for kafka messages compressed with snappy.
type Codec struct {
// An optional framing to apply to snappy compression.
//
// Default to Framed.
Framing Framing
// Compression level.
Compression Compression
}
// Code implements the compress.Codec interface.
func (c *Codec) Code() int8 { return 2 }
// Name implements the compress.Codec interface.
func (c *Codec) Name() string { return "snappy" }
// NewReader implements the compress.Codec interface.
func (c *Codec) NewReader(r io.Reader) io.ReadCloser {
x, _ := readerPool.Get().(*xerialReader)
if x != nil {
x.Reset(r)
} else {
x = &xerialReader{
reader: r,
decode: snappy.Decode,
}
}
return &reader{xerialReader: x}
}
// NewWriter implements the compress.Codec interface.
func (c *Codec) NewWriter(w io.Writer) io.WriteCloser {
x, _ := writerPool.Get().(*xerialWriter)
if x != nil {
x.Reset(w)
} else {
x = &xerialWriter{writer: w}
}
x.framed = c.Framing == Framed
switch c.Compression {
case FasterCompression:
x.encode = s2.EncodeSnappy
case BetterCompression:
x.encode = s2.EncodeSnappyBetter
case BestCompression:
x.encode = s2.EncodeSnappyBest
default:
x.encode = snappy.Encode // aka. s2.EncodeSnappyBetter
}
return &writer{xerialWriter: x}
}
type reader struct{ *xerialReader }
func (r *reader) Close() (err error) {
if x := r.xerialReader; x != nil {
r.xerialReader = nil
x.Reset(nil)
readerPool.Put(x)
}
return
}
type writer struct{ *xerialWriter }
func (w *writer) Close() (err error) {
if x := w.xerialWriter; x != nil {
w.xerialWriter = nil
err = x.Flush()
x.Reset(nil)
writerPool.Put(x)
}
return
}
================================================
FILE: compress/snappy/xerial.go
================================================
package snappy
import (
"bytes"
"encoding/binary"
"errors"
"io"
"github.com/klauspost/compress/snappy"
)
const defaultBufferSize = 32 * 1024
// An implementation of io.Reader which consumes a stream of xerial-framed
// snappy-encoeded data. The framing is optional, if no framing is detected
// the reader will simply forward the bytes from its underlying stream.
type xerialReader struct {
reader io.Reader
header [16]byte
input []byte
output []byte
offset int64
nbytes int64
decode func([]byte, []byte) ([]byte, error)
}
func (x *xerialReader) Reset(r io.Reader) {
x.reader = r
x.input = x.input[:0]
x.output = x.output[:0]
x.header = [16]byte{}
x.offset = 0
x.nbytes = 0
}
func (x *xerialReader) Read(b []byte) (int, error) {
for {
if x.offset < int64(len(x.output)) {
n := copy(b, x.output[x.offset:])
x.offset += int64(n)
return n, nil
}
n, err := x.readChunk(b)
if err != nil {
return 0, err
}
if n > 0 {
return n, nil
}
}
}
func (x *xerialReader) WriteTo(w io.Writer) (int64, error) {
wn := int64(0)
for {
for x.offset < int64(len(x.output)) {
n, err := w.Write(x.output[x.offset:])
wn += int64(n)
x.offset += int64(n)
if err != nil {
return wn, err
}
}
if _, err := x.readChunk(nil); err != nil {
if errors.Is(err, io.EOF) {
err = nil
}
return wn, err
}
}
}
func (x *xerialReader) readChunk(dst []byte) (int, error) {
x.output = x.output[:0]
x.offset = 0
prefix := 0
if x.nbytes == 0 {
n, err := x.readFull(x.header[:])
if err != nil && n == 0 {
return 0, err
}
prefix = n
}
if isXerialHeader(x.header[:]) {
if cap(x.input) < 4 {
x.input = make([]byte, 4, defaultBufferSize)
} else {
x.input = x.input[:4]
}
_, err := x.readFull(x.input)
if err != nil {
return 0, err
}
frame := int(binary.BigEndian.Uint32(x.input))
if cap(x.input) < frame {
x.input = make([]byte, frame, align(frame, defaultBufferSize))
} else {
x.input = x.input[:frame]
}
if _, err := x.readFull(x.input); err != nil {
return 0, err
}
} else {
if cap(x.input) == 0 {
x.input = make([]byte, 0, defaultBufferSize)
} else {
x.input = x.input[:0]
}
if prefix > 0 {
x.input = append(x.input, x.header[:prefix]...)
}
for {
if len(x.input) == cap(x.input) {
b := make([]byte, len(x.input), 2*cap(x.input))
copy(b, x.input)
x.input = b
}
n, err := x.read(x.input[len(x.input):cap(x.input)])
x.input = x.input[:len(x.input)+n]
if err != nil {
if errors.Is(err, io.EOF) && len(x.input) > 0 {
break
}
return 0, err
}
}
}
var n int
var err error
if x.decode == nil {
x.output, x.input, err = x.input, x.output, nil
} else if n, err = snappy.DecodedLen(x.input); n <= len(dst) && err == nil {
// If the output buffer is large enough to hold the decode value,
// write it there directly instead of using the intermediary output
// buffer.
_, err = x.decode(dst, x.input)
} else {
var b []byte
n = 0
b, err = x.decode(x.output[:cap(x.output)], x.input)
if err == nil {
x.output = b
}
}
return n, err
}
func (x *xerialReader) read(b []byte) (int, error) {
n, err := x.reader.Read(b)
x.nbytes += int64(n)
return n, err
}
func (x *xerialReader) readFull(b []byte) (int, error) {
n, err := io.ReadFull(x.reader, b)
x.nbytes += int64(n)
return n, err
}
// An implementation of a xerial-framed snappy-encoded output stream.
// Each Write made to the writer is framed with a xerial header.
type xerialWriter struct {
writer io.Writer
header [16]byte
input []byte
output []byte
nbytes int64
framed bool
encode func([]byte, []byte) []byte
}
func (x *xerialWriter) Reset(w io.Writer) {
x.writer = w
x.input = x.input[:0]
x.output = x.output[:0]
x.nbytes = 0
}
func (x *xerialWriter) ReadFrom(r io.Reader) (int64, error) {
wn := int64(0)
if cap(x.input) == 0 {
x.input = make([]byte, 0, defaultBufferSize)
}
for {
if x.full() {
x.grow()
}
n, err := r.Read(x.input[len(x.input):cap(x.input)])
wn += int64(n)
x.input = x.input[:len(x.input)+n]
if x.fullEnough() {
if err := x.Flush(); err != nil {
return wn, err
}
}
if err != nil {
if errors.Is(err, io.EOF) {
err = nil
}
return wn, err
}
}
}
func (x *xerialWriter) Write(b []byte) (int, error) {
wn := 0
if cap(x.input) == 0 {
x.input = make([]byte, 0, defaultBufferSize)
}
for len(b) > 0 {
if x.full() {
x.grow()
}
n := copy(x.input[len(x.input):cap(x.input)], b)
b = b[n:]
wn += n
x.input = x.input[:len(x.input)+n]
if x.fullEnough() {
if err := x.Flush(); err != nil {
return wn, err
}
}
}
return wn, nil
}
func (x *xerialWriter) Flush() error {
if len(x.input) == 0 {
return nil
}
var b []byte
if x.encode == nil {
b = x.input
} else {
x.output = x.encode(x.output[:cap(x.output)], x.input)
b = x.output
}
x.input = x.input[:0]
x.output = x.output[:0]
if x.framed && x.nbytes == 0 {
writeXerialHeader(x.header[:])
_, err := x.write(x.header[:])
if err != nil {
return err
}
}
if x.framed {
writeXerialFrame(x.header[:4], len(b))
_, err := x.write(x.header[:4])
if err != nil {
return err
}
}
_, err := x.write(b)
return err
}
func (x *xerialWriter) write(b []byte) (int, error) {
n, err := x.writer.Write(b)
x.nbytes += int64(n)
return n, err
}
func (x *xerialWriter) full() bool {
return len(x.input) == cap(x.input)
}
func (x *xerialWriter) fullEnough() bool {
return x.framed && (cap(x.input)-len(x.input)) < 1024
}
func (x *xerialWriter) grow() {
tmp := make([]byte, len(x.input), 2*cap(x.input))
copy(tmp, x.input)
x.input = tmp
}
func align(n, a int) int {
if (n % a) == 0 {
return n
}
return ((n / a) + 1) * a
}
var (
xerialHeader = [...]byte{130, 83, 78, 65, 80, 80, 89, 0}
xerialVersionInfo = [...]byte{0, 0, 0, 1, 0, 0, 0, 1}
)
func isXerialHeader(src []byte) bool {
return len(src) >= 16 && bytes.Equal(src[:8], xerialHeader[:])
}
func writeXerialHeader(b []byte) {
copy(b[:8], xerialHeader[:])
copy(b[8:], xerialVersionInfo[:])
}
func writeXerialFrame(b []byte, n int) {
binary.BigEndian.PutUint32(b, uint32(n))
}
================================================
FILE: compress/snappy/xerial_test.go
================================================
package snappy
import (
"bytes"
"crypto/rand"
"io"
"testing"
"github.com/klauspost/compress/snappy"
goxerialsnappy "github.com/segmentio/kafka-go/compress/snappy/go-xerial-snappy"
)
// Wrap an io.Reader or io.Writer to disable all copy optimizations like
// io.WriterTo or io.ReaderFrom.
// We use this to ensure writes are chunked by io.Copy's internal buffer
// in the tests.
type simpleReader struct{ io.Reader }
type simpleWriter struct{ io.Writer }
func TestXerialReaderSnappy(t *testing.T) {
rawData := new(bytes.Buffer)
rawData.Grow(1024 * 1024)
io.CopyN(rawData, rand.Reader, 1024*1024)
compressedRawData := bytes.NewReader(snappy.Encode(nil, rawData.Bytes()))
decompressedData := new(bytes.Buffer)
io.Copy(decompressedData,
&xerialReader{reader: compressedRawData, decode: snappy.Decode})
b0 := rawData.Bytes()
b1 := decompressedData.Bytes()
if !bytes.Equal(b0, b1) {
t.Error("data mismatch")
}
}
func TestXerialReaderWriter(t *testing.T) {
rawData := new(bytes.Buffer)
rawData.Grow(1024 * 1024)
io.CopyN(rawData, rand.Reader, 1024*1024)
framedData := new(bytes.Buffer)
framedData.Grow(rawData.Len() + 1024)
w := simpleWriter{&xerialWriter{writer: framedData}}
r := simpleReader{bytes.NewReader(rawData.Bytes())}
io.Copy(w, r)
w.Writer.(*xerialWriter).Flush()
unframedData := new(bytes.Buffer)
unframedData.Grow(rawData.Len())
io.Copy(unframedData, &xerialReader{reader: framedData})
b0 := rawData.Bytes()
b1 := unframedData.Bytes()
if !bytes.Equal(b0, b1) {
t.Error("data mismatch")
}
}
func TestXerialFramedCompression(t *testing.T) {
rawData := new(bytes.Buffer)
rawData.Grow(1024 * 1024)
io.CopyN(rawData, rand.Reader, 1024*1024)
framedAndCompressedData := new(bytes.Buffer)
framedAndCompressedData.Grow(rawData.Len())
w := simpleWriter{&xerialWriter{writer: framedAndCompressedData, framed: true, encode: snappy.Encode}}
r := simpleReader{bytes.NewReader(rawData.Bytes())}
io.Copy(w, r)
w.Writer.(*xerialWriter).Flush()
unframedAndDecompressedData := new(bytes.Buffer)
unframedAndDecompressedData.Grow(rawData.Len())
io.Copy(unframedAndDecompressedData,
simpleReader{&xerialReader{reader: framedAndCompressedData, decode: snappy.Decode}})
b0 := rawData.Bytes()
b1 := unframedAndDecompressedData.Bytes()
if !bytes.Equal(b0, b1) {
t.Error("data mismatch")
}
}
func TestXerialFramedCompressionOptimized(t *testing.T) {
rawData := new(bytes.Buffer)
rawData.Grow(1024 * 1024)
io.CopyN(rawData, rand.Reader, 1024*1024)
framedAndCompressedData := new(bytes.Buffer)
framedAndCompressedData.Grow(rawData.Len())
w := &xerialWriter{writer: framedAndCompressedData, framed: true, encode: snappy.Encode}
r := simpleReader{bytes.NewReader(rawData.Bytes())}
io.Copy(w, r)
w.Flush()
unframedAndDecompressedData := new(bytes.Buffer)
unframedAndDecompressedData.Grow(rawData.Len())
io.Copy(unframedAndDecompressedData,
&xerialReader{reader: framedAndCompressedData, decode: snappy.Decode})
b0 := rawData.Bytes()
b1 := unframedAndDecompressedData.Bytes()
if !bytes.Equal(b0, b1) {
t.Error("data mismatch")
}
}
func TestXerialReaderAgainstGoXerialSnappy(t *testing.T) {
rawData := new(bytes.Buffer)
rawData.Grow(1024 * 1024)
io.CopyN(rawData, rand.Reader, 1024*1024)
rawBytes := rawData.Bytes()
framedAndCompressedData := []byte{}
const chunkSize = 999
for i := 0; i < len(rawBytes); i += chunkSize {
j := i + chunkSize
if j > len(rawBytes) {
j = len(rawBytes)
}
framedAndCompressedData = goxerialsnappy.EncodeStream(framedAndCompressedData, rawBytes[i:j])
}
unframedAndDecompressedData := new(bytes.Buffer)
unframedAndDecompressedData.Grow(rawData.Len())
io.Copy(unframedAndDecompressedData,
&xerialReader{reader: bytes.NewReader(framedAndCompressedData), decode: snappy.Decode})
b0 := rawBytes
b1 := unframedAndDecompressedData.Bytes()
if !bytes.Equal(b0, b1) {
t.Error("data mismatch")
}
}
func TestXerialWriterAgainstGoXerialSnappy(t *testing.T) {
rawData := new(bytes.Buffer)
rawData.Grow(1024 * 1024)
io.CopyN(rawData, rand.Reader, 1024*1024)
framedAndCompressedData := new(bytes.Buffer)
framedAndCompressedData.Grow(rawData.Len())
w := &xerialWriter{writer: framedAndCompressedData, framed: true, encode: snappy.Encode}
r := simpleReader{bytes.NewReader(rawData.Bytes())}
io.Copy(w, r)
w.Flush()
unframedAndDecompressedData, err := goxerialsnappy.Decode(framedAndCompressedData.Bytes())
if err != nil {
t.Error(err)
}
b0 := rawData.Bytes()
b1 := unframedAndDecompressedData
if !bytes.Equal(b0, b1) {
t.Error("data mismatch")
}
}
================================================
FILE: compress/zstd/zstd.go
================================================
// Package zstd implements Zstandard compression.
package zstd
import (
"io"
"sync"
"github.com/klauspost/compress/zstd"
)
// Codec is the implementation of a compress.Codec which supports creating
// readers and writers for kafka messages compressed with zstd.
type Codec struct {
// The compression level configured on writers created by the codec.
//
// Default to 3.
Level int
encoderPool sync.Pool // *encoder
}
// Code implements the compress.Codec interface.
func (c *Codec) Code() int8 { return 4 }
// Name implements the compress.Codec interface.
func (c *Codec) Name() string { return "zstd" }
// NewReader implements the compress.Codec interface.
func (c *Codec) NewReader(r io.Reader) io.ReadCloser {
p := new(reader)
if p.dec, _ = decoderPool.Get().(*zstd.Decoder); p.dec != nil {
p.dec.Reset(r)
} else {
z, err := zstd.NewReader(r,
zstd.WithDecoderConcurrency(1),
)
if err != nil {
p.err = err
} else {
p.dec = z
}
}
return p
}
func (c *Codec) level() int {
if c.Level != 0 {
return c.Level
}
return 3
}
func (c *Codec) zstdLevel() zstd.EncoderLevel {
return zstd.EncoderLevelFromZstd(c.level())
}
var decoderPool sync.Pool // *zstd.Decoder
type reader struct {
dec *zstd.Decoder
err error
}
// Close implements the io.Closer interface.
func (r *reader) Close() error {
if r.dec != nil {
r.dec.Reset(devNull{}) // don't retain the underlying reader
decoderPool.Put(r.dec)
r.dec = nil
r.err = io.ErrClosedPipe
}
return nil
}
// Read implements the io.Reader interface.
func (r *reader) Read(p []byte) (int, error) {
if r.err != nil {
return 0, r.err
}
if r.dec == nil {
return 0, io.EOF
}
return r.dec.Read(p)
}
// WriteTo implements the io.WriterTo interface.
func (r *reader) WriteTo(w io.Writer) (int64, error) {
if r.err != nil {
return 0, r.err
}
if r.dec == nil {
return 0, io.ErrClosedPipe
}
return r.dec.WriteTo(w)
}
// NewWriter implements the compress.Codec interface.
func (c *Codec) NewWriter(w io.Writer) io.WriteCloser {
p := new(writer)
if enc, _ := c.encoderPool.Get().(*zstd.Encoder); enc == nil {
z, err := zstd.NewWriter(w,
zstd.WithEncoderLevel(c.zstdLevel()),
zstd.WithEncoderConcurrency(1),
zstd.WithZeroFrames(true),
)
if err != nil {
p.err = err
} else {
p.enc = z
}
} else {
p.enc = enc
p.enc.Reset(w)
}
p.c = c
return p
}
type writer struct {
c *Codec
enc *zstd.Encoder
err error
}
// Close implements the io.Closer interface.
func (w *writer) Close() error {
if w.enc != nil {
// Close needs to be called to write the end of stream marker and flush
// the buffers. The zstd package documents that the encoder is re-usable
// after being closed.
err := w.enc.Close()
if err != nil {
w.err = err
}
w.enc.Reset(devNull{}) // don't retain the underlying writer
w.c.encoderPool.Put(w.enc)
w.enc = nil
return err
}
return w.err
}
// WriteTo implements the io.WriterTo interface.
func (w *writer) Write(p []byte) (int, error) {
if w.err != nil {
return 0, w.err
}
if w.enc == nil {
return 0, io.ErrClosedPipe
}
return w.enc.Write(p)
}
// ReadFrom implements the io.ReaderFrom interface.
func (w *writer) ReadFrom(r io.Reader) (int64, error) {
if w.err != nil {
return 0, w.err
}
if w.enc == nil {
return 0, io.ErrClosedPipe
}
return w.enc.ReadFrom(r)
}
type devNull struct{}
func (devNull) Read([]byte) (int, error) { return 0, io.EOF }
func (devNull) Write([]byte) (int, error) { return 0, nil }
================================================
FILE: compression.go
================================================
package kafka
import (
"errors"
"github.com/segmentio/kafka-go/compress"
)
type Compression = compress.Compression
const (
Gzip Compression = compress.Gzip
Snappy Compression = compress.Snappy
Lz4 Compression = compress.Lz4
Zstd Compression = compress.Zstd
)
type CompressionCodec = compress.Codec
var (
errUnknownCodec = errors.New("the compression code is invalid or its codec has not been imported")
)
// resolveCodec looks up a codec by Code().
func resolveCodec(code int8) (CompressionCodec, error) {
codec := compress.Compression(code).Codec()
if codec == nil {
return nil, errUnknownCodec
}
return codec, nil
}
================================================
FILE: conn.go
================================================
package kafka
import (
"bufio"
"errors"
"fmt"
"io"
"math"
"net"
"os"
"path/filepath"
"sync"
"sync/atomic"
"time"
)
var (
errInvalidWriteTopic = errors.New("writes must NOT set Topic on kafka.Message")
errInvalidWritePartition = errors.New("writes must NOT set Partition on kafka.Message")
)
// Conn represents a connection to a kafka broker.
//
// Instances of Conn are safe to use concurrently from multiple goroutines.
type Conn struct {
// base network connection
conn net.Conn
// number of inflight requests on the connection.
inflight int32
// offset management (synchronized on the mutex field)
mutex sync.Mutex
offset int64
// read buffer (synchronized on rlock)
rlock sync.Mutex
rbuf bufio.Reader
// write buffer (synchronized on wlock)
wlock sync.Mutex
wbuf bufio.Writer
wb writeBuffer
// deadline management
wdeadline connDeadline
rdeadline connDeadline
// immutable values of the connection object
clientID string
topic string
partition int32
fetchMaxBytes int32
fetchMinSize int32
broker int32
rack string
// correlation ID generator (synchronized on wlock)
correlationID int32
// number of replica acks required when publishing to a partition
requiredAcks int32
// lazily loaded API versions used by this connection
apiVersions atomic.Value // apiVersionMap
transactionalID *string
}
type apiVersionMap map[apiKey]ApiVersion
func (v apiVersionMap) negotiate(key apiKey, sortedSupportedVersions ...apiVersion) apiVersion {
x := v[key]
for i := len(sortedSupportedVersions) - 1; i >= 0; i-- {
s := sortedSupportedVersions[i]
if apiVersion(x.MaxVersion) >= s {
return s
}
}
return -1
}
// ConnConfig is a configuration object used to create new instances of Conn.
type ConnConfig struct {
ClientID string
Topic string
Partition int
Broker int
Rack string
// The transactional id to use for transactional delivery. Idempotent
// deliver should be enabled if transactional id is configured.
// For more details look at transactional.id description here: http://kafka.apache.org/documentation.html#producerconfigs
// Empty string means that this connection can't be transactional.
TransactionalID string
}
// ReadBatchConfig is a configuration object used for reading batches of messages.
type ReadBatchConfig struct {
// MinBytes indicates to the broker the minimum batch size that the consumer
// will accept. Setting a high minimum when consuming from a low-volume topic
// may result in delayed delivery when the broker does not have enough data to
// satisfy the defined minimum.
MinBytes int
// MaxBytes indicates to the broker the maximum batch size that the consumer
// will accept. The broker will truncate a message to satisfy this maximum, so
// choose a value that is high enough for your largest message size.
MaxBytes int
// IsolationLevel controls the visibility of transactional records.
// ReadUncommitted makes all records visible. With ReadCommitted only
// non-transactional and committed records are visible.
IsolationLevel IsolationLevel
// MaxWait is the amount of time for the broker while waiting to hit the
// min/max byte targets. This setting is independent of any network-level
// timeouts or deadlines.
//
// For backward compatibility, when this field is left zero, kafka-go will
// infer the max wait from the connection's read deadline.
MaxWait time.Duration
}
type IsolationLevel int8
const (
ReadUncommitted IsolationLevel = 0
ReadCommitted IsolationLevel = 1
)
var (
// DefaultClientID is the default value used as ClientID of kafka
// connections.
DefaultClientID string
)
func init() {
progname := filepath.Base(os.Args[0])
hostname, _ := os.Hostname()
DefaultClientID = fmt.Sprintf("%s@%s (github.com/segmentio/kafka-go)", progname, hostname)
}
// NewConn returns a new kafka connection for the given topic and partition.
func NewConn(conn net.
gitextract_308eihdc/
├── .circleci/
│ └── config.yml
├── .gitattributes
├── .github/
│ └── ISSUE_TEMPLATE/
│ ├── bug_report.md
│ └── feature_request.md
├── .gitignore
├── .golangci.yml
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── LICENSE
├── Makefile
├── README.md
├── addoffsetstotxn.go
├── addoffsetstotxn_test.go
├── addpartitionstotxn.go
├── addpartitionstotxn_test.go
├── address.go
├── address_test.go
├── alterclientquotas.go
├── alterclientquotas_test.go
├── alterconfigs.go
├── alterconfigs_test.go
├── alterpartitionreassignments.go
├── alterpartitionreassignments_test.go
├── alteruserscramcredentials.go
├── alteruserscramcredentials_test.go
├── apiversions.go
├── apiversions_test.go
├── balancer.go
├── balancer_test.go
├── batch.go
├── batch_test.go
├── buffer.go
├── builder_test.go
├── client.go
├── client_test.go
├── commit.go
├── commit_test.go
├── compress/
│ ├── compress.go
│ ├── compress_test.go
│ ├── gzip/
│ │ └── gzip.go
│ ├── lz4/
│ │ └── lz4.go
│ ├── snappy/
│ │ ├── go-xerial-snappy/
│ │ │ ├── LICENSE
│ │ │ ├── README.md
│ │ │ ├── corpus/
│ │ │ │ ├── 020dfb19a68cbcf99dc93dc1030068d4c9968ad0-2
│ │ │ │ ├── 05979b224be0294bf350310d4ba5257c9bb815db-3
│ │ │ │ ├── 0e64ca2823923c5efa03ff2bd6e0aa1018eeca3b-9
│ │ │ │ ├── 1
│ │ │ │ ├── 361a1c6d2a8f80780826c3d83ad391d0475c922f-4
│ │ │ │ ├── 4117af68228fa64339d362cf980c68ffadff96c8-12
│ │ │ │ ├── 4142249be82c8a617cf838eef05394ece39becd3-9
│ │ │ │ ├── 41ea8c7d904f1cd913b52e9ead4a96c639d76802-10
│ │ │ │ ├── 44083e1447694980c0ee682576e32358c9ee883f-2
│ │ │ │ ├── 4d6b359bd538feaa7d36c89235d07d0a443797ac-1
│ │ │ │ ├── 521e7e67b6063a75e0eeb24b0d1dd20731d34ad8-4
│ │ │ │ ├── 526e6f85d1b8777f0d9f70634c9f8b77fbdccdff-7
│ │ │ │ ├── 581b8fe7088f921567811fdf30e1f527c9f48e5e
│ │ │ │ ├── 60cd10738158020f5843b43960158c3d116b3a71-11
│ │ │ │ ├── 652b031b4b9d601235f86ef62523e63d733b8623-3
│ │ │ │ ├── 684a011f6fdfc7ae9863e12381165e82d2a2e356-9
│ │ │ │ ├── 72e42fc8e5eaed6a8a077f420fc3bd1f9a7c0919-1
│ │ │ │ ├── 80881d1b911b95e0203b3b0e7dc6360c35f7620f-7
│ │ │ │ ├── 8484b3082d522e0a1f315db1fa1b2a5118be7cc3-8
│ │ │ │ ├── 9635bb09260f100bc4a2ee4e3b980fecc5b874ce-1
│ │ │ │ ├── 99d36b0b5b1be7151a508dd440ec725a2576c41c-1
│ │ │ │ ├── 9d339eddb4e2714ea319c3fb571311cb95fdb067-6
│ │ │ │ ├── b2419fcb7a9aef359de67cb6bd2b8a8c1f5c100f-4
│ │ │ │ ├── c1951b29109ec1017f63535ce3699630f46f54e1-5
│ │ │ │ ├── cb806bc4f67316af02d6ae677332a3b6005a18da-5
│ │ │ │ ├── cd7dd228703739e9252c7ea76f1c5f82ab44686a-10
│ │ │ │ ├── ce3671e91907349cea04fc3f2a4b91c65b99461d-3
│ │ │ │ ├── ce3c6f4c31f74d72fbf74c17d14a8d29aa62059e-6
│ │ │ │ ├── da39a3ee5e6b4b0d3255bfef95601890afd80709-1
│ │ │ │ ├── e2230aa0ecaebb9b890440effa13f501a89247b2-1
│ │ │ │ ├── efa11d676fb2a77afb8eac3d7ed30e330a7c2efe-11
│ │ │ │ ├── f0445ac39e03978bbc8011316ac8468015ddb72c-1
│ │ │ │ └── f241da53c6bc1fe3368c55bf28db86ce15a2c784-2
│ │ │ ├── fuzz.go
│ │ │ ├── snappy.go
│ │ │ └── snappy_test.go
│ │ ├── snappy.go
│ │ ├── xerial.go
│ │ └── xerial_test.go
│ └── zstd/
│ └── zstd.go
├── compression.go
├── conn.go
├── conn_test.go
├── consumergroup.go
├── consumergroup_test.go
├── crc32.go
├── crc32_test.go
├── createacls.go
├── createacls_test.go
├── createpartitions.go
├── createpartitions_test.go
├── createtopics.go
├── createtopics_test.go
├── deleteacls.go
├── deleteacls_test.go
├── deletegroups.go
├── deletegroups_test.go
├── deletetopics.go
├── deletetopics_test.go
├── describeacls.go
├── describeacls_test.go
├── describeclientquotas.go
├── describeconfigs.go
├── describeconfigs_test.go
├── describegroups.go
├── describegroups_test.go
├── describeuserscramcredentials.go
├── describeuserscramcredentials_test.go
├── dialer.go
├── dialer_test.go
├── discard.go
├── discard_test.go
├── docker-compose.yml
├── docker_compose_versions/
│ ├── README.md
│ ├── docker-compose-270.yml
│ ├── docker-compose-370.yml
│ └── docker-compose-400.yml
├── electleaders.go
├── electleaders_test.go
├── endtxn.go
├── error.go
├── error_test.go
├── example_consumergroup_test.go
├── example_groupbalancer_test.go
├── example_writer_test.go
├── examples/
│ ├── .gitignore
│ ├── consumer-logger/
│ │ ├── Dockerfile
│ │ ├── go.mod
│ │ ├── go.sum
│ │ └── main.go
│ ├── consumer-mongo-db/
│ │ ├── Dockerfile
│ │ ├── go.mod
│ │ ├── go.sum
│ │ └── main.go
│ ├── docker-compose.yaml
│ ├── kafka/
│ │ └── kafka-variables.env
│ ├── producer-api/
│ │ ├── Dockerfile
│ │ ├── go.mod
│ │ ├── go.sum
│ │ ├── main.go
│ │ └── test.http
│ └── producer-random/
│ ├── Dockerfile
│ ├── go.mod
│ ├── go.sum
│ └── main.go
├── fetch.go
├── fetch_test.go
├── findcoordinator.go
├── findcoordinator_test.go
├── fixtures/
│ ├── v1-v1.hex
│ ├── v1-v1.pcapng
│ ├── v1-v1c-v2-v2c-v2b-v2b-v2b-v2bc-v1b-v1bc.hex
│ ├── v1-v1c-v2-v2c-v2b-v2b-v2b-v2bc-v1b-v1bc.pcapng
│ ├── v1c-v1-v1c.hex
│ ├── v1c-v1-v1c.pcapng
│ ├── v1c-v1c.hex
│ ├── v1c-v1c.pcapng
│ ├── v2-v2.hex
│ ├── v2-v2.pcapng
│ ├── v2b-v1.hex
│ ├── v2b-v1.pcapng
│ ├── v2bc-v1-v1c.hex
│ ├── v2bc-v1-v1c.pcapng
│ ├── v2bc-v1.hex
│ ├── v2bc-v1.pcapng
│ ├── v2bc-v1c.hex
│ ├── v2bc-v1c.pcapng
│ ├── v2c-v2-v2c.hex
│ ├── v2c-v2-v2c.pcapng
│ ├── v2c-v2c.hex
│ └── v2c-v2c.pcapng
├── go.mod
├── go.sum
├── groupbalancer.go
├── groupbalancer_test.go
├── gzip/
│ └── gzip.go
├── heartbeat.go
├── heartbeat_test.go
├── incrementalalterconfigs.go
├── incrementalalterconfigs_test.go
├── initproducerid.go
├── initproducerid_test.go
├── joingroup.go
├── joingroup_test.go
├── kafka.go
├── kafka_test.go
├── leavegroup.go
├── leavegroup_test.go
├── listgroups.go
├── listgroups_test.go
├── listoffset.go
├── listoffset_test.go
├── listpartitionreassignments.go
├── listpartitionreassignments_test.go
├── logger.go
├── lz4/
│ └── lz4.go
├── message.go
├── message_reader.go
├── message_test.go
├── metadata.go
├── metadata_test.go
├── offsetcommit.go
├── offsetcommit_test.go
├── offsetdelete.go
├── offsetdelete_test.go
├── offsetfetch.go
├── offsetfetch_test.go
├── produce.go
├── produce_test.go
├── protocol/
│ ├── addoffsetstotxn/
│ │ ├── addoffsetstotxn.go
│ │ └── addoffsetstotxn_test.go
│ ├── addpartitionstotxn/
│ │ ├── addpartitionstotxn.go
│ │ └── addpartitionstotxn_test.go
│ ├── alterclientquotas/
│ │ ├── alterclientquotas.go
│ │ └── alterclientquotas_test.go
│ ├── alterconfigs/
│ │ ├── alterconfigs.go
│ │ └── alterconfigs_test.go
│ ├── alterpartitionreassignments/
│ │ ├── alterpartitionreassignments.go
│ │ └── alterpartitionreassignments_test.go
│ ├── alteruserscramcredentials/
│ │ ├── alteruserscramcredentials.go
│ │ └── alteruserscramcredentials_test.go
│ ├── apiversions/
│ │ ├── apiversions.go
│ │ └── apiversions_test.go
│ ├── buffer.go
│ ├── buffer_test.go
│ ├── cluster.go
│ ├── conn.go
│ ├── consumer/
│ │ ├── consumer.go
│ │ └── consumer_test.go
│ ├── createacls/
│ │ ├── createacls.go
│ │ └── createacls_test.go
│ ├── createpartitions/
│ │ ├── createpartitions.go
│ │ └── createpartitions_test.go
│ ├── createtopics/
│ │ └── createtopics.go
│ ├── decode.go
│ ├── deleteacls/
│ │ ├── deleteacls.go
│ │ └── deleteacls_test.go
│ ├── deletegroups/
│ │ ├── deletegroups.go
│ │ └── deletegroups_test.go
│ ├── deletetopics/
│ │ ├── deletetopics.go
│ │ └── deletetopics_test.go
│ ├── describeacls/
│ │ ├── describeacls.go
│ │ └── describeacls_test.go
│ ├── describeclientquotas/
│ │ ├── describeclientquotas.go
│ │ └── describeclientquotas_test.go
│ ├── describeconfigs/
│ │ ├── describeconfigs.go
│ │ └── describeconfigs_test.go
│ ├── describegroups/
│ │ ├── describegroups.go
│ │ └── describegroups_test.go
│ ├── describeuserscramcredentials/
│ │ ├── describeuserscramcredentials.go
│ │ └── describeuserscramcredentials_test.go
│ ├── electleaders/
│ │ ├── electleaders.go
│ │ └── electleaders_test.go
│ ├── encode.go
│ ├── endtxn/
│ │ ├── endtxn.go
│ │ └── endtxn_test.go
│ ├── error.go
│ ├── fetch/
│ │ ├── fetch.go
│ │ └── fetch_test.go
│ ├── findcoordinator/
│ │ └── findcoordinator.go
│ ├── heartbeat/
│ │ ├── heartbeat.go
│ │ └── heartbeat_test.go
│ ├── incrementalalterconfigs/
│ │ ├── incrementalalterconfigs.go
│ │ └── incrementalalterconfigs_test.go
│ ├── initproducerid/
│ │ ├── initproducerid.go
│ │ └── initproducerid_test.go
│ ├── joingroup/
│ │ ├── joingroup.go
│ │ └── joingroup_test.go
│ ├── leavegroup/
│ │ ├── leavegroup.go
│ │ └── leavegroup_test.go
│ ├── listgroups/
│ │ └── listgroups.go
│ ├── listoffsets/
│ │ ├── listoffsets.go
│ │ └── listoffsets_test.go
│ ├── listpartitionreassignments/
│ │ ├── listpartitionreassignments.go
│ │ └── listpartitionreassignments_test.go
│ ├── metadata/
│ │ ├── metadata.go
│ │ └── metadata_test.go
│ ├── offsetcommit/
│ │ ├── offsetcommit.go
│ │ └── offsetcommit_test.go
│ ├── offsetdelete/
│ │ ├── offsetdelete.go
│ │ └── offsetdelete_test.go
│ ├── offsetfetch/
│ │ └── offsetfetch.go
│ ├── produce/
│ │ ├── produce.go
│ │ └── produce_test.go
│ ├── protocol.go
│ ├── protocol_test.go
│ ├── prototest/
│ │ ├── bytes.go
│ │ ├── prototest.go
│ │ ├── reflect.go
│ │ ├── request.go
│ │ └── response.go
│ ├── rawproduce/
│ │ ├── rawproduce.go
│ │ └── rawproduce_test.go
│ ├── record.go
│ ├── record_batch.go
│ ├── record_batch_test.go
│ ├── record_v1.go
│ ├── record_v2.go
│ ├── reflect.go
│ ├── reflect_unsafe.go
│ ├── request.go
│ ├── response.go
│ ├── response_test.go
│ ├── roundtrip.go
│ ├── saslauthenticate/
│ │ └── saslauthenticate.go
│ ├── saslhandshake/
│ │ └── saslhandshake.go
│ ├── size.go
│ ├── syncgroup/
│ │ ├── syncgroup.go
│ │ └── syncgroup_test.go
│ └── txnoffsetcommit/
│ ├── txnoffsetcommit.go
│ └── txnoffsetcommit_test.go
├── protocol.go
├── protocol_test.go
├── rawproduce.go
├── rawproduce_test.go
├── read.go
├── read_test.go
├── reader.go
├── reader_test.go
├── record.go
├── recordbatch.go
├── resolver.go
├── resource.go
├── resource_test.go
├── sasl/
│ ├── aws_msk_iam/
│ │ ├── go.mod
│ │ ├── go.sum
│ │ ├── msk_iam.go
│ │ └── msk_iam_test.go
│ ├── aws_msk_iam_v2/
│ │ ├── README.md
│ │ ├── example_test.go
│ │ ├── go.mod
│ │ ├── go.sum
│ │ ├── msk_iam.go
│ │ └── msk_iam_test.go
│ ├── plain/
│ │ └── plain.go
│ ├── sasl.go
│ ├── sasl_test.go
│ └── scram/
│ └── scram.go
├── saslauthenticate.go
├── saslauthenticate_test.go
├── saslhandshake.go
├── saslhandshake_test.go
├── scripts/
│ └── wait-for-kafka.sh
├── sizeof.go
├── snappy/
│ └── snappy.go
├── stats.go
├── syncgroup.go
├── syncgroup_test.go
├── testing/
│ ├── conn.go
│ ├── version.go
│ └── version_test.go
├── time.go
├── topics/
│ ├── list_topics.go
│ └── list_topics_test.go
├── transport.go
├── transport_test.go
├── txnoffsetcommit.go
├── txnoffsetcommit_test.go
├── write.go
├── write_test.go
├── writer.go
├── writer_test.go
└── zstd/
└── zstd.go
Showing preview only (251K chars total). Download the full file or copy to clipboard to get everything.
SYMBOL INDEX (2948 symbols across 269 files)
FILE: addoffsetstotxn.go
type AddOffsetsToTxnRequest (line 13) | type AddOffsetsToTxnRequest struct
type AddOffsetsToTxnResponse (line 32) | type AddOffsetsToTxnResponse struct
method AddOffsetsToTxn (line 45) | func (c *Client) AddOffsetsToTxn(
FILE: addoffsetstotxn_test.go
function TestClientAddOffsetsToTxn (line 13) | func TestClientAddOffsetsToTxn(t *testing.T) {
FILE: addpartitionstotxn.go
type AddPartitionToTxn (line 14) | type AddPartitionToTxn struct
type AddPartitionsToTxnRequest (line 20) | type AddPartitionsToTxnRequest struct
type AddPartitionsToTxnResponse (line 39) | type AddPartitionsToTxnResponse struct
type AddPartitionToTxnPartition (line 49) | type AddPartitionToTxnPartition struct
method AddPartitionsToTxn (line 62) | func (c *Client) AddPartitionsToTxn(
FILE: addpartitionstotxn_test.go
function TestClientAddPartitionsToTxn (line 13) | func TestClientAddPartitionsToTxn(t *testing.T) {
FILE: address.go
function TCP (line 9) | func TCP(address ...string) net.Addr { return makeNetAddr("tcp", address) }
function makeNetAddr (line 11) | func makeNetAddr(network string, addresses []string) net.Addr {
function makeAddr (line 22) | func makeAddr(network, address string) net.Addr {
function makeMultiAddr (line 29) | func makeMultiAddr(network string, addresses []string) net.Addr {
type networkAddress (line 37) | type networkAddress struct
method Network (line 42) | func (a *networkAddress) Network() string { return a.network }
method String (line 44) | func (a *networkAddress) String() string { return a.address }
type multiAddr (line 46) | type multiAddr
method Network (line 48) | func (m multiAddr) Network() string { return m.join(net.Addr.Network) }
method String (line 50) | func (m multiAddr) String() string { return m.join(net.Addr.String) }
method join (line 52) | func (m multiAddr) join(f func(net.Addr) string) string {
FILE: address_test.go
function TestNetworkAddress (line 8) | func TestNetworkAddress(t *testing.T) {
FILE: alterclientquotas.go
type AlterClientQuotasRequest (line 14) | type AlterClientQuotasRequest struct
type AlterClientQuotaEntry (line 25) | type AlterClientQuotaEntry struct
type AlterClientQuotaEntity (line 33) | type AlterClientQuotaEntity struct
type AlterClientQuotaOps (line 41) | type AlterClientQuotaOps struct
type AlterClientQuotaResponseQuotas (line 52) | type AlterClientQuotaResponseQuotas struct
type AlterClientQuotasResponse (line 63) | type AlterClientQuotasResponse struct
method AlterClientQuotas (line 73) | func (c *Client) AlterClientQuotas(ctx context.Context, req *AlterClient...
FILE: alterclientquotas_test.go
function TestClientAlterClientQuotas (line 12) | func TestClientAlterClientQuotas(t *testing.T) {
FILE: alterconfigs.go
type AlterConfigsRequest (line 13) | type AlterConfigsRequest struct
type AlterConfigRequestResource (line 25) | type AlterConfigRequestResource struct
type AlterConfigRequestConfig (line 36) | type AlterConfigRequestConfig struct
type AlterConfigsResponse (line 45) | type AlterConfigsResponse struct
type AlterConfigsResponseResource (line 59) | type AlterConfigsResponseResource struct
method AlterConfigs (line 66) | func (c *Client) AlterConfigs(ctx context.Context, req *AlterConfigsRequ...
FILE: alterconfigs_test.go
function TestClientAlterConfigs (line 11) | func TestClientAlterConfigs(t *testing.T) {
FILE: alterpartitionreassignments.go
type AlterPartitionReassignmentsRequest (line 12) | type AlterPartitionReassignmentsRequest struct
type AlterPartitionReassignmentsRequestAssignment (line 29) | type AlterPartitionReassignmentsRequestAssignment struct
type AlterPartitionReassignmentsResponse (line 41) | type AlterPartitionReassignmentsResponse struct
type AlterPartitionReassignmentsResponsePartitionResult (line 52) | type AlterPartitionReassignmentsResponsePartitionResult struct
method AlterPartitionReassignments (line 64) | func (c *Client) AlterPartitionReassignments(
FILE: alterpartitionreassignments_test.go
function TestClientAlterPartitionReassignments (line 11) | func TestClientAlterPartitionReassignments(t *testing.T) {
function TestClientAlterPartitionReassignmentsMultiTopics (line 62) | func TestClientAlterPartitionReassignmentsMultiTopics(t *testing.T) {
FILE: alteruserscramcredentials.go
type AlterUserScramCredentialsRequest (line 14) | type AlterUserScramCredentialsRequest struct
type ScramMechanism (line 25) | type ScramMechanism
constant ScramMechanismUnknown (line 28) | ScramMechanismUnknown ScramMechanism = iota
constant ScramMechanismSha256 (line 29) | ScramMechanismSha256
constant ScramMechanismSha512 (line 30) | ScramMechanismSha512
type UserScramCredentialsDeletion (line 33) | type UserScramCredentialsDeletion struct
type UserScramCredentialsUpsertion (line 38) | type UserScramCredentialsUpsertion struct
type AlterUserScramCredentialsResponse (line 48) | type AlterUserScramCredentialsResponse struct
type AlterUserScramCredentialsResponseUser (line 56) | type AlterUserScramCredentialsResponseUser struct
method AlterUserScramCredentials (line 63) | func (c *Client) AlterUserScramCredentials(ctx context.Context, req *Alt...
FILE: alteruserscramcredentials_test.go
function TestAlterUserScramCredentials (line 10) | func TestAlterUserScramCredentials(t *testing.T) {
FILE: apiversions.go
type ApiVersionsRequest (line 12) | type ApiVersionsRequest struct
type ApiVersionsResponse (line 18) | type ApiVersionsResponse struct
type ApiVersionsResponseApiKey (line 27) | type ApiVersionsResponseApiKey struct
method ApiVersions (line 41) | func (c *Client) ApiVersions(
FILE: apiversions_test.go
function TestClientApiVersions (line 8) | func TestClientApiVersions(t *testing.T) {
FILE: balancer.go
type Balancer (line 17) | type Balancer interface
type BalancerFunc (line 30) | type BalancerFunc
method Balance (line 33) | func (f BalancerFunc) Balance(msg Message, partitions ...int) int {
type RoundRobin (line 41) | type RoundRobin struct
method Balance (line 51) | func (rr *RoundRobin) Balance(msg Message, partitions ...int) int {
method balance (line 55) | func (rr *RoundRobin) balance(partitions []int) int {
type LeastBytes (line 76) | type LeastBytes struct
method Balance (line 87) | func (lb *LeastBytes) Balance(msg Message, partitions ...int) int {
method makeCounters (line 111) | func (lb *LeastBytes) makeCounters(partitions ...int) (counters []leas...
type leastBytesCounter (line 81) | type leastBytesCounter struct
type Hash (line 143) | type Hash struct
method Balance (line 153) | func (h *Hash) Balance(msg Message, partitions ...int) int {
type ReferenceHash (line 194) | type ReferenceHash struct
method Balance (line 204) | func (h *ReferenceHash) Balance(msg Message, partitions ...int) int {
type randomBalancer (line 230) | type randomBalancer struct
method Balance (line 234) | func (b randomBalancer) Balance(msg Message, partitions ...int) (parti...
type CRC32Balancer (line 256) | type CRC32Balancer struct
method Balance (line 261) | func (b CRC32Balancer) Balance(msg Message, partitions ...int) (partit...
type Murmur2Balancer (line 293) | type Murmur2Balancer struct
method Balance (line 298) | func (b Murmur2Balancer) Balance(msg Message, partitions ...int) (part...
function murmur2 (line 311) | func murmur2(data []byte) uint32 {
FILE: balancer_test.go
function TestHashBalancer (line 10) | func TestHashBalancer(t *testing.T) {
function TestReferenceHashBalancer (line 66) | func TestReferenceHashBalancer(t *testing.T) {
function TestCRC32Balancer (line 125) | func TestCRC32Balancer(t *testing.T) {
function TestMurmur2 (line 200) | func TestMurmur2(t *testing.T) {
function TestMurmur2Balancer (line 231) | func TestMurmur2Balancer(t *testing.T) {
function TestLeastBytes (line 340) | func TestLeastBytes(t *testing.T) {
function TestRoundRobin (line 415) | func TestRoundRobin(t *testing.T) {
FILE: batch.go
type Batch (line 20) | type Batch struct
method Throttle (line 45) | func (batch *Batch) Throttle() time.Duration {
method HighWaterMark (line 50) | func (batch *Batch) HighWaterMark() int64 {
method Partition (line 55) | func (batch *Batch) Partition() int {
method Offset (line 60) | func (batch *Batch) Offset() int64 {
method Close (line 69) | func (batch *Batch) Close() error {
method close (line 76) | func (batch *Batch) close() (err error) {
method Err (line 128) | func (batch *Batch) Err() error { return batch.err }
method Read (line 140) | func (batch *Batch) Read(b []byte) (int, error) {
method ReadMessage (line 192) | func (batch *Batch) ReadMessage() (Message, error) {
method readMessage (line 239) | func (batch *Batch) readMessage(
function checkTimeoutErr (line 306) | func checkTimeoutErr(deadline time.Time) (err error) {
FILE: batch_test.go
function TestBatchDontExpectEOF (line 12) | func TestBatchDontExpectEOF(t *testing.T) {
FILE: buffer.go
function newBuffer (line 12) | func newBuffer() *bytes.Buffer {
function acquireBuffer (line 18) | func acquireBuffer() *bytes.Buffer {
function releaseBuffer (line 22) | func releaseBuffer(b *bytes.Buffer) {
FILE: builder_test.go
type fetchResponseBuilder (line 18) | type fetchResponseBuilder struct
method messages (line 36) | func (b *fetchResponseBuilder) messages() (res []Message) {
method bytes (line 43) | func (b *fetchResponseBuilder) bytes() []byte {
method Len (line 68) | func (b *fetchResponseBuilder) Len() int {
type fetchResponseHeader (line 24) | type fetchResponseHeader struct
type messageSetBuilder (line 72) | type messageSetBuilder interface
type v0MessageSetBuilder (line 77) | type v0MessageSetBuilder struct
method messages (line 82) | func (f v0MessageSetBuilder) messages() []Message {
method bytes (line 86) | func (f v0MessageSetBuilder) bytes() []byte {
type v1MessageSetBuilder (line 118) | type v1MessageSetBuilder struct
method messages (line 123) | func (f v1MessageSetBuilder) messages() []Message {
method bytes (line 127) | func (f v1MessageSetBuilder) bytes() []byte {
type v2MessageSetBuilder (line 165) | type v2MessageSetBuilder struct
method messages (line 170) | func (f v2MessageSetBuilder) messages() []Message {
method bytes (line 174) | func (f v2MessageSetBuilder) bytes() []byte {
type kafkaWriteBuffer (line 226) | type kafkaWriteBuffer struct
method Bytes (line 237) | func (f *kafkaWriteBuffer) Bytes() []byte {
method call (line 245) | func (f *kafkaWriteBuffer) call(cb func(wb *kafkaWriteBuffer)) []byte {
function newWB (line 231) | func newWB() *kafkaWriteBuffer {
function mustCompress (line 254) | func mustCompress(bs []byte, codec compress.Codec) (res []byte) {
FILE: client.go
constant defaultCreateTopicsTimeout (line 14) | defaultCreateTopicsTimeout = 2 * time.Second
constant defaultDeleteTopicsTimeout (line 15) | defaultDeleteTopicsTimeout = 2 * time.Second
constant defaultCreatePartitionsTimeout (line 16) | defaultCreatePartitionsTimeout = 2 * time.Second
constant defaultProduceTimeout (line 17) | defaultProduceTimeout = 500 * time.Millisecond
constant defaultMaxWait (line 18) | defaultMaxWait = 500 * time.Millisecond
type Client (line 28) | type Client struct
method ConsumerOffsets (line 65) | func (c *Client) ConsumerOffsets(ctx context.Context, tg TopicAndGroup...
method roundTrip (line 102) | func (c *Client) roundTrip(ctx context.Context, addr net.Addr, msg pro...
method transport (line 118) | func (c *Client) transport() RoundTripper {
method timeout (line 125) | func (c *Client) timeout(ctx context.Context, defaultTimeout time.Dura...
method timeoutMs (line 144) | func (c *Client) timeoutMs(ctx context.Context, defaultTimeout time.Du...
type TopicAndGroup (line 55) | type TopicAndGroup struct
FILE: client_test.go
function newLocalClientAndTopic (line 17) | func newLocalClientAndTopic() (*Client, string, func()) {
function newLocalClientWithTopic (line 23) | func newLocalClientWithTopic(topic string, partitions int) (*Client, fun...
function clientCreateTopic (line 37) | func clientCreateTopic(client *Client, topic string, partitions int) err...
function clientEndTxn (line 71) | func clientEndTxn(client *Client, req *EndTxnRequest) error {
function newLocalClient (line 82) | func newLocalClient() (*Client, func()) {
function newClient (line 86) | func newClient(addr net.Addr) (*Client, func()) {
function TestClient (line 105) | func TestClient(t *testing.T) {
function testConsumerGroupFetchOffsets (line 130) | func testConsumerGroupFetchOffsets(t *testing.T, ctx context.Context, cl...
function TestClientProduceAndConsume (line 194) | func TestClientProduceAndConsume(t *testing.T) {
FILE: commit.go
type commit (line 5) | type commit struct
function makeCommit (line 13) | func makeCommit(msg Message) commit {
function makeCommits (line 24) | func makeCommits(msgs ...Message) []commit {
type commitRequest (line 36) | type commitRequest struct
FILE: commit_test.go
function TestMakeCommit (line 5) | func TestMakeCommit(t *testing.T) {
FILE: compress/compress.go
type Compression (line 17) | type Compression
method Codec (line 27) | func (c Compression) Codec() Codec {
method String (line 34) | func (c Compression) String() string {
method MarshalText (line 41) | func (c Compression) MarshalText() ([]byte, error) {
method UnmarshalText (line 45) | func (c *Compression) UnmarshalText(b []byte) error {
constant None (line 20) | None Compression = 0
constant Gzip (line 21) | Gzip Compression = 1
constant Snappy (line 22) | Snappy Compression = 2
constant Lz4 (line 23) | Lz4 Compression = 3
constant Zstd (line 24) | Zstd Compression = 4
type Codec (line 89) | type Codec interface
FILE: compress/compress_test.go
function init (line 28) | func init() {
function TestCodecs (line 34) | func TestCodecs(t *testing.T) {
function TestCompression (line 44) | func TestCompression(t *testing.T) {
function compress (line 57) | func compress(codec pkg.Codec, src []byte) ([]byte, error) {
function decompress (line 71) | func decompress(codec pkg.Codec, src []byte) ([]byte, error) {
function testEncodeDecode (line 84) | func testEncodeDecode(t *testing.T, m kafka.Message, codec pkg.Codec) {
function TestCompressedMessages (line 138) | func TestCompressedMessages(t *testing.T) {
function testCompressedMessages (line 148) | func testCompressedMessages(t *testing.T, codec pkg.Codec) {
function TestMixedCompressedMessages (line 218) | func TestMixedCompressedMessages(t *testing.T) {
type noopCodec (line 293) | type noopCodec struct
method Code (line 295) | func (noopCodec) Code() int8 {
method Name (line 299) | func (noopCodec) Name() string {
method NewReader (line 303) | func (noopCodec) NewReader(r io.Reader) io.ReadCloser {
method NewWriter (line 307) | func (noopCodec) NewWriter(w io.Writer) io.WriteCloser {
type nopWriteCloser (line 311) | type nopWriteCloser struct
method Close (line 313) | func (nopWriteCloser) Close() error { return nil }
function BenchmarkCompression (line 315) | func BenchmarkCompression(b *testing.B) {
function benchmarkCompression (line 381) | func benchmarkCompression(b *testing.B, codec pkg.Codec, buf *bytes.Buff...
function init (line 444) | func init() {
function makeTopic (line 448) | func makeTopic() string {
function newLocalClientAndTopic (line 452) | func newLocalClientAndTopic() (*kafka.Client, string, func()) {
function newLocalClient (line 492) | func newLocalClient() (*kafka.Client, func()) {
function newClient (line 496) | func newClient(addr net.Addr) (*kafka.Client, func()) {
FILE: compress/gzip/gzip.go
type Codec (line 16) | type Codec struct
method Code (line 27) | func (c *Codec) Code() int8 { return 1 }
method Name (line 30) | func (c *Codec) Name() string { return "gzip" }
method NewReader (line 33) | func (c *Codec) NewReader(r io.Reader) io.ReadCloser {
method NewWriter (line 51) | func (c *Codec) NewWriter(w io.Writer) io.WriteCloser {
method level (line 66) | func (c *Codec) level() int {
type reader (line 73) | type reader struct
method Close (line 75) | func (r *reader) Close() (err error) {
type writer (line 92) | type writer struct
method Close (line 97) | func (w *writer) Close() (err error) {
type emptyReader (line 107) | type emptyReader struct
method ReadByte (line 109) | func (emptyReader) ReadByte() (byte, error) { return 0, io.EOF }
method Read (line 111) | func (emptyReader) Read([]byte) (int, error) { return 0, io.EOF }
type errorReader (line 113) | type errorReader struct
method Close (line 115) | func (r *errorReader) Close() error { return r.err }
method Read (line 117) | func (r *errorReader) Read([]byte) (int, error) { return 0, r.err }
type errorWriter (line 119) | type errorWriter struct
method Close (line 121) | func (w *errorWriter) Close() error { return w.err }
method Write (line 123) | func (w *errorWriter) Write([]byte) (int, error) { return 0, w.err }
FILE: compress/lz4/lz4.go
type Codec (line 17) | type Codec struct
method Code (line 20) | func (c *Codec) Code() int8 { return 3 }
method Name (line 23) | func (c *Codec) Name() string { return "lz4" }
method NewReader (line 26) | func (c *Codec) NewReader(r io.Reader) io.ReadCloser {
method NewWriter (line 37) | func (c *Codec) NewWriter(w io.Writer) io.WriteCloser {
type reader (line 47) | type reader struct
method Close (line 49) | func (r *reader) Close() (err error) {
type writer (line 58) | type writer struct
method Close (line 60) | func (w *writer) Close() (err error) {
FILE: compress/snappy/go-xerial-snappy/fuzz.go
function Fuzz (line 5) | func Fuzz(data []byte) int {
FILE: compress/snappy/go-xerial-snappy/snappy.go
constant sizeOffset (line 12) | sizeOffset = 16
constant sizeBytes (line 13) | sizeBytes = 4
function min (line 27) | func min(x, y int) int {
function Encode (line 35) | func Encode(src []byte) []byte {
function EncodeStream (line 43) | func EncodeStream(dst, src []byte) []byte {
function Decode (line 76) | func Decode(src []byte) ([]byte, error) {
function DecodeInto (line 85) | func DecodeInto(dst, src []byte) ([]byte, error) {
FILE: compress/snappy/go-xerial-snappy/snappy_test.go
constant largeString (line 9) | largeString = `Sed ut perspiciatis unde omnis iste natus error sit volup...
function makeMassive (line 23) | func makeMassive(input string, numCopies int) string {
function TestSnappyEncode (line 33) | func TestSnappyEncode(t *testing.T) {
function TestSnappyEncodeStream (line 42) | func TestSnappyEncodeStream(t *testing.T) {
function TestSnappyLargeStringEncodeStream (line 57) | func TestSnappyLargeStringEncodeStream(t *testing.T) {
function TestSnappyDecode (line 69) | func TestSnappyDecode(t *testing.T) {
function TestSnappyDecodeStreams (line 80) | func TestSnappyDecodeStreams(t *testing.T) {
function TestSnappyDecodeMalformedTruncatedHeader (line 91) | func TestSnappyDecodeMalformedTruncatedHeader(t *testing.T) {
function TestSnappyDecodeMalformedTruncatedSize (line 102) | func TestSnappyDecodeMalformedTruncatedSize(t *testing.T) {
function TestSnappyDecodeMalformedBNoData (line 114) | func TestSnappyDecodeMalformedBNoData(t *testing.T) {
function TestSnappyMasterDecodeFailed (line 125) | func TestSnappyMasterDecodeFailed(t *testing.T) {
function BenchmarkSnappyDecode (line 137) | func BenchmarkSnappyDecode(b *testing.B) {
function BenchmarkSnappyDecodeInto (line 154) | func BenchmarkSnappyDecodeInto(b *testing.B) {
function BenchmarkSnappyStreamDecode (line 177) | func BenchmarkSnappyStreamDecode(b *testing.B) {
function BenchmarkSnappyStreamDecodeInto (line 194) | func BenchmarkSnappyStreamDecodeInto(b *testing.B) {
function BenchmarkSnappyStreamDecodeMassive (line 215) | func BenchmarkSnappyStreamDecodeMassive(b *testing.B) {
function BenchmarkSnappyStreamDecodeIntoMassive (line 231) | func BenchmarkSnappyStreamDecodeIntoMassive(b *testing.B) {
FILE: compress/snappy/snappy.go
type Framing (line 13) | type Framing
constant Framed (line 16) | Framed Framing = iota
constant Unframed (line 17) | Unframed
type Compression (line 21) | type Compression
constant DefaultCompression (line 24) | DefaultCompression Compression = iota
constant FasterCompression (line 25) | FasterCompression
constant BetterCompression (line 26) | BetterCompression
constant BestCompression (line 27) | BestCompression
type Codec (line 37) | type Codec struct
method Code (line 48) | func (c *Codec) Code() int8 { return 2 }
method Name (line 51) | func (c *Codec) Name() string { return "snappy" }
method NewReader (line 54) | func (c *Codec) NewReader(r io.Reader) io.ReadCloser {
method NewWriter (line 68) | func (c *Codec) NewWriter(w io.Writer) io.WriteCloser {
type reader (line 89) | type reader struct
method Close (line 91) | func (r *reader) Close() (err error) {
type writer (line 100) | type writer struct
method Close (line 102) | func (w *writer) Close() (err error) {
FILE: compress/snappy/xerial.go
constant defaultBufferSize (line 12) | defaultBufferSize = 32 * 1024
type xerialReader (line 17) | type xerialReader struct
method Reset (line 27) | func (x *xerialReader) Reset(r io.Reader) {
method Read (line 36) | func (x *xerialReader) Read(b []byte) (int, error) {
method WriteTo (line 54) | func (x *xerialReader) WriteTo(w io.Writer) (int64, error) {
method readChunk (line 76) | func (x *xerialReader) readChunk(dst []byte) (int, error) {
method read (line 162) | func (x *xerialReader) read(b []byte) (int, error) {
method readFull (line 168) | func (x *xerialReader) readFull(b []byte) (int, error) {
type xerialWriter (line 176) | type xerialWriter struct
method Reset (line 186) | func (x *xerialWriter) Reset(w io.Writer) {
method ReadFrom (line 193) | func (x *xerialWriter) ReadFrom(r io.Reader) (int64, error) {
method Write (line 224) | func (x *xerialWriter) Write(b []byte) (int, error) {
method Flush (line 251) | func (x *xerialWriter) Flush() error {
method write (line 287) | func (x *xerialWriter) write(b []byte) (int, error) {
method full (line 293) | func (x *xerialWriter) full() bool {
method fullEnough (line 297) | func (x *xerialWriter) fullEnough() bool {
method grow (line 301) | func (x *xerialWriter) grow() {
function align (line 307) | func align(n, a int) int {
function isXerialHeader (line 319) | func isXerialHeader(src []byte) bool {
function writeXerialHeader (line 323) | func writeXerialHeader(b []byte) {
function writeXerialFrame (line 328) | func writeXerialFrame(b []byte, n int) {
FILE: compress/snappy/xerial_test.go
type simpleReader (line 17) | type simpleReader struct
type simpleWriter (line 18) | type simpleWriter struct
function TestXerialReaderSnappy (line 20) | func TestXerialReaderSnappy(t *testing.T) {
function TestXerialReaderWriter (line 39) | func TestXerialReaderWriter(t *testing.T) {
function TestXerialFramedCompression (line 63) | func TestXerialFramedCompression(t *testing.T) {
function TestXerialFramedCompressionOptimized (line 88) | func TestXerialFramedCompressionOptimized(t *testing.T) {
function TestXerialReaderAgainstGoXerialSnappy (line 113) | func TestXerialReaderAgainstGoXerialSnappy(t *testing.T) {
function TestXerialWriterAgainstGoXerialSnappy (line 142) | func TestXerialWriterAgainstGoXerialSnappy(t *testing.T) {
FILE: compress/zstd/zstd.go
type Codec (line 13) | type Codec struct
method Code (line 23) | func (c *Codec) Code() int8 { return 4 }
method Name (line 26) | func (c *Codec) Name() string { return "zstd" }
method NewReader (line 29) | func (c *Codec) NewReader(r io.Reader) io.ReadCloser {
method level (line 46) | func (c *Codec) level() int {
method zstdLevel (line 53) | func (c *Codec) zstdLevel() zstd.EncoderLevel {
method NewWriter (line 98) | func (c *Codec) NewWriter(w io.Writer) io.WriteCloser {
type reader (line 59) | type reader struct
method Close (line 65) | func (r *reader) Close() error {
method Read (line 76) | func (r *reader) Read(p []byte) (int, error) {
method WriteTo (line 87) | func (r *reader) WriteTo(w io.Writer) (int64, error) {
type writer (line 119) | type writer struct
method Close (line 126) | func (w *writer) Close() error {
method Write (line 144) | func (w *writer) Write(p []byte) (int, error) {
method ReadFrom (line 155) | func (w *writer) ReadFrom(r io.Reader) (int64, error) {
type devNull (line 165) | type devNull struct
method Read (line 167) | func (devNull) Read([]byte) (int, error) { return 0, io.EOF }
method Write (line 168) | func (devNull) Write([]byte) (int, error) { return 0, nil }
FILE: compression.go
constant Gzip (line 12) | Gzip Compression = compress.Gzip
constant Snappy (line 13) | Snappy Compression = compress.Snappy
constant Lz4 (line 14) | Lz4 Compression = compress.Lz4
constant Zstd (line 15) | Zstd Compression = compress.Zstd
function resolveCodec (line 25) | func resolveCodec(code int8) (CompressionCodec, error) {
FILE: conn.go
type Conn (line 25) | type Conn struct
method negotiateVersion (line 205) | func (c *Conn) negotiateVersion(key apiKey, sortedSupportedVersions .....
method loadVersions (line 217) | func (c *Conn) loadVersions() (apiVersionMap, error) {
method Broker (line 240) | func (c *Conn) Broker() Broker {
method Controller (line 252) | func (c *Conn) Controller() (broker Broker, err error) {
method Brokers (line 279) | func (c *Conn) Brokers() ([]Broker, error) {
method DeleteTopics (line 308) | func (c *Conn) DeleteTopics(topics ...string) error {
method findCoordinator (line 318) | func (c *Conn) findCoordinator(request findCoordinatorRequestV0) (find...
method heartbeat (line 345) | func (c *Conn) heartbeat(request heartbeatRequestV0) (heartbeatRespons...
method joinGroup (line 371) | func (c *Conn) joinGroup(request joinGroupRequest) (joinGroupResponse,...
method leaveGroup (line 402) | func (c *Conn) leaveGroup(request leaveGroupRequestV0) (leaveGroupResp...
method listGroups (line 428) | func (c *Conn) listGroups(request listGroupsRequestV1) (listGroupsResp...
method offsetCommit (line 454) | func (c *Conn) offsetCommit(request offsetCommitRequestV2) (offsetComm...
method offsetFetch (line 485) | func (c *Conn) offsetFetch(request offsetFetchRequestV1) (offsetFetchR...
method syncGroup (line 515) | func (c *Conn) syncGroup(request syncGroupRequestV0) (syncGroupRespons...
method Close (line 539) | func (c *Conn) Close() error {
method LocalAddr (line 544) | func (c *Conn) LocalAddr() net.Addr {
method RemoteAddr (line 549) | func (c *Conn) RemoteAddr() net.Addr {
method SetDeadline (line 563) | func (c *Conn) SetDeadline(t time.Time) error {
method SetReadDeadline (line 572) | func (c *Conn) SetReadDeadline(t time.Time) error {
method SetWriteDeadline (line 582) | func (c *Conn) SetWriteDeadline(t time.Time) error {
method Offset (line 592) | func (c *Conn) Offset() (offset int64, whence int) {
method Seek (line 629) | func (c *Conn) Seek(offset int64, whence int) (int64, error) {
method Read (line 709) | func (c *Conn) Read(b []byte) (int, error) {
method ReadMessage (line 730) | func (c *Conn) ReadMessage(maxBytes int) (Message, error) {
method ReadBatch (line 749) | func (c *Conn) ReadBatch(minBytes, maxBytes int) *Batch {
method ReadBatchWith (line 758) | func (c *Conn) ReadBatchWith(cfg ReadBatchConfig) *Batch {
method ReadOffset (line 896) | func (c *Conn) ReadOffset(t time.Time) (int64, error) {
method ReadFirstOffset (line 901) | func (c *Conn) ReadFirstOffset() (int64, error) {
method ReadLastOffset (line 906) | func (c *Conn) ReadLastOffset() (int64, error) {
method ReadOffsets (line 912) | func (c *Conn) ReadOffsets() (first, last int64, err error) {
method readOffset (line 926) | func (c *Conn) readOffset(t int64) (offset int64, err error) {
method ReadPartitions (line 966) | func (c *Conn) ReadPartitions(topics ...string) (partitions []Partitio...
method readPartitionsResponse (line 1000) | func (c *Conn) readPartitionsResponse(metadataVersion apiVersion, size...
method readTopicMetadatav1 (line 1032) | func (c *Conn) readTopicMetadatav1(brokers map[int32]Broker, topicMeta...
method readTopicMetadatav6 (line 1054) | func (c *Conn) readTopicMetadatav6(brokers map[int32]Broker, topicMeta...
method Write (line 1100) | func (c *Conn) Write(b []byte) (int, error) {
method WriteMessages (line 1107) | func (c *Conn) WriteMessages(msgs ...Message) (int, error) {
method WriteCompressedMessages (line 1116) | func (c *Conn) WriteCompressedMessages(codec CompressionCodec, msgs .....
method WriteCompressedMessagesAt (line 1127) | func (c *Conn) WriteCompressedMessagesAt(codec CompressionCodec, msgs ...
method writeCompressedMessages (line 1131) | func (c *Conn) writeCompressedMessages(codec CompressionCodec, msgs .....
method SetRequiredAcks (line 1277) | func (c *Conn) SetRequiredAcks(n int) error {
method writeRequest (line 1287) | func (c *Conn) writeRequest(apiKey apiKey, apiVersion apiVersion, corr...
method readResponse (line 1295) | func (c *Conn) readResponse(size int, res interface{}) error {
method peekResponseSizeAndID (line 1306) | func (c *Conn) peekResponseSizeAndID() (int32, int32, error) {
method skipResponseSizeAndID (line 1315) | func (c *Conn) skipResponseSizeAndID() {
method readDeadline (line 1319) | func (c *Conn) readDeadline() time.Time {
method writeDeadline (line 1323) | func (c *Conn) writeDeadline() time.Time {
method readOperation (line 1327) | func (c *Conn) readOperation(write func(time.Time, int32) error, read ...
method writeOperation (line 1331) | func (c *Conn) writeOperation(write func(time.Time, int32) error, read...
method enter (line 1335) | func (c *Conn) enter() {
method leave (line 1339) | func (c *Conn) leave() {
method concurrency (line 1343) | func (c *Conn) concurrency() int {
method do (line 1347) | func (c *Conn) do(d *connDeadline, write func(time.Time, int32) error,...
method doRequest (line 1370) | func (c *Conn) doRequest(d *connDeadline, write func(time.Time, int32)...
method waitResponse (line 1390) | func (c *Conn) waitResponse(d *connDeadline, id int32) (deadline time....
method requestHeader (line 1432) | func (c *Conn) requestHeader(apiKey apiKey, apiVersion apiVersion, cor...
method ApiVersions (line 1441) | func (c *Conn) ApiVersions() ([]ApiVersion, error) {
method saslHandshake (line 1574) | func (c *Conn) saslHandshake(mechanism string) error {
method saslAuthenticate (line 1605) | func (c *Conn) saslAuthenticate(data []byte) ([]byte, error) {
type apiVersionMap (line 70) | type apiVersionMap
method negotiate (line 72) | func (v apiVersionMap) negotiate(key apiKey, sortedSupportedVersions ....
type ConnConfig (line 87) | type ConnConfig struct
type ReadBatchConfig (line 102) | type ReadBatchConfig struct
type IsolationLevel (line 128) | type IsolationLevel
constant ReadUncommitted (line 131) | ReadUncommitted IsolationLevel = 0
constant ReadCommitted (line 132) | ReadCommitted IsolationLevel = 1
function init (line 141) | func init() {
function NewConn (line 148) | func NewConn(conn net.Conn, topic string, partition int) *Conn {
function emptyToNullable (line 155) | func emptyToNullable(transactionalID string) (result *string) {
function NewConnWith (line 164) | func NewConnWith(conn net.Conn, config ConnConfig) *Conn {
constant SeekStart (line 611) | SeekStart = 0
constant SeekAbsolute (line 612) | SeekAbsolute = 1
constant SeekEnd (line 613) | SeekEnd = 2
constant SeekCurrent (line 614) | SeekCurrent = 3
constant SeekDontCheck (line 620) | SeekDontCheck = 1 << 30
function readBrokerMetadata (line 1019) | func readBrokerMetadata(brokerMetadata []brokerMetadataV1) map[int32]Bro...
function makeBrokers (line 1076) | func makeBrokers(brokers map[int32]Broker, ids ...int32) []Broker {
type connDeadline (line 1504) | type connDeadline struct
method deadline (line 1511) | func (d *connDeadline) deadline() time.Time {
method setDeadline (line 1518) | func (d *connDeadline) setDeadline(t time.Time) {
method setConnReadDeadline (line 1533) | func (d *connDeadline) setConnReadDeadline(conn net.Conn) time.Time {
method setConnWriteDeadline (line 1542) | func (d *connDeadline) setConnWriteDeadline(conn net.Conn) time.Time {
method unsetConnReadDeadline (line 1551) | func (d *connDeadline) unsetConnReadDeadline() {
method unsetConnWriteDeadline (line 1557) | func (d *connDeadline) unsetConnWriteDeadline() {
FILE: conn_test.go
type timeout (line 21) | type timeout struct
method Error (line 23) | func (*timeout) Error() string { return "timeout" }
method Temporary (line 24) | func (*timeout) Temporary() bool { return true }
method Timeout (line 25) | func (*timeout) Timeout() bool { return true }
type connPipe (line 29) | type connPipe struct
method Close (line 34) | func (c *connPipe) Close() error {
method Read (line 43) | func (c *connPipe) Read(b []byte) (int, error) {
method Write (line 57) | func (c *connPipe) Write(b []byte) (int, error) {
method LocalAddr (line 76) | func (c *connPipe) LocalAddr() net.Addr {
method RemoteAddr (line 80) | func (c *connPipe) RemoteAddr() net.Addr {
method SetDeadline (line 84) | func (c *connPipe) SetDeadline(t time.Time) error {
method SetReadDeadline (line 90) | func (c *connPipe) SetReadDeadline(t time.Time) error {
method SetWriteDeadline (line 94) | func (c *connPipe) SetWriteDeadline(t time.Time) error {
function init (line 98) | func init() {
function makeTopic (line 102) | func makeTopic() string {
function makeGroupID (line 106) | func makeGroupID() string {
function makeTransactionalID (line 110) | func makeTransactionalID() string {
function TestConn (line 114) | func TestConn(t *testing.T) {
function testConnClose (line 364) | func testConnClose(t *testing.T, conn *Conn) {
function testConnFirstOffset (line 370) | func testConnFirstOffset(t *testing.T, conn *Conn) {
function testConnWrite (line 378) | func testConnWrite(t *testing.T, conn *Conn) {
function testConnCloseAndWrite (line 390) | func testConnCloseAndWrite(t *testing.T, conn *Conn) {
function testConnSeekFirstOffset (line 402) | func testConnSeekFirstOffset(t *testing.T, conn *Conn) {
function testConnSeekLastOffset (line 419) | func testConnSeekLastOffset(t *testing.T, conn *Conn) {
function testConnSeekCurrentOffset (line 436) | func testConnSeekCurrentOffset(t *testing.T, conn *Conn) {
function testConnSeekRandomOffset (line 462) | func testConnSeekRandomOffset(t *testing.T, conn *Conn) {
function testConnSeekDontCheck (line 479) | func testConnSeekDontCheck(t *testing.T, conn *Conn) {
function testConnWriteReadSequentially (line 500) | func testConnWriteReadSequentially(t *testing.T, conn *Conn) {
function testConnWriteBatchReadSequentially (line 524) | func testConnWriteBatchReadSequentially(t *testing.T, conn *Conn) {
function testConnReadWatermarkFromBatch (line 549) | func testConnReadWatermarkFromBatch(t *testing.T, conn *Conn) {
function testConnReadBatchWithNoMinMaxBytes (line 577) | func testConnReadBatchWithNoMinMaxBytes(t *testing.T, conn *Conn) {
function testConnReadBatchWithMaxWait (line 608) | func testConnReadBatchWithMaxWait(t *testing.T, conn *Conn) {
function waitForCoordinator (line 657) | func waitForCoordinator(t *testing.T, conn *Conn, groupID string) {
function createGroup (line 680) | func createGroup(t *testing.T, conn *Conn, groupID string) (generationID...
function testConnFindCoordinator (line 744) | func testConnFindCoordinator(t *testing.T, conn *Conn) {
function testConnJoinGroupInvalidGroupID (line 773) | func testConnJoinGroupInvalidGroupID(t *testing.T, conn *Conn) {
function testConnJoinGroupInvalidSessionTimeout (line 780) | func testConnJoinGroupInvalidSessionTimeout(t *testing.T, conn *Conn) {
function testConnJoinGroupInvalidRefreshTimeout (line 792) | func testConnJoinGroupInvalidRefreshTimeout(t *testing.T, conn *Conn) {
function testConnHeartbeatErr (line 805) | func testConnHeartbeatErr(t *testing.T, conn *Conn) {
function testConnLeaveGroupErr (line 817) | func testConnLeaveGroupErr(t *testing.T, conn *Conn) {
function testConnSyncGroupErr (line 829) | func testConnSyncGroupErr(t *testing.T, conn *Conn) {
function testConnListGroupsReturnsGroups (line 841) | func testConnListGroupsReturnsGroups(t *testing.T, conn *Conn) {
function testConnFetchAndCommitOffsets (line 873) | func testConnFetchAndCommitOffsets(t *testing.T, conn *Conn) {
function testConnWriteReadConcurrently (line 942) | func testConnWriteReadConcurrently(t *testing.T, conn *Conn) {
function testConnReadShortBuffer (line 985) | func testConnReadShortBuffer(t *testing.T, conn *Conn) {
function testConnReadEmptyWithDeadline (line 1011) | func testConnReadEmptyWithDeadline(t *testing.T, conn *Conn) {
function testDeleteTopics (line 1029) | func testDeleteTopics(t *testing.T, conn *Conn) {
function testDeleteTopicsInvalidTopic (line 1061) | func testDeleteTopicsInvalidTopic(t *testing.T, conn *Conn) {
function testController (line 1087) | func testController(t *testing.T, conn *Conn) {
function testBrokers (line 1107) | func testBrokers(t *testing.T, conn *Conn) {
function testConnBroker (line 1122) | func testConnBroker(t *testing.T, conn *Conn) {
function TestReadPartitionsNoTopic (line 1139) | func TestReadPartitionsNoTopic(t *testing.T) {
function TestUnsupportedSASLMechanism (line 1156) | func TestUnsupportedSASLMechanism(t *testing.T) {
constant benchmarkMessageCount (line 1173) | benchmarkMessageCount = 100
function BenchmarkConn (line 1175) | func BenchmarkConn(b *testing.B) {
function benchmarkConnSeek (line 1232) | func benchmarkConnSeek(b *testing.B, conn *Conn, _ []byte) {
function benchmarkConnRead (line 1241) | func benchmarkConnRead(b *testing.B, conn *Conn, a []byte) {
function benchmarkConnReadBatch (line 1266) | func benchmarkConnReadBatch(b *testing.B, conn *Conn, a []byte) {
function benchmarkConnReadOffsets (line 1295) | func benchmarkConnReadOffsets(b *testing.B, conn *Conn, _ []byte) {
function benchmarkConnWrite (line 1305) | func benchmarkConnWrite(b *testing.B, conn *Conn, _ []byte) {
function TestEmptyToNullableReturnsNil (line 1323) | func TestEmptyToNullableReturnsNil(t *testing.T) {
function TestEmptyToNullableLeavesStringsIntact (line 1329) | func TestEmptyToNullableLeavesStringsIntact(t *testing.T) {
function TestMakeBrokersAllPresent (line 1337) | func TestMakeBrokersAllPresent(t *testing.T) {
function TestMakeBrokersOneMissing (line 1354) | func TestMakeBrokersOneMissing(t *testing.T) {
FILE: consumergroup.go
constant defaultProtocolType (line 31) | defaultProtocolType = "consumer"
constant defaultHeartbeatInterval (line 39) | defaultHeartbeatInterval = 3 * time.Second
constant defaultSessionTimeout (line 43) | defaultSessionTimeout = 30 * time.Second
constant defaultRebalanceTimeout (line 47) | defaultRebalanceTimeout = 30 * time.Second
constant defaultJoinGroupBackoff (line 51) | defaultJoinGroupBackoff = 5 * time.Second
constant defaultRetentionTime (line 55) | defaultRetentionTime = -1 * time.Millisecond
constant defaultPartitionWatchTime (line 59) | defaultPartitionWatchTime = 5 * time.Second
constant defaultTimeout (line 63) | defaultTimeout = 5 * time.Second
type ConsumerGroupConfig (line 68) | type ConsumerGroupConfig struct
method Validate (line 170) | func (config *ConsumerGroupConfig) Validate() error {
type PartitionAssignment (line 264) | type PartitionAssignment struct
type genCtx (line 279) | type genCtx struct
method Done (line 283) | func (c genCtx) Done() <-chan struct{} {
method Err (line 287) | func (c genCtx) Err() error {
method Deadline (line 296) | func (c genCtx) Deadline() (time.Time, bool) {
method Value (line 300) | func (c genCtx) Value(interface{}) interface{} {
type Generation (line 308) | type Generation struct
method close (line 344) | func (g *Generation) close() {
method Start (line 375) | func (g *Generation) Start(fn func(ctx context.Context)) {
method CommitOffsets (line 418) | func (g *Generation) CommitOffsets(offsets map[string]map[int]int64) e...
method heartbeatLoop (line 464) | func (g *Generation) heartbeatLoop(interval time.Duration) {
method partitionWatcher (line 500) | func (g *Generation) partitionWatcher(interval time.Duration, topic st...
type coordinator (line 555) | type coordinator interface
type timeoutCoordinator (line 573) | type timeoutCoordinator struct
method Close (line 580) | func (t *timeoutCoordinator) Close() error {
method findCoordinator (line 584) | func (t *timeoutCoordinator) findCoordinator(req findCoordinatorReques...
method joinGroup (line 591) | func (t *timeoutCoordinator) joinGroup(req joinGroupRequest) (joinGrou...
method syncGroup (line 600) | func (t *timeoutCoordinator) syncGroup(req syncGroupRequestV0) (syncGr...
method leaveGroup (line 609) | func (t *timeoutCoordinator) leaveGroup(req leaveGroupRequestV0) (leav...
method heartbeat (line 616) | func (t *timeoutCoordinator) heartbeat(req heartbeatRequestV0) (heartb...
method offsetFetch (line 623) | func (t *timeoutCoordinator) offsetFetch(req offsetFetchRequestV1) (of...
method offsetCommit (line 630) | func (t *timeoutCoordinator) offsetCommit(req offsetCommitRequestV2) (...
method readPartitions (line 637) | func (t *timeoutCoordinator) readPartitions(topics ...string) ([]Parti...
function NewConsumerGroup (line 648) | func NewConsumerGroup(config ConsumerGroupConfig) (*ConsumerGroup, error) {
type ConsumerGroup (line 672) | type ConsumerGroup struct
method Close (line 685) | func (cg *ConsumerGroup) Close() error {
method Next (line 701) | func (cg *ConsumerGroup) Next(ctx context.Context) (*Generation, error) {
method run (line 714) | func (cg *ConsumerGroup) run() {
method nextGeneration (line 776) | func (cg *ConsumerGroup) nextGeneration(memberID string) (string, erro...
method coordinator (line 898) | func (cg *ConsumerGroup) coordinator() (coordinator, error) {
method joinGroup (line 934) | func (cg *ConsumerGroup) joinGroup(conn coordinator, memberID string) ...
method makeJoinGroupRequest (line 981) | func (cg *ConsumerGroup) makeJoinGroupRequest(memberID string) (joinGr...
method assignTopicPartitions (line 1010) | func (cg *ConsumerGroup) assignTopicPartitions(conn coordinator, group...
method makeMemberProtocolMetadata (line 1053) | func (cg *ConsumerGroup) makeMemberProtocolMetadata(in []joinGroupResp...
method syncGroup (line 1081) | func (cg *ConsumerGroup) syncGroup(conn coordinator, memberID string, ...
method makeSyncGroupRequestV0 (line 1110) | func (cg *ConsumerGroup) makeSyncGroupRequestV0(memberID string, gener...
method fetchOffsets (line 1146) | func (cg *ConsumerGroup) fetchOffsets(conn coordinator, subs map[strin...
method makeAssignments (line 1182) | func (cg *ConsumerGroup) makeAssignments(assignments map[string][]int3...
method leaveGroup (line 1205) | func (cg *ConsumerGroup) leaveGroup(memberID string) error {
method withLogger (line 1240) | func (cg *ConsumerGroup) withLogger(do func(Logger)) {
method withErrorLogger (line 1246) | func (cg *ConsumerGroup) withErrorLogger(do func(Logger)) {
function makeConnect (line 878) | func makeConnect(config ConsumerGroupConfig) func(dialer *Dialer, broker...
FILE: consumergroup_test.go
type mockCoordinator (line 16) | type mockCoordinator struct
method Close (line 28) | func (c mockCoordinator) Close() error {
method findCoordinator (line 35) | func (c mockCoordinator) findCoordinator(req findCoordinatorRequestV0)...
method joinGroup (line 42) | func (c mockCoordinator) joinGroup(req joinGroupRequest) (joinGroupRes...
method syncGroup (line 49) | func (c mockCoordinator) syncGroup(req syncGroupRequestV0) (syncGroupR...
method leaveGroup (line 56) | func (c mockCoordinator) leaveGroup(req leaveGroupRequestV0) (leaveGro...
method heartbeat (line 63) | func (c mockCoordinator) heartbeat(req heartbeatRequestV0) (heartbeatR...
method offsetFetch (line 70) | func (c mockCoordinator) offsetFetch(req offsetFetchRequestV1) (offset...
method offsetCommit (line 77) | func (c mockCoordinator) offsetCommit(req offsetCommitRequestV2) (offs...
method readPartitions (line 84) | func (c mockCoordinator) readPartitions(topics ...string) ([]Partition...
function TestValidateConsumerGroupConfig (line 91) | func TestValidateConsumerGroupConfig(t *testing.T) {
function TestReaderAssignTopicPartitions (line 120) | func TestReaderAssignTopicPartitions(t *testing.T) {
function TestConsumerGroup (line 244) | func TestConsumerGroup(t *testing.T) {
function TestConsumerGroupErrors (line 346) | func TestConsumerGroupErrors(t *testing.T) {
function TestGenerationExitsOnPartitionChange (line 599) | func TestGenerationExitsOnPartitionChange(t *testing.T) {
function TestGenerationStartsFunctionAfterClosed (line 661) | func TestGenerationStartsFunctionAfterClosed(t *testing.T) {
FILE: crc32.go
type crc32Writer (line 8) | type crc32Writer struct
method update (line 14) | func (w *crc32Writer) update(b []byte) {
method writeInt8 (line 18) | func (w *crc32Writer) writeInt8(i int8) {
method writeInt16 (line 23) | func (w *crc32Writer) writeInt16(i int16) {
method writeInt32 (line 28) | func (w *crc32Writer) writeInt32(i int32) {
method writeInt64 (line 33) | func (w *crc32Writer) writeInt64(i int64) {
method writeBytes (line 38) | func (w *crc32Writer) writeBytes(b []byte) {
method Write (line 47) | func (w *crc32Writer) Write(b []byte) (int, error) {
method WriteString (line 52) | func (w *crc32Writer) WriteString(s string) (int, error) {
FILE: crc32_test.go
function TestMessageCRC32 (line 9) | func TestMessageCRC32(t *testing.T) {
FILE: createacls.go
type CreateACLsRequest (line 15) | type CreateACLsRequest struct
type CreateACLsResponse (line 25) | type CreateACLsResponse struct
type ACLPermissionType (line 37) | type ACLPermissionType
method String (line 46) | func (apt ACLPermissionType) String() string {
method MarshalText (line 61) | func (apt ACLPermissionType) MarshalText() ([]byte, error) {
method UnmarshalText (line 66) | func (apt *ACLPermissionType) UnmarshalText(text []byte) error {
constant ACLPermissionTypeUnknown (line 40) | ACLPermissionTypeUnknown ACLPermissionType = 0
constant ACLPermissionTypeAny (line 41) | ACLPermissionTypeAny ACLPermissionType = 1
constant ACLPermissionTypeDeny (line 42) | ACLPermissionTypeDeny ACLPermissionType = 2
constant ACLPermissionTypeAllow (line 43) | ACLPermissionTypeAllow ACLPermissionType = 3
type ACLOperationType (line 83) | type ACLOperationType
method String (line 101) | func (aot ACLOperationType) String() string {
method MarshalText (line 125) | func (aot ACLOperationType) MarshalText() ([]byte, error) {
method UnmarshalText (line 130) | func (aot *ACLOperationType) UnmarshalText(text []byte) error {
constant ACLOperationTypeUnknown (line 86) | ACLOperationTypeUnknown ACLOperationType = 0
constant ACLOperationTypeAny (line 87) | ACLOperationTypeAny ACLOperationType = 1
constant ACLOperationTypeAll (line 88) | ACLOperationTypeAll ACLOperationType = 2
constant ACLOperationTypeRead (line 89) | ACLOperationTypeRead ACLOperationType = 3
constant ACLOperationTypeWrite (line 90) | ACLOperationTypeWrite ACLOperationType = 4
constant ACLOperationTypeCreate (line 91) | ACLOperationTypeCreate ACLOperationType = 5
constant ACLOperationTypeDelete (line 92) | ACLOperationTypeDelete ACLOperationType = 6
constant ACLOperationTypeAlter (line 93) | ACLOperationTypeAlter ACLOperationType = 7
constant ACLOperationTypeDescribe (line 94) | ACLOperationTypeDescribe ACLOperationType = 8
constant ACLOperationTypeClusterAction (line 95) | ACLOperationTypeClusterAction ACLOperationType = 9
constant ACLOperationTypeDescribeConfigs (line 96) | ACLOperationTypeDescribeConfigs ACLOperationType = 10
constant ACLOperationTypeAlterConfigs (line 97) | ACLOperationTypeAlterConfigs ACLOperationType = 11
constant ACLOperationTypeIdempotentWrite (line 98) | ACLOperationTypeIdempotentWrite ACLOperationType = 12
type ACLEntry (line 157) | type ACLEntry struct
method CreateACLs (line 169) | func (c *Client) CreateACLs(ctx context.Context, req *CreateACLsRequest)...
FILE: createacls_test.go
function TestClientCreateACLs (line 10) | func TestClientCreateACLs(t *testing.T) {
function TestACLPermissionTypeMarshal (line 54) | func TestACLPermissionTypeMarshal(t *testing.T) {
function TestACLOperationTypeMarshal (line 71) | func TestACLOperationTypeMarshal(t *testing.T) {
FILE: createpartitions.go
type CreatePartitionsRequest (line 14) | type CreatePartitionsRequest struct
type CreatePartitionsResponse (line 28) | type CreatePartitionsResponse struct
method CreatePartitions (line 42) | func (c *Client) CreatePartitions(ctx context.Context, req *CreatePartit...
type TopicPartitionsConfig (line 76) | type TopicPartitionsConfig struct
method assignments (line 87) | func (t *TopicPartitionsConfig) assignments() []createpartitions.Reque...
type TopicPartitionAssignment (line 100) | type TopicPartitionAssignment struct
FILE: createpartitions_test.go
function TestClientCreatePartitions (line 10) | func TestClientCreatePartitions(t *testing.T) {
function TestClientCreatePartitionsNoAssignments (line 45) | func TestClientCreatePartitionsNoAssignments(t *testing.T) {
FILE: createtopics.go
type CreateTopicsRequest (line 15) | type CreateTopicsRequest struct
type CreateTopicsResponse (line 32) | type CreateTopicsResponse struct
method CreateTopics (line 49) | func (c *Client) CreateTopics(ctx context.Context, req *CreateTopicsRequ...
type ConfigEntry (line 84) | type ConfigEntry struct
method toCreateTopicsRequestV0ConfigEntry (line 89) | func (c ConfigEntry) toCreateTopicsRequestV0ConfigEntry() createTopics...
type createTopicsRequestV0ConfigEntry (line 93) | type createTopicsRequestV0ConfigEntry struct
method size (line 98) | func (t createTopicsRequestV0ConfigEntry) size() int32 {
method writeTo (line 103) | func (t createTopicsRequestV0ConfigEntry) writeTo(wb *writeBuffer) {
type ReplicaAssignment (line 108) | type ReplicaAssignment struct
method partitionIndex (line 121) | func (a *ReplicaAssignment) partitionIndex() int32 {
method brokerIDs (line 125) | func (a *ReplicaAssignment) brokerIDs() []int32 {
method toCreateTopicsRequestV0ReplicaAssignment (line 136) | func (a ReplicaAssignment) toCreateTopicsRequestV0ReplicaAssignment() ...
type createTopicsRequestV0ReplicaAssignment (line 143) | type createTopicsRequestV0ReplicaAssignment struct
method size (line 148) | func (t createTopicsRequestV0ReplicaAssignment) size() int32 {
method writeTo (line 153) | func (t createTopicsRequestV0ReplicaAssignment) writeTo(wb *writeBuffe...
type TopicConfig (line 161) | type TopicConfig struct
method assignments (line 179) | func (t *TopicConfig) assignments() []createtopics.RequestAssignment {
method configs (line 193) | func (t *TopicConfig) configs() []createtopics.RequestConfig {
method toCreateTopicsRequestV0Topic (line 207) | func (t TopicConfig) toCreateTopicsRequestV0Topic() createTopicsReques...
type createTopicsRequestV0Topic (line 230) | type createTopicsRequestV0Topic struct
method size (line 248) | func (t createTopicsRequestV0Topic) size() int32 {
method writeTo (line 256) | func (t createTopicsRequestV0Topic) writeTo(wb *writeBuffer) {
type createTopicsRequest (line 265) | type createTopicsRequest struct
method size (line 281) | func (t createTopicsRequest) size() int32 {
method writeTo (line 290) | func (t createTopicsRequest) writeTo(wb *writeBuffer) {
type createTopicsResponseTopicError (line 298) | type createTopicsResponseTopicError struct
method size (line 311) | func (t createTopicsResponseTopicError) size() int32 {
method writeTo (line 320) | func (t createTopicsResponseTopicError) writeTo(wb *writeBuffer) {
method readFrom (line 328) | func (t *createTopicsResponseTopicError) readFrom(r *bufio.Reader, siz...
type createTopicsResponse (line 344) | type createTopicsResponse struct
method size (line 351) | func (t createTopicsResponse) size() int32 {
method writeTo (line 359) | func (t createTopicsResponse) writeTo(wb *writeBuffer) {
method readFrom (line 366) | func (t *createTopicsResponse) readFrom(r *bufio.Reader, size int) (re...
method createTopics (line 388) | func (c *Conn) createTopics(request createTopicsRequest) (createTopicsRe...
method CreateTopics (line 430) | func (c *Conn) CreateTopics(topics ...TopicConfig) error {
FILE: createtopics_test.go
function TestConnCreateTopics (line 14) | func TestConnCreateTopics(t *testing.T) {
function TestClientCreateTopics (line 87) | func TestClientCreateTopics(t *testing.T) {
function TestCreateTopicsResponse (line 163) | func TestCreateTopicsResponse(t *testing.T) {
FILE: deleteacls.go
type DeleteACLsRequest (line 14) | type DeleteACLsRequest struct
type DeleteACLsFilter (line 22) | type DeleteACLsFilter struct
type DeleteACLsResponse (line 34) | type DeleteACLsResponse struct
type DeleteACLsResult (line 42) | type DeleteACLsResult struct
type DeleteACLsMatchingACLs (line 47) | type DeleteACLsMatchingACLs struct
method DeleteACLs (line 60) | func (c *Client) DeleteACLs(ctx context.Context, req *DeleteACLsRequest)...
FILE: deleteacls_test.go
function TestClientDeleteACLs (line 11) | func TestClientDeleteACLs(t *testing.T) {
FILE: deletegroups.go
type DeleteGroupsRequest (line 14) | type DeleteGroupsRequest struct
type DeleteGroupsResponse (line 24) | type DeleteGroupsResponse struct
method DeleteGroups (line 37) | func (c *Client) DeleteGroups(
FILE: deletegroups_test.go
function TestClientDeleteGroups (line 12) | func TestClientDeleteGroups(t *testing.T) {
FILE: deletetopics.go
type DeleteTopicsRequest (line 15) | type DeleteTopicsRequest struct
type DeleteTopicsResponse (line 25) | type DeleteTopicsResponse struct
method DeleteTopics (line 42) | func (c *Client) DeleteTopics(ctx context.Context, req *DeleteTopicsRequ...
type deleteTopicsRequest (line 70) | type deleteTopicsRequest struct
method size (line 80) | func (t deleteTopicsRequest) size() int32 {
method writeTo (line 85) | func (t deleteTopicsRequest) writeTo(wb *writeBuffer) {
type deleteTopicsResponse (line 90) | type deleteTopicsResponse struct
method size (line 98) | func (t deleteTopicsResponse) size() int32 {
method readFrom (line 106) | func (t *deleteTopicsResponse) readFrom(r *bufio.Reader, size int) (re...
method writeTo (line 127) | func (t deleteTopicsResponse) writeTo(wb *writeBuffer) {
type deleteTopicsResponseV0TopicErrorCode (line 134) | type deleteTopicsResponseV0TopicErrorCode struct
method size (line 142) | func (t deleteTopicsResponseV0TopicErrorCode) size() int32 {
method readFrom (line 147) | func (t *deleteTopicsResponseV0TopicErrorCode) readFrom(r *bufio.Reade...
method writeTo (line 157) | func (t deleteTopicsResponseV0TopicErrorCode) writeTo(wb *writeBuffer) {
method deleteTopics (line 165) | func (c *Conn) deleteTopics(request deleteTopicsRequest) (deleteTopicsRe...
FILE: deletetopics_test.go
function TestClientDeleteTopics (line 11) | func TestClientDeleteTopics(t *testing.T) {
function TestDeleteTopicsResponseV1 (line 31) | func TestDeleteTopicsResponseV1(t *testing.T) {
FILE: describeacls.go
type DescribeACLsRequest (line 14) | type DescribeACLsRequest struct
type ACLFilter (line 22) | type ACLFilter struct
type DescribeACLsResponse (line 35) | type DescribeACLsResponse struct
type ACLResource (line 47) | type ACLResource struct
type ACLDescription (line 54) | type ACLDescription struct
method DescribeACLs (line 61) | func (c *Client) DescribeACLs(ctx context.Context, req *DescribeACLsRequ...
FILE: describeacls_test.go
function TestClientDescribeACLs (line 11) | func TestClientDescribeACLs(t *testing.T) {
FILE: describeclientquotas.go
type DescribeClientQuotasRequest (line 14) | type DescribeClientQuotasRequest struct
type DescribeClientQuotasRequestComponent (line 26) | type DescribeClientQuotasRequestComponent struct
type DescribeClientQuotasResponse (line 39) | type DescribeClientQuotasResponse struct
type DescribeClientQuotasEntity (line 51) | type DescribeClientQuotasEntity struct
type DescribeClientQuotasValue (line 59) | type DescribeClientQuotasValue struct
type DescribeClientQuotasResponseQuotas (line 67) | type DescribeClientQuotasResponseQuotas struct
method DescribeClientQuotas (line 77) | func (c *Client) DescribeClientQuotas(ctx context.Context, req *Describe...
FILE: describeconfigs.go
type DescribeConfigsRequest (line 13) | type DescribeConfigsRequest struct
type DescribeConfigRequestResource (line 27) | type DescribeConfigRequestResource struct
type DescribeConfigsResponse (line 39) | type DescribeConfigsResponse struct
type DescribeConfigResponseResource (line 48) | type DescribeConfigResponseResource struct
type DescribeConfigResponseConfigEntry (line 63) | type DescribeConfigResponseConfigEntry struct
type DescribeConfigResponseConfigSynonym (line 87) | type DescribeConfigResponseConfigSynonym struct
method DescribeConfigs (line 100) | func (c *Client) DescribeConfigs(ctx context.Context, req *DescribeConfi...
FILE: describeconfigs_test.go
function TestClientDescribeConfigs (line 11) | func TestClientDescribeConfigs(t *testing.T) {
FILE: describegroups.go
type DescribeGroupsRequest (line 14) | type DescribeGroupsRequest struct
type DescribeGroupsResponse (line 23) | type DescribeGroupsResponse struct
type DescribeGroupsResponseGroup (line 29) | type DescribeGroupsResponseGroup struct
type DescribeGroupsResponseMember (line 45) | type DescribeGroupsResponseMember struct
type DescribeGroupsResponseMemberMetadata (line 63) | type DescribeGroupsResponseMemberMetadata struct
type DescribeGroupsResponseMemberMetadataOwnedPartition (line 78) | type DescribeGroupsResponseMemberMetadataOwnedPartition struct
type DescribeGroupsResponseAssignments (line 87) | type DescribeGroupsResponseAssignments struct
type GroupMemberTopic (line 100) | type GroupMemberTopic struct
method DescribeGroups (line 111) | func (c *Client) DescribeGroups(
function decodeMemberMetadata (line 164) | func decodeMemberMetadata(rawMetadata []byte) (DescribeGroupsResponseMem...
function decodeMemberAssignments (line 227) | func decodeMemberAssignments(rawAssignments []byte) (DescribeGroupsRespo...
function readInt32Array (line 282) | func readInt32Array(r *bufio.Reader, sz int, v *[]int32) (remain int, er...
FILE: describegroups_test.go
function TestClientDescribeGroups (line 13) | func TestClientDescribeGroups(t *testing.T) {
FILE: describeuserscramcredentials.go
type DescribeUserScramCredentialsRequest (line 14) | type DescribeUserScramCredentialsRequest struct
type UserScramCredentialsUser (line 22) | type UserScramCredentialsUser struct
type DescribeUserScramCredentialsResponse (line 28) | type DescribeUserScramCredentialsResponse struct
type DescribeUserScramCredentialsResponseResult (line 43) | type DescribeUserScramCredentialsResponseResult struct
type DescribeUserScramCredentialsCredentialInfo (line 49) | type DescribeUserScramCredentialsCredentialInfo struct
method DescribeUserScramCredentials (line 56) | func (c *Client) DescribeUserScramCredentials(ctx context.Context, req *...
FILE: describeuserscramcredentials_test.go
function TestDescribeUserScramCredentials (line 12) | func TestDescribeUserScramCredentials(t *testing.T) {
FILE: dialer.go
type Dialer (line 19) | type Dialer struct
method Dial (line 95) | func (d *Dialer) Dial(network string, address string) (*Conn, error) {
method DialContext (line 112) | func (d *Dialer) DialContext(ctx context.Context, network string, addr...
method DialLeader (line 132) | func (d *Dialer) DialLeader(ctx context.Context, network string, addre...
method DialPartition (line 143) | func (d *Dialer) DialPartition(ctx context.Context, network string, ad...
method LookupLeader (line 156) | func (d *Dialer) LookupLeader(ctx context.Context, network string, add...
method LookupPartition (line 162) | func (d *Dialer) LookupPartition(ctx context.Context, network string, ...
method LookupPartitions (line 212) | func (d *Dialer) LookupPartitions(ctx context.Context, network string,...
method connectTLS (line 241) | func (d *Dialer) connectTLS(ctx context.Context, conn net.Conn, config...
method connect (line 265) | func (d *Dialer) connect(ctx context.Context, network, address string,...
method authenticateSASL (line 309) | func (d *Dialer) authenticateSASL(ctx context.Context, conn *Conn) err...
method dialContext (line 341) | func (d *Dialer) dialContext(ctx context.Context, network string, addr...
function Dial (line 389) | func Dial(network string, address string) (*Conn, error) {
function DialContext (line 394) | func DialContext(ctx context.Context, network string, address string) (*...
function DialLeader (line 399) | func DialLeader(ctx context.Context, network string, address string, top...
function DialPartition (line 404) | func DialPartition(ctx context.Context, network string, address string, ...
function LookupPartition (line 409) | func LookupPartition(ctx context.Context, network string, address string...
function LookupPartitions (line 414) | func LookupPartitions(ctx context.Context, network string, address strin...
function sleep (line 418) | func sleep(ctx context.Context, duration time.Duration) bool {
function backoff (line 437) | func backoff(attempt int, min time.Duration, max time.Duration) time.Dur...
function canonicalAddress (line 445) | func canonicalAddress(s string) string {
function splitHostPort (line 449) | func splitHostPort(s string) (host string, port string) {
function splitHostPortNumber (line 458) | func splitHostPortNumber(s string) (host string, portNumber int, err err...
function lookupHost (line 467) | func lookupHost(ctx context.Context, address string, resolver Resolver) ...
FILE: dialer_test.go
function TestDialer (line 17) | func TestDialer(t *testing.T) {
function testDialerLookupPartitions (line 39) | func testDialerLookupPartitions(t *testing.T, ctx context.Context, d *Di...
function tlsConfig (line 77) | func tlsConfig(t *testing.T) *tls.Config {
function TestDialerTLS (line 173) | func TestDialerTLS(t *testing.T) {
type MockConn (line 247) | type MockConn struct
method Read (line 253) | func (m *MockConn) Read(b []byte) (n int, err error) {
method Write (line 263) | func (m *MockConn) Write(b []byte) (n int, err error) {
method Close (line 273) | func (m *MockConn) Close() error {
method ReadPartitions (line 282) | func (m *MockConn) ReadPartitions(topics ...string) (partitions []Part...
function TestDialerConnectTLSHonorsContext (line 286) | func TestDialerConnectTLSHonorsContext(t *testing.T) {
function TestDialerResolver (line 306) | func TestDialerResolver(t *testing.T) {
type mockResolver (line 397) | type mockResolver struct
method LookupHost (line 401) | func (mr *mockResolver) LookupHost(ctx context.Context, host string) (...
FILE: discard.go
function discardN (line 5) | func discardN(r *bufio.Reader, sz int, n int) (int, error) {
function discardInt32 (line 18) | func discardInt32(r *bufio.Reader, sz int) (int, error) {
function discardString (line 22) | func discardString(r *bufio.Reader, sz int) (int, error) {
function discardBytes (line 31) | func discardBytes(r *bufio.Reader, sz int) (int, error) {
FILE: discard_test.go
function TestDiscardN (line 11) | func TestDiscardN(t *testing.T) {
FILE: electleaders.go
type ElectLeadersRequest (line 12) | type ElectLeadersRequest struct
type ElectLeadersResponse (line 27) | type ElectLeadersResponse struct
type ElectLeadersResponsePartitionResult (line 36) | type ElectLeadersResponsePartitionResult struct
method ElectLeaders (line 45) | func (c *Client) ElectLeaders(
FILE: electleaders_test.go
function TestClientElectLeaders (line 10) | func TestClientElectLeaders(t *testing.T) {
FILE: endtxn.go
type EndTxnRequest (line 13) | type EndTxnRequest struct
type EndTxnResponse (line 31) | type EndTxnResponse struct
method EndTxn (line 42) | func (c *Client) EndTxn(ctx context.Context, req *EndTxnRequest) (*EndTx...
FILE: error.go
type Error (line 12) | type Error
method Error (line 125) | func (e Error) Error() string {
method Timeout (line 130) | func (e Error) Timeout() bool {
method Temporary (line 138) | func (e Error) Temporary() bool {
method Title (line 176) | func (e Error) Title() string {
method Description (line 387) | func (e Error) Description() string {
constant Unknown (line 15) | Unknown Error = -1
constant OffsetOutOfRange (line 16) | OffsetOutOfRange Error = 1
constant InvalidMessage (line 17) | InvalidMessage Error = 2
constant UnknownTopicOrPartition (line 18) | UnknownTopicOrPartition Error = 3
constant InvalidMessageSize (line 19) | InvalidMessageSize Error = 4
constant LeaderNotAvailable (line 20) | LeaderNotAvailable Error = 5
constant NotLeaderForPartition (line 21) | NotLeaderForPartition Error = 6
constant RequestTimedOut (line 22) | RequestTimedOut Error = 7
constant BrokerNotAvailable (line 23) | BrokerNotAvailable Error = 8
constant ReplicaNotAvailable (line 24) | ReplicaNotAvailable Error = 9
constant MessageSizeTooLarge (line 25) | MessageSizeTooLarge Error = 10
constant StaleControllerEpoch (line 26) | StaleControllerEpoch Error = 11
constant OffsetMetadataTooLarge (line 27) | OffsetMetadataTooLarge Error = 12
constant NetworkException (line 28) | NetworkException Error = 13
constant GroupLoadInProgress (line 29) | GroupLoadInProgress Error = 14
constant GroupCoordinatorNotAvailable (line 30) | GroupCoordinatorNotAvailable Error = 15
constant NotCoordinatorForGroup (line 31) | NotCoordinatorForGroup Error = 16
constant InvalidTopic (line 32) | InvalidTopic Error = 17
constant RecordListTooLarge (line 33) | RecordListTooLarge Error = 18
constant NotEnoughReplicas (line 34) | NotEnoughReplicas Error = 19
constant NotEnoughReplicasAfterAppend (line 35) | NotEnoughReplicasAfterAppend Error = 20
constant InvalidRequiredAcks (line 36) | InvalidRequiredAcks Error = 21
constant IllegalGeneration (line 37) | IllegalGeneration Error = 22
constant InconsistentGroupProtocol (line 38) | InconsistentGroupProtocol Error = 23
constant InvalidGroupId (line 39) | InvalidGroupId Error = 24
constant UnknownMemberId (line 40) | UnknownMemberId Error = 25
constant InvalidSessionTimeout (line 41) | InvalidSessionTimeout Error = 26
constant RebalanceInProgress (line 42) | RebalanceInProgress Error = 27
constant InvalidCommitOffsetSize (line 43) | InvalidCommitOffsetSize Error = 28
constant TopicAuthorizationFailed (line 44) | TopicAuthorizationFailed Error = 29
constant GroupAuthorizationFailed (line 45) | GroupAuthorizationFailed Error = 30
constant ClusterAuthorizationFailed (line 46) | ClusterAuthorizationFailed Error = 31
constant InvalidTimestamp (line 47) | InvalidTimestamp Error = 32
constant UnsupportedSASLMechanism (line 48) | UnsupportedSASLMechanism Error = 33
constant IllegalSASLState (line 49) | IllegalSASLState Error = 34
constant UnsupportedVersion (line 50) | UnsupportedVersion Error = 35
constant TopicAlreadyExists (line 51) | TopicAlreadyExists Error = 36
constant InvalidPartitionNumber (line 52) | InvalidPartitionNumber Error = 37
constant InvalidReplicationFactor (line 53) | InvalidReplicationFactor Error = 38
constant InvalidReplicaAssignment (line 54) | InvalidReplicaAssignment Error = 39
constant InvalidConfiguration (line 55) | InvalidConfiguration Error = 40
constant NotController (line 56) | NotController Error = 41
constant InvalidRequest (line 57) | InvalidRequest Error = 42
constant UnsupportedForMessageFormat (line 58) | UnsupportedForMessageFormat Error = 43
constant PolicyViolation (line 59) | PolicyViolation Error = 44
constant OutOfOrderSequenceNumber (line 60) | OutOfOrderSequenceNumber Error = 45
constant DuplicateSequenceNumber (line 61) | DuplicateSequenceNumber Error = 46
constant InvalidProducerEpoch (line 62) | InvalidProducerEpoch Error = 47
constant InvalidTransactionState (line 63) | InvalidTransactionState Error = 48
constant InvalidProducerIDMapping (line 64) | InvalidProducerIDMapping Error = 49
constant InvalidTransactionTimeout (line 65) | InvalidTransactionTimeout Error = 50
constant ConcurrentTransactions (line 66) | ConcurrentTransactions Error = 51
constant TransactionCoordinatorFenced (line 67) | TransactionCoordinatorFenced Error = 52
constant TransactionalIDAuthorizationFailed (line 68) | TransactionalIDAuthorizationFailed Error = 53
constant SecurityDisabled (line 69) | SecurityDisabled Error = 54
constant BrokerAuthorizationFailed (line 70) | BrokerAuthorizationFailed Error = 55
constant KafkaStorageError (line 71) | KafkaStorageError Error = 56
constant LogDirNotFound (line 72) | LogDirNotFound Error = 57
constant SASLAuthenticationFailed (line 73) | SASLAuthenticationFailed Error = 58
constant UnknownProducerId (line 74) | UnknownProducerId Error = 59
constant ReassignmentInProgress (line 75) | ReassignmentInProgress Error = 60
constant DelegationTokenAuthDisabled (line 76) | DelegationTokenAuthDisabled Error = 61
constant DelegationTokenNotFound (line 77) | DelegationTokenNotFound Error = 62
constant DelegationTokenOwnerMismatch (line 78) | DelegationTokenOwnerMismatch Error = 63
constant DelegationTokenRequestNotAllowed (line 79) | DelegationTokenRequestNotAllowed Error = 64
constant DelegationTokenAuthorizationFailed (line 80) | DelegationTokenAuthorizationFailed Error = 65
constant DelegationTokenExpired (line 81) | DelegationTokenExpired Error = 66
constant InvalidPrincipalType (line 82) | InvalidPrincipalType Error = 67
constant NonEmptyGroup (line 83) | NonEmptyGroup Error = 68
constant GroupIdNotFound (line 84) | GroupIdNotFound Error = 69
constant FetchSessionIDNotFound (line 85) | FetchSessionIDNotFound Error = 70
constant InvalidFetchSessionEpoch (line 86) | InvalidFetchSessionEpoch Error = 71
constant ListenerNotFound (line 87) | ListenerNotFound Error = 72
constant TopicDeletionDisabled (line 88) | TopicDeletionDisabled Error = 73
constant FencedLeaderEpoch (line 89) | FencedLeaderEpoch Error = 74
constant UnknownLeaderEpoch (line 90) | UnknownLeaderEpoch Error = 75
constant UnsupportedCompressionType (line 91) | UnsupportedCompressionType Error = 76
constant StaleBrokerEpoch (line 92) | StaleBrokerEpoch Error = 77
constant OffsetNotAvailable (line 93) | OffsetNotAvailable Error = 78
constant MemberIDRequired (line 94) | MemberIDRequired Error = 79
constant PreferredLeaderNotAvailable (line 95) | PreferredLeaderNotAvailable Error = 80
constant GroupMaxSizeReached (line 96) | GroupMaxSizeReached Error = 81
constant FencedInstanceID (line 97) | FencedInstanceID Error = 82
constant EligibleLeadersNotAvailable (line 98) | EligibleLeadersNotAvailable Error = 83
constant ElectionNotNeeded (line 99) | ElectionNotNeeded Error = 84
constant NoReassignmentInProgress (line 100) | NoReassignmentInProgress Error = 85
constant GroupSubscribedToTopic (line 101) | GroupSubscribedToTopic Error = 86
constant InvalidRecord (line 102) | InvalidRecord Error = 87
constant UnstableOffsetCommit (line 103) | UnstableOffsetCommit Error = 88
constant ThrottlingQuotaExceeded (line 104) | ThrottlingQuotaExceeded Error = 89
constant ProducerFenced (line 105) | ProducerFenced Error = 90
constant ResourceNotFound (line 106) | ResourceNotFound Error = 91
constant DuplicateResource (line 107) | DuplicateResource Error = 92
constant UnacceptableCredential (line 108) | UnacceptableCredential Error = 93
constant InconsistentVoterSet (line 109) | InconsistentVoterSet Error = 94
constant InvalidUpdateVersion (line 110) | InvalidUpdateVersion Error = 95
constant FeatureUpdateFailed (line 111) | FeatureUpdateFailed Error = 96
constant PrincipalDeserializationFailure (line 112) | PrincipalDeserializationFailure Error = 97
constant SnapshotNotFound (line 113) | SnapshotNotFound Error = 98
constant PositionOutOfRange (line 114) | PositionOutOfRange Error = 99
constant UnknownTopicID (line 115) | UnknownTopicID Error = 100
constant DuplicateBrokerRegistration (line 116) | DuplicateBrokerRegistration Error = 101
constant BrokerIDNotRegistered (line 117) | BrokerIDNotRegistered Error = 102
constant InconsistentTopicID (line 118) | InconsistentTopicID Error = 103
constant InconsistentClusterID (line 119) | InconsistentClusterID Error = 104
constant TransactionalIDNotFound (line 120) | TransactionalIDNotFound Error = 105
constant FetchSessionTopicIDError (line 121) | FetchSessionTopicIDError Error = 106
function isTimeout (line 597) | func isTimeout(err error) bool {
function isTemporary (line 605) | func isTemporary(err error) bool {
function isTransientNetworkError (line 613) | func isTransientNetworkError(err error) bool {
function silentEOF (line 620) | func silentEOF(err error) error {
function dontExpectEOF (line 627) | func dontExpectEOF(err error) error {
function coalesceErrors (line 634) | func coalesceErrors(errs ...error) error {
type MessageTooLargeError (line 644) | type MessageTooLargeError struct
method Error (line 659) | func (e MessageTooLargeError) Error() string {
method Unwrap (line 663) | func (e MessageTooLargeError) Unwrap() error {
function messageTooLarge (line 649) | func messageTooLarge(msgs []Message, i int) MessageTooLargeError {
function makeError (line 667) | func makeError(code int16, message string) error {
type WriteErrors (line 696) | type WriteErrors
method Count (line 699) | func (err WriteErrors) Count() int {
method Error (line 711) | func (err WriteErrors) Error() string {
FILE: error_test.go
function TestError (line 10) | func TestError(t *testing.T) {
FILE: example_consumergroup_test.go
function ExampleGeneration_Start_consumerGroupParallelReaders (line 12) | func ExampleGeneration_Start_consumerGroupParallelReaders() {
function ExampleGeneration_CommitOffsets_overwriteOffsets (line 66) | func ExampleGeneration_CommitOffsets_overwriteOffsets() {
FILE: example_groupbalancer_test.go
function ExampleNewReader_rackAffinity (line 17) | func ExampleNewReader_rackAffinity() {
function findRack (line 37) | func findRack() string {
constant ecsContainerMetadataURI (line 47) | ecsContainerMetadataURI = "ECS_CONTAINER_METADATA_URI"
function whereAmI (line 50) | func whereAmI() string {
function ecsAvailabilityZone (line 76) | func ecsAvailabilityZone() string {
function ec2AvailabilityZone (line 103) | func ec2AvailabilityZone() string {
FILE: example_writer_test.go
function ExampleWriter (line 9) | func ExampleWriter() {
FILE: examples/consumer-logger/main.go
function getKafkaReader (line 13) | func getKafkaReader(kafkaURL, topic, groupID string) *kafka.Reader {
function main (line 24) | func main() {
FILE: examples/consumer-mongo-db/main.go
function getMongoCollection (line 13) | func getMongoCollection(mongoURL, dbName, collectionName string) *mongo....
function getKafkaReader (line 32) | func getKafkaReader(kafkaURL, topic, groupID string) *kafka.Reader {
function main (line 42) | func main() {
FILE: examples/producer-api/main.go
function producerHandler (line 13) | func producerHandler(kafkaWriter *kafka.Writer) func(http.ResponseWriter...
function getKafkaWriter (line 32) | func getKafkaWriter(kafkaURL, topic string) *kafka.Writer {
function main (line 40) | func main() {
FILE: examples/producer-random/main.go
function newKafkaWriter (line 13) | func newKafkaWriter(kafkaURL, topic string) *kafka.Writer {
function main (line 21) | func main() {
FILE: fetch.go
type FetchRequest (line 16) | type FetchRequest struct
method maxWait (line 186) | func (req *FetchRequest) maxWait() time.Duration {
type FetchResponse (line 43) | type FetchResponse struct
method Fetch (line 88) | func (c *Client) Fetch(ctx context.Context, req *FetchRequest) (*FetchRe...
type fetchRequestV2 (line 193) | type fetchRequestV2 struct
method size (line 200) | func (r fetchRequestV2) size() int32 {
method writeTo (line 204) | func (r fetchRequestV2) writeTo(wb *writeBuffer) {
type fetchRequestTopicV2 (line 211) | type fetchRequestTopicV2 struct
method size (line 216) | func (t fetchRequestTopicV2) size() int32 {
method writeTo (line 221) | func (t fetchRequestTopicV2) writeTo(wb *writeBuffer) {
type fetchRequestPartitionV2 (line 226) | type fetchRequestPartitionV2 struct
method size (line 232) | func (p fetchRequestPartitionV2) size() int32 {
method writeTo (line 236) | func (p fetchRequestPartitionV2) writeTo(wb *writeBuffer) {
type fetchResponseV2 (line 242) | type fetchResponseV2 struct
method size (line 247) | func (r fetchResponseV2) size() int32 {
method writeTo (line 251) | func (r fetchResponseV2) writeTo(wb *writeBuffer) {
type fetchResponseTopicV2 (line 256) | type fetchResponseTopicV2 struct
method size (line 261) | func (t fetchResponseTopicV2) size() int32 {
method writeTo (line 266) | func (t fetchResponseTopicV2) writeTo(wb *writeBuffer) {
type fetchResponsePartitionV2 (line 271) | type fetchResponsePartitionV2 struct
method size (line 279) | func (p fetchResponsePartitionV2) size() int32 {
method writeTo (line 283) | func (p fetchResponsePartitionV2) writeTo(wb *writeBuffer) {
FILE: fetch_test.go
function produceRecords (line 16) | func produceRecords(t *testing.T, n int, addr net.Addr, topic string, co...
function TestClientFetch (line 49) | func TestClientFetch(t *testing.T) {
function TestClientFetchCompressed (line 76) | func TestClientFetchCompressed(t *testing.T) {
function assertFetchResponse (line 103) | func assertFetchResponse(t *testing.T, found, expected *FetchResponse) {
type memoryRecord (line 135) | type memoryRecord struct
function assertRecords (line 142) | func assertRecords(t *testing.T, found, expected []memoryRecord) {
function readRecords (line 170) | func readRecords(records RecordReader) ([]memoryRecord, error) {
function TestClientPipeline (line 209) | func TestClientPipeline(t *testing.T) {
FILE: findcoordinator.go
type CoordinatorKeyType (line 14) | type CoordinatorKeyType
constant CoordinatorKeyTypeConsumer (line 18) | CoordinatorKeyTypeConsumer CoordinatorKeyType = 0
constant CoordinatorKeyTypeTransaction (line 21) | CoordinatorKeyTypeTransaction CoordinatorKeyType = 1
type FindCoordinatorRequest (line 25) | type FindCoordinatorRequest struct
type FindCoordinatorResponseCoordinator (line 37) | type FindCoordinatorResponseCoordinator struct
type FindCoordinatorResponse (line 49) | type FindCoordinatorResponse struct
method FindCoordinator (line 65) | func (c *Client) FindCoordinator(ctx context.Context, req *FindCoordinat...
type findCoordinatorRequestV0 (line 94) | type findCoordinatorRequestV0 struct
method size (line 100) | func (t findCoordinatorRequestV0) size() int32 {
method writeTo (line 104) | func (t findCoordinatorRequestV0) writeTo(wb *writeBuffer) {
type findCoordinatorResponseCoordinatorV0 (line 108) | type findCoordinatorResponseCoordinatorV0 struct
method size (line 119) | func (t findCoordinatorResponseCoordinatorV0) size() int32 {
method writeTo (line 125) | func (t findCoordinatorResponseCoordinatorV0) writeTo(wb *writeBuffer) {
method readFrom (line 131) | func (t *findCoordinatorResponseCoordinatorV0) readFrom(r *bufio.Reade...
type findCoordinatorResponseV0 (line 144) | type findCoordinatorResponseV0 struct
method size (line 152) | func (t findCoordinatorResponseV0) size() int32 {
method writeTo (line 157) | func (t findCoordinatorResponseV0) writeTo(wb *writeBuffer) {
method readFrom (line 162) | func (t *findCoordinatorResponseV0) readFrom(r *bufio.Reader, size int...
FILE: findcoordinator_test.go
function TestFindCoordinatorResponseV0 (line 14) | func TestFindCoordinatorResponseV0(t *testing.T) {
function TestClientFindCoordinator (line 44) | func TestClientFindCoordinator(t *testing.T) {
function waitForCoordinatorIndefinitely (line 65) | func waitForCoordinatorIndefinitely(ctx context.Context, c *Client, req ...
function shouldRetryfindingCoordinator (line 77) | func shouldRetryfindingCoordinator(resp *FindCoordinatorResponse, err er...
FILE: groupbalancer.go
type GroupMember (line 8) | type GroupMember struct
type GroupMemberAssignments (line 21) | type GroupMemberAssignments
type GroupBalancer (line 24) | type GroupBalancer interface
type RangeGroupBalancer (line 52) | type RangeGroupBalancer struct
method ProtocolName (line 54) | func (r RangeGroupBalancer) ProtocolName() string {
method UserData (line 58) | func (r RangeGroupBalancer) UserData() ([]byte, error) {
method AssignGroups (line 62) | func (r RangeGroupBalancer) AssignGroups(members []GroupMember, topicP...
type RoundRobinGroupBalancer (line 103) | type RoundRobinGroupBalancer struct
method ProtocolName (line 105) | func (r RoundRobinGroupBalancer) ProtocolName() string {
method UserData (line 109) | func (r RoundRobinGroupBalancer) UserData() ([]byte, error) {
method AssignGroups (line 113) | func (r RoundRobinGroupBalancer) AssignGroups(members []GroupMember, t...
type RackAffinityGroupBalancer (line 153) | type RackAffinityGroupBalancer struct
method ProtocolName (line 160) | func (r RackAffinityGroupBalancer) ProtocolName() string {
method AssignGroups (line 164) | func (r RackAffinityGroupBalancer) AssignGroups(members []GroupMember,...
method UserData (line 192) | func (r RackAffinityGroupBalancer) UserData() ([]byte, error) {
method assignTopic (line 196) | func (r *RackAffinityGroupBalancer) assignTopic(members []GroupMember,...
function findPartitions (line 290) | func findPartitions(topic string, partitions []Partition) []int {
function findMembersByTopic (line 301) | func findMembersByTopic(members []GroupMember) map[string][]GroupMember {
function findGroupBalancer (line 332) | func findGroupBalancer(protocolName string, balancers []GroupBalancer) (...
FILE: groupbalancer_test.go
function TestFindMembersByTopic (line 11) | func TestFindMembersByTopic(t *testing.T) {
function TestRangeAssignGroups (line 78) | func TestRangeAssignGroups(t *testing.T) {
function TestRangeAssignGroupsUnbalanced (line 198) | func TestRangeAssignGroupsUnbalanced(t *testing.T) {
function TestRoundRobinAssignGroups (line 226) | func TestRoundRobinAssignGroups(t *testing.T) {
function TestFindMembersByTopicSortsByMemberID (line 339) | func TestFindMembersByTopicSortsByMemberID(t *testing.T) {
function TestRackAffinityGroupBalancer (line 379) | func TestRackAffinityGroupBalancer(t *testing.T) {
FILE: gzip/gzip.go
constant Code (line 12) | Code = 1
constant DefaultCompressionLevel (line 13) | DefaultCompressionLevel = gz.DefaultCompression
function NewCompressionCodec (line 18) | func NewCompressionCodec() *CompressionCodec {
function NewCompressionCodecLevel (line 22) | func NewCompressionCodecLevel(level int) *CompressionCodec {
FILE: heartbeat.go
type HeartbeatRequest (line 14) | type HeartbeatRequest struct
type HeartbeatResponse (line 32) | type HeartbeatResponse struct
type heartbeatRequestV0 (line 43) | type heartbeatRequestV0 struct
method size (line 79) | func (t heartbeatRequestV0) size() int32 {
method writeTo (line 85) | func (t heartbeatRequestV0) writeTo(wb *writeBuffer) {
method Heartbeat (line 55) | func (c *Client) Heartbeat(ctx context.Context, req *HeartbeatRequest) (...
type heartbeatResponseV0 (line 91) | type heartbeatResponseV0 struct
method size (line 96) | func (t heartbeatResponseV0) size() int32 {
method writeTo (line 100) | func (t heartbeatResponseV0) writeTo(wb *writeBuffer) {
method readFrom (line 104) | func (t *heartbeatResponseV0) readFrom(r *bufio.Reader, sz int) (remai...
FILE: heartbeat_test.go
function TestClientHeartbeat (line 14) | func TestClientHeartbeat(t *testing.T) {
function TestHeartbeatRequestV0 (line 59) | func TestHeartbeatRequestV0(t *testing.T) {
FILE: incrementalalterconfigs.go
type ConfigOperation (line 10) | type ConfigOperation
constant ConfigOperationSet (line 13) | ConfigOperationSet ConfigOperation = 0
constant ConfigOperationDelete (line 14) | ConfigOperationDelete ConfigOperation = 1
constant ConfigOperationAppend (line 15) | ConfigOperationAppend ConfigOperation = 2
constant ConfigOperationSubtract (line 16) | ConfigOperationSubtract ConfigOperation = 3
type IncrementalAlterConfigsRequest (line 20) | type IncrementalAlterConfigsRequest struct
type IncrementalAlterConfigsRequestResource (line 34) | type IncrementalAlterConfigsRequestResource struct
type IncrementalAlterConfigsRequestConfig (line 47) | type IncrementalAlterConfigsRequestConfig struct
type IncrementalAlterConfigsResponse (line 59) | type IncrementalAlterConfigsResponse struct
type IncrementalAlterConfigsResponseResource (line 66) | type IncrementalAlterConfigsResponseResource struct
method IncrementalAlterConfigs (line 78) | func (c *Client) IncrementalAlterConfigs(
FILE: incrementalalterconfigs_test.go
function TestClientIncrementalAlterConfigs (line 11) | func TestClientIncrementalAlterConfigs(t *testing.T) {
FILE: initproducerid.go
type InitProducerIDRequest (line 13) | type InitProducerIDRequest struct
type ProducerSession (line 36) | type ProducerSession struct
type InitProducerIDResponse (line 45) | type InitProducerIDResponse struct
method InitProducerID (line 61) | func (c *Client) InitProducerID(ctx context.Context, req *InitProducerID...
FILE: initproducerid_test.go
function TestClientInitProducerId (line 14) | func TestClientInitProducerId(t *testing.T) {
FILE: joingroup.go
type JoinGroupRequest (line 17) | type JoinGroupRequest struct
type GroupProtocol (line 45) | type GroupProtocol struct
type GroupProtocolSubscription (line 53) | type GroupProtocolSubscription struct
type JoinGroupResponse (line 65) | type JoinGroupResponse struct
type JoinGroupResponseMember (line 95) | type JoinGroupResponseMember struct
method JoinGroup (line 107) | func (c *Client) JoinGroup(ctx context.Context, req *JoinGroupRequest) (...
type groupMetadata (line 192) | type groupMetadata struct
method size (line 198) | func (t groupMetadata) size() int32 {
method writeTo (line 204) | func (t groupMetadata) writeTo(wb *writeBuffer) {
method bytes (line 210) | func (t groupMetadata) bytes() []byte {
method readFrom (line 216) | func (t *groupMetadata) readFrom(r *bufio.Reader, size int) (remain in...
type joinGroupRequestGroupProtocolV1 (line 229) | type joinGroupRequestGroupProtocolV1 struct
method size (line 234) | func (t joinGroupRequestGroupProtocolV1) size() int32 {
method writeTo (line 239) | func (t joinGroupRequestGroupProtocolV1) writeTo(wb *writeBuffer) {
type joinGroupRequest (line 244) | type joinGroupRequest struct
method size (line 267) | func (t joinGroupRequest) size() int32 {
method writeTo (line 276) | func (t joinGroupRequest) writeTo(wb *writeBuffer) {
type joinGroupResponseMember (line 285) | type joinGroupResponseMember struct
method size (line 291) | func (t joinGroupResponseMember) size() int32 {
method writeTo (line 296) | func (t joinGroupResponseMember) writeTo(wb *writeBuffer) {
method readFrom (line 301) | func (t *joinGroupResponseMember) readFrom(r *bufio.Reader, size int) ...
type joinGroupResponse (line 311) | type joinGroupResponse struct
method size (line 333) | func (t joinGroupResponse) size() int32 {
method writeTo (line 346) | func (t joinGroupResponse) writeTo(wb *writeBuffer) {
method readFrom (line 358) | func (t *joinGroupResponse) readFrom(r *bufio.Reader, size int) (remai...
FILE: joingroup_test.go
function TestClientJoinGroup (line 15) | func TestClientJoinGroup(t *testing.T) {
function TestSaramaCompatibility (line 127) | func TestSaramaCompatibility(t *testing.T) {
function TestMemberMetadata (line 193) | func TestMemberMetadata(t *testing.T) {
function TestJoinGroupResponse (line 220) | func TestJoinGroupResponse(t *testing.T) {
FILE: kafka.go
type Broker (line 6) | type Broker struct
type Topic (line 14) | type Topic struct
type Partition (line 34) | type Partition struct
function Marshal (line 74) | func Marshal(v interface{}) ([]byte, error) {
function Unmarshal (line 81) | func Unmarshal(b []byte, v interface{}) error {
type Version (line 86) | type Version
method Marshal (line 91) | func (n Version) Marshal(v interface{}) ([]byte, error) {
method Unmarshal (line 98) | func (n Version) Unmarshal(b []byte, v interface{}) error {
FILE: kafka_test.go
function TestMarshalUnmarshal (line 11) | func TestMarshalUnmarshal(t *testing.T) {
function TestVersionMarshalUnmarshal (line 75) | func TestVersionMarshalUnmarshal(t *testing.T) {
type Struct (line 120) | type Struct struct
function BenchmarkMarshal (line 138) | func BenchmarkMarshal(b *testing.B) {
function BenchmarkUnmarshal (line 151) | func BenchmarkUnmarshal(b *testing.B) {
type testKafkaLogger (line 175) | type testKafkaLogger struct
method Printf (line 184) | func (l *testKafkaLogger) Printf(msg string, args ...interface{}) {
function newTestKafkaLogger (line 180) | func newTestKafkaLogger(t *testing.T, prefix string) Logger {
FILE: leavegroup.go
type LeaveGroupRequest (line 14) | type LeaveGroupRequest struct
type LeaveGroupRequestMember (line 26) | type LeaveGroupRequestMember struct
type LeaveGroupResponse (line 35) | type LeaveGroupResponse struct
type LeaveGroupResponseMember (line 50) | type LeaveGroupResponseMember struct
method LeaveGroup (line 64) | func (c *Client) LeaveGroup(ctx context.Context, req *LeaveGroupRequest)...
type leaveGroupRequestV0 (line 113) | type leaveGroupRequestV0 struct
method size (line 122) | func (t leaveGroupRequestV0) size() int32 {
method writeTo (line 126) | func (t leaveGroupRequestV0) writeTo(wb *writeBuffer) {
type leaveGroupResponseV0 (line 131) | type leaveGroupResponseV0 struct
method size (line 136) | func (t leaveGroupResponseV0) size() int32 {
method writeTo (line 140) | func (t leaveGroupResponseV0) writeTo(wb *writeBuffer) {
method readFrom (line 144) | func (t *leaveGroupResponseV0) readFrom(r *bufio.Reader, size int) (re...
FILE: leavegroup_test.go
function TestClientLeaveGroup (line 13) | func TestClientLeaveGroup(t *testing.T) {
function TestLeaveGroupResponseV0 (line 193) | func TestLeaveGroupResponseV0(t *testing.T) {
FILE: listgroups.go
type ListGroupsRequest (line 12) | type ListGroupsRequest struct
type ListGroupsResponse (line 18) | type ListGroupsResponse struct
type ListGroupsResponseGroup (line 28) | type ListGroupsResponseGroup struct
method ListGroups (line 39) | func (c *Client) ListGroups(
type listGroupsRequestV1 (line 64) | type listGroupsRequestV1 struct
method size (line 67) | func (t listGroupsRequestV1) size() int32 {
method writeTo (line 71) | func (t listGroupsRequestV1) writeTo(wb *writeBuffer) {
type listGroupsResponseGroupV1 (line 74) | type listGroupsResponseGroupV1 struct
method size (line 80) | func (t listGroupsResponseGroupV1) size() int32 {
method writeTo (line 84) | func (t listGroupsResponseGroupV1) writeTo(wb *writeBuffer) {
method readFrom (line 89) | func (t *listGroupsResponseGroupV1) readFrom(r *bufio.Reader, size int...
type listGroupsResponseV1 (line 99) | type listGroupsResponseV1 struct
method size (line 110) | func (t listGroupsResponseV1) size() int32 {
method writeTo (line 116) | func (t listGroupsResponseV1) writeTo(wb *writeBuffer) {
method readFrom (line 122) | func (t *listGroupsResponseV1) readFrom(r *bufio.Reader, size int) (re...
FILE: listgroups_test.go
function TestListGroupsResponseV1 (line 13) | func TestListGroupsResponseV1(t *testing.T) {
function TestClientListGroups (line 44) | func TestClientListGroups(t *testing.T) {
FILE: listoffset.go
type OffsetRequest (line 14) | type OffsetRequest struct
function FirstOffsetOf (line 21) | func FirstOffsetOf(partition int) OffsetRequest {
function LastOffsetOf (line 27) | func LastOffsetOf(partition int) OffsetRequest {
function TimeOffsetOf (line 33) | func TimeOffsetOf(partition int, at time.Time) OffsetRequest {
type PartitionOffsets (line 39) | type PartitionOffsets struct
type ListOffsetsRequest (line 49) | type ListOffsetsRequest struct
type ListOffsetsResponse (line 68) | type ListOffsetsResponse struct
method ListOffsets (line 79) | func (c *Client) ListOffsets(ctx context.Context, req *ListOffsetsReques...
type listOffsetRequestV1 (line 186) | type listOffsetRequestV1 struct
method size (line 191) | func (r listOffsetRequestV1) size() int32 {
method writeTo (line 195) | func (r listOffsetRequestV1) writeTo(wb *writeBuffer) {
type listOffsetRequestTopicV1 (line 200) | type listOffsetRequestTopicV1 struct
method size (line 205) | func (t listOffsetRequestTopicV1) size() int32 {
method writeTo (line 210) | func (t listOffsetRequestTopicV1) writeTo(wb *writeBuffer) {
type listOffsetRequestPartitionV1 (line 215) | type listOffsetRequestPartitionV1 struct
method size (line 220) | func (p listOffsetRequestPartitionV1) size() int32 {
method writeTo (line 224) | func (p listOffsetRequestPartitionV1) writeTo(wb *writeBuffer) {
type listOffsetResponseV1 (line 229) | type listOffsetResponseV1
method size (line 231) | func (r listOffsetResponseV1) size() int32 {
method writeTo (line 235) | func (r listOffsetResponseV1) writeTo(wb *writeBuffer) {
type listOffsetResponseTopicV1 (line 239) | type listOffsetResponseTopicV1 struct
method size (line 244) | func (t listOffsetResponseTopicV1) size() int32 {
method writeTo (line 249) | func (t listOffsetResponseTopicV1) writeTo(wb *writeBuffer) {
type partitionOffsetV1 (line 254) | type partitionOffsetV1 struct
method size (line 261) | func (p partitionOffsetV1) size() int32 {
method writeTo (line 265) | func (p partitionOffsetV1) writeTo(wb *writeBuffer) {
method readFrom (line 272) | func (p *partitionOffsetV1) readFrom(r *bufio.Reader, sz int) (remain ...
FILE: listoffset_test.go
function TestClientListOffsets (line 9) | func TestClientListOffsets(t *testing.T) {
FILE: listpartitionreassignments.go
type ListPartitionReassignmentsRequest (line 12) | type ListPartitionReassignmentsRequest struct
type ListPartitionReassignmentsRequestTopic (line 25) | type ListPartitionReassignmentsRequestTopic struct
type ListPartitionReassignmentsResponse (line 31) | type ListPartitionReassignmentsResponse struct
type ListPartitionReassignmentsResponseTopic (line 42) | type ListPartitionReassignmentsResponseTopic struct
type ListPartitionReassignmentsResponsePartition (line 49) | type ListPartitionReassignmentsResponsePartition struct
method ListPartitionReassignments (line 63) | func (c *Client) ListPartitionReassignments(
function intToInt32Array (line 115) | func intToInt32Array(arr []int) []int32 {
function int32ToIntArray (line 126) | func int32ToIntArray(arr []int32) []int {
FILE: listpartitionreassignments_test.go
function TestClientListPartitionReassignments (line 10) | func TestClientListPartitionReassignments(t *testing.T) {
FILE: logger.go
type Logger (line 4) | type Logger interface
type LoggerFunc (line 15) | type LoggerFunc
method Printf (line 17) | func (f LoggerFunc) Printf(msg string, args ...interface{}) { f(msg, a...
FILE: lz4/lz4.go
constant Code (line 9) | Code = 3
function NewCompressionCodec (line 14) | func NewCompressionCodec() *CompressionCodec {
FILE: message.go
type Message (line 8) | type Message struct
method message (line 33) | func (msg Message) message(cw *crc32Writer) message {
method size (line 48) | func (msg *Message) size() int32 {
method headerSize (line 52) | func (msg *Message) headerSize() int {
method totalSize (line 59) | func (msg *Message) totalSize() int32 {
constant timestampSize (line 46) | timestampSize = 8
type message (line 63) | type message struct
method crc32 (line 72) | func (m message) crc32(cw *crc32Writer) int32 {
method size (line 84) | func (m message) size() int32 {
method writeTo (line 92) | func (m message) writeTo(wb *writeBuffer) {
type messageSetItem (line 103) | type messageSetItem struct
method size (line 109) | func (m messageSetItem) size() int32 {
method writeTo (line 113) | func (m messageSetItem) writeTo(wb *writeBuffer) {
type messageSet (line 119) | type messageSet
method size (line 121) | func (s messageSet) size() (size int32) {
method writeTo (line 128) | func (s messageSet) writeTo(wb *writeBuffer) {
FILE: message_reader.go
type readBytesFunc (line 11) | type readBytesFunc
type messageSetReader (line 16) | type messageSetReader struct
method remaining (line 96) | func (r *messageSetReader) remaining() (remain int) {
method discard (line 106) | func (r *messageSetReader) discard() (err error) {
method readMessage (line 122) | func (r *messageSetReader) readMessage(min int64, key readBytesFunc, v...
method readMessageV1 (line 145) | func (r *messageSetReader) readMessageV1(min int64, key readBytesFunc,...
method readMessageV2 (line 252) | func (r *messageSetReader) readMessageV2(_ int64, key readBytesFunc, v...
method discardBytes (line 341) | func (r *messageSetReader) discardBytes() (err error) {
method discardN (line 346) | func (r *messageSetReader) discardN(sz int) (err error) {
method markRead (line 351) | func (r *messageSetReader) markRead() {
method unwindStack (line 362) | func (r *messageSetReader) unwindStack() {
method readMessageHeader (line 377) | func (r *messageSetReader) readMessageHeader(header *Header) (err erro...
method runFunc (line 395) | func (r *messageSetReader) runFunc(rbFunc readBytesFunc) (err error) {
method readHeader (line 406) | func (r *messageSetReader) readHeader() (err error) {
method readNewBytes (line 495) | func (r *messageSetReader) readNewBytes(len int) (res []byte, err erro...
method readNewString (line 500) | func (r *messageSetReader) readNewString(len int) (res string, err err...
method readInt8 (line 505) | func (r *messageSetReader) readInt8(val *int8) (err error) {
method readInt16 (line 510) | func (r *messageSetReader) readInt16(val *int16) (err error) {
method readInt32 (line 515) | func (r *messageSetReader) readInt32(val *int32) (err error) {
method readInt64 (line 520) | func (r *messageSetReader) readInt64(val *int64) (err error) {
method readVarInt (line 525) | func (r *messageSetReader) readVarInt(val *int64) (err error) {
method readBytesWith (line 530) | func (r *messageSetReader) readBytesWith(fn readBytesFunc) (err error) {
method log (line 535) | func (r *messageSetReader) log(msg string, args ...interface{}) {
type readerStack (line 28) | type readerStack struct
type messagesHeader (line 38) | type messagesHeader struct
method compression (line 62) | func (h messagesHeader) compression() (codec CompressionCodec, err err...
method badMagic (line 80) | func (h messagesHeader) badMagic() error {
function newMessageSetReader (line 84) | func newMessageSetReader(reader *bufio.Reader, remain int) (*messageSetR...
function extractOffset (line 539) | func extractOffset(base int64, msgSet []byte) (offset int64, err error) {
FILE: message_test.go
function TestV1BatchOffsets (line 25) | func TestV1BatchOffsets(t *testing.T) {
function TestMessageSetReader (line 239) | func TestMessageSetReader(t *testing.T) {
function TestMessageSetReaderEmpty (line 538) | func TestMessageSetReaderEmpty(t *testing.T) {
function TestMessageFixtures (line 568) | func TestMessageFixtures(t *testing.T) {
function TestMessageSize (line 679) | func TestMessageSize(t *testing.T) {
function randate (line 699) | func randate() time.Time {
type readerHelper (line 710) | type readerHelper struct
method readMessageErr (line 730) | func (r *readerHelper) readMessageErr() (msg Message, err error) {
method readMessage (line 751) | func (r *readerHelper) readMessage() (msg Message) {
function newReaderHelper (line 716) | func newReaderHelper(t *testing.T, bs []byte) (r *readerHelper, err erro...
FILE: metadata.go
type MetadataRequest (line 14) | type MetadataRequest struct
type MetadataResponse (line 24) | type MetadataResponse struct
method Metadata (line 42) | func (c *Client) Metadata(ctx context.Context, req *MetadataRequest) (*M...
type topicMetadataRequestV1 (line 110) | type topicMetadataRequestV1
method size (line 112) | func (r topicMetadataRequestV1) size() int32 {
method writeTo (line 116) | func (r topicMetadataRequestV1) writeTo(wb *writeBuffer) {
type metadataResponseV1 (line 127) | type metadataResponseV1 struct
method size (line 133) | func (r metadataResponseV1) size() int32 {
method writeTo (line 139) | func (r metadataResponseV1) writeTo(wb *writeBuffer) {
type brokerMetadataV1 (line 145) | type brokerMetadataV1 struct
method size (line 152) | func (b brokerMetadataV1) size() int32 {
method writeTo (line 156) | func (b brokerMetadataV1) writeTo(wb *writeBuffer) {
type topicMetadataV1 (line 163) | type topicMetadataV1 struct
method size (line 170) | func (t topicMetadataV1) size() int32 {
method writeTo (line 176) | func (t topicMetadataV1) writeTo(wb *writeBuffer) {
type partitionMetadataV1 (line 183) | type partitionMetadataV1 struct
method size (line 191) | func (p partitionMetadataV1) size() int32 {
method writeTo (line 195) | func (p partitionMetadataV1) writeTo(wb *writeBuffer) {
type topicMetadataRequestV6 (line 203) | type topicMetadataRequestV6 struct
method size (line 208) | func (r topicMetadataRequestV6) size() int32 {
method writeTo (line 212) | func (r topicMetadataRequestV6) writeTo(wb *writeBuffer) {
type metadataResponseV6 (line 224) | type metadataResponseV6 struct
method size (line 232) | func (r metadataResponseV6) size() int32 {
method writeTo (line 239) | func (r metadataResponseV6) writeTo(wb *writeBuffer) {
type topicMetadataV6 (line 247) | type topicMetadataV6 struct
method size (line 254) | func (t topicMetadataV6) size() int32 {
method writeTo (line 260) | func (t topicMetadataV6) writeTo(wb *writeBuffer) {
type partitionMetadataV6 (line 267) | type partitionMetadataV6 struct
method size (line 276) | func (p partitionMetadataV6) size() int32 {
method writeTo (line 280) | func (p partitionMetadataV6) writeTo(wb *writeBuffer) {
FILE: metadata_test.go
function TestClientMetadata (line 8) | func TestClientMetadata(t *testing.T) {
FILE: offsetcommit.go
type OffsetCommit (line 18) | type OffsetCommit struct
type OffsetCommitRequest (line 26) | type OffsetCommitRequest struct
type OffsetCommitResponse (line 51) | type OffsetCommitResponse struct
type OffsetCommitPartition (line 62) | type OffsetCommitPartition struct
method OffsetCommit (line 77) | func (c *Client) OffsetCommit(ctx context.Context, req *OffsetCommitRequ...
type offsetCommitRequestV2Partition (line 140) | type offsetCommitRequestV2Partition struct
method size (line 151) | func (t offsetCommitRequestV2Partition) size() int32 {
method writeTo (line 157) | func (t offsetCommitRequestV2Partition) writeTo(wb *writeBuffer) {
type offsetCommitRequestV2Topic (line 163) | type offsetCommitRequestV2Topic struct
method size (line 171) | func (t offsetCommitRequestV2Topic) size() int32 {
method writeTo (line 176) | func (t offsetCommitRequestV2Topic) writeTo(wb *writeBuffer) {
type offsetCommitRequestV2 (line 181) | type offsetCommitRequestV2 struct
method size (line 198) | func (t offsetCommitRequestV2) size() int32 {
method writeTo (line 206) | func (t offsetCommitRequestV2) writeTo(wb *writeBuffer) {
type offsetCommitResponseV2PartitionResponse (line 214) | type offsetCommitResponseV2PartitionResponse struct
method size (line 221) | func (t offsetCommitResponseV2PartitionResponse) size() int32 {
method writeTo (line 226) | func (t offsetCommitResponseV2PartitionResponse) writeTo(wb *writeBuff...
method readFrom (line 231) | func (t *offsetCommitResponseV2PartitionResponse) readFrom(r *bufio.Re...
type offsetCommitResponseV2Response (line 241) | type offsetCommitResponseV2Response struct
method size (line 246) | func (t offsetCommitResponseV2Response) size() int32 {
method writeTo (line 251) | func (t offsetCommitResponseV2Response) writeTo(wb *writeBuffer) {
method readFrom (line 256) | func (t *offsetCommitResponseV2Response) readFrom(r *bufio.Reader, siz...
type offsetCommitResponseV2 (line 276) | type offsetCommitResponseV2 struct
method size (line 280) | func (t offsetCommitResponseV2) size() int32 {
method writeTo (line 284) | func (t offsetCommitResponseV2) writeTo(wb *writeBuffer) {
method readFrom (line 288) | func (t *offsetCommitResponseV2) readFrom(r *bufio.Reader, size int) (...
FILE: offsetcommit_test.go
function TestOffsetCommitResponseV2 (line 15) | func TestOffsetCommitResponseV2(t *testing.T) {
function TestClientOffsetCommit (line 50) | func TestClientOffsetCommit(t *testing.T) {
FILE: offsetdelete.go
type OffsetDelete (line 14) | type OffsetDelete struct
type OffsetDeleteRequest (line 21) | type OffsetDeleteRequest struct
type OffsetDeleteResponse (line 34) | type OffsetDeleteResponse struct
type OffsetDeletePartition (line 48) | type OffsetDeletePartition struct
method OffsetDelete (line 59) | func (c *Client) OffsetDelete(ctx context.Context, req *OffsetDeleteRequ...
FILE: offsetdelete_test.go
function TestClientDeleteOffset (line 14) | func TestClientDeleteOffset(t *testing.T) {
FILE: offsetfetch.go
type OffsetFetchRequest (line 15) | type OffsetFetchRequest struct
type OffsetFetchResponse (line 28) | type OffsetFetchResponse struct
type OffsetFetchPartition (line 46) | type OffsetFetchPartition struct
method OffsetFetch (line 68) | func (c *Client) OffsetFetch(ctx context.Context, req *OffsetFetchReques...
type offsetFetchRequestV1Topic (line 127) | type offsetFetchRequestV1Topic struct
method size (line 135) | func (t offsetFetchRequestV1Topic) size() int32 {
method writeTo (line 140) | func (t offsetFetchRequestV1Topic) writeTo(wb *writeBuffer) {
type offsetFetchRequestV1 (line 145) | type offsetFetchRequestV1 struct
method size (line 153) | func (t offsetFetchRequestV1) size() int32 {
method writeTo (line 158) | func (t offsetFetchRequestV1) writeTo(wb *writeBuffer) {
type offsetFetchResponseV1PartitionResponse (line 163) | type offsetFetchResponseV1PartitionResponse struct
method size (line 177) | func (t offsetFetchResponseV1PartitionResponse) size() int32 {
method writeTo (line 184) | func (t offsetFetchResponseV1PartitionResponse) writeTo(wb *writeBuffe...
method readFrom (line 191) | func (t *offsetFetchResponseV1PartitionResponse) readFrom(r *bufio.Rea...
type offsetFetchResponseV1Response (line 207) | type offsetFetchResponseV1Response struct
method size (line 215) | func (t offsetFetchResponseV1Response) size() int32 {
method writeTo (line 220) | func (t offsetFetchResponseV1Response) writeTo(wb *writeBuffer) {
method readFrom (line 225) | func (t *offsetFetchResponseV1Response) readFrom(r *bufio.Reader, size...
type offsetFetchResponseV1 (line 245) | type offsetFetchResponseV1 struct
method size (line 250) | func (t offsetFetchResponseV1) size() int32 {
method writeTo (line 254) | func (t offsetFetchResponseV1) writeTo(wb *writeBuffer) {
method readFrom (line 258) | func (t *offsetFetchResponseV1) readFrom(r *bufio.Reader, size int) (r...
FILE: offsetfetch_test.go
function TestOffsetFetchResponseV1 (line 14) | func TestOffsetFetchResponseV1(t *testing.T) {
function TestOffsetFetchRequestWithNoTopic (line 51) | func TestOffsetFetchRequestWithNoTopic(t *testing.T) {
function TestOffsetFetchRequestWithOneTopic (line 112) | func TestOffsetFetchRequestWithOneTopic(t *testing.T) {
FILE: produce.go
type RequiredAcks (line 17) | type RequiredAcks
method String (line 25) | func (acks RequiredAcks) String() string {
method MarshalText (line 38) | func (acks RequiredAcks) MarshalText() ([]byte, error) {
method UnmarshalText (line 42) | func (acks *RequiredAcks) UnmarshalText(b []byte) error {
constant RequireNone (line 20) | RequireNone RequiredAcks = 0
constant RequireOne (line 21) | RequireOne RequiredAcks = 1
constant RequireAll (line 22) | RequireAll RequiredAcks = -1
type ProduceRequest (line 68) | type ProduceRequest struct
type ProduceResponse (line 101) | type ProduceResponse struct
method Produce (line 145) | func (c *Client) Produce(ctx context.Context, req *ProduceRequest) (*Pro...
type produceRequestV2 (line 205) | type produceRequestV2 struct
method size (line 211) | func (r produceRequestV2) size() int32 {
method writeTo (line 215) | func (r produceRequestV2) writeTo(wb *writeBuffer) {
type produceRequestTopicV2 (line 221) | type produceRequestTopicV2 struct
method size (line 226) | func (t produceRequestTopicV2) size() int32 {
method writeTo (line 231) | func (t produceRequestTopicV2) writeTo(wb *writeBuffer) {
type produceRequestPartitionV2 (line 236) | type produceRequestPartitionV2 struct
method size (line 242) | func (p produceRequestPartitionV2) size() int32 {
method writeTo (line 246) | func (p produceRequestPartitionV2) writeTo(wb *writeBuffer) {
type produceResponsePartitionV2 (line 252) | type produceResponsePartitionV2 struct
method size (line 259) | func (p produceResponsePartitionV2) size() int32 {
method writeTo (line 263) | func (p produceResponsePartitionV2) writeTo(wb *writeBuffer) {
method readFrom (line 270) | func (p *produceResponsePartitionV2) readFrom(r *bufio.Reader, sz int)...
type produceResponsePartitionV7 (line 286) | type produceResponsePartitionV7 struct
method size (line 294) | func (p produceResponsePartitionV7) size() int32 {
method writeTo (line 298) | func (p produceResponsePartitionV7) writeTo(wb *writeBuffer) {
method readFrom (line 306) | func (p *produceResponsePartitionV7) readFrom(r *bufio.Reader, sz int)...
FILE: produce_test.go
function TestRequiredAcks (line 12) | func TestRequiredAcks(t *testing.T) {
function TestClientProduce (line 44) | func TestClientProduce(t *testing.T) {
function TestClientProduceCompressed (line 74) | func TestClientProduceCompressed(t *testing.T) {
function TestClientProduceNilRecords (line 105) | func TestClientProduceNilRecords(t *testing.T) {
function TestClientProduceEmptyRecords (line 121) | func TestClientProduceEmptyRecords(t *testing.T) {
FILE: protocol.go
type ApiVersion (line 9) | type ApiVersion struct
method Format (line 15) | func (v ApiVersion) Format(w fmt.State, r rune) {
type apiKey (line 42) | type apiKey
method String (line 95) | func (k apiKey) String() string {
constant produce (line 45) | produce apiKey = 0
constant fetch (line 46) | fetch apiKey = 1
constant listOffsets (line 47) | listOffsets apiKey = 2
constant metadata (line 48) | metadata apiKey = 3
constant leaderAndIsr (line 49) | leaderAndIsr apiKey = 4
constant stopReplica (line 50) | stopReplica apiKey = 5
constant updateMetadata (line 51) | updateMetadata apiKey = 6
constant controlledShutdown (line 52) | controlledShutdown apiKey = 7
constant offsetCommit (line 53) | offsetCommit apiKey = 8
constant offsetFetch (line 54) | offsetFetch apiKey = 9
constant findCoordinator (line 55) | findCoordinator apiKey = 10
constant joinGroup (line 56) | joinGroup apiKey = 11
constant heartbeat (line 57) | heartbeat apiKey = 12
constant leaveGroup (line 58) | leaveGroup apiKey = 13
constant syncGroup (line 59) | syncGroup apiKey = 14
constant describeGroups (line 60) | describeGroups apiKey = 15
constant listGroups (line 61) | listGroups apiKey = 16
constant saslHandshake (line 62) | saslHandshake apiKey = 17
constant apiVersions (line 63) | apiVersions apiKey = 18
constant createTopics (line 64) | createTopics apiKey = 19
constant deleteTopics (line 65) | deleteTopics apiKey = 20
constant deleteRecords (line 66) | deleteRecords apiKey = 21
constant initProducerId (line 67) | initProducerId apiKey = 22
constant offsetForLeaderEpoch (line 68) | offsetForLeaderEpoch apiKey = 23
constant addPartitionsToTxn (line 69) | addPartitionsToTxn apiKey = 24
constant addOffsetsToTxn (line 70) | addOffsetsToTxn apiKey = 25
constant endTxn (line 71) | endTxn apiKey = 26
constant writeTxnMarkers (line 72) | writeTxnMarkers apiKey = 27
constant txnOffsetCommit (line 73) | txnOffsetCommit apiKey = 28
constant describeAcls (line 74) | describeAcls apiKey = 29
constant createAcls (line 75) | createAcls apiKey = 30
constant deleteAcls (line 76) | deleteAcls apiKey = 31
constant describeConfigs (line 77) | describeConfigs apiKey = 32
constant alterConfigs (line 78) | alterConfigs apiKey = 33
constant alterReplicaLogDirs (line 79) | alterReplicaLogDirs apiKey = 34
constant describeLogDirs (line 80) | describeLogDirs apiKey = 35
constant saslAuthenticate (line 81) | saslAuthenticate apiKey = 36
constant createPartitions (line 82) | createPartitions apiKey = 37
constant createDelegationToken (line 83) | createDelegationToken apiKey = 38
constant renewDelegationToken (line 84) | renewDelegationToken apiKey = 39
constant expireDelegationToken (line 85) | expireDelegationToken apiKey = 40
constant describeDelegationToken (line 86) | describeDelegationToken apiKey = 41
constant deleteGroups (line 87) | deleteGroups apiKey = 42
constant electLeaders (line 88) | electLeaders apiKey = 43
constant incrementalAlterConfigs (line 89) | incrementalAlterConfigs apiKey = 44
constant alterPartitionReassignments (line 90) | alterPartitionReassignments apiKey = 45
constant listPartitionReassignments (line 91) | listPartitionReassignments apiKey = 46
constant offsetDelete (line 92) | offsetDelete apiKey = 47
type apiVersion (line 102) | type apiVersion
constant v0 (line 105) | v0 = 0
constant v1 (line 106) | v1 = 1
constant v2 (line 107) | v2 = 2
constant v3 (line 108) | v3 = 3
constant v5 (line 109) | v5 = 5
constant v6 (line 110) | v6 = 6
constant v7 (line 111) | v7 = 7
constant v10 (line 112) | v10 = 10
type requestHeader (line 168) | type requestHeader struct
method size (line 176) | func (h requestHeader) size() int32 {
method writeTo (line 180) | func (h requestHeader) writeTo(wb *writeBuffer) {
type request (line 188) | type request interface
function makeInt8 (line 193) | func makeInt8(b []byte) int8 {
function makeInt16 (line 197) | func makeInt16(b []byte) int16 {
function makeInt32 (line 201) | func makeInt32(b []byte) int32 {
function makeInt64 (line 205) | func makeInt64(b []byte) int64 {
function expectZeroSize (line 209) | func expectZeroSize(sz int, err error) error {
FILE: protocol/addoffsetstotxn/addoffsetstotxn.go
function init (line 5) | func init() {
type Request (line 9) | type Request struct
method ApiKey (line 20) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.AddOffset...
method Transaction (line 22) | func (r *Request) Transaction() string { return r.TransactionalID }
type Response (line 26) | type Response struct
method ApiKey (line 35) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.AddOffse...
FILE: protocol/addoffsetstotxn/addoffsetstotxn_test.go
function TestAddOffsetsToTxnRequest (line 10) | func TestAddOffsetsToTxnRequest(t *testing.T) {
function TestAddOffsetsToTxnResponse (line 21) | func TestAddOffsetsToTxnResponse(t *testing.T) {
FILE: protocol/addpartitionstotxn/addpartitionstotxn.go
function init (line 5) | func init() {
type Request (line 9) | type Request struct
method ApiKey (line 29) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.AddPartit...
method Transaction (line 31) | func (r *Request) Transaction() string { return r.TransactionalID }
type RequestTopic (line 20) | type RequestTopic struct
type Response (line 35) | type Response struct
method ApiKey (line 62) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.AddParti...
type ResponseResult (line 44) | type ResponseResult struct
type ResponsePartition (line 53) | type ResponsePartition struct
FILE: protocol/addpartitionstotxn/addpartitionstotxn_test.go
function TestAddPartitionsToTxnRequest (line 10) | func TestAddPartitionsToTxnRequest(t *testing.T) {
function TestAddPartitionsToTxnResponse (line 30) | func TestAddPartitionsToTxnResponse(t *testing.T) {
FILE: protocol/alterclientquotas/alterclientquotas.go
function init (line 5) | func init() {
type Request (line 10) | type Request struct
method ApiKey (line 18) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.AlterClie...
method Broker (line 20) | func (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, e...
type Entry (line 24) | type Entry struct
type Entity (line 32) | type Entity struct
type Ops (line 40) | type Ops struct
type Response (line 49) | type Response struct
method ApiKey (line 57) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.AlterCli...
type ResponseQuotas (line 59) | type ResponseQuotas struct
FILE: protocol/alterclientquotas/alterclientquotas_test.go
constant v0 (line 11) | v0 = 0
constant v1 (line 12) | v1 = 1
function TestAlterClientQuotasRequest (line 15) | func TestAlterClientQuotasRequest(t *testing.T) {
function TestAlterClientQuotasResponse (line 59) | func TestAlterClientQuotasResponse(t *testing.T) {
FILE: protocol/alterconfigs/alterconfigs.go
function init (line 5) | func init() {
type Request (line 10) | type Request struct
method ApiKey (line 15) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.AlterConf...
method Broker (line 17) | func (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, e...
type RequestResources (line 21) | type RequestResources struct
type RequestConfig (line 27) | type RequestConfig struct
type Response (line 32) | type Response struct
method ApiKey (line 37) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.AlterCon...
type ResponseResponses (line 39) | type ResponseResponses struct
FILE: protocol/alterconfigs/alterconfigs_test.go
constant v0 (line 11) | v0 = 0
constant v1 (line 12) | v1 = 1
function TestAlterConfigsRequest (line 15) | func TestAlterConfigsRequest(t *testing.T) {
function TestAlterConfigsResponse (line 49) | func TestAlterConfigsResponse(t *testing.T) {
FILE: protocol/alterpartitionreassignments/alterpartitionreassignments.go
function init (line 5) | func init() {
type Request (line 10) | type Request struct
method ApiKey (line 29) | func (r *Request) ApiKey() protocol.ApiKey {
method Broker (line 33) | func (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, e...
type RequestTopic (line 19) | type RequestTopic struct
type RequestPartition (line 24) | type RequestPartition struct
type Response (line 37) | type Response struct
method ApiKey (line 59) | func (r *Response) ApiKey() protocol.ApiKey {
type ResponseResult (line 48) | type ResponseResult struct
type ResponsePartition (line 53) | type ResponsePartition struct
FILE: protocol/alterpartitionreassignments/alterpartitionreassignments_test.go
constant v0 (line 11) | v0 = 0
function TestAlterPartitionReassignmentsRequest (line 14) | func TestAlterPartitionReassignmentsRequest(t *testing.T) {
function TestAlterPartitionReassignmentsResponse (line 34) | func TestAlterPartitionReassignmentsResponse(t *testing.T) {
FILE: protocol/alteruserscramcredentials/alteruserscramcredentials.go
function init (line 5) | func init() {
type Request (line 9) | type Request struct
method ApiKey (line 18) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.AlterUser...
method Broker (line 20) | func (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, e...
type RequestUserScramCredentialsDeletion (line 24) | type RequestUserScramCredentialsDeletion struct
type RequestUserScramCredentialsUpsertion (line 33) | type RequestUserScramCredentialsUpsertion struct
type Response (line 45) | type Response struct
method ApiKey (line 54) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.AlterUse...
type ResponseUserScramCredentials (line 56) | type ResponseUserScramCredentials struct
FILE: protocol/alteruserscramcredentials/alteruserscramcredentials_test.go
constant v0 (line 11) | v0 = 0
function TestAlterUserScramCredentialsRequest (line 14) | func TestAlterUserScramCredentialsRequest(t *testing.T) {
function TestAlterUserScramCredentialsResponse (line 34) | func TestAlterUserScramCredentialsResponse(t *testing.T) {
FILE: protocol/apiversions/apiversions.go
function init (line 5) | func init() {
type Request (line 9) | type Request struct
method ApiKey (line 13) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.ApiVersio...
type Response (line 15) | type Response struct
method ApiKey (line 21) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.ApiVersi...
type ApiKeyResponse (line 23) | type ApiKeyResponse struct
FILE: protocol/apiversions/apiversions_test.go
constant v0 (line 11) | v0 = 0
constant v1 (line 12) | v1 = 1
constant v2 (line 13) | v2 = 2
function TestApiversionsRequest (line 16) | func TestApiversionsRequest(t *testing.T) {
function TestApiversionsResponse (line 24) | func TestApiversionsResponse(t *testing.T) {
FILE: protocol/buffer.go
type Bytes (line 24) | type Bytes interface
function NewBytes (line 37) | func NewBytes(b []byte) Bytes {
function ReadAll (line 50) | func ReadAll(b Bytes) ([]byte, error) {
type bytesReader (line 59) | type bytesReader struct
method Close (line 61) | func (*bytesReader) Close() error { return nil }
type refCount (line 63) | type refCount
method ref (line 65) | func (rc *refCount) ref() { atomic.AddUintptr((*uintptr)(rc), 1) }
method unref (line 67) | func (rc *refCount) unref(onZero func()) {
constant pageSize (line 79) | pageSize = 65536
type page (line 82) | type page struct
method ref (line 105) | func (p *page) ref() { p.refc.ref() }
method unref (line 107) | func (p *page) unref() { p.refc.unref(func() { pagePool.Put(p) }) }
method slice (line 109) | func (p *page) slice(begin, end int64) []byte {
method Cap (line 131) | func (p *page) Cap() int { return pageSize }
method Len (line 133) | func (p *page) Len() int { return p.length }
method Size (line 135) | func (p *page) Size() int64 { return int64(p.length) }
method Truncate (line 137) | func (p *page) Truncate(n int) {
method ReadAt (line 143) | func (p *page) ReadAt(b []byte, off int64) (int, error) {
method ReadFrom (line 153) | func (p *page) ReadFrom(r io.Reader) (int64, error) {
method WriteAt (line 162) | func (p *page) WriteAt(b []byte, off int64) (int, error) {
method Write (line 173) | func (p *page) Write(b []byte) (int, error) {
function newPage (line 89) | func newPage(offset int64) *page {
type pageBuffer (line 184) | type pageBuffer struct
method refTo (line 205) | func (pb *pageBuffer) refTo(ref *pageRef, begin, end int64) {
method ref (line 218) | func (pb *pageBuffer) ref(begin, end int64) *pageRef {
method unref (line 224) | func (pb *pageBuffer) unref() {
method newPage (line 234) | func (pb *pageBuffer) newPage() *page {
method Close (line 238) | func (pb *pageBuffer) Close() error {
method Len (line 242) | func (pb *pageBuffer) Len() int {
method Size (line 246) | func (pb *pageBuffer) Size() int64 {
method Discard (line 250) | func (pb *pageBuffer) Discard(n int) (int, error) {
method Truncate (line 259) | func (pb *pageBuffer) Truncate(n int) {
method Seek (line 284) | func (pb *pageBuffer) Seek(offset int64, whence int) (int64, error) {
method ReadByte (line 293) | func (pb *pageBuffer) ReadByte() (byte, error) {
method Read (line 299) | func (pb *pageBuffer) Read(b []byte) (int, error) {
method ReadAt (line 308) | func (pb *pageBuffer) ReadAt(b []byte, off int64) (int, error) {
method ReadFrom (line 312) | func (pb *pageBuffer) ReadFrom(r io.Reader) (int64, error) {
method WriteString (line 338) | func (pb *pageBuffer) WriteString(s string) (int, error) {
method Write (line 342) | func (pb *pageBuffer) Write(b []byte) (int, error) {
method WriteAt (line 372) | func (pb *pageBuffer) WriteAt(b []byte, off int64) (int, error) {
method WriteTo (line 383) | func (pb *pageBuffer) WriteTo(w io.Writer) (int64, error) {
function newPageBuffer (line 191) | func newPageBuffer() *pageBuffer {
type contiguousPages (line 408) | type contiguousPages
method ref (line 410) | func (pages contiguousPages) ref() {
method unref (line 416) | func (pages contiguousPages) unref() {
method clear (line 422) | func (pages contiguousPages) clear() {
method ReadAt (line 428) | func (pages contiguousPages) ReadAt(b []byte, off int64) (int, error) {
method WriteAt (line 441) | func (pages contiguousPages) WriteAt(b []byte, off int64) (int, error) {
method slice (line 454) | func (pages contiguousPages) slice(begin, end int64) contiguousPages {
method indexOf (line 463) | func (pages contiguousPages) indexOf(offset int64) int {
method scan (line 470) | func (pages contiguousPages) scan(begin, end int64, f func([]byte) boo...
type pageRef (line 483) | type pageRef struct
method unref (line 492) | func (ref *pageRef) unref() {
method Len (line 503) | func (ref *pageRef) Len() int { return int(ref.Size() - ref.cursor) }
method Size (line 505) | func (ref *pageRef) Size() int64 { return int64(ref.length) }
method Close (line 507) | func (ref *pageRef) Close() error { ref.unref(); return nil }
method String (line 509) | func (ref *pageRef) String() string {
method Seek (line 513) | func (ref *pageRef) Seek(offset int64, whence int) (int64, error) {
method ReadByte (line 522) | func (ref *pageRef) ReadByte() (byte, error) {
method Read (line 537) | func (ref *pageRef) Read(b []byte) (int, error) {
method ReadAt (line 546) | func (ref *pageRef) ReadAt(b []byte, off int64) (int, error) {
method WriteTo (line 569) | func (ref *pageRef) WriteTo(w io.Writer) (wn int64, err error) {
method scan (line 580) | func (ref *pageRef) scan(off int64, f func([]byte) bool) {
type pageRefAllocator (line 594) | type pageRefAllocator struct
method newPageRef (line 600) | func (a *pageRefAllocator) newPageRef() *pageRef {
function seek (line 610) | func seek(cursor, limit, offset int64, whence int) (int64, error) {
function closeBytes (line 630) | func closeBytes(b Bytes) {
FILE: protocol/buffer_test.go
function TestPageBufferWriteReadSeek (line 10) | func TestPageBufferWriteReadSeek(t *testing.T) {
function TestPageRefWriteReadSeek (line 48) | func TestPageRefWriteReadSeek(t *testing.T) {
function TestPageRefReadByte (line 89) | func TestPageRefReadByte(t *testing.T) {
FILE: protocol/cluster.go
type Cluster (line 10) | type Cluster struct
method BrokerIDs (line 17) | func (c Cluster) BrokerIDs() []int32 {
method TopicNames (line 28) | func (c Cluster) TopicNames() []string {
method IsZero (line 37) | func (c Cluster) IsZero() bool {
method Format (line 41) | func (c Cluster) Format(w fmt.State, _ rune) {
function formatBrokerIDs (line 108) | func formatBrokerIDs(brokerIDs []int32, leader int32) string {
FILE: protocol/conn.go
type Conn (line 11) | type Conn struct
method String (line 27) | func (c *Conn) String() string {
method Close (line 31) | func (c *Conn) Close() error {
method Discard (line 35) | func (c *Conn) Discard(n int) (int, error) {
method Peek (line 39) | func (c *Conn) Peek(n int) ([]byte, error) {
method Read (line 43) | func (c *Conn) Read(b []byte) (int, error) {
method Write (line 47) | func (c *Conn) Write(b []byte) (int, error) {
method LocalAddr (line 51) | func (c *Conn) LocalAddr() net.Addr {
method RemoteAddr (line 55) | func (c *Conn) RemoteAddr() net.Addr {
method SetDeadline (line 59) | func (c *Conn) SetDeadline(t time.Time) error {
method SetReadDeadline (line 63) | func (c *Conn) SetReadDeadline(t time.Time) error {
method SetWriteDeadline (line 67) | func (c *Conn) SetWriteDeadline(t time.Time) error {
method SetVersions (line 71) | func (c *Conn) SetVersions(versions map[ApiKey]int16) {
method RoundTrip (line 81) | func (c *Conn) RoundTrip(msg Message) (Message, error) {
function NewConn (line 19) | func NewConn(conn net.Conn, clientID string) *Conn {
FILE: protocol/consumer/consumer.go
constant MaxVersionSupported (line 3) | MaxVersionSupported = 1
type Subscription (line 5) | type Subscription struct
type Assignment (line 12) | type Assignment struct
type TopicPartition (line 18) | type TopicPartition struct
FILE: protocol/consumer/consumer_test.go
function TestSubscription (line 11) | func TestSubscription(t *testing.T) {
FILE: protocol/createacls/createacls.go
function init (line 5) | func init() {
type Request (line 9) | type Request struct
method ApiKey (line 17) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.CreateAcls }
method Broker (line 19) | func (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, e...
type RequestACLs (line 23) | type RequestACLs struct
type Response (line 37) | type Response struct
method ApiKey (line 46) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.CreateAc...
type ResponseACLs (line 48) | type ResponseACLs struct
FILE: protocol/createacls/createacls_test.go
constant v0 (line 11) | v0 = 0
constant v1 (line 12) | v1 = 1
constant v2 (line 13) | v2 = 2
constant v3 (line 14) | v3 = 3
function TestCreateACLsRequest (line 17) | func TestCreateACLsRequest(t *testing.T) {
function TestCreateACLsResponse (line 74) | func TestCreateACLsResponse(t *testing.T) {
FILE: protocol/createpartitions/createpartitions.go
function init (line 5) | func init() {
type Request (line 11) | type Request struct
method ApiKey (line 17) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.CreatePar...
method Broker (line 19) | func (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, e...
type RequestTopic (line 23) | type RequestTopic struct
type RequestAssignment (line 29) | type RequestAssignment struct
type Response (line 33) | type Response struct
method ApiKey (line 38) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.CreatePa...
type ResponseResult (line 40) | type ResponseResult struct
FILE: protocol/createpartitions/createpartitions_test.go
constant v0 (line 11) | v0 = 0
constant v1 (line 12) | v1 = 1
function TestCreatePartitionsRequest (line 15) | func TestCreatePartitionsRequest(t *testing.T) {
function TestCreatePartitionsResponse (line 49) | func TestCreatePartitionsResponse(t *testing.T) {
FILE: protocol/createtopics/createtopics.go
function init (line 5) | func init() {
type Request (line 9) | type Request struct
method ApiKey (line 19) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.CreateTop...
method Broker (line 21) | func (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, e...
type RequestTopic (line 25) | type RequestTopic struct
type RequestAssignment (line 33) | type RequestAssignment struct
type RequestConfig (line 38) | type RequestConfig struct
type Response (line 43) | type Response struct
method ApiKey (line 52) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.CreateTo...
type ResponseTopic (line 54) | type ResponseTopic struct
type ResponseTopicConfig (line 64) | type ResponseTopicConfig struct
FILE: protocol/decode.go
type discarder (line 16) | type discarder interface
type decoder (line 20) | type decoder struct
method Reset (line 29) | func (d *decoder) Reset(r io.Reader, n int) {
method Read (line 38) | func (d *decoder) Read(b []byte) (int, error) {
method ReadByte (line 56) | func (d *decoder) ReadByte() (byte, error) {
method done (line 61) | func (d *decoder) done() bool {
method setCRC (line 65) | func (d *decoder) setCRC(table *crc32.Table) {
method decodeBool (line 69) | func (d *decoder) decodeBool(v value) {
method decodeInt8 (line 73) | func (d *decoder) decodeInt8(v value) {
method decodeInt16 (line 77) | func (d *decoder) decodeInt16(v value) {
method decodeInt32 (line 81) | func (d *decoder) decodeInt32(v value) {
method decodeInt64 (line 85) | func (d *decoder) decodeInt64(v value) {
method decodeFloat64 (line 89) | func (d *decoder) decodeFloat64(v value) {
method decodeString (line 93) | func (d *decoder) decodeString(v value) {
method decodeCompactString (line 97) | func (d *decoder) decodeCompactString(v value) {
method decodeBytes (line 101) | func (d *decoder) decodeBytes(v value) {
method decodeCompactBytes (line 105) | func (d *decoder) decodeCompactBytes(v value) {
method decodeArray (line 109) | func (d *decoder) decodeArray(v value, elemType reflect.Type, decodeEl...
method decodeCompactArray (line 121) | func (d *decoder) decodeCompactArray(v value, elemType reflect.Type, d...
method discardAll (line 133) | func (d *decoder) discardAll() {
method discard (line 137) | func (d *decoder) discard(n int) {
method read (line 151) | func (d *decoder) read(n int) []byte {
method writeTo (line 159) | func (d *decoder) writeTo(w io.Writer, n int) {
method setError (line 172) | func (d *decoder) setError(err error) {
method readFull (line 179) | func (d *decoder) readFull(b []byte) bool {
method readByte (line 185) | func (d *decoder) readByte() byte {
method readBool (line 192) | func (d *decoder) readBool() bool {
method readInt8 (line 196) | func (d *decoder) readInt8() int8 {
method readInt16 (line 203) | func (d *decoder) readInt16() int16 {
method readInt32 (line 210) | func (d *decoder) readInt32() int32 {
method readInt64 (line 217) | func (d *decoder) readInt64() int64 {
method readFloat64 (line 224) | func (d *decoder) readFloat64() float64 {
method readString (line 231) | func (d *decoder) readString() string {
method readVarString (line 239) | func (d *decoder) readVarString() string {
method readCompactString (line 247) | func (d *decoder) readCompactString() string {
method readBytes (line 255) | func (d *decoder) readBytes() []byte {
method readVarBytes (line 263) | func (d *decoder) readVarBytes() []byte {
method readCompactBytes (line 271) | func (d *decoder) readCompactBytes() []byte {
method readVarInt (line 279) | func (d *decoder) readVarInt() int64 {
method readUnsignedVarInt (line 306) | func (d *decoder) readUnsignedVarInt() uint64 {
type decodeFunc (line 333) | type decodeFunc
function decodeFuncOf (line 342) | func decodeFuncOf(typ reflect.Type, version int16, flexible bool, tag st...
function stringDecodeFuncOf (line 373) | func stringDecodeFuncOf(flexible bool, tag structTag) decodeFunc {
function bytesDecodeFuncOf (line 381) | func bytesDecodeFuncOf(flexible bool, tag structTag) decodeFunc {
function structDecodeFuncOf (line 389) | func structDecodeFuncOf(typ reflect.Type, version int16, flexible bool) ...
function arrayDecodeFuncOf (line 447) | func arrayDecodeFuncOf(typ reflect.Type, version int16, flexible bool, t...
function readerDecodeFuncOf (line 458) | func readerDecodeFuncOf(typ reflect.Type) decodeFunc {
function readInt8 (line 470) | func readInt8(b []byte) int8 {
function readInt16 (line 474) | func readInt16(b []byte) int16 {
function readInt32 (line 478) | func readInt32(b []byte) int32 {
function readInt64 (line 482) | func readInt64(b []byte) int64 {
function readFloat64 (line 486) | func readFloat64(b []byte) float64 {
function Unmarshal (line 490) | func Unmarshal(data []byte, version int16, value interface{}) error {
FILE: protocol/deleteacls/deleteacls.go
function init (line 5) | func init() {
type Request (line 9) | type Request struct
method ApiKey (line 17) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.DeleteAcls }
method Broker (line 19) | func (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, e...
type RequestFilter (line 23) | type RequestFilter struct
type Response (line 37) | type Response struct
method ApiKey (line 46) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.DeleteAc...
type FilterResult (line 48) | type FilterResult struct
type MatchingACL (line 58) | type MatchingACL struct
FILE: protocol/deleteacls/deleteacls_test.go
constant v0 (line 11) | v0 = 0
constant v1 (line 12) | v1 = 1
constant v2 (line 13) | v2 = 2
constant v3 (line 14) | v3 = 3
function TestDeleteACLsRequest (line 17) | func TestDeleteACLsRequest(t *testing.T) {
function TestDeleteACLsResponse (line 74) | func TestDeleteACLsResponse(t *testing.T) {
FILE: protocol/deletegroups/deletegroups.go
function init (line 5) | func init() {
type Request (line 9) | type Request struct
method Group (line 17) | func (r *Request) Group() string {
method ApiKey (line 25) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.DeleteGro...
type Response (line 31) | type Response struct
method ApiKey (line 40) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.DeleteGr...
type ResponseGroup (line 42) | type ResponseGroup struct
FILE: protocol/deletegroups/deletegroups_test.go
function TestDeleteGroupsRequest (line 10) | func TestDeleteGroupsRequest(t *testing.T) {
function TestDeleteGroupsResponse (line 18) | func TestDeleteGroupsResponse(t *testing.T) {
FILE: protocol/deletetopics/deletetopics.go
function init (line 5) | func init() {
type Request (line 9) | type Request struct
method ApiKey (line 14) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.DeleteTop...
method Broker (line 16) | func (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, e...
type Response (line 20) | type Response struct
method ApiKey (line 25) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.DeleteTo...
type ResponseTopic (line 27) | type ResponseTopic struct
FILE: protocol/deletetopics/deletetopics_test.go
constant v0 (line 11) | v0 = 0
constant v1 (line 12) | v1 = 1
constant v3 (line 13) | v3 = 3
function TestDeleteTopicsRequest (line 16) | func TestDeleteTopicsRequest(t *testing.T) {
function TestDeleteTopicsResponse (line 33) | func TestDeleteTopicsResponse(t *testing.T) {
FILE: protocol/describeacls/describeacls.go
function init (line 5) | func init() {
type Request (line 9) | type Request struct
method ApiKey (line 17) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.DescribeA...
method Broker (line 19) | func (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, e...
type ACLFilter (line 23) | type ACLFilter struct
type Response (line 37) | type Response struct
method ApiKey (line 48) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.Describe...
type Resource (line 50) | type Resource struct
type ResponseACL (line 61) | type ResponseACL struct
FILE: protocol/describeacls/describeacls_test.go
constant v0 (line 11) | v0 = 0
constant v1 (line 12) | v1 = 1
constant v2 (line 13) | v2 = 2
constant v3 (line 14) | v3 = 3
function TestDescribeACLsRequest (line 17) | func TestDescribeACLsRequest(t *testing.T) {
function TestDescribeACLsResponse (line 66) | func TestDescribeACLsResponse(t *testing.T) {
FILE: protocol/describeclientquotas/describeclientquotas.go
function init (line 5) | func init() {
type Request (line 9) | type Request struct
method ApiKey (line 17) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.DescribeC...
method Broker (line 19) | func (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, e...
type Component (line 23) | type Component struct
type Response (line 32) | type Response struct
method ApiKey (line 42) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.Describe...
type Entity (line 44) | type Entity struct
type Value (line 52) | type Value struct
type ResponseQuotas (line 60) | type ResponseQuotas struct
FILE: protocol/describeclientquotas/describeclientquotas_test.go
constant v0 (line 11) | v0 = 0
constant v1 (line 12) | v1 = 1
function TestDescribeClientQuotasRequest (line 15) | func TestDescribeClientQuotasRequest(t *testing.T) {
function TestDescribeClientQuotasResponse (line 39) | func TestDescribeClientQuotasResponse(t *testing.T) {
FILE: protocol/describeconfigs/describeconfigs.go
constant resourceTypeBroker (line 10) | resourceTypeBroker int8 = 4
function init (line 13) | func init() {
type Request (line 18) | type Request struct
method ApiKey (line 24) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.DescribeC...
method Broker (line 26) | func (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, e...
method Split (line 42) | func (r *Request) Split(cluster protocol.Cluster) (
type RequestResource (line 70) | type RequestResource struct
type Response (line 76) | type Response struct
method ApiKey (line 81) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.Describe...
method Merge (line 83) | func (r *Response) Merge(requests []protocol.Message, results []interf...
type ResponseResource (line 103) | type ResponseResource struct
type ResponseConfigEntry (line 111) | type ResponseConfigEntry struct
type ResponseConfigSynonym (line 123) | type ResponseConfigSynonym struct
FILE: protocol/describeconfigs/describeconfigs_test.go
function TestResponse_Merge (line 14) | func TestResponse_Merge(t *testing.T) {
FILE: protocol/describegroups/describegroups.go
function init (line 7) | func init() {
type Request (line 12) | type Request struct
method ApiKey (line 20) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.DescribeG...
method Group (line 22) | func (r *Request) Group() string {
method Split (line 26) | func (r *Request) Split(cluster protocol.Cluster) (
type Response (line 47) | type Response struct
method ApiKey (line 80) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.Describe...
method Merge (line 82) | func (r *Response) Merge(requests []protocol.Message, results []interf...
type ResponseGroup (line 55) | type ResponseGroup struct
type ResponseGroupMember (line 68) | type ResponseGroupMember struct
FILE: protocol/describegroups/describegroups_test.go
constant v0 (line 11) | v0 = 0
constant v1 (line 12) | v1 = 1
constant v2 (line 13) | v2 = 2
constant v3 (line 14) | v3 = 3
constant v4 (line 15) | v4 = 4
constant v5 (line 16) | v5 = 5
function TestDescribeGroupsRequest (line 19) | func TestDescribeGroupsRequest(t *testing.T) {
function TestDescribeGroupsResponse (line 48) | func TestDescribeGroupsResponse(t *testing.T) {
FILE: protocol/describeuserscramcredentials/describeuserscramcredentials.go
function init (line 5) | func init() {
type Request (line 9) | type Request struct
method ApiKey (line 17) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.DescribeU...
method Broker (line 19) | func (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, e...
type RequestUser (line 23) | type RequestUser struct
type Response (line 31) | type Response struct
method ApiKey (line 42) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.Describe...
type ResponseResult (line 44) | type ResponseResult struct
type CredentialInfo (line 55) | type CredentialInfo struct
FILE: protocol/describeuserscramcredentials/describeuserscramcredentials_test.go
constant v0 (line 11) | v0 = 0
function TestDescribeUserScramCredentialsRequest (line 14) | func TestDescribeUserScramCredentialsRequest(t *testing.T) {
function TestDescribeUserScramCredentialsResponse (line 24) | func TestDescribeUserScramCredentialsResponse(t *testing.T) {
FILE: protocol/electleaders/electleaders.go
function init (line 5) | func init() {
type Request (line 10) | type Request struct
method ApiKey (line 21) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.ElectLead...
method Broker (line 23) | func (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, e...
type RequestTopicPartitions (line 16) | type RequestTopicPartitions struct
type Response (line 27) | type Response struct
method ApiKey (line 44) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.ElectLea...
type ResponseReplicaElectionResult (line 33) | type ResponseReplicaElectionResult struct
type ResponsePartitionResult (line 38) | type ResponsePartitionResult struct
FILE: protocol/electleaders/electleaders_test.go
constant v0 (line 11) | v0 = 0
constant v1 (line 12) | v1 = 1
function TestElectLeadersRequest (line 15) | func TestElectLeadersRequest(t *testing.T) {
function TestElectLeadersResponse (line 38) | func TestElectLeadersResponse(t *testing.T) {
FILE: protocol/encode.go
type encoder (line 15) | type encoder struct
method Reset (line 36) | func (e *encoder) Reset(w io.Writer) {
method ReadFrom (line 44) | func (e *encoder) ReadFrom(r io.Reader) (int64, error) {
method Write (line 54) | func (e *encoder) Write(b []byte) (int, error) {
method WriteByte (line 68) | func (e *encoder) WriteByte(b byte) error {
method WriteString (line 74) | func (e *encoder) WriteString(s string) (int, error) {
method setCRC (line 99) | func (e *encoder) setCRC(table *crc32.Table) {
method update (line 103) | func (e *encoder) update(b []byte) {
method encodeBool (line 109) | func (e *encoder) encodeBool(v value) {
method encodeInt8 (line 117) | func (e *encoder) encodeInt8(v value) {
method encodeInt16 (line 121) | func (e *encoder) encodeInt16(v value) {
method encodeInt32 (line 125) | func (e *encoder) encodeInt32(v value) {
method encodeInt64 (line 129) | func (e *encoder) encodeInt64(v value) {
method encodeFloat64 (line 133) | func (e *encoder) encodeFloat64(v value) {
method encodeString (line 137) | func (e *encoder) encodeString(v value) {
method encodeCompactString (line 141) | func (e *encoder) encodeCompactString(v value) {
method encodeNullString (line 145) | func (e *encoder) encodeNullString(v value) {
method encodeCompactNullString (line 149) | func (e *encoder) encodeCompactNullString(v value) {
method encodeBytes (line 153) | func (e *encoder) encodeBytes(v value) {
method encodeCompactBytes (line 157) | func (e *encoder) encodeCompactBytes(v value) {
method encodeNullBytes (line 161) | func (e *encoder) encodeNullBytes(v value) {
method encodeCompactNullBytes (line 165) | func (e *encoder) encodeCompactNullBytes(v value) {
method encodeArray (line 169) | func (e *encoder) encodeArray(v value, elemType reflect.Type, encodeEl...
method encodeCompactArray (line 179) | func (e *encoder) encodeCompactArray(v value, elemType reflect.Type, e...
method encodeNullArray (line 189) | func (e *encoder) encodeNullArray(v value, elemType reflect.Type, enco...
method encodeCompactNullArray (line 204) | func (e *encoder) encodeCompactNullArray(v value, elemType reflect.Typ...
method writeInt8 (line 218) | func (e *encoder) writeInt8(i int8) {
method writeInt16 (line 223) | func (e *encoder) writeInt16(i int16) {
method writeInt32 (line 228) | func (e *encoder) writeInt32(i int32) {
method writeInt64 (line 233) | func (e *encoder) writeInt64(i int64) {
method writeFloat64 (line 238) | func (e *encoder) writeFloat64(f float64) {
method writeString (line 243) | func (e *encoder) writeString(s string) {
method writeVarString (line 248) | func (e *encoder) writeVarString(s string) {
method writeCompactString (line 253) | func (e *encoder) writeCompactString(s string) {
method writeNullString (line 258) | func (e *encoder) writeNullString(s string) {
method writeCompactNullString (line 267) | func (e *encoder) writeCompactNullString(s string) {
method writeBytes (line 276) | func (e *encoder) writeBytes(b []byte) {
method writeCompactBytes (line 281) | func (e *encoder) writeCompactBytes(b []byte) {
method writeNullBytes (line 286) | func (e *encoder) writeNullBytes(b []byte) {
method writeVarNullBytes (line 295) | func (e *encoder) writeVarNullBytes(b []byte) {
method writeCompactNullBytes (line 304) | func (e *encoder) writeCompactNullBytes(b []byte) {
method writeNullBytesFrom (line 313) | func (e *encoder) writeNullBytesFrom(b Bytes) error {
method writeVarNullBytesFrom (line 328) | func (e *encoder) writeVarNullBytesFrom(b Bytes) error {
method writeVarInt (line 343) | func (e *encoder) writeVarInt(i int64) {
method writeUnsignedVarInt (line 347) | func (e *encoder) writeUnsignedVarInt(i uint64) {
type encoderChecksum (line 23) | type encoderChecksum struct
method Read (line 28) | func (e *encoderChecksum) Read(b []byte) (int, error) {
type encodeFunc (line 365) | type encodeFunc
function encodeFuncOf (line 376) | func encodeFuncOf(typ reflect.Type, version int16, flexible bool, tag st...
function stringEncodeFuncOf (line 407) | func stringEncodeFuncOf(flexible bool, tag structTag) encodeFunc {
function bytesEncodeFuncOf (line 422) | func bytesEncodeFuncOf(flexible bool, tag structTag) encodeFunc {
function structEncodeFuncOf (line 437) | func structEncodeFuncOf(typ reflect.Type, version int16, flexible bool) ...
function arrayEncodeFuncOf (line 496) | func arrayEncodeFuncOf(typ reflect.Type, version int16, flexible bool, t...
function writerEncodeFuncOf (line 513) | func writerEncodeFuncOf(typ reflect.Type) encodeFunc {
function writeInt8 (line 529) | func writeInt8(b []byte, i int8) {
function writeInt16 (line 533) | func writeInt16(b []byte, i int16) {
function writeInt32 (line 537) | func writeInt32(b []byte, i int32) {
function writeInt64 (line 541) | func writeInt64(b []byte, i int64) {
function writeFloat64 (line 545) | func writeFloat64(b []byte, f float64) {
function Marshal (line 549) | func Marshal(version int16, value interface{}) ([]byte, error) {
type versionedType (line 598) | type versionedType struct
FILE: protocol/endtxn/endtxn.go
function init (line 5) | func init() {
type Request (line 9) | type Request struct
method ApiKey (line 20) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.EndTxn }
method Transaction (line 22) | func (r *Request) Transaction() string { return r.TransactionalID }
type Response (line 26) | type Response struct
method ApiKey (line 35) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.EndTxn }
FILE: protocol/endtxn/endtxn_test.go
function TestEndTxnRequest (line 10) | func TestEndTxnRequest(t *testing.T) {
function TestEndTxnResponse (line 21) | func TestEndTxnResponse(t *testing.T) {
FILE: protocol/error.go
type Error (line 8) | type Error
method Error (line 10) | func (e Error) Error() string { return string(e) }
function Errorf (line 12) | func Errorf(msg string, args ...interface{}) Error {
constant ErrNoTopic (line 18) | ErrNoTopic Error = "topic not found"
constant ErrNoPartition (line 22) | ErrNoPartition Error = "topic partition not found"
constant ErrNoLeader (line 27) | ErrNoLeader Error = "topic partition has no leader"
constant ErrNoRecord (line 35) | ErrNoRecord Error = "record set contains no records"
constant ErrNoReset (line 39) | ErrNoReset Error = "record sequence does not support reset"
type TopicError (line 42) | type TopicError struct
method Error (line 55) | func (e *TopicError) Error() string {
method Unwrap (line 59) | func (e *TopicError) Unwrap() error {
function NewTopicError (line 47) | func NewTopicError(topic string, err error) *TopicError {
function NewErrNoTopic (line 51) | func NewErrNoTopic(topic string) *TopicError {
type TopicPartitionError (line 63) | type TopicPartitionError struct
method Error (line 85) | func (e *TopicPartitionError) Error() string {
method Unwrap (line 89) | func (e *TopicPartitionError) Unwrap() error {
function NewTopicPartitionError (line 69) | func NewTopicPartitionError(topic string, partition int32, err error) *T...
function NewErrNoPartition (line 77) | func NewErrNoPartition(topic string, partition int32) *TopicPartitionErr...
function NewErrNoLeader (line 81) | func NewErrNoLeader(topic string, partition int32) *TopicPartitionError {
FILE: protocol/fetch/fetch.go
function init (line 9) | func init() {
type Request (line 13) | type Request struct
method ApiKey (line 26) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.Fetch }
method Broker (line 28) | func (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, e...
type RequestTopic (line 60) | type RequestTopic struct
type RequestPartition (line 65) | type RequestPartition struct
type RequestForgottenTopic (line 73) | type RequestForgottenTopic struct
type Response (line 78) | type Response struct
method ApiKey (line 85) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.Fetch }
type ResponseTopic (line 87) | type ResponseTopic struct
type ResponsePartition (line 92) | type ResponsePartition struct
type ResponseTransaction (line 103) | type ResponseTransaction struct
type Error (line 112) | type Error struct
method Error (line 120) | func (e *Error) Error() string {
method Unwrap (line 124) | func (e *Error) Unwrap() error {
function NewError (line 116) | func NewError(err error) *Error {
FILE: protocol/fetch/fetch_test.go
constant v0 (line 13) | v0 = 0
constant v11 (line 14) | v11 = 11
function TestFetchRequest (line 17) | func TestFetchRequest(t *testing.T) {
function TestFetchResponse (line 37) | func TestFetchResponse(t *testing.T) {
function BenchmarkFetchResponse (line 93) | func BenchmarkFetchResponse(b *testing.B) {
FILE: protocol/findcoordinator/findcoordinator.go
function init (line 5) | func init() {
type Request (line 9) | type Request struct
method ApiKey (line 14) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.FindCoord...
type Response (line 16) | type Response struct
method ApiKey (line 25) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.FindCoor...
FILE: protocol/heartbeat/heartbeat.go
function init (line 5) | func init() {
type Request (line 10) | type Request struct
method ApiKey (line 21) | func (r *Request) ApiKey() protocol.ApiKey {
type Response (line 25) | type Response struct
method ApiKey (line 34) | func (r *Response) ApiKey() protocol.ApiKey {
FILE: protocol/heartbeat/heartbeat_test.go
function TestHeartbeatRequest (line 10) | func TestHeartbeatRequest(t *testing.T) {
function TestHeartbeatResponse (line 30) | func TestHeartbeatResponse(t *testing.T) {
FILE: protocol/incrementalalterconfigs/incrementalalterconfigs.go
constant resourceTypeBroker (line 11) | resourceTypeBroker int8 = 4
function init (line 14) | func init() {
type Request (line 19) | type Request struct
method ApiKey (line 36) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.Increment...
method Broker (line 38) | func (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, e...
type RequestResource (line 24) | type RequestResource struct
type RequestConfig (line 30) | type RequestConfig struct
type Response (line 67) | type Response struct
method ApiKey (line 79) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.Incremen...
type ResponseAlterResponse (line 72) | type ResponseAlterResponse struct
FILE: protocol/incrementalalterconfigs/incrementalalterconfigs_test.go
constant resourceTypeTopic (line 11) | resourceTypeTopic int8 = 2
constant resourceTypeBroker (line 12) | resourceTypeBroker int8 = 4
function TestMetadataRequestBroker (line 15) | func TestMetadataRequestBroker(t *testing.T) {
FILE: protocol/initproducerid/initproducerid.go
function init (line 5) | func init() {
type Request (line 9) | type Request struct
method ApiKey (line 20) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.InitProdu...
method Transaction (line 22) | func (r *Request) Transaction() string { return r.TransactionalID }
type Response (line 26) | type Response struct
method ApiKey (line 37) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.InitProd...
FILE: protocol/initproducerid/initproducerid_test.go
function TestInitProducerIDRequest (line 10) | func TestInitProducerIDRequest(t *testing.T) {
function TestInitProducerIDResponse (line 31) | func TestInitProducerIDResponse(t *testing.T) {
FILE: protocol/joingroup/joingroup.go
function init (line 5) | func init() {
type Request (line 9) | type Request struct
method ApiKey (line 32) | func (r *Request) ApiKey() protocol.ApiKey {
method Group (line 36) | func (r *Request) Group() string { return r.GroupID }
type RequestProtocol (line 23) | type RequestProtocol struct
type Response (line 40) | type Response struct
method ApiKey (line 67) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.JoinGroup }
type ResponseMember (line 55) | type ResponseMember struct
type ResponseMemberMetadata (line 65) | type ResponseMemberMetadata struct
FILE: protocol/joingroup/joingroup_test.go
function TestJoinGroupReq (line 10) | func TestJoinGroupReq(t *testing.T) {
function TestJoinGroupResp (line 64) | func TestJoinGroupResp(t *testing.T) {
FILE: protocol/leavegroup/leavegroup.go
function init (line 5) | func init() {
type Request (line 9) | type Request struct
method Prepare (line 19) | func (r *Request) Prepare(apiVersion int16) {
method ApiKey (line 36) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.LeaveGroup }
method Group (line 38) | func (r *Request) Group() string { return r.GroupID }
type RequestMember (line 27) | type RequestMember struct
type Response (line 45) | type Response struct
method ApiKey (line 65) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.LeaveGro...
type ResponseMember (line 55) | type ResponseMember struct
FILE: protocol/leavegroup/leavegroup_test.go
function TestLeaveGroupReq (line 10) | func TestLeaveGroupReq(t *testing.T) {
function TestLeaveGroupResp (line 35) | func TestLeaveGroupResp(t *testing.T) {
FILE: protocol/listgroups/listgroups.go
function init (line 7) | func init() {
type Request (line 12) | type Request struct
method ApiKey (line 17) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.ListGroups }
method Broker (line 19) | func (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, e...
method Split (line 23) | func (r *Request) Split(cluster protocol.Cluster) (
type Response (line 37) | type Response struct
method ApiKey (line 51) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.ListGrou...
method Merge (line 53) | func (r *Response) Merge(requests []protocol.Message, results []interf...
type ResponseGroup (line 43) | type ResponseGroup struct
FILE: protocol/listoffsets/listoffsets.go
function init (line 9) | func init() {
type Request (line 13) | type Request struct
method ApiKey (line 35) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.ListOffse...
method Broker (line 37) | func (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, e...
method Split (line 52) | func (r *Request) Split(cluster protocol.Cluster) ([]protocol.Message,...
type RequestTopic (line 19) | type RequestTopic struct
type RequestPartition (line 24) | type RequestPartition struct
type Response (line 94) | type Response struct
method ApiKey (line 112) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.ListOffs...
method Merge (line 114) | func (r *Response) Merge(requests []protocol.Message, results []interf...
type ResponseTopic (line 99) | type ResponseTopic struct
type ResponsePartition (line 104) | type ResponsePartition struct
FILE: protocol/listoffsets/listoffsets_test.go
constant v1 (line 11) | v1 = 1
constant v4 (line 12) | v4 = 4
function TestListOffsetsRequest (line 15) | func TestListOffsetsRequest(t *testing.T) {
function TestListOffsetsResponse (line 54) | func TestListOffsetsResponse(t *testing.T) {
FILE: protocol/listpartitionreassignments/listpartitionreassignments.go
function init (line 5) | func init() {
type Request (line 11) | type Request struct
method ApiKey (line 29) | func (r *Request) ApiKey() protocol.ApiKey {
method Broker (line 33) | func (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, e...
type RequestTopic (line 20) | type RequestTopic struct
type Response (line 37) | type Response struct
method ApiKey (line 68) | func (r *Response) ApiKey() protocol.ApiKey {
type ResponseTopic (line 48) | type ResponseTopic struct
type ResponsePartition (line 57) | type ResponsePartition struct
FILE: protocol/listpartitionreassignments/listpartitionreassignments_test.go
constant v0 (line 11) | v0 = 0
function TestListPartitionReassignmentsRequest (line 14) | func TestListPartitionReassignmentsRequest(t *testing.T) {
function TestListPartitionReassignmentsResponse (line 25) | func TestListPartitionReassignmentsResponse(t *testing.T) {
FILE: protocol/metadata/metadata.go
function init (line 5) | func init() {
type Request (line 9) | type Request struct
method ApiKey (line 16) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.Metadata }
type Response (line 18) | type Response struct
method ApiKey (line 27) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.Metadata }
type ResponseBroker (line 29) | type ResponseBroker struct
type ResponseTopic (line 36) | type ResponseTopic struct
type ResponsePartition (line 44) | type ResponsePartition struct
FILE: protocol/metadata/metadata_test.go
constant v0 (line 11) | v0 = 0
constant v1 (line 12) | v1 = 1
constant v4 (line 13) | v4 = 4
constant v8 (line 14) | v8 = 8
function TestMetadataRequest (line 17) | func TestMetadataRequest(t *testing.T) {
function TestMetadataResponse (line 35) | func TestMetadataResponse(t *testing.T) {
function BenchmarkMetadataRequest (line 146) | func BenchmarkMetadataRequest(b *testing.B) {
function BenchmarkMetadataResponse (line 155) | func BenchmarkMetadataResponse(b *testing.B) {
FILE: protocol/offsetcommit/offsetcommit.go
function init (line 5) | func init() {
type Request (line 9) | type Request struct
method ApiKey (line 18) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.OffsetCom...
method Group (line 20) | func (r *Request) Group() string { return r.GroupID }
type RequestTopic (line 22) | type RequestTopic struct
type RequestPartition (line 27) | type RequestPartition struct
type Response (line 39) | type Response struct
method ApiKey (line 44) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.OffsetCo...
type ResponseTopic (line 46) | type ResponseTopic struct
type ResponsePartition (line 51) | type ResponsePartition struct
FILE: protocol/offsetcommit/offsetcommit_test.go
function TestOffsetCommitRequest (line 10) | func TestOffsetCommitRequest(t *testing.T) {
function TestOffsetCommitResponse (line 183) | func TestOffsetCommitResponse(t *testing.T) {
FILE: protocol/offsetdelete/offsetdelete.go
function init (line 5) | func init() {
type Request (line 9) | type Request struct
method ApiKey (line 14) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.OffsetDel...
method Group (line 16) | func (r *Request) Group() string { return r.GroupID }
type RequestTopic (line 18) | type RequestTopic struct
type RequestPartition (line 23) | type RequestPartition struct
type Response (line 31) | type Response struct
method ApiKey (line 37) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.OffsetDe...
type ResponseTopic (line 39) | type ResponseTopic struct
type ResponsePartition (line 44) | type ResponsePartition struct
FILE: protocol/offsetdelete/offsetdelete_test.go
function TestOffsetDeleteRequest (line 10) | func TestOffsetDeleteRequest(t *testing.T) {
function TestOffsetDeleteResponse (line 31) | func TestOffsetDeleteResponse(t *testing.T) {
FILE: protocol/offsetfetch/offsetfetch.go
function init (line 5) | func init() {
type Request (line 9) | type Request struct
method ApiKey (line 14) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.OffsetFet...
method Group (line 16) | func (r *Request) Group() string { return r.GroupID }
type RequestTopic (line 18) | type RequestTopic struct
type Response (line 27) | type Response struct
method ApiKey (line 33) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.OffsetFe...
type ResponseTopic (line 35) | type ResponseTopic struct
type ResponsePartition (line 40) | type ResponsePartition struct
FILE: protocol/produce/produce.go
function init (line 9) | func init() {
type Request (line 13) | type Request struct
method ApiKey (line 20) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.Produce }
method Broker (line 22) | func (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, e...
method Prepare (line 54) | func (r *Request) Prepare(apiVersion int16) {
method HasResponse (line 87) | func (r *Request) HasResponse() bool {
type RequestTopic (line 91) | type RequestTopic struct
type RequestPartition (line 96) | type RequestPartition struct
type Response (line 101) | type Response struct
method ApiKey (line 106) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.Produce }
type ResponseTopic (line 108) | type ResponseTopic struct
type ResponsePartition (line 113) | type ResponsePartition struct
type ResponseError (line 123) | type ResponseError struct
type Error (line 133) | type Error struct
method Error (line 141) | func (e *Error) Error() string {
method Unwrap (line 145) | func (e *Error) Unwrap() error {
function NewError (line 137) | func NewError(err error) *Error {
FILE: protocol/produce/produce_test.go
constant v0 (line 13) | v0 = 0
constant v3 (line 14) | v3 = 3
constant v5 (line 15) | v5 = 5
constant v8 (line 16) | v8 = 8
function TestProduceRequest (line 19) | func TestProduceRequest(t *testing.T) {
function TestProduceResponse (line 156) | func TestProduceResponse(t *testing.T) {
function BenchmarkProduceRequest (line 206) | func BenchmarkProduceRequest(b *testing.B) {
FILE: protocol/protocol.go
type Message (line 18) | type Message interface
type ApiKey (line 22) | type ApiKey
method String (line 24) | func (k ApiKey) String() string {
method MinVersion (line 31) | func (k ApiKey) MinVersion() int16 { return k.apiType().minVersion() }
method MaxVersion (line 33) | func (k ApiKey) MaxVersion() int16 { return k.apiType().maxVersion() }
method SelectVersion (line 35) | func (k ApiKey) SelectVersion(minVersion, maxVersion int16) int16 {
method apiType (line 48) | func (k ApiKey) apiType() apiType {
constant Produce (line 56) | Produce ApiKey = 0
constant Fetch (line 57) | Fetch ApiKey = 1
constant ListOffsets (line 58) | ListOffsets ApiKey = 2
constant Metadata (line 59) | Metadata ApiKey = 3
constant LeaderAndIsr (line 60) | LeaderAndIsr ApiKey = 4
constant StopReplica (line 61) | StopReplica ApiKey = 5
constant UpdateMetadata (line 62) | UpdateMetadata ApiKey = 6
constant ControlledShutdown (line 63) | ControlledShutdown ApiKey = 7
constant OffsetCommit (line 64) | OffsetCommit ApiKey = 8
constant OffsetFetch (line 65) | OffsetFetch ApiKey = 9
constant FindCoordinator (line 66) | FindCoordinator ApiKey = 10
constant JoinGroup (line 67) | JoinGroup ApiKey = 11
constant Heartbeat (line 68) | Heartbeat ApiKey = 12
constant LeaveGroup (line 69) | LeaveGroup ApiKey = 13
constant SyncGroup (line 70) | SyncGroup ApiKey = 14
constant DescribeGroups (line 71) | DescribeGroups ApiKey = 15
constant ListGroups (line 72) | ListGroups ApiKey = 16
constant SaslHandshake (line 73) | SaslHandshake ApiKey = 17
constant ApiVersions (line 74) | ApiVersions ApiKey = 18
constant CreateTopics (line 75) | CreateTopics ApiKey = 19
constant DeleteTopics (line 76) | DeleteTopics ApiKey = 20
constant DeleteRecords (line 77) | DeleteRecords ApiKey = 21
constant InitProducerId (line 78) | InitProducerId ApiKey = 22
constant OffsetForLeaderEpoch (line 79) | OffsetForLeaderEpoch ApiKey = 23
constant AddPartitionsToTxn (line 80) | AddPartitionsToTxn ApiKey = 24
constant AddOffsetsToTxn (line 81) | AddOffsetsToTxn ApiKey = 25
constant EndTxn (line 82) | EndTxn ApiKey = 26
constant WriteTxnMarkers (line 83) | WriteTxnMarkers ApiKey = 27
constant TxnOffsetCommit (line 84) | TxnOffsetCommit ApiKey = 28
constant DescribeAcls (line 85) | DescribeAcls ApiKey = 29
constant CreateAcls (line 86) | CreateAcls ApiKey = 30
constant DeleteAcls (line 87) | DeleteAcls ApiKey = 31
constant DescribeConfigs (line 88) | DescribeConfigs ApiKey = 32
constant AlterConfigs (line 89) | AlterConfigs ApiKey = 33
constant AlterReplicaLogDirs (line 90) | AlterReplicaLogDirs ApiKey = 34
constant DescribeLogDirs (line 91) | DescribeLogDirs ApiKey = 35
constant SaslAuthenticate (line 92) | SaslAuthenticate ApiKey = 36
constant CreatePartitions (line 93) | CreatePartitions ApiKey = 37
constant CreateDelegationToken (line 94) | CreateDelegationToken ApiKey = 38
constant RenewDelegationToken (line 95) | RenewDelegationToken ApiKey = 39
constant ExpireDelegationToken (line 96) | ExpireDelegationToken ApiKey = 40
constant DescribeDelegationToken (line 97) | DescribeDelegationToken ApiKey = 41
constant DeleteGroups (line 98) | DeleteGroups ApiKey = 42
constant ElectLeaders (line 99) | ElectLeaders ApiKey = 43
constant IncrementalAlterConfigs (line 100) | IncrementalAlterConfigs ApiKey = 44
constant AlterPartitionReassignments (line 101) | AlterPartitionReassignments ApiKey = 45
constant ListPartitionReassignments (line 102) | ListPartitionReassignments ApiKey = 46
constant OffsetDelete (line 103) | OffsetDelete ApiKey = 47
constant DescribeClientQuotas (line 104) | DescribeClientQuotas ApiKey = 48
constant AlterClientQuotas (line 105) | AlterClientQuotas ApiKey = 49
constant DescribeUserScramCredentials (line 106) | DescribeUserScramCredentials ApiKey = 50
constant AlterUserScramCredentials (line 107) | AlterUserScramCredentials ApiKey = 51
constant numApis (line 109) | numApis = 52
type messageType (line 167) | type messageType struct
method new (line 175) | func (t *messageType) new() Message {
type apiType (line 179) | type apiType struct
method minVersion (line 184) | func (t apiType) minVersion() int16 {
method maxVersion (line 191) | func (t apiType) maxVersion() int16 {
function Register (line 202) | func Register(req, res Message) {
type OverrideTypeMessage (line 218) | type OverrideTypeMessage interface
type OverrideTypeKey (line 222) | type OverrideTypeKey
constant RawProduceOverride (line 225) | RawProduceOverride OverrideTypeKey = 0
function RegisterOverride (line 230) | func RegisterOverride(req, res Message, key OverrideTypeKey) {
function typesOf (line 247) | func typesOf(v interface{}) []messageType {
function makeTypes (line 251) | func makeTypes(t reflect.Type) []messageType {
type structTag (line 291) | type structTag struct
function forEachStructTag (line 299) | func forEachStructTag(tag string, do func(structTag) bool) {
function forEach (line 356) | func forEach(s string, sep byte, do func(string) bool) bool {
function forEachStructField (line 372) | func forEachStructField(t reflect.Type, do func(reflect.Type, index, str...
function parseVersion (line 389) | func parseVersion(s string) (int16, error) {
function dontExpectEOF (line 403) | func dontExpectEOF(err error) error {
type Broker (line 415) | type Broker struct
method String (line 422) | func (b Broker) String() string {
method Format (line 426) | func (b Broker) Format(w fmt.State, v rune) {
function itoa (line 443) | func itoa(i int32) string {
type Topic (line 447) | type Topic struct
type Partition (line 453) | type Partition struct
type RawExchanger (line 467) | type RawExchanger interface
type BrokerMessage (line 480) | type BrokerMessage interface
type GroupMessage (line 489) | type GroupMessage interface
type TransactionalMessage (line 497) | type TransactionalMessage interface
type PreparedMessage (line 505) | type PreparedMessage interface
type Splitter (line 513) | type Splitter interface
type Merger (line 523) | type Merger interface
function Result (line 532) | func Result(r interface{}) (Message, error) {
FILE: protocol/protocol_test.go
type testType (line 10) | type testType struct
type testSubType (line 20) | type testSubType struct
function TestMakeFlexibleTypes (line 24) | func TestMakeFlexibleTypes(t *testing.T) {
function TestEncodeDecodeFlexibleType (line 51) | func TestEncodeDecodeFlexibleType(t *testing.T) {
function TestVarInts (line 153) | func TestVarInts(t *testing.T) {
function TestFloat64 (line 284) | func TestFloat64(t *testing.T) {
FILE: protocol/prototest/bytes.go
function Bytes (line 8) | func Bytes(b []byte) protocol.Bytes {
function String (line 13) | func String(s string) protocol.Bytes {
FILE: protocol/prototest/prototest.go
function deepEqual (line 13) | func deepEqual(x1, x2 interface{}) bool {
function deepEqualValue (line 38) | func deepEqualValue(v1, v2 reflect.Value) bool {
function deepEqualPtr (line 66) | func deepEqualPtr(v1, v2 reflect.Value) bool {
function deepEqualStruct (line 73) | func deepEqualStruct(v1, v2 reflect.Value) bool {
function deepEqualSlice (line 95) | func deepEqualSlice(v1, v2 reflect.Value) bool {
function deepEqualBytes (line 122) | func deepEqualBytes(s1, s2 protocol.Bytes) bool {
function deepEqualRecords (line 152) | func deepEqualRecords(r1, r2 protocol.RecordReader) bool {
function deepEqualRecord (line 167) | func deepEqualRecord(r1, r2 *protocol.Record) bool {
FILE: protocol/prototest/reflect.go
function closeMessage (line 17) | func closeMessage(m protocol.Message) {
function load (line 37) | func load(v interface{}) (reset func()) {
function loadValue (line 41) | func loadValue(v reflect.Value) (reset func()) {
function forEachField (line 70) | func forEachField(v reflect.Value, do func(reflect.Value)) {
type memoryRecord (line 94) | type memoryRecord struct
method Record (line 102) | func (m *memoryRecord) Record() protocol.Record {
function makeRecords (line 112) | func makeRecords(memoryRecords []memoryRecord) []protocol.Record {
function loadRecords (line 120) | func loadRecords(r protocol.RecordReader) []memoryRecord {
function readAll (line 141) | func readAll(bytes protocol.Bytes) []byte {
FILE: protocol/prototest/request.go
function TestRequest (line 14) | func TestRequest(t *testing.T, version int16, msg protocol.Message) {
function TestRequestWithOverride (line 51) | func TestRequestWithOverride(t *testing.T, version int16, msg protocol.M...
function BenchmarkRequest (line 82) | func BenchmarkRequest(b *testing.B, version int16, msg protocol.Message) {
FILE: protocol/prototest/response.go
function TestResponse (line 14) | func TestResponse(t *testing.T, version int16, msg protocol.Message) {
function BenchmarkResponse (line 44) | func BenchmarkResponse(b *testing.B, version int16, msg protocol.Message) {
FILE: protocol/rawproduce/rawproduce.go
function init (line 10) | func init() {
type Request (line 16) | type Request struct
method ApiKey (line 23) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.Produce }
method TypeKey (line 25) | func (r *Request) TypeKey() protocol.OverrideTypeKey { return protocol...
method Broker (line 27) | func (r *Request) Broker(cluster protocol.Cluster) (protocol.Broker, e...
method HasResponse (line 59) | func (r *Request) HasResponse() bool {
type RequestTopic (line 63) | type RequestTopic struct
type RequestPartition (line 68) | type RequestPartition struct
type Error (line 77) | type Error struct
method Error (line 85) | func (e *Error) Error() string {
method Unwrap (line 89) | func (e *Error) Unwrap() error {
function NewError (line 81) | func NewError(err error) *Error {
FILE: protocol/rawproduce/rawproduce_test.go
constant v0 (line 14) | v0 = 0
constant v3 (line 15) | v3 = 3
constant v5 (line 16) | v5 = 5
function TestRawProduceRequest (line 19) | func TestRawProduceRequest(t *testing.T) {
function NewRawRecordSet (line 133) | func NewRawRecordSet(reader protocol.RecordReader, version int8, attr pr...
function BenchmarkProduceRequest (line 143) | func BenchmarkProduceRequest(b *testing.B) {
FILE: protocol/record.go
type Attributes (line 14) | type Attributes
method Compression (line 25) | func (a Attributes) Compression() compress.Compression {
method Transactional (line 29) | func (a Attributes) Transactional() bool {
method Control (line 33) | func (a Attributes) Control() bool {
method String (line 37) | func (a Attributes) String() string {
constant Gzip (line 17) | Gzip Attributes = Attributes(compress.Gzip)
constant Snappy (line 18) | Snappy Attributes = Attributes(compress.Snappy)
constant Lz4 (line 19) | Lz4 Attributes = Attributes(compress.Lz4)
constant Zstd (line 20) | Zstd Attributes = Attributes(compress.Zstd)
constant Transactional (line 21) | Transactional Attributes = 1 << 4
constant Control (line 22) | Control Attributes = 1 << 5
type Header (line 49) | type Header struct
type Record (line 57) | type Record struct
type RecordSet (line 86) | type RecordSet struct
method ReadFrom (line 137) | func (rs *RecordSet) ReadFrom(r io.Reader) (int64, error) {
method WriteTo (line 243) | func (rs *RecordSet) WriteTo(w io.Writer) (int64, error) {
type bufferedReader (line 118) | type bufferedReader interface
type bytesBuffer (line 126) | type bytesBuffer interface
constant magicByteOffset (line 132) | magicByteOffset = 16
type RawRecordSet (line 297) | type RawRecordSet struct
method ReadFrom (line 309) | func (rrs *RawRecordSet) ReadFrom(r io.Reader) (int64, error) {
method WriteTo (line 327) | func (rrs *RawRecordSet) WriteTo(w io.Writer) (int64, error) {
function makeTime (line 335) | func makeTime(t int64) time.Time {
function timestamp (line 339) | func timestamp(t time.Time) int64 {
function packUint32 (line 346) | func packUint32(u uint32) (b [4]byte) {
function packUint64 (line 351) | func packUint64(u uint64) (b [8]byte) {
FILE: protocol/record_batch.go
type RecordReader (line 14) | type RecordReader interface
function NewRecordReader (line 25) | func NewRecordReader(records ...Record) RecordReader {
function MultiRecordReader (line 37) | func MultiRecordReader(batches ...RecordReader) RecordReader {
function forEachRecord (line 50) | func forEachRecord(r RecordReader, f func(int, *Record) error) error {
function handleRecord (line 67) | func handleRecord(i int, r *Record, f func(int, *Record) error) error {
type recordReader (line 77) | type recordReader struct
method ReadRecord (line 82) | func (r *recordReader) ReadRecord() (*Record, error) {
type multiRecordReader (line 90) | type multiRecordReader struct
method ReadRecord (line 95) | func (m *multiRecordReader) ReadRecord() (*Record, error) {
type optimizedRecordReader (line 113) | type optimizedRecordReader struct
method ReadRecord (line 120) | func (r *optimizedRecordReader) ReadRecord() (*Record, error) {
type optimizedRecord (line 138) | type optimizedRecord struct
method time (line 145) | func (r *optimizedRecord) time() time.Time {
method key (line 149) | func (r *optimizedRecord) key() Bytes {
method value (line 153) | func (r *optimizedRecord) value() Bytes {
function makeBytes (line 157) | func makeBytes(ref *pageRef) Bytes {
type emptyRecordReader (line 164) | type emptyRecordReader struct
method ReadRecord (line 166) | func (emptyRecordReader) ReadRecord() (*Record, error) { return nil, i...
type ControlRecord (line 169) | type ControlRecord struct
method Key (line 214) | func (cr *ControlRecord) Key() Bytes {
method Value (line 221) | func (cr *ControlRecord) Value() Bytes {
method Record (line 225) | func (cr *ControlRecord) Record() Record {
function ReadControlRecord (line 178) | func ReadControlRecord(r *Record) (*ControlRecord, error) {
type ControlBatch (line 237) | type ControlBatch struct
method ReadRecord (line 259) | func (c *ControlBatch) ReadRecord() (*Record, error) {
method ReadControlRecord (line 263) | func (c *ControlBatch) ReadControlRecord() (*ControlRecord, error) {
method Offset (line 277) | func (c *ControlBatch) Offset() int64 {
method Version (line 281) | func (c *ControlBatch) Version() int {
function NewControlBatch (line 249) | func NewControlBatch(records ...ControlRecord) *ControlBatch {
type RecordBatch (line 287) | type RecordBatch struct
method ReadRecord (line 297) | func (r *RecordBatch) ReadRecord() (*Record, error) {
method Offset (line 301) | func (r *RecordBatch) Offset() int64 {
method Version (line 305) | func (r *RecordBatch) Version() int {
type MessageSet (line 311) | type MessageSet struct
method ReadRecord (line 317) | func (m *MessageSet) ReadRecord() (*Record, error) {
method Offset (line 321) | func (m *MessageSet) Offset() int64 {
method Version (line 325) | func (m *MessageSet) Version() int {
type RecordStream (line 332) | type RecordStream struct
method ReadRecord (line 337) | func (s *RecordStream) ReadRecord() (*Record, error) {
FILE: protocol/record_batch_test.go
type memoryRecord (line 11) | type memoryRecord struct
method Record (line 19) | func (m *memoryRecord) Record() Record {
function makeRecords (line 29) | func makeRecords(memoryRecords []memoryRecord) []Record {
function TestRecordReader (line 37) | func TestRecordReader(t *testing.T) {
function TestMultiRecordReader (line 67) | func TestMultiRecordReader(t *testing.T) {
function TestControlRecord (line 100) | func TestControlRecord(t *testing.T) {
function assertRecords (line 141) | func assertRecords(t *testing.T, r1, r2 RecordReader) {
function equalRecords (line 165) | func equalRecords(r1, r2 *Record) bool {
function readAll (line 191) | func readAll(bytes Bytes) []byte {
FILE: protocol/record_v1.go
function readMessage (line 11) | func readMessage(b *pageBuffer, d *decoder) (attributes int8, baseOffset...
method readFromVersion1 (line 55) | func (rs *RecordSet) readFromVersion1(d *decoder) error {
method writeToVersion1 (line 152) | func (rs *RecordSet) writeToVersion1(buffer *pageBuffer, bufferOffset in...
type message (line 232) | type message struct
method ReadRecord (line 237) | func (m *message) ReadRecord() (*Record, error) {
FILE: protocol/record_v2.go
method readFromVersion2 (line 10) | func (rs *RecordSet) readFromVersion2(d *decoder) error {
method writeToVersion2 (line 191) | func (rs *RecordSet) writeToVersion2(buffer *pageBuffer, bufferOffset in...
FILE: protocol/reflect.go
type index (line 10) | type index
type _type (line 12) | type _type struct
function typeOf (line 14) | func typeOf(x interface{}) _type {
function elemTypeOf (line 18) | func elemTypeOf(x interface{}) _type {
function makeType (line 22) | func makeType(t reflect.Type) _type {
type value (line 26) | type value struct
method bool (line 38) | func (v value) bool() bool { return v.val.Bool() }
method int8 (line 40) | func (v value) int8() int8 { return int8(v.int64()) }
method int16 (line 42) | func (v value) int16() int16 { return int16(v.int64()) }
method int32 (line 44) | func (v value) int32() int32 { return int32(v.int64()) }
method int64 (line 46) | func (v value) int64() int64 { return v.val.Int() }
method float64 (line 48) | func (v value) float64() float64 { return v.val.Float() }
method string (line 50) | func (v value) string() string { return v.val.String() }
method bytes (line 52) | func (v value) bytes() []byte { return v.val.Bytes() }
method iface (line 54) | func (v value) iface(t reflect.Type) interface{} { return v.val.Addr()...
method array (line 56) | func (v value) array(t reflect.Type) array { return array(v) }
method setBool (line 58) | func (v value) setBool(b bool) { v.val.SetBool(b) }
method setInt8 (line 60) | func (v value) setInt8(i int8) { v.setInt64(int64(i)) }
method setInt16 (line 62) | func (v value) setInt16(i int16) { v.setInt64(int64(i)) }
method setInt32 (line 64) | func (v value) setInt32(i int32) { v.setInt64(int64(i)) }
method setInt64 (line 66) | func (v value) setInt64(i int64) { v.val.SetInt(i) }
method setFloat64 (line 68) | func (v value) setFloat64(f float64) { v.val.SetFloat(f) }
method setString (line 70) | func (v value) setString(s string) { v.val.SetString(s) }
method setBytes (line 72) | func (v value) setBytes(b []byte) { v.val.SetBytes(b) }
method setArray (line 74) | func (v value) setArray(a array) {
method fieldByIndex (line 82) | func (v value) fieldByIndex(i index) value {
function nonAddressableValueOf (line 30) | func nonAddressableValueOf(x interface{}) value {
function valueOf (line 34) | func valueOf(x interface{}) value {
type array (line 86) | type array struct
method index (line 94) | func (a array) index(i int) value { return value{val: a.val.Index(i)} }
method length (line 96) | func (a array) length() int { return a.val.Len() }
method isNil (line 98) | func (a array) isNil() bool { return a.val.IsNil() }
function makeArray (line 90) | func makeArray(t reflect.Type, n int) array {
function indexOf (line 100) | func indexOf(s reflect.StructField) index { return index(s.Index) }
function bytesToString (line 102) | func bytesToString(b []byte) string { return string(b) }
FILE: protocol/reflect_unsafe.go
type iface (line 11) | type iface struct
type slice (line 16) | type slice struct
type index (line 22) | type index
type _type (line 24) | type _type struct
function typeOf (line 28) | func typeOf(x interface{}) _type {
function elemTypeOf (line 32) | func elemTypeOf(x interface{}) _type {
function makeType (line 36) | func makeType(t reflect.Type) _type {
type value (line 40) | type value struct
method bool (line 56) | func (v value) bool() bool { return *(*bool)(v.ptr) }
method int8 (line 58) | func (v value) int8() int8 { return *(*int8)(v.ptr) }
method int16 (line 60) | func (v value) int16() int16 { return *(*int16)(v.ptr) }
method int32 (line 62) | func (v value) int32() int32 { return *(*int32)(v.ptr) }
method int64 (line 64) | func (v value) int64() int64 { return *(*int64)(v.ptr) }
method float64 (line 66) | func (v value) float64() float64 { return *(*float64)(v.ptr) }
method string (line 68) | func (v value) string() string { return *(*string)(v.ptr) }
method bytes (line 70) | func (v value) bytes() []byte { return *(*[]byte)(v.ptr) }
method iface (line 72) | func (v value) iface(t reflect.Type) interface{} {
method array (line 79) | func (v value) array(t reflect.Type) array {
method setBool (line 87) | func (v value) setBool(b bool) { *(*bool)(v.ptr) = b }
method setInt8 (line 89) | func (v value) setInt8(i int8) { *(*int8)(v.ptr) = i }
method setInt16 (line 91) | func (v value) setInt16(i int16) { *(*int16)(v.ptr) = i }
method setInt32 (line 93) | func (v value) setInt32(i int32) { *(*int32)(v.ptr) = i }
method setInt64 (line 95) | func (v value) setInt64(i int64) { *(*int64)(v.ptr) = i }
method setFloat64 (line 97) | func (v value) setFloat64(f float64) { *(*float64)(v.ptr) = f }
method setString (line 99) | func (v value) setString(s string) { *(*string)(v.ptr) = s }
method setBytes (line 101) | func (v value) setBytes(b []byte) { *(*[]byte)(v.ptr) = b }
method setArray (line 103) | func (v value) setArray(a array) { *(*slice)(v.ptr) = slice{ptr: a.ele...
method fieldByIndex (line 105) | func (v value) fieldByIndex(i index) value {
function nonAddressableValueOf (line 44) | func nonAddressableValueOf(x interface{}) value {
function valueOf (line 48) | func valueOf(x interface{}) value {
function makeValue (line 52) | func makeValue(t reflect.Type) value {
type array (line 109) | type array struct
method index (line 130) | func (a array) index(i int) value {
method length (line 134) | func (a array) length() int { return a.len }
method isNil (line 136) | func (a array) isNil() bool { return a.elem == nil }
function makeArray (line 119) | func makeArray(t reflect.Type, n int) array {
function indexOf (line 138) | func indexOf(s reflect.StructField) index { return index(s.Offset) }
function bytesToString (line 140) | func bytesToString(b []byte) string { return *(*string)(unsafe.Pointer(&...
function unsafe_NewArray (line 143) | func unsafe_NewArray(rtype unsafe.Pointer, length int) unsafe.Pointer
FILE: protocol/request.go
function ReadRequest (line 8) | func ReadRequest(r io.Reader) (apiVersion int16, correlationID int32, cl...
function WriteRequest (line 72) | func WriteRequest(w io.Writer, apiVersion int16, correlationID int32, cl...
FILE: protocol/response.go
function ReadResponse (line 11) | func ReadResponse(r io.Reader, apiKey ApiKey, apiVersion int16) (correla...
function WriteResponse (line 86) | func WriteResponse(w io.Writer, apiVersion int16, correlationID int32, m...
constant tlsAlertByte (line 142) | tlsAlertByte byte = 0x15
function looksLikeUnexpectedTLS (line 148) | func looksLikeUnexpectedTLS(size int32) bool {
FILE: protocol/response_test.go
function TestReadResponseUnexpectedTLSDetection (line 11) | func TestReadResponseUnexpectedTLSDetection(t *testing.T) {
FILE: protocol/roundtrip.go
function RoundTrip (line 8) | func RoundTrip(rw io.ReadWriter, apiVersion int16, correlationID int32, ...
function hasResponse (line 25) | func hasResponse(msg Message) bool {
FILE: protocol/saslauthenticate/saslauthenticate.go
function init (line 10) | func init() {
type Request (line 14) | type Request struct
method RawExchange (line 18) | func (r *Request) RawExchange(rw io.ReadWriter) (protocol.Message, err...
method Required (line 25) | func (*Request) Required(versions map[protocol.ApiKey]int16) bool {
method writeTo (line 30) | func (r *Request) writeTo(w io.Writer) error {
method readResp (line 39) | func (r *Request) readResp(read io.Reader) (protocol.Message, error) {
method ApiKey (line 55) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.SaslAuthe...
type Response (line 57) | type Response struct
method ApiKey (line 64) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.SaslAuth...
FILE: protocol/saslhandshake/saslhandshake.go
function init (line 5) | func init() {
type Request (line 9) | type Request struct
method ApiKey (line 13) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.SaslHands...
type Response (line 15) | type Response struct
method ApiKey (line 20) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.SaslHand...
FILE: protocol/size.go
function sizeOfVarString (line 7) | func sizeOfVarString(s string) int {
function sizeOfVarNullBytes (line 11) | func sizeOfVarNullBytes(b []byte) int {
function sizeOfVarNullBytesIface (line 19) | func sizeOfVarNullBytesIface(b Bytes) int {
function sizeOfVarInt (line 27) | func sizeOfVarInt(i int64) int {
function sizeOfUnsignedVarInt (line 31) | func sizeOfUnsignedVarInt(i uint64) int {
FILE: protocol/syncgroup/syncgroup.go
function init (line 5) | func init() {
type Request (line 9) | type Request struct
method ApiKey (line 32) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.SyncGroup }
method Group (line 34) | func (r *Request) Group() string { return r.GroupID }
type RequestAssignment (line 23) | type RequestAssignment struct
type Response (line 38) | type Response struct
method ApiKey (line 50) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.SyncGroup }
FILE: protocol/syncgroup/syncgroup_test.go
function TestSyncGroupReq (line 10) | func TestSyncGroupReq(t *testing.T) {
function TestSyncGroupResp (line 63) | func TestSyncGroupResp(t *testing.T) {
FILE: protocol/txnoffsetcommit/txnoffsetcommit.go
function init (line 5) | func init() {
type Request (line 9) | type Request struct
method ApiKey (line 44) | func (r *Request) ApiKey() protocol.ApiKey { return protocol.TxnOffset...
method Group (line 46) | func (r *Request) Group() string { return r.GroupID }
type RequestTopic (line 24) | type RequestTopic struct
type RequestPartition (line 33) | type RequestPartition struct
type Response (line 50) | type Response struct
method ApiKey (line 77) | func (r *Response) ApiKey() protocol.ApiKey { return protocol.TxnOffse...
type ResponseTopic (line 59) | type ResponseTopic struct
type ResponsePartition (line 68) | type ResponsePartition struct
FILE: protocol/txnoffsetcommit/txnoffsetcommit_test.go
function TestTxnOffsetCommitRequest (line 10) | func TestTxnOffsetCommitRequest(t *testing.T) {
function TestTxnOffsetCommitResponse (line 152) | func TestTxnOffsetCommitResponse(t *testing.T) {
FILE: protocol_test.go
function TestApiVersionsFormat (line 11) | func TestApiVersionsFormat(t *testing.T) {
function TestProtocol (line 34) | func TestProtocol(t *testing.T) {
FILE: rawproduce.go
type RawProduceRequest (line 16) | type RawProduceRequest struct
method RawProduce (line 50) | func (c *Client) RawProduce(ctx context.Context, req *RawProduceRequest)...
FILE: rawproduce_test.go
function TestClientRawProduce (line 13) | func TestClientRawProduce(t *testing.T) {
function TestClientRawProduceCompressed (line 48) | func TestClientRawProduceCompressed(t *testing.T) {
function TestClientRawProduceNilRecords (line 83) | func TestClientRawProduceNilRecords(t *testing.T) {
function TestClientRawProduc
Condensed preview — 366 files, each showing path, character count, and a content snippet. Download the .json file or copy for the full structured content (1,529K chars).
[
{
"path": ".circleci/config.yml",
"chars": 8380,
"preview": "version: 2\njobs:\n lint:\n docker:\n - image: golangci/golangci-lint:v1.45-alpine\n steps:\n - checkout\n "
},
{
"path": ".gitattributes",
"chars": 22,
"preview": "fixtures/*.hex binary\n"
},
{
"path": ".github/ISSUE_TEMPLATE/bug_report.md",
"chars": 1416,
"preview": "---\nname: Bug report\nabout: Create a report to help us improve\ntitle: ''\nlabels: bug\nassignees: ''\n\n---\n\n**Describe the "
},
{
"path": ".github/ISSUE_TEMPLATE/feature_request.md",
"chars": 320,
"preview": "---\nname: Feature request\nabout: Suggest an idea for this project\ntitle: ''\nlabels: enhancement\nassignees: ''\n\n---\n\n**De"
},
{
"path": ".gitignore",
"chars": 357,
"preview": "# Compiled Object files, Static and Dynamic libs (Shared Objects)\n*.o\n*.a\n*.so\n\n# Folders\n_obj\n_test\n\n# Architecture spe"
},
{
"path": ".golangci.yml",
"chars": 285,
"preview": "linters:\n enable:\n - bodyclose\n - errorlint\n - goconst\n - godot\n - gofmt\n - goimports\n - prealloc\n"
},
{
"path": "CODE_OF_CONDUCT.md",
"chars": 3409,
"preview": "# Contributor Covenant Code of Conduct\n\n## Our Pledge\n\nIn the interest of fostering an open and welcoming environment, w"
},
{
"path": "CONTRIBUTING.md",
"chars": 4742,
"preview": "# Contributing to kafka-go\n\nkafka-go is an open source project. We welcome contributions to kafka-go of any kind includ"
},
{
"path": "LICENSE",
"chars": 1064,
"preview": "MIT License\n\nCopyright (c) 2017 Segment\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof"
},
{
"path": "Makefile",
"chars": 112,
"preview": "test:\n\tKAFKA_SKIP_NETTEST=1 \\\n\tKAFKA_VERSION=2.3.1 \\\n\tgo test -race -cover ./...\n\ndocker:\n\tdocker compose up -d\n"
},
{
"path": "README.md",
"chars": 23199,
"preview": "# kafka-go [](https://circleci.com/gh/segmentio/"
},
{
"path": "addoffsetstotxn.go",
"chars": 1824,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/addoffsetstotxn\"\n)\n\n/"
},
{
"path": "addoffsetstotxn_test.go",
"chars": 2958,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"log\"\n\t\"os\"\n\t\"testing\"\n\t\"time\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n"
},
{
"path": "addpartitionstotxn.go",
"chars": 3210,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/addpartitionstotxn\"\n)"
},
{
"path": "addpartitionstotxn_test.go",
"chars": 2996,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"net\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\tktesting \"github.com/segmentio/kafka-go/testin"
},
{
"path": "address.go",
"chars": 1357,
"preview": "package kafka\n\nimport (\n\t\"net\"\n\t\"strings\"\n)\n\n// TCP constructs an address with the network set to \"tcp\".\nfunc TCP(addres"
},
{
"path": "address_test.go",
"chars": 1033,
"preview": "package kafka\n\nimport (\n\t\"net\"\n\t\"testing\"\n)\n\nfunc TestNetworkAddress(t *testing.T) {\n\ttests := []struct {\n\t\taddr net."
},
{
"path": "alterclientquotas.go",
"chars": 3643,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/alterclientquotas\"\n)\n"
},
{
"path": "alterclientquotas_test.go",
"chars": 2099,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n\t\"time\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n\t\"github.com/st"
},
{
"path": "alterconfigs.go",
"chars": 2816,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/alterconfigs\"\n)\n\n// A"
},
{
"path": "alterconfigs_test.go",
"chars": 1485,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n\t\"github.com/stretchr/t"
},
{
"path": "alterpartitionreassignments.go",
"chars": 4006,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/alterpartitionreassignments\""
},
{
"path": "alterpartitionreassignments_test.go",
"chars": 2375,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n\t\"time\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc TestCli"
},
{
"path": "alteruserscramcredentials.go",
"chars": 3216,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/alteruserscramcredent"
},
{
"path": "alteruserscramcredentials_test.go",
"chars": 1756,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc TestAlterUserSc"
},
{
"path": "apiversions.go",
"chars": 1711,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"net\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n\t\"github.com/segmentio/kafka-go/pro"
},
{
"path": "apiversions_test.go",
"chars": 520,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n)\n\nfunc TestClientApiVersions(t *testing.T) {\n\tctx := context.Background()"
},
{
"path": "balancer.go",
"chars": 11136,
"preview": "package kafka\n\nimport (\n\t\"hash\"\n\t\"hash/crc32\"\n\t\"hash/fnv\"\n\t\"math/rand\"\n\t\"sort\"\n\t\"sync\"\n)\n\n// The Balancer interface prov"
},
{
"path": "balancer_test.go",
"chars": 11980,
"preview": "package kafka\n\nimport (\n\t\"fmt\"\n\t\"hash\"\n\t\"hash/crc32\"\n\t\"testing\"\n)\n\nfunc TestHashBalancer(t *testing.T) {\n\ttestCases := m"
},
{
"path": "batch.go",
"chars": 9419,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"errors\"\n\t\"io\"\n\t\"sync\"\n\t\"time\"\n)\n\n// A Batch is an iterator over a sequence of message"
},
{
"path": "batch_test.go",
"chars": 929,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"io\"\n\t\"net\"\n\t\"strconv\"\n\t\"testing\"\n)\n\nfunc TestBatchDontExpectEOF(t *testin"
},
{
"path": "buffer.go",
"chars": 380,
"preview": "package kafka\n\nimport (\n\t\"bytes\"\n\t\"sync\"\n)\n\nvar bufferPool = sync.Pool{\n\tNew: func() interface{} { return newBuffer() },"
},
{
"path": "builder_test.go",
"chars": 8330,
"preview": "package kafka\n\nimport (\n\t\"bytes\"\n\t\"fmt\"\n\t\"io\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/compress\"\n)\n\n// This file defines"
},
{
"path": "client.go",
"chars": 4102,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n)\n\nconst ("
},
{
"path": "client_test.go",
"chars": 7335,
"preview": "package kafka\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"io\"\n\t\"math/rand\"\n\t\"net\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/segment"
},
{
"path": "commit.go",
"chars": 1031,
"preview": "package kafka\n\n// A commit represents the instruction of publishing an update of the last\n// offset read by a program fo"
},
{
"path": "commit_test.go",
"chars": 513,
"preview": "package kafka\n\nimport \"testing\"\n\nfunc TestMakeCommit(t *testing.T) {\n\tmsg := Message{\n\t\tTopic: \"blah\",\n\t\tPartition: "
},
{
"path": "compress/compress.go",
"chars": 2733,
"preview": "package compress\n\nimport (\n\t\"encoding\"\n\t\"fmt\"\n\t\"io\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/segmentio/kafka-go/compress/gzip"
},
{
"path": "compress/compress_test.go",
"chars": 12207,
"preview": "package compress_test\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"io\"\n\t\"io/ioutil\"\n\t\"math/rand\"\n\t\"net\"\n\t\"os\"\n\t\"path/filepath\""
},
{
"path": "compress/gzip/gzip.go",
"chars": 2754,
"preview": "package gzip\n\nimport (\n\t\"io\"\n\t\"sync\"\n\n\t\"github.com/klauspost/compress/gzip\"\n)\n\nvar (\n\treaderPool sync.Pool\n)\n\n// Codec i"
},
{
"path": "compress/lz4/lz4.go",
"chars": 1313,
"preview": "package lz4\n\nimport (\n\t\"io\"\n\t\"sync\"\n\n\t\"github.com/pierrec/lz4/v4\"\n)\n\nvar (\n\treaderPool sync.Pool\n\twriterPool sync.Pool\n)"
},
{
"path": "compress/snappy/go-xerial-snappy/LICENSE",
"chars": 1076,
"preview": "The MIT License (MIT)\n\nCopyright (c) 2016 Evan Huus\n\nPermission is hereby granted, free of charge, to any person obtaini"
},
{
"path": "compress/snappy/go-xerial-snappy/README.md",
"chars": 528,
"preview": "# go-xerial-snappy\n\n[](https://travis-c"
},
{
"path": "compress/snappy/go-xerial-snappy/corpus/05979b224be0294bf350310d4ba5257c9bb815db-3",
"chars": 4,
"preview": "���Y"
},
{
"path": "compress/snappy/go-xerial-snappy/corpus/521e7e67b6063a75e0eeb24b0d1dd20731d34ad8-4",
"chars": 5,
"preview": "����Y"
},
{
"path": "compress/snappy/go-xerial-snappy/corpus/581b8fe7088f921567811fdf30e1f527c9f48e5e",
"chars": 8,
"preview": "package "
},
{
"path": "compress/snappy/go-xerial-snappy/corpus/80881d1b911b95e0203b3b0e7dc6360c35f7620f-7",
"chars": 10,
"preview": "墳￩��￩￩���꿽"
},
{
"path": "compress/snappy/go-xerial-snappy/corpus/b2419fcb7a9aef359de67cb6bd2b8a8c1f5c100f-4",
"chars": 3,
"preview": "��Y"
},
{
"path": "compress/snappy/go-xerial-snappy/corpus/cb806bc4f67316af02d6ae677332a3b6005a18da-5",
"chars": 3,
"preview": "墳�濽"
},
{
"path": "compress/snappy/go-xerial-snappy/corpus/ce3c6f4c31f74d72fbf74c17d14a8d29aa62059e-6",
"chars": 6,
"preview": "墳￩���꿽"
},
{
"path": "compress/snappy/go-xerial-snappy/corpus/da39a3ee5e6b4b0d3255bfef95601890afd80709-1",
"chars": 0,
"preview": ""
},
{
"path": "compress/snappy/go-xerial-snappy/fuzz.go",
"chars": 211,
"preview": "// +build gofuzz\n\npackage snappy\n\nfunc Fuzz(data []byte) int {\n\tdecode, err := Decode(data)\n\tif decode == nil && err == "
},
{
"path": "compress/snappy/go-xerial-snappy/snappy.go",
"chars": 3139,
"preview": "package snappy\n\nimport (\n\t\"bytes\"\n\t\"encoding/binary\"\n\t\"errors\"\n\n\tmaster \"github.com/klauspost/compress/snappy\"\n)\n\nconst "
},
{
"path": "compress/snappy/go-xerial-snappy/snappy_test.go",
"chars": 11232,
"preview": "package snappy\n\nimport (\n\t\"bytes\"\n\t\"errors\"\n\t\"testing\"\n)\n\nconst largeString = `Sed ut perspiciatis unde omnis iste natus"
},
{
"path": "compress/snappy/snappy.go",
"chars": 2219,
"preview": "package snappy\n\nimport (\n\t\"io\"\n\t\"sync\"\n\n\t\"github.com/klauspost/compress/s2\"\n\t\"github.com/klauspost/compress/snappy\"\n)\n\n/"
},
{
"path": "compress/snappy/xerial.go",
"chars": 6214,
"preview": "package snappy\n\nimport (\n\t\"bytes\"\n\t\"encoding/binary\"\n\t\"errors\"\n\t\"io\"\n\n\t\"github.com/klauspost/compress/snappy\"\n)\n\nconst d"
},
{
"path": "compress/snappy/xerial_test.go",
"chars": 4591,
"preview": "package snappy\n\nimport (\n\t\"bytes\"\n\t\"crypto/rand\"\n\t\"io\"\n\t\"testing\"\n\n\t\"github.com/klauspost/compress/snappy\"\n\tgoxerialsnap"
},
{
"path": "compress/zstd/zstd.go",
"chars": 3507,
"preview": "// Package zstd implements Zstandard compression.\npackage zstd\n\nimport (\n\t\"io\"\n\t\"sync\"\n\n\t\"github.com/klauspost/compress/"
},
{
"path": "compression.go",
"chars": 645,
"preview": "package kafka\n\nimport (\n\t\"errors\"\n\n\t\"github.com/segmentio/kafka-go/compress\"\n)\n\ntype Compression = compress.Compression\n"
},
{
"path": "conn.go",
"chars": 46888,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"net\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"t"
},
{
"path": "conn_test.go",
"chars": 31780,
"preview": "package kafka\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math/rand\"\n\t\"net\"\n\t\"os\"\n\t\"strconv\"\n\t\"testing\"\n\t\"tim"
},
{
"path": "consumergroup.go",
"chars": 41221,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"net\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\""
},
{
"path": "consumergroup_test.go",
"chars": 20631,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n)\n\nvar _ coordi"
},
{
"path": "crc32.go",
"chars": 1010,
"preview": "package kafka\n\nimport (\n\t\"encoding/binary\"\n\t\"hash/crc32\"\n)\n\ntype crc32Writer struct {\n\ttable *crc32.Table\n\tbuffer [8]by"
},
{
"path": "crc32_test.go",
"chars": 517,
"preview": "package kafka\n\nimport (\n\t\"bytes\"\n\t\"hash/crc32\"\n\t\"testing\"\n)\n\nfunc TestMessageCRC32(t *testing.T) {\n\tm := message{\n\t\tMagi"
},
{
"path": "createacls.go",
"chars": 6277,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/createacls"
},
{
"path": "createacls_test.go",
"chars": 2021,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc TestClientCreat"
},
{
"path": "createpartitions.go",
"chars": 2812,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/createpartitions\"\n)\n\n"
},
{
"path": "createpartitions_test.go",
"chars": 1284,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc TestClientCreat"
},
{
"path": "createtopics.go",
"chars": 12482,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/createtopics"
},
{
"path": "createtopics_test.go",
"chars": 3809,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"net\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"testing\"\n)\n\nfunc TestConnC"
},
{
"path": "deleteacls.go",
"chars": 3423,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/deleteacls\"\n)\n\n// Del"
},
{
"path": "deleteacls_test.go",
"chars": 2747,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n\t\"github.com/stretchr/t"
},
{
"path": "deletegroups.go",
"chars": 1662,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/deletegroups\"\n)\n\n// D"
},
{
"path": "deletegroups_test.go",
"chars": 1620,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"testing\"\n\t\"time\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfu"
},
{
"path": "deletetopics.go",
"chars": 5259,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/deletetopics"
},
{
"path": "deletetopics_test.go",
"chars": 997,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"reflect\"\n\t\"testing\"\n)\n\nfunc TestClientDeleteTopics(t *testing.T) "
},
{
"path": "describeacls.go",
"chars": 3114,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/describeacls\"\n)\n\n// D"
},
{
"path": "describeacls_test.go",
"chars": 2050,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n\t\"github.com/stretchr/t"
},
{
"path": "describeclientquotas.go",
"chars": 3735,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/describeclientquotas\""
},
{
"path": "describeconfigs.go",
"chars": 4205,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/describeconfigs\"\n)\n\n/"
},
{
"path": "describeconfigs_test.go",
"chars": 1488,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n\t\"github.com/stretchr/t"
},
{
"path": "describegroups.go",
"chars": 8272,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\n\t\"github.com/segmentio/kafka-go/protocol/describegro"
},
{
"path": "describegroups_test.go",
"chars": 2431,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"os\"\n\t\"reflect\"\n\t\"sort\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestClientDescribeGroups(t"
},
{
"path": "describeuserscramcredentials.go",
"chars": 3125,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/describeuserscramcred"
},
{
"path": "describeuserscramcredentials_test.go",
"chars": 3500,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"testing\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n\t\"github.com/"
},
{
"path": "dialer.go",
"chars": 15389,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.c"
},
{
"path": "dialer_test.go",
"chars": 11692,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"crypto/x509\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"net\"\n\t\"reflect\"\n\t\"sort\"\n\t\"testi"
},
{
"path": "discard.go",
"chars": 757,
"preview": "package kafka\n\nimport \"bufio\"\n\nfunc discardN(r *bufio.Reader, sz int, n int) (int, error) {\n\tvar err error\n\tif n <= sz {"
},
{
"path": "discard_test.go",
"chars": 2354,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"errors\"\n\t\"io\"\n\t\"testing\"\n)\n\nfunc TestDiscardN(t *testing.T) {\n\ttests := []st"
},
{
"path": "docker-compose.yml",
"chars": 1947,
"preview": "# See https://hub.docker.com/r/bitnami/kafka/tags for the complete list.\nversion: '3'\nservices:\n zookeeper:\n contain"
},
{
"path": "docker_compose_versions/README.md",
"chars": 7809,
"preview": "# Bitnami Kafka\n\nThis document outlines how to create a docker-compose file for a specific Bitnami Kafka version.\n\n\n## S"
},
{
"path": "docker_compose_versions/docker-compose-270.yml",
"chars": 1953,
"preview": "# See https://hub.docker.com/r/bitnamilegacy/kafka/tags for the complete list.\nversion: '3'\nservices:\n zookeeper:\n c"
},
{
"path": "docker_compose_versions/docker-compose-370.yml",
"chars": 1953,
"preview": "# See https://hub.docker.com/r/bitnamilegacy/kafka/tags for the complete list.\nversion: '3'\nservices:\n zookeeper:\n c"
},
{
"path": "docker_compose_versions/docker-compose-400.yml",
"chars": 2060,
"preview": "# See https://hub.docker.com/r/bitnamilegacy/kafka/tags for the complete list.\nversion: '3'\nservices:\n kafka:\n conta"
},
{
"path": "electleaders.go",
"chars": 2277,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/electleaders\"\n)\n\n// ElectLea"
},
{
"path": "electleaders_test.go",
"chars": 866,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc TestClientElect"
},
{
"path": "endtxn.go",
"chars": 1659,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/endtxn\"\n)\n\n// EndTxnR"
},
{
"path": "error.go",
"chars": 27360,
"preview": "package kafka\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"syscall\"\n)\n\n// Error represents the different error codes that may be r"
},
{
"path": "error_test.go",
"chars": 3115,
"preview": "package kafka\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestError(t *testing.T) {\n\terro"
},
{
"path": "example_consumergroup_test.go",
"chars": 2271,
"preview": "package kafka_test\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"fmt\"\n\t\"os\"\n\n\t\"github.com/segmentio/kafka-go\"\n)\n\nfunc ExampleGenerati"
},
{
"path": "example_groupbalancer_test.go",
"chars": 3188,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"io/ioutil\"\n\t\"net/http\"\n\t\"os\"\n\t\"strings\"\n\t\"time\"\n)\n\n// ExampleNewRe"
},
{
"path": "example_writer_test.go",
"chars": 322,
"preview": "package kafka_test\n\nimport (\n\t\"context\"\n\n\t\"github.com/segmentio/kafka-go\"\n)\n\nfunc ExampleWriter() {\n\tw := &kafka.Writer{"
},
{
"path": "examples/.gitignore",
"chars": 8,
"preview": "kafka-go"
},
{
"path": "examples/consumer-logger/Dockerfile",
"chars": 639,
"preview": "#####################################\n# STEP 1 build executable binary #\n#####################################\nFROM g"
},
{
"path": "examples/consumer-logger/go.mod",
"chars": 173,
"preview": "module github.com/segmentio/kafka-go/example/consumer-logger\n\ngo 1.15\n\nrequire (\n\tgithub.com/klauspost/compress v1.12.2 "
},
{
"path": "examples/consumer-logger/go.sum",
"chars": 4398,
"preview": "github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=\ngithub.com/davecgh/go-spew v1.1.0/go.m"
},
{
"path": "examples/consumer-logger/main.go",
"chars": 899,
"preview": "package main\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\t\"strings\"\n\n\tkafka \"github.com/segmentio/kafka-go\"\n)\n\nfunc getKafk"
},
{
"path": "examples/consumer-mongo-db/Dockerfile",
"chars": 639,
"preview": "#####################################\n# STEP 1 build executable binary #\n#####################################\nFROM g"
},
{
"path": "examples/consumer-mongo-db/go.mod",
"chars": 376,
"preview": "module github.com/segmentio/kafka-go/example/consumer-mongo-db\n\ngo 1.15\n\nrequire (\n\tgithub.com/go-stack/stack v1.8.0 // "
},
{
"path": "examples/consumer-mongo-db/go.sum",
"chars": 5132,
"preview": "github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=\ngithub.com/davecgh/go-spew v1.1.0/go.m"
},
{
"path": "examples/consumer-mongo-db/main.go",
"chars": 1629,
"preview": "package main\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\n\t\"github.com/mongodb/mongo-go-driver/mongo\"\n\tkafka \"github.com/se"
},
{
"path": "examples/docker-compose.yaml",
"chars": 1625,
"preview": "version: '2.3'\nservices:\n\n zookeeper:\n hostname: zookeeper\n image: bitnamilegacy/zookeeper:latest\n restart: al"
},
{
"path": "examples/kafka/kafka-variables.env",
"chars": 998,
"preview": "\nKAFKA_CFG_ADVERTISED_HOST_NAME=kafka\nKAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181\nKAFKA_CFG_CONNECT_BOOTSTRAP_SERVERS=loc"
},
{
"path": "examples/producer-api/Dockerfile",
"chars": 639,
"preview": "#####################################\n# STEP 1 build executable binary #\n#####################################\nFROM g"
},
{
"path": "examples/producer-api/go.mod",
"chars": 170,
"preview": "module github.com/segmentio/kafka-go/example/producer-api\n\ngo 1.15\n\nrequire (\n\tgithub.com/klauspost/compress v1.12.2 // "
},
{
"path": "examples/producer-api/go.sum",
"chars": 4398,
"preview": "github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=\ngithub.com/davecgh/go-spew v1.1.0/go.m"
},
{
"path": "examples/producer-api/main.go",
"chars": 1183,
"preview": "package main\n\nimport (\n\t\"fmt\"\n\t\"io/ioutil\"\n\t\"log\"\n\t\"net/http\"\n\t\"os\"\n\n\tkafka \"github.com/segmentio/kafka-go\"\n)\n\nfunc prod"
},
{
"path": "examples/producer-api/test.http",
"chars": 189,
"preview": "### send data text\nPOST http://localhost:8080\nContent-Type: text/plain\n\n\"Hello-api\"\n\n### send data json\nPOST http://loca"
},
{
"path": "examples/producer-random/Dockerfile",
"chars": 639,
"preview": "#####################################\n# STEP 1 build executable binary #\n#####################################\nFROM g"
},
{
"path": "examples/producer-random/go.mod",
"chars": 251,
"preview": "module github.com/segmentio/kafka-go/example/producer-random\n\ngo 1.15\n\nrequire (\n\tgithub.com/davecgh/go-spew v1.1.1 // i"
},
{
"path": "examples/producer-random/go.sum",
"chars": 4650,
"preview": "github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/davecgh/go-spew v1.1"
},
{
"path": "examples/producer-random/main.go",
"chars": 855,
"preview": "package main\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"os\"\n\t\"time\"\n\n\t\"github.com/google/uuid\"\n\tkafka \"github.com/segmentio/kafka-go\""
},
{
"path": "fetch.go",
"chars": 7904,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"math\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n\tfetchAPI \"g"
},
{
"path": "fetch_test.go",
"chars": 5668,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"io\"\n\t\"io/ioutil\"\n\t\"net\"\n\t\"reflect\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/segme"
},
{
"path": "findcoordinator.go",
"chars": 4538,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/findcoordina"
},
{
"path": "findcoordinator_test.go",
"chars": 2157,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"reflect\"\n\t\"strings\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestFind"
},
{
"path": "fixtures/v1-v1.hex",
"chars": 724,
"preview": "000001660000000a00000000000015c79861000000010009746573742d65646779000000010000000000000000000000000004000000000000000400"
},
{
"path": "fixtures/v1-v1c-v2-v2c-v2b-v2b-v2b-v2bc-v1b-v1bc.hex",
"chars": 3440,
"preview": "000006b40000000a00000000000021f08796000000010007746573742d38380000000100000000000000000000000000140000000000000014000000"
},
{
"path": "fixtures/v1c-v1-v1c.hex",
"chars": 1138,
"preview": "000002350000000a0000000000003d15acfe00000001000b746573742d627265657a7900000001000000000000000000000000000600000000000000"
},
{
"path": "fixtures/v1c-v1c.hex",
"chars": 844,
"preview": "000001a20000000a0000000000001abffa5700000001000a746573742d67757473790000000100000000000000000000000000040000000000000004"
},
{
"path": "fixtures/v2-v2.hex",
"chars": 756,
"preview": "000001760000000a0000000000001163921100000001000a746573742d6c756369640000000100000000000000000000000000040000000000000004"
},
{
"path": "fixtures/v2b-v1.hex",
"chars": 740,
"preview": "0000016e0000000a00000000000023f24a1a00000001000b746573742d66656973747900000001000000000000000000000000000400000000000000"
},
{
"path": "fixtures/v2bc-v1-v1c.hex",
"chars": 980,
"preview": "000001e60000000a000000000000530076a100000001000a746573742d68617264790000000100000000000000000000000000060000000000000006"
},
{
"path": "fixtures/v2bc-v1.hex",
"chars": 706,
"preview": "0000015d0000000a0000000000006d36526200000001000a746573742d68617264790000000100000000000000000000000000040000000000000004"
},
{
"path": "fixtures/v2bc-v1c.hex",
"chars": 684,
"preview": "000001520000000a0000000000004aa4215500000001000b746573742d6b61726d696300000001000000000000000000000000000400000000000000"
},
{
"path": "fixtures/v2c-v2-v2c.hex",
"chars": 996,
"preview": "000001ee0000000a000000000000670352ac00000001000a746573742d6e617474790000000100000000000000000000000000060000000000000006"
},
{
"path": "fixtures/v2c-v2c.hex",
"chars": 692,
"preview": "000001560000000a0000000000005698dc5100000001000c746573742d6f6e6569726963000000010000000000000000000000000004000000000000"
},
{
"path": "go.mod",
"chars": 535,
"preview": "module github.com/segmentio/kafka-go\n\ngo 1.23\n\nrequire (\n\tgithub.com/klauspost/compress v1.15.9\n\tgithub.com/pierrec/lz4/"
},
{
"path": "go.sum",
"chars": 4946,
"preview": "github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/davecgh/go-spew v1.1"
},
{
"path": "groupbalancer.go",
"chars": 10351,
"preview": "package kafka\n\nimport (\n\t\"sort\"\n)\n\n// GroupMember describes a single participant in a consumer group.\ntype GroupMember s"
},
{
"path": "groupbalancer_test.go",
"chars": 15537,
"preview": "package kafka\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"testing\"\n)\n\nfunc TestFindMembersByTopic(t *tes"
},
{
"path": "gzip/gzip.go",
"chars": 647,
"preview": "// Package gzip does nothing, it's kept for backward compatibility to avoid\n// breaking the majority of programs that im"
},
{
"path": "heartbeat.go",
"chars": 2602,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\theartbeatAPI \"github.com/segmentio/kafka-go/protocol"
},
{
"path": "heartbeat_test.go",
"chars": 1606,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"log\"\n\t\"os\"\n\t\"reflect\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestClientHeartb"
},
{
"path": "incrementalalterconfigs.go",
"chars": 3657,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"net\"\n\n\t\"github.com/segmentio/kafka-go/protocol/incrementalalterconfigs\"\n)\n\ntype Con"
},
{
"path": "incrementalalterconfigs_test.go",
"chars": 1835,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"reflect\"\n\t\"testing\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc Test"
},
{
"path": "initproducerid.go",
"chars": 2571,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/initproducerid\"\n)\n\n//"
},
{
"path": "initproducerid_test.go",
"chars": 2813,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"net\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\tktesting \"github.com/segmentio/kafka"
},
{
"path": "joingroup.go",
"chars": 10827,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n\t\""
},
{
"path": "joingroup_test.go",
"chars": 6010,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"reflect\"\n\t\"testing\"\n\t\"time\"\n\n\tktesting \"github.com/segm"
},
{
"path": "kafka.go",
"chars": 3115,
"preview": "package kafka\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\n// Broker represents a kafka broker in a kafka cluster.\n"
},
{
"path": "kafka_test.go",
"chars": 3236,
"preview": "package kafka\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"testing\"\n)\n\nfunc TestMarshalUnmarshal(t *testing.T) {\n\tva"
},
{
"path": "leavegroup.go",
"chars": 3920,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/leavegroup\"\n"
},
{
"path": "leavegroup_test.go",
"chars": 4923,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"errors\"\n\t\"reflect\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestClientLeaveGrou"
},
{
"path": "listgroups.go",
"chars": 3682,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"net\"\n\n\t\"github.com/segmentio/kafka-go/protocol/listgroups\"\n)\n\n// ListGroup"
},
{
"path": "listgroups_test.go",
"chars": 1981,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"fmt\"\n\t\"reflect\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestListGroupsResponse"
},
{
"path": "listoffset.go",
"chars": 7323,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/listoffsets\""
},
{
"path": "listoffset_test.go",
"chars": 1997,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestClientListOffsets(t *testing.T) {\n\tclient, topic, shut"
},
{
"path": "listpartitionreassignments.go",
"chars": 3823,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/listpartitionreassignments\"\n"
},
{
"path": "listpartitionreassignments_test.go",
"chars": 988,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n\n\tktesting \"github.com/segmentio/kafka-go/testing\"\n)\n\nfunc TestClientListP"
},
{
"path": "logger.go",
"chars": 504,
"preview": "package kafka\n\n// Logger interface API for log.Logger.\ntype Logger interface {\n\tPrintf(string, ...interface{})\n}\n\n// Log"
},
{
"path": "lz4/lz4.go",
"chars": 395,
"preview": "// Package lz4 does nothing, it's kept for backward compatibility to avoid\n// breaking the majority of programs that imp"
},
{
"path": "message.go",
"chars": 2811,
"preview": "package kafka\n\nimport (\n\t\"time\"\n)\n\n// Message is a data structure representing kafka messages.\ntype Message struct {\n\t//"
},
{
"path": "message_reader.go",
"chars": 14982,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n)\n\ntype readBytesFunc func(*bufio.Reader, int, int) (int, "
},
{
"path": "message_test.go",
"chars": 19287,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"encoding/hex\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math/rand\"\n\t\"os\"\n\t\"testing\"\n\t\"time\"\n\n"
},
{
"path": "metadata.go",
"chars": 7540,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\tmetadataAPI \"github.com/segmentio/kafka-go/protocol/metadata\""
},
{
"path": "metadata_test.go",
"chars": 1906,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"testing\"\n)\n\nfunc TestClientMetadata(t *testing.T) {\n\tclient, topic, shutdown := new"
},
{
"path": "offsetcommit.go",
"chars": 8586,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/offsetcommit"
},
{
"path": "offsetcommit_test.go",
"chars": 3338,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"log\"\n\t\"os\"\n\t\"reflect\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestO"
},
{
"path": "offsetdelete.go",
"chars": 2843,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/offsetdelete\"\n)\n\n// O"
},
{
"path": "offsetdelete_test.go",
"chars": 3367,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"log\"\n\t\"os\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\tktesting \"github.com/segmentio/kafka-go/"
},
{
"path": "offsetfetch.go",
"chars": 7453,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/protocol/offsetfetch\""
},
{
"path": "offsetfetch_test.go",
"chars": 3653,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"context\"\n\t\"reflect\"\n\t\"testing\"\n\t\"time\"\n\n\tktesting \"github.com/segmentio/kafk"
},
{
"path": "produce.go",
"chars": 8545,
"preview": "package kafka\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"encoding\"\n\t\"errors\"\n\t\"fmt\"\n\t\"net\"\n\t\"strconv\"\n\t\"time\"\n\n\t\"github.com/segment"
},
{
"path": "produce_test.go",
"chars": 2850,
"preview": "package kafka\n\nimport (\n\t\"context\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/segmentio/kafka-go/compress\"\n)\n\nfunc Test"
},
{
"path": "protocol/addoffsetstotxn/addoffsetstotxn.go",
"chars": 1068,
"preview": "package addoffsetstotxn\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &"
},
{
"path": "protocol/addoffsetstotxn/addoffsetstotxn_test.go",
"chars": 670,
"preview": "package addoffsetstotxn_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/addoffsetstotxn\"\n\t\"github.co"
},
{
"path": "protocol/addpartitionstotxn/addpartitionstotxn.go",
"chars": 1941,
"preview": "package addpartitionstotxn\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}"
},
{
"path": "protocol/addpartitionstotxn/addpartitionstotxn_test.go",
"chars": 1296,
"preview": "package addpartitionstotxn_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/addpartitionstotxn\"\n\t\"git"
},
{
"path": "protocol/alterclientquotas/alterclientquotas.go",
"chars": 2332,
"preview": "package alterclientquotas\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{},"
},
{
"path": "protocol/alterclientquotas/alterclientquotas_test.go",
"chars": 1751,
"preview": "package alterclientquotas_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/alterclientquotas\"\n\t\"githu"
},
{
"path": "protocol/alterconfigs/alterconfigs.go",
"chars": 1378,
"preview": "package alterconfigs\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Res"
},
{
"path": "protocol/alterconfigs/alterconfigs_test.go",
"chars": 1341,
"preview": "package alterconfigs_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/alterconfigs\"\n\t\"github.com/segm"
},
{
"path": "protocol/alterpartitionreassignments/alterpartitionreassignments.go",
"chars": 1832,
"preview": "package alterpartitionreassignments\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&"
},
{
"path": "protocol/alterpartitionreassignments/alterpartitionreassignments_test.go",
"chars": 1148,
"preview": "package alterpartitionreassignments_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/alterpartitionre"
},
{
"path": "protocol/alteruserscramcredentials/alteruserscramcredentials.go",
"chars": 2152,
"preview": "package alteruserscramcredentials\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Re"
},
{
"path": "protocol/alteruserscramcredentials/alteruserscramcredentials_test.go",
"chars": 1052,
"preview": "package alteruserscramcredentials_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/alteruserscramcred"
},
{
"path": "protocol/apiversions/apiversions.go",
"chars": 699,
"preview": "package apiversions\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Resp"
},
{
"path": "protocol/apiversions/apiversions_test.go",
"chars": 1059,
"preview": "package apiversions_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/apiversions\"\n\t\"github.com/segmen"
},
{
"path": "protocol/buffer.go",
"chars": 12129,
"preview": "package protocol\n\nimport (\n\t\"bytes\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"sync\"\n\t\"sync/atomic\"\n)\n\n// Bytes is an interface im"
},
{
"path": "protocol/buffer_test.go",
"chars": 2002,
"preview": "package protocol\n\nimport (\n\t\"bytes\"\n\t\"io\"\n\t\"io/ioutil\"\n\t\"testing\"\n)\n\nfunc TestPageBufferWriteReadSeek(t *testing.T) {\n\tb"
},
{
"path": "protocol/cluster.go",
"chars": 2912,
"preview": "package protocol\n\nimport (\n\t\"fmt\"\n\t\"sort\"\n\t\"strings\"\n\t\"text/tabwriter\"\n)\n\ntype Cluster struct {\n\tClusterID string\n\tCont"
},
{
"path": "protocol/conn.go",
"chars": 1951,
"preview": "package protocol\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"net\"\n\t\"sync/atomic\"\n\t\"time\"\n)\n\ntype Conn struct {\n\tbuffer *bufio.Reader\n\t"
},
{
"path": "protocol/consumer/consumer.go",
"chars": 659,
"preview": "package consumer\n\nconst MaxVersionSupported = 1\n\ntype Subscription struct {\n\tVersion int16 `kafka:\"mi"
},
{
"path": "protocol/consumer/consumer_test.go",
"chars": 953,
"preview": "package consumer_test\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol\"\n\t\"github.com/segmentio/"
},
{
"path": "protocol/createacls/createacls.go",
"chars": 1807,
"preview": "package createacls\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, &Respo"
},
{
"path": "protocol/createacls/createacls_test.go",
"chars": 2310,
"preview": "package createacls_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/createacls\"\n\t\"github.com/segmenti"
},
{
"path": "protocol/createpartitions/createpartitions.go",
"chars": 1385,
"preview": "package createpartitions\n\nimport \"github.com/segmentio/kafka-go/protocol\"\n\nfunc init() {\n\tprotocol.Register(&Request{}, "
},
{
"path": "protocol/createpartitions/createpartitions_test.go",
"chars": 1353,
"preview": "package createpartitions_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/segmentio/kafka-go/protocol/createpartitions\"\n\t\"github."
}
]
// ... and 166 more files (download for full content)
About this extraction
This page contains the full source code of the segmentio/kafka-go GitHub repository, extracted and formatted as plain text for AI agents and large language models (LLMs). The extraction includes 366 files (1.3 MB), approximately 408.1k tokens, and a symbol index with 2948 extracted functions, classes, methods, constants, and types. Use this with OpenClaw, Claude, ChatGPT, Cursor, Windsurf, or any other AI tool that accepts text input. You can copy the full output to your clipboard or download it as a .txt file.
Extracted by GitExtract — free GitHub repo to text converter for AI. Built by Nikandr Surkov.