\n\n```\n\n```\n\n
\n\nimport org.apache.kafka.common.security.scram.internals.ScramFormatter;\nimport org.apache.kafka.common.security.scram.internals.ScramMechanism;\nimport java.nio.charset.StandardCharsets;\n\npublic class App {\n\n public static String bytesToHex(byte[] in) {\n final StringBuilder builder = new StringBuilder();\n for(byte b : in) {\n builder.append(String.format(\"0x%02x, \", b));\n }\n return builder.toString();\n }\n\n\tpublic static void main(String[] args) throws NoSuchAlgorithmException, InvalidKeyException {\n\t int digestIterations = 4096;\n\t String password = \"hello\";\n\t byte[] salt = \"world\".getBytes(StandardCharsets.UTF_8);\n\t byte[] saltedPassword = new ScramFormatter(ScramMechanism.SCRAM_SHA_256)\n\t\t\t .saltedPassword(password, salt, digestIterations);\n\t System.out.println(bytesToHex(saltedPassword));\n\t}\n}\n\n*/\n\nfunc TestScramSaltedPasswordSha512(t *testing.T) {\n\tpassword := []byte(\"hello\")\n\tsalt := []byte(\"world\")\n\n\tformatter := scramFormatter{mechanism: SCRAM_MECHANISM_SHA_512}\n\tresult, _ := formatter.saltedPassword(password, salt, 4096)\n\n\t// calculated using ScramFormatter (see comment above)\n\texpected := []byte{\n\t\t0x35, 0x0c, 0x77, 0x84, 0x8a, 0x63, 0x06, 0x92, 0x00,\n\t\t0x6e, 0xc6, 0x6a, 0x0c, 0x39, 0xeb, 0xb0, 0x00, 0xd3,\n\t\t0xf8, 0x8a, 0x94, 0xae, 0x7f, 0x8c, 0xcd, 0x1d, 0x92,\n\t\t0x52, 0x6c, 0x5b, 0x16, 0x15, 0x86, 0x3b, 0xde, 0xa1,\n\t\t0x6c, 0x12, 0x9a, 0x7b, 0x09, 0xed, 0x0e, 0x38, 0xf2,\n\t\t0x07, 0x4d, 0x2f, 0xe2, 0x9f, 0x0f, 0x41, 0xe1, 0xfb,\n\t\t0x00, 0xc1, 0xd3, 0xbd, 0xd3, 0xfd, 0x51, 0x0b, 0xa9,\n\t\t0x8f,\n\t}\n\n\tif !bytes.Equal(result, expected) {\n\t\tt.Errorf(\"saltedPassword SHA-512 failed, expected: %v, result: %v\", expected, result)\n\t}\n}\n\nfunc TestScramSaltedPasswordSha256(t *testing.T) {\n\tpassword := []byte(\"hello\")\n\tsalt := []byte(\"world\")\n\n\tformatter := scramFormatter{mechanism: SCRAM_MECHANISM_SHA_256}\n\tresult, _ := formatter.saltedPassword(password, salt, 4096)\n\n\t// calculated using ScramFormatter (see comment above)\n\texpected := []byte{\n\t\t0xc1, 0x55, 0x53, 0x03, 0xda, 0x30, 0x9f, 0x6b, 0x7d,\n\t\t0x1e, 0x8f, 0xe4, 0x56, 0x36, 0xbf, 0xdd, 0xdc, 0x4b,\n\t\t0xf5, 0x64, 0x05, 0xe7, 0xe9, 0x4e, 0x9d, 0x15, 0xf0,\n\t\t0xe7, 0xb9, 0xcb, 0xd3, 0x80,\n\t}\n\n\tif !bytes.Equal(result, expected) {\n\t\tt.Errorf(\"saltedPassword SHA-256 failed, expected: %v, result: %v\", expected, result)\n\t}\n}\n"
},
{
"path": "server.properties",
"content": "# Licensed to the Apache Software Foundation (ASF) under one or more\n# contributor license agreements. See the NOTICE file distributed with\n# this work for additional information regarding copyright ownership.\n# The ASF licenses this file to You under the Apache License, Version 2.0\n# (the \"License\"); you may not use this file except in compliance with\n# the License. You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\n#\n# This configuration file is intended for use in ZK-based mode, where Apache ZooKeeper is required.\n# See kafka.server.KafkaConfig for additional details and defaults\n#\n\n############################# Server Basics #############################\n\n# The id of the broker. This must be set to a unique integer for each broker.\nbroker.id=0\n\n############################# Socket Server Settings #############################\n\n# The address the socket server listens on. If not configured, the host name will be equal to the value of\n# java.net.InetAddress.getCanonicalHostName(), with PLAINTEXT listener name, and port 9092.\n# FORMAT:\n# listeners = listener_name://host_name:port\n# EXAMPLE:\n# listeners = PLAINTEXT://your.host.name:9092\n#listeners=PLAINTEXT://:9092\n\n# Listener name, hostname and port the broker will advertise to clients.\n# If not set, it uses the value for \"listeners\".\n#advertised.listeners=PLAINTEXT://your.host.name:9092\n\n# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details\n#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL\n\n# The number of threads that the server uses for receiving requests from the network and sending responses to the network\nnum.network.threads=3\n\n# The number of threads that the server uses for processing requests, which may include disk I/O\nnum.io.threads=8\n\n# The send buffer (SO_SNDBUF) used by the socket server\nsocket.send.buffer.bytes=102400\n\n# The receive buffer (SO_RCVBUF) used by the socket server\nsocket.receive.buffer.bytes=102400\n\n# The maximum size of a request that the socket server will accept (protection against OOM)\nsocket.request.max.bytes=104857600\n\n\n############################# Log Basics #############################\n\n# A comma separated list of directories under which to store log files\nlog.dirs=/tmp/kafka-logs\n\n# The default number of log partitions per topic. More partitions allow greater\n# parallelism for consumption, but this will also result in more files across\n# the brokers.\nnum.partitions=1\n\n# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown.\n# This value is recommended to be increased for installations with data dirs located in RAID array.\nnum.recovery.threads.per.data.dir=1\n\n############################# Internal Topic Settings #############################\n# The replication factor for the group metadata internal topics \"__consumer_offsets\" and \"__transaction_state\"\n# For anything other than development testing, a value greater than 1 is recommended to ensure availability such as 3.\noffsets.topic.replication.factor=1\ntransaction.state.log.replication.factor=1\ntransaction.state.log.min.isr=1\n\n############################# Log Flush Policy #############################\n\n# Messages are immediately written to the filesystem but by default we only fsync() to sync\n# the OS cache lazily. The following configurations control the flush of data to disk.\n# There are a few important trade-offs here:\n# 1. Durability: Unflushed data may be lost if you are not using replication.\n# 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush.\n# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks.\n# The settings below allow one to configure the flush policy to flush data after a period of time or\n# every N messages (or both). This can be done globally and overridden on a per-topic basis.\n\n# The number of messages to accept before forcing a flush of data to disk\n#log.flush.interval.messages=10000\n\n# The maximum amount of time a message can sit in a log before we force a flush\n#log.flush.interval.ms=1000\n\n############################# Log Retention Policy #############################\n\n# The following configurations control the disposal of log segments. The policy can\n# be set to delete segments after a period of time, or after a given size has accumulated.\n# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens\n# from the end of the log.\n\n# The minimum age of a log file to be eligible for deletion due to age\nlog.retention.hours=168\n\n# A size-based retention policy for logs. Segments are pruned from the log unless the remaining\n# segments drop below log.retention.bytes. Functions independently of log.retention.hours.\n#log.retention.bytes=1073741824\n\n# The maximum size of a log segment file. When this size is reached a new log segment will be created.\n#log.segment.bytes=1073741824\n\n# The interval at which log segments are checked to see if they can be deleted according\n# to the retention policies\nlog.retention.check.interval.ms=300000\n\n############################# Zookeeper #############################\n\n# Zookeeper connection string (see zookeeper docs for details).\n# This is a comma separated host:port pairs, each corresponding to a zk\n# server. e.g. \"127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002\".\n# You can also append an optional chroot string to the urls to specify the\n# root directory for all kafka znodes.\nzookeeper.connect=localhost:2181\n\n# Timeout in ms for connecting to zookeeper\nzookeeper.connection.timeout.ms=18000\n\n\n############################# Group Coordinator Settings #############################\n\n# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance.\n# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms.\n# The default value for this is 3 seconds.\n# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing.\n# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup.\ngroup.initial.rebalance.delay.ms=0\n"
},
{
"path": "sticky_assignor_user_data.go",
"content": "package sarama\n\ntype topicPartitionAssignment struct {\n\tTopic string\n\tPartition int32\n}\n\ntype StickyAssignorUserData interface {\n\tpartitions() []topicPartitionAssignment\n\thasGeneration() bool\n\tgeneration() int\n}\n\n// StickyAssignorUserDataV0 holds topic partition information for an assignment\ntype StickyAssignorUserDataV0 struct {\n\tTopics map[string][]int32\n\n\ttopicPartitions []topicPartitionAssignment\n}\n\nfunc (m *StickyAssignorUserDataV0) encode(pe packetEncoder) error {\n\tif err := pe.putArrayLength(len(m.Topics)); err != nil {\n\t\treturn err\n\t}\n\n\tfor topic, partitions := range m.Topics {\n\t\tif err := pe.putString(topic); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := pe.putInt32Array(partitions); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\treturn nil\n}\n\nfunc (m *StickyAssignorUserDataV0) decode(pd packetDecoder) (err error) {\n\tvar topicLen int\n\tif topicLen, err = pd.getArrayLength(); err != nil {\n\t\treturn\n\t}\n\n\tm.Topics = make(map[string][]int32, topicLen)\n\tfor i := 0; i < topicLen; i++ {\n\t\tvar topic string\n\t\tif topic, err = pd.getString(); err != nil {\n\t\t\treturn\n\t\t}\n\t\tif m.Topics[topic], err = pd.getInt32Array(); err != nil {\n\t\t\treturn\n\t\t}\n\t}\n\tm.topicPartitions = populateTopicPartitions(m.Topics)\n\treturn nil\n}\n\nfunc (m *StickyAssignorUserDataV0) partitions() []topicPartitionAssignment { return m.topicPartitions }\nfunc (m *StickyAssignorUserDataV0) hasGeneration() bool { return false }\nfunc (m *StickyAssignorUserDataV0) generation() int { return defaultGeneration }\n\n// StickyAssignorUserDataV1 holds topic partition information for an assignment\ntype StickyAssignorUserDataV1 struct {\n\tTopics map[string][]int32\n\tGeneration int32\n\n\ttopicPartitions []topicPartitionAssignment\n}\n\nfunc (m *StickyAssignorUserDataV1) encode(pe packetEncoder) error {\n\tif err := pe.putArrayLength(len(m.Topics)); err != nil {\n\t\treturn err\n\t}\n\n\tfor topic, partitions := range m.Topics {\n\t\tif err := pe.putString(topic); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := pe.putInt32Array(partitions); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tpe.putInt32(m.Generation)\n\treturn nil\n}\n\nfunc (m *StickyAssignorUserDataV1) decode(pd packetDecoder) (err error) {\n\tvar topicLen int\n\tif topicLen, err = pd.getArrayLength(); err != nil {\n\t\treturn\n\t}\n\n\tm.Topics = make(map[string][]int32, topicLen)\n\tfor i := 0; i < topicLen; i++ {\n\t\tvar topic string\n\t\tif topic, err = pd.getString(); err != nil {\n\t\t\treturn\n\t\t}\n\t\tif m.Topics[topic], err = pd.getInt32Array(); err != nil {\n\t\t\treturn\n\t\t}\n\t}\n\n\tm.Generation, err = pd.getInt32()\n\tif err != nil {\n\t\treturn err\n\t}\n\tm.topicPartitions = populateTopicPartitions(m.Topics)\n\treturn nil\n}\n\nfunc (m *StickyAssignorUserDataV1) partitions() []topicPartitionAssignment { return m.topicPartitions }\nfunc (m *StickyAssignorUserDataV1) hasGeneration() bool { return true }\nfunc (m *StickyAssignorUserDataV1) generation() int { return int(m.Generation) }\n\nfunc populateTopicPartitions(topics map[string][]int32) []topicPartitionAssignment {\n\ttopicPartitions := make([]topicPartitionAssignment, 0)\n\tfor topic, partitions := range topics {\n\t\tfor _, partition := range partitions {\n\t\t\ttopicPartitions = append(topicPartitions, topicPartitionAssignment{Topic: topic, Partition: partition})\n\t\t}\n\t}\n\treturn topicPartitions\n}\n"
},
{
"path": "sticky_assignor_user_data_test.go",
"content": "//go:build !functional\n\npackage sarama\n\nimport (\n\t\"encoding/base64\"\n\t\"testing\"\n)\n\nfunc TestStickyAssignorUserDataV0(t *testing.T) {\n\t// Single topic with deterministic ordering across encode-decode\n\treq := &StickyAssignorUserDataV0{}\n\tdata := decodeUserDataBytes(t, \"AAAAAQADdDAzAAAAAQAAAAU=\")\n\ttestDecodable(t, \"\", req, data)\n\ttestEncodable(t, \"\", req, data)\n\n\t// Multiple partitions\n\treq = &StickyAssignorUserDataV0{}\n\tdata = decodeUserDataBytes(t, \"AAAAAQADdDE4AAAAEgAAAAAAAAABAAAAAgAAAAMAAAAEAAAABQAAAAYAAAAHAAAACAAAAAkAAAAKAAAACwAAAAwAAAANAAAADgAAAA8AAAAQAAAAEQ==\")\n\ttestDecodable(t, \"\", req, data)\n\n\t// Multiple topics and partitions\n\treq = &StickyAssignorUserDataV0{}\n\tdata = decodeUserDataBytes(t, \"AAAABQADdDEyAAAAAgAAAAIAAAAKAAN0MTEAAAABAAAABAADdDE0AAAAAQAAAAgAA3QxMwAAAAEAAAANAAN0MDkAAAABAAAABQ==\")\n\ttestDecodable(t, \"\", req, data)\n}\n\nfunc TestStickyAssignorUserDataV1(t *testing.T) {\n\t// Single topic with deterministic ordering across encode-decode\n\treq := &StickyAssignorUserDataV1{}\n\tdata := decodeUserDataBytes(t, \"AAAAAQADdDA2AAAAAgAAAAAAAAAE/////w==\")\n\ttestDecodable(t, \"\", req, data)\n\ttestEncodable(t, \"\", req, data)\n\n\t// Multiple topics and partitions\n\treq = &StickyAssignorUserDataV1{}\n\tdata = decodeUserDataBytes(t, \"AAAABgADdDEwAAAAAgAAAAIAAAAJAAN0MTIAAAACAAAAAwAAAAsAA3QxNAAAAAEAAAAEAAN0MTMAAAABAAAACwADdDE1AAAAAQAAAAwAA3QwOQAAAAEAAAAG/////w==\")\n\ttestDecodable(t, \"\", req, data)\n\n\t// Generation is populated\n\treq = &StickyAssignorUserDataV1{}\n\tdata = decodeUserDataBytes(t, \"AAAAAQAHdG9waWMwMQAAAAMAAAAAAAAAAQAAAAIAAAAB\")\n\ttestDecodable(t, \"\", req, data)\n}\n\nfunc decodeUserDataBytes(t *testing.T, base64Data string) []byte {\n\tdata, err := base64.StdEncoding.DecodeString(base64Data)\n\tif err != nil {\n\t\tt.Errorf(\"Error decoding data: %v\", err)\n\t\tt.FailNow()\n\t}\n\treturn data\n}\n"
},
{
"path": "sync_group_request.go",
"content": "package sarama\n\ntype SyncGroupRequestAssignment struct {\n\t// MemberId contains the ID of the member to assign.\n\tMemberId string\n\t// Assignment contains the member assignment.\n\tAssignment []byte\n}\n\nfunc (a *SyncGroupRequestAssignment) encode(pe packetEncoder, version int16) (err error) {\n\tif err := pe.putString(a.MemberId); err != nil {\n\t\treturn err\n\t}\n\n\tif err := pe.putBytes(a.Assignment); err != nil {\n\t\treturn err\n\t}\n\n\tpe.putEmptyTaggedFieldArray()\n\treturn nil\n}\n\nfunc (a *SyncGroupRequestAssignment) decode(pd packetDecoder, version int16) (err error) {\n\tif a.MemberId, err = pd.getString(); err != nil {\n\t\treturn err\n\t}\n\n\tif a.Assignment, err = pd.getBytes(); err != nil {\n\t\treturn err\n\t}\n\n\t_, err = pd.getEmptyTaggedFieldArray()\n\treturn err\n}\n\ntype SyncGroupRequest struct {\n\t// Version defines the protocol version to use for encode and decode\n\tVersion int16\n\t// GroupId contains the unique group identifier.\n\tGroupId string\n\t// GenerationId contains the generation of the group.\n\tGenerationId int32\n\t// MemberId contains the member ID assigned by the group.\n\tMemberId string\n\t// GroupInstanceId contains the unique identifier of the consumer instance provided by end user.\n\tGroupInstanceId *string\n\t// GroupAssignments contains each assignment.\n\tGroupAssignments []SyncGroupRequestAssignment\n}\n\nfunc (s *SyncGroupRequest) setVersion(v int16) {\n\ts.Version = v\n}\n\nfunc (s *SyncGroupRequest) encode(pe packetEncoder) (err error) {\n\tif err := pe.putString(s.GroupId); err != nil {\n\t\treturn err\n\t}\n\n\tpe.putInt32(s.GenerationId)\n\n\tif err := pe.putString(s.MemberId); err != nil {\n\t\treturn err\n\t}\n\n\tif s.Version >= 3 {\n\t\tif err := pe.putNullableString(s.GroupInstanceId); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif err := pe.putArrayLength(len(s.GroupAssignments)); err != nil {\n\t\treturn err\n\t}\n\tfor _, block := range s.GroupAssignments {\n\t\tif err := block.encode(pe, s.Version); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tpe.putEmptyTaggedFieldArray()\n\treturn nil\n}\n\nfunc (s *SyncGroupRequest) decode(pd packetDecoder, version int16) (err error) {\n\ts.Version = version\n\tif s.GroupId, err = pd.getString(); err != nil {\n\t\treturn err\n\t}\n\n\tif s.GenerationId, err = pd.getInt32(); err != nil {\n\t\treturn err\n\t}\n\n\tif s.MemberId, err = pd.getString(); err != nil {\n\t\treturn err\n\t}\n\n\tif s.Version >= 3 {\n\t\tif s.GroupInstanceId, err = pd.getNullableString(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif numAssignments, err := pd.getArrayLength(); err != nil {\n\t\treturn err\n\t} else if numAssignments > 0 {\n\t\ts.GroupAssignments = make([]SyncGroupRequestAssignment, numAssignments)\n\t\tfor i := 0; i < numAssignments; i++ {\n\t\t\tvar block SyncGroupRequestAssignment\n\t\t\tif err := block.decode(pd, s.Version); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\ts.GroupAssignments[i] = block\n\t\t}\n\t}\n\n\t_, err = pd.getEmptyTaggedFieldArray()\n\treturn err\n}\n\nfunc (r *SyncGroupRequest) key() int16 {\n\treturn apiKeySyncGroup\n}\n\nfunc (r *SyncGroupRequest) version() int16 {\n\treturn r.Version\n}\n\nfunc (r *SyncGroupRequest) headerVersion() int16 {\n\tif r.Version >= 4 {\n\t\treturn 2\n\t}\n\treturn 1\n}\n\nfunc (r *SyncGroupRequest) isValidVersion() bool {\n\treturn r.Version >= 0 && r.Version <= 4\n}\n\nfunc (r *SyncGroupRequest) isFlexible() bool {\n\treturn r.isFlexibleVersion(r.Version)\n}\n\nfunc (r *SyncGroupRequest) isFlexibleVersion(version int16) bool {\n\treturn version >= 4\n}\n\nfunc (r *SyncGroupRequest) requiredVersion() KafkaVersion {\n\tswitch r.Version {\n\tcase 4:\n\t\treturn V2_4_0_0\n\tcase 3:\n\t\treturn V2_3_0_0\n\tcase 2:\n\t\treturn V2_0_0_0\n\tcase 1:\n\t\treturn V0_11_0_0\n\tcase 0:\n\t\treturn V0_9_0_0\n\tdefault:\n\t\treturn V2_3_0_0\n\t}\n}\n\nfunc (r *SyncGroupRequest) AddGroupAssignment(memberId string, memberAssignment []byte) {\n\tr.GroupAssignments = append(r.GroupAssignments, SyncGroupRequestAssignment{\n\t\tMemberId: memberId,\n\t\tAssignment: memberAssignment,\n\t})\n}\n\nfunc (r *SyncGroupRequest) AddGroupAssignmentMember(\n\tmemberId string,\n\tmemberAssignment *ConsumerGroupMemberAssignment,\n) error {\n\tbin, err := encode(memberAssignment, nil)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tr.AddGroupAssignment(memberId, bin)\n\treturn nil\n}\n"
},
{
"path": "sync_group_request_test.go",
"content": "//go:build !functional\n\npackage sarama\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n)\n\nvar (\n\temptySyncGroupRequest = []byte{\n\t\t0, 3, 'f', 'o', 'o', // Group ID\n\t\t0x00, 0x01, 0x02, 0x03, // Generation ID\n\t\t0, 3, 'b', 'a', 'z', // Member ID\n\t\t0, 0, 0, 0, // no assignments\n\t}\n\n\tpopulatedSyncGroupRequest = []byte{\n\t\t0, 3, 'f', 'o', 'o', // Group ID\n\t\t0x00, 0x01, 0x02, 0x03, // Generation ID\n\t\t0, 3, 'b', 'a', 'z', // Member ID\n\t\t0, 0, 0, 1, // one assignment\n\t\t0, 3, 'b', 'a', 'z', // Member ID\n\t\t0, 0, 0, 3, 'f', 'o', 'o', // Member assignment\n\t}\n)\n\nfunc TestSyncGroupRequest(t *testing.T) {\n\tvar request *SyncGroupRequest\n\n\trequest = new(SyncGroupRequest)\n\trequest.GroupId = \"foo\"\n\trequest.GenerationId = 66051\n\trequest.MemberId = \"baz\"\n\ttestRequest(t, \"empty\", request, emptySyncGroupRequest)\n\n\trequest = new(SyncGroupRequest)\n\trequest.GroupId = \"foo\"\n\trequest.GenerationId = 66051\n\trequest.MemberId = \"baz\"\n\trequest.AddGroupAssignment(\"baz\", []byte(\"foo\"))\n\ttestRequest(t, \"populated\", request, populatedSyncGroupRequest)\n}\n\nvar (\n\tpopulatedSyncGroupRequestV3 = []byte{\n\t\t0, 3, 'f', 'o', 'o', // Group ID\n\t\t0x00, 0x01, 0x02, 0x03, // Generation ID\n\t\t0, 3, 'b', 'a', 'z', // Member ID\n\t\t0, 3, 'g', 'i', 'd', // GroupInstance ID\n\t\t0, 0, 0, 1, // one assignment\n\t\t0, 3, 'b', 'a', 'z', // Member ID\n\t\t0, 0, 0, 3, 'f', 'o', 'o', // Member assignment\n\t}\n\tpopulatedSyncGroupRequestV4 = []byte{\n\t\t4, 'f', 'o', 'o', // Group ID\n\t\t0x00, 0x01, 0x02, 0x03, // Generation ID\n\t\t4, 'b', 'a', 'z', // Member ID\n\t\t4, 'g', 'i', 'd', // GroupInstance ID\n\t\t2, // 1 + one assignment\n\t\t4, 'b', 'a', 'z', // Member ID\n\t\t4, 'f', 'o', 'o', // Member assignment\n\t\t0, // empty tagged fields\n\t\t0, // empty tagged fields\n\t}\n)\n\nfunc TestSyncGroupRequestV3AndPlus(t *testing.T) {\n\tgroupInstanceId := \"gid\"\n\ttests := []struct {\n\t\tCaseName string\n\t\tVersion int16\n\t\tMessageBytes []byte\n\t\tMessage *SyncGroupRequest\n\t}{\n\t\t{\n\t\t\t\"v3\",\n\t\t\t3,\n\t\t\tpopulatedSyncGroupRequestV3,\n\t\t\t&SyncGroupRequest{\n\t\t\t\tVersion: 3,\n\t\t\t\tGroupId: \"foo\",\n\t\t\t\tGenerationId: 0x00010203,\n\t\t\t\tMemberId: \"baz\",\n\t\t\t\tGroupInstanceId: &groupInstanceId,\n\t\t\t\tGroupAssignments: []SyncGroupRequestAssignment{\n\t\t\t\t\t{\n\t\t\t\t\t\tMemberId: \"baz\",\n\t\t\t\t\t\tAssignment: []byte(\"foo\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t\"v4\",\n\t\t\t4,\n\t\t\tpopulatedSyncGroupRequestV4,\n\t\t\t&SyncGroupRequest{\n\t\t\t\tVersion: 4,\n\t\t\t\tGroupId: \"foo\",\n\t\t\t\tGenerationId: 0x00010203,\n\t\t\t\tMemberId: \"baz\",\n\t\t\t\tGroupInstanceId: &groupInstanceId,\n\t\t\t\tGroupAssignments: []SyncGroupRequestAssignment{\n\t\t\t\t\t{\n\t\t\t\t\t\tMemberId: \"baz\",\n\t\t\t\t\t\tAssignment: []byte(\"foo\"),\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t},\n\t\t},\n\t}\n\tfor _, c := range tests {\n\t\trequest := new(SyncGroupRequest)\n\t\ttestVersionDecodable(t, c.CaseName, request, c.MessageBytes, c.Version)\n\t\tif !reflect.DeepEqual(c.Message, request) {\n\t\t\tt.Errorf(\"case %s decode failed, expected:%+v got %+v\", c.CaseName, c.Message, request)\n\t\t}\n\t\ttestEncodable(t, c.CaseName, c.Message, c.MessageBytes)\n\t}\n}\n"
},
{
"path": "sync_group_response.go",
"content": "package sarama\n\nimport \"time\"\n\ntype SyncGroupResponse struct {\n\t// Version defines the protocol version to use for encode and decode\n\tVersion int16\n\t// ThrottleTime contains the duration in milliseconds for which the\n\t// request was throttled due to a quota violation, or zero if the request\n\t// did not violate any quota.\n\tThrottleTime int32\n\t// Err contains the error code, or 0 if there was no error.\n\tErr KError\n\t// MemberAssignment contains the member assignment.\n\tMemberAssignment []byte\n}\n\nfunc (r *SyncGroupResponse) setVersion(v int16) {\n\tr.Version = v\n}\n\nfunc (r *SyncGroupResponse) GetMemberAssignment() (*ConsumerGroupMemberAssignment, error) {\n\tassignment := new(ConsumerGroupMemberAssignment)\n\terr := decode(r.MemberAssignment, assignment, nil)\n\treturn assignment, err\n}\n\nfunc (r *SyncGroupResponse) encode(pe packetEncoder) error {\n\tif r.Version >= 1 {\n\t\tpe.putInt32(r.ThrottleTime)\n\t}\n\tpe.putKError(r.Err)\n\tif err := pe.putBytes(r.MemberAssignment); err != nil {\n\t\treturn err\n\t}\n\n\tpe.putEmptyTaggedFieldArray()\n\treturn nil\n}\n\nfunc (r *SyncGroupResponse) decode(pd packetDecoder, version int16) (err error) {\n\tr.Version = version\n\tif r.Version >= 1 {\n\t\tif r.ThrottleTime, err = pd.getInt32(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\tr.Err, err = pd.getKError()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tr.MemberAssignment, err = pd.getBytes()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t_, err = pd.getEmptyTaggedFieldArray()\n\treturn err\n}\n\nfunc (r *SyncGroupResponse) key() int16 {\n\treturn apiKeySyncGroup\n}\n\nfunc (r *SyncGroupResponse) version() int16 {\n\treturn r.Version\n}\n\nfunc (r *SyncGroupResponse) headerVersion() int16 {\n\tif r.Version >= 4 {\n\t\treturn 1\n\t}\n\treturn 0\n}\n\nfunc (r *SyncGroupResponse) isValidVersion() bool {\n\treturn r.Version >= 0 && r.Version <= 4\n}\n\nfunc (r *SyncGroupResponse) isFlexible() bool {\n\treturn r.isFlexibleVersion(r.Version)\n}\n\nfunc (r *SyncGroupResponse) isFlexibleVersion(version int16) bool {\n\treturn version >= 4\n}\n\nfunc (r *SyncGroupResponse) requiredVersion() KafkaVersion {\n\tswitch r.Version {\n\tcase 4:\n\t\treturn V2_4_0_0\n\tcase 3:\n\t\treturn V2_3_0_0\n\tcase 2:\n\t\treturn V2_0_0_0\n\tcase 1:\n\t\treturn V0_11_0_0\n\tcase 0:\n\t\treturn V0_9_0_0\n\tdefault:\n\t\treturn V2_3_0_0\n\t}\n}\n\nfunc (r *SyncGroupResponse) throttleTime() time.Duration {\n\treturn time.Duration(r.ThrottleTime) * time.Millisecond\n}\n"
},
{
"path": "sync_group_response_test.go",
"content": "//go:build !functional\n\npackage sarama\n\nimport (\n\t\"reflect\"\n\t\"testing\"\n)\n\nvar (\n\tsyncGroupResponseV0NoError = []byte{\n\t\t0x00, 0x00, // No error\n\t\t0, 0, 0, 3, 0x01, 0x02, 0x03, // Member assignment data\n\t}\n\n\tsyncGroupResponseV0WithError = []byte{\n\t\t0, 27, // ErrRebalanceInProgress\n\t\t0, 0, 0, 0, // No member assignment data\n\t}\n\n\tsyncGroupResponseV1NoError = []byte{\n\t\t0, 0, 0, 100, // ThrottleTimeMs\n\t\t0x00, 0x00, // No error\n\t\t0, 0, 0, 3, 0x01, 0x02, 0x03, // Member assignment data\n\t}\n\n\tsyncGroupResponseV4NoError = []byte{\n\t\t0, 0, 0, 100, // ThrottleTimeMs\n\t\t0x00, 0x00, // No error\n\t\t4, 0x01, 0x02, 0x03, // Member assignment data\n\t\t0, // empty tagged fields\n\t}\n)\n\nfunc TestSyncGroupResponse(t *testing.T) {\n\ttests := []struct {\n\t\tCaseName string\n\t\tVersion int16\n\t\tMessageBytes []byte\n\t\tMessage *SyncGroupResponse\n\t}{\n\t\t{\n\t\t\t\"v0-noErr\",\n\t\t\t0,\n\t\t\tsyncGroupResponseV0NoError,\n\t\t\t&SyncGroupResponse{\n\t\t\t\tVersion: 0,\n\t\t\t\tErr: ErrNoError,\n\t\t\t\tMemberAssignment: []byte{1, 2, 3},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t\"v0-Err\",\n\t\t\t0,\n\t\t\tsyncGroupResponseV0WithError,\n\t\t\t&SyncGroupResponse{\n\t\t\t\tVersion: 0,\n\t\t\t\tErr: ErrRebalanceInProgress,\n\t\t\t\tMemberAssignment: []byte{},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t\"v1-noErr\",\n\t\t\t1,\n\t\t\tsyncGroupResponseV1NoError,\n\t\t\t&SyncGroupResponse{\n\t\t\t\tThrottleTime: 100,\n\t\t\t\tVersion: 1,\n\t\t\t\tErr: ErrNoError,\n\t\t\t\tMemberAssignment: []byte{1, 2, 3},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\t\"v4-noErr\",\n\t\t\t4,\n\t\t\tsyncGroupResponseV4NoError,\n\t\t\t&SyncGroupResponse{\n\t\t\t\tThrottleTime: 100,\n\t\t\t\tVersion: 4,\n\t\t\t\tErr: ErrNoError,\n\t\t\t\tMemberAssignment: []byte{1, 2, 3},\n\t\t\t},\n\t\t},\n\t}\n\tfor _, c := range tests {\n\t\tresponse := new(SyncGroupResponse)\n\t\ttestVersionDecodable(t, c.CaseName, response, c.MessageBytes, c.Version)\n\t\tif !reflect.DeepEqual(c.Message, response) {\n\t\t\tt.Errorf(\"case %s decode failed, expected:%+v got %+v\", c.CaseName, c.Message, response)\n\t\t}\n\t\ttestEncodable(t, c.CaseName, c.Message, c.MessageBytes)\n\t}\n}\n"
},
{
"path": "sync_producer.go",
"content": "package sarama\n\nimport \"sync\"\n\nvar expectationsPool = sync.Pool{\n\tNew: func() interface{} {\n\t\treturn make(chan *ProducerError, 1)\n\t},\n}\n\n// SyncProducer publishes Kafka messages, blocking until they have been acknowledged. It routes messages to the correct\n// broker, refreshing metadata as appropriate, and parses responses for errors. You must call Close() on a producer\n// to avoid leaks, it may not be garbage-collected automatically when it passes out of scope.\n//\n// The SyncProducer comes with two caveats: it will generally be less efficient than the AsyncProducer, and the actual\n// durability guarantee provided when a message is acknowledged depend on the configured value of `Producer.RequiredAcks`.\n// There are configurations where a message acknowledged by the SyncProducer can still sometimes be lost.\n//\n// For implementation reasons, the SyncProducer requires `Producer.Return.Errors` and `Producer.Return.Successes` to\n// be set to true in its configuration.\ntype SyncProducer interface {\n\n\t// SendMessage produces a given message, and returns only when it either has\n\t// succeeded or failed to produce. It will return the partition and the offset\n\t// of the produced message, or an error if the message failed to produce.\n\tSendMessage(msg *ProducerMessage) (partition int32, offset int64, err error)\n\n\t// SendMessages produces a given set of messages, and returns only when all\n\t// messages in the set have either succeeded or failed. Note that messages\n\t// can succeed and fail individually; if some succeed and some fail,\n\t// SendMessages will return an error.\n\tSendMessages(msgs []*ProducerMessage) error\n\n\t// Close shuts down the producer; you must call this function before a producer\n\t// object passes out of scope, as it may otherwise leak memory.\n\t// You must call this before calling Close on the underlying client.\n\tClose() error\n\n\t// TxnStatus return current producer transaction status.\n\tTxnStatus() ProducerTxnStatusFlag\n\n\t// IsTransactional return true when current producer is transactional.\n\tIsTransactional() bool\n\n\t// BeginTxn mark current transaction as ready.\n\tBeginTxn() error\n\n\t// CommitTxn commit current transaction.\n\tCommitTxn() error\n\n\t// AbortTxn abort current transaction.\n\tAbortTxn() error\n\n\t// AddOffsetsToTxn add associated offsets to current transaction.\n\tAddOffsetsToTxn(offsets map[string][]*PartitionOffsetMetadata, groupId string) error\n\n\t// AddMessageToTxn add message offsets to current transaction.\n\tAddMessageToTxn(msg *ConsumerMessage, groupId string, metadata *string) error\n}\n\ntype syncProducer struct {\n\tproducer *asyncProducer\n\twg sync.WaitGroup\n}\n\n// NewSyncProducer creates a new SyncProducer using the given broker addresses and configuration.\nfunc NewSyncProducer(addrs []string, config *Config) (SyncProducer, error) {\n\tif config == nil {\n\t\tconfig = NewConfig()\n\t\tconfig.Producer.Return.Successes = true\n\t}\n\n\tif err := verifyProducerConfig(config); err != nil {\n\t\treturn nil, err\n\t}\n\n\tp, err := NewAsyncProducer(addrs, config)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn newSyncProducerFromAsyncProducer(p.(*asyncProducer)), nil\n}\n\n// NewSyncProducerFromClient creates a new SyncProducer using the given client. It is still\n// necessary to call Close() on the underlying client when shutting down this producer.\nfunc NewSyncProducerFromClient(client Client) (SyncProducer, error) {\n\tif err := verifyProducerConfig(client.Config()); err != nil {\n\t\treturn nil, err\n\t}\n\n\tp, err := NewAsyncProducerFromClient(client)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn newSyncProducerFromAsyncProducer(p.(*asyncProducer)), nil\n}\n\nfunc newSyncProducerFromAsyncProducer(p *asyncProducer) *syncProducer {\n\tsp := &syncProducer{producer: p}\n\n\tsp.wg.Add(2)\n\tgo withRecover(sp.handleSuccesses)\n\tgo withRecover(sp.handleErrors)\n\n\treturn sp\n}\n\nfunc verifyProducerConfig(config *Config) error {\n\tif !config.Producer.Return.Errors {\n\t\treturn ConfigurationError(\"Producer.Return.Errors must be true to be used in a SyncProducer\")\n\t}\n\tif !config.Producer.Return.Successes {\n\t\treturn ConfigurationError(\"Producer.Return.Successes must be true to be used in a SyncProducer\")\n\t}\n\treturn nil\n}\n\nfunc (sp *syncProducer) SendMessage(msg *ProducerMessage) (partition int32, offset int64, err error) {\n\texpectation := expectationsPool.Get().(chan *ProducerError)\n\tmsg.expectation = expectation\n\tsp.producer.Input() <- msg\n\tpErr := <-expectation\n\tmsg.expectation = nil\n\texpectationsPool.Put(expectation)\n\tif pErr != nil {\n\t\treturn -1, -1, pErr.Err\n\t}\n\n\treturn msg.Partition, msg.Offset, nil\n}\n\nfunc (sp *syncProducer) SendMessages(msgs []*ProducerMessage) error {\n\tindices := make(chan int, len(msgs))\n\tgo func() {\n\t\tfor i, msg := range msgs {\n\t\t\texpectation := expectationsPool.Get().(chan *ProducerError)\n\t\t\tmsg.expectation = expectation\n\t\t\tsp.producer.Input() <- msg\n\t\t\tindices <- i\n\t\t}\n\t\tclose(indices)\n\t}()\n\n\tvar errors ProducerErrors\n\tfor i := range indices {\n\t\texpectation := msgs[i].expectation\n\t\tpErr := <-expectation\n\t\tmsgs[i].expectation = nil\n\t\texpectationsPool.Put(expectation)\n\t\tif pErr != nil {\n\t\t\terrors = append(errors, pErr)\n\t\t}\n\t}\n\n\tif len(errors) > 0 {\n\t\treturn errors\n\t}\n\treturn nil\n}\n\nfunc (sp *syncProducer) handleSuccesses() {\n\tdefer sp.wg.Done()\n\tfor msg := range sp.producer.Successes() {\n\t\texpectation := msg.expectation\n\t\texpectation <- nil\n\t}\n}\n\nfunc (sp *syncProducer) handleErrors() {\n\tdefer sp.wg.Done()\n\tfor err := range sp.producer.Errors() {\n\t\texpectation := err.Msg.expectation\n\t\texpectation <- err\n\t}\n}\n\nfunc (sp *syncProducer) Close() error {\n\tsp.producer.AsyncClose()\n\tsp.wg.Wait()\n\treturn nil\n}\n\nfunc (sp *syncProducer) IsTransactional() bool {\n\treturn sp.producer.IsTransactional()\n}\n\nfunc (sp *syncProducer) BeginTxn() error {\n\treturn sp.producer.BeginTxn()\n}\n\nfunc (sp *syncProducer) CommitTxn() error {\n\treturn sp.producer.CommitTxn()\n}\n\nfunc (sp *syncProducer) AbortTxn() error {\n\treturn sp.producer.AbortTxn()\n}\n\nfunc (sp *syncProducer) AddOffsetsToTxn(offsets map[string][]*PartitionOffsetMetadata, groupId string) error {\n\treturn sp.producer.AddOffsetsToTxn(offsets, groupId)\n}\n\nfunc (sp *syncProducer) AddMessageToTxn(msg *ConsumerMessage, groupId string, metadata *string) error {\n\treturn sp.producer.AddMessageToTxn(msg, groupId, metadata)\n}\n\nfunc (p *syncProducer) TxnStatus() ProducerTxnStatusFlag {\n\treturn p.producer.TxnStatus()\n}\n"
},
{
"path": "sync_producer_test.go",
"content": "//go:build !functional\n\npackage sarama\n\nimport (\n\t\"errors\"\n\t\"log\"\n\t\"sync\"\n\t\"testing\"\n)\n\nfunc TestSyncProducer(t *testing.T) {\n\tseedBroker := NewMockBroker(t, 1)\n\tleader := NewMockBroker(t, 2)\n\n\tmetadataResponse := new(MetadataResponse)\n\tmetadataResponse.AddBroker(leader.Addr(), leader.BrokerID())\n\tmetadataResponse.AddTopicPartition(\"my_topic\", 0, leader.BrokerID(), nil, nil, nil, ErrNoError)\n\tseedBroker.Returns(metadataResponse)\n\n\tprodSuccess := new(ProduceResponse)\n\tprodSuccess.AddTopicPartition(\"my_topic\", 0, ErrNoError)\n\tfor i := 0; i < 10; i++ {\n\t\tleader.Returns(prodSuccess)\n\t}\n\n\tconfig := NewTestConfig()\n\tconfig.Producer.Return.Successes = true\n\tproducer, err := NewSyncProducer([]string{seedBroker.Addr()}, config)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tfor i := 0; i < 10; i++ {\n\t\tmsg := &ProducerMessage{\n\t\t\tTopic: \"my_topic\",\n\t\t\tValue: StringEncoder(TestMessage),\n\t\t\tMetadata: \"test\",\n\t\t}\n\n\t\tpartition, offset, err := producer.SendMessage(msg)\n\n\t\tif partition != 0 || msg.Partition != partition {\n\t\t\tt.Error(\"Unexpected partition\")\n\t\t}\n\t\tif offset != 0 || msg.Offset != offset {\n\t\t\tt.Error(\"Unexpected offset\")\n\t\t}\n\t\tif str, ok := msg.Metadata.(string); !ok || str != \"test\" {\n\t\t\tt.Error(\"Unexpected metadata\")\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}\n\n\tsafeClose(t, producer)\n\tleader.Close()\n\tseedBroker.Close()\n}\n\nfunc TestSyncProducerTransactional(t *testing.T) {\n\tseedBroker := NewMockBroker(t, 1)\n\tdefer seedBroker.Close()\n\tleader := NewMockBroker(t, 2)\n\tdefer leader.Close()\n\n\tconfig := NewTestConfig()\n\tconfig.Version = V0_11_0_0\n\tconfig.Producer.RequiredAcks = WaitForAll\n\tconfig.Producer.Return.Successes = true\n\tconfig.Producer.Transaction.ID = \"test\"\n\tconfig.Producer.Idempotent = true\n\tconfig.Producer.Retry.Max = 5\n\tconfig.Net.MaxOpenRequests = 1\n\n\tmetadataResponse := new(MetadataResponse)\n\tmetadataResponse.Version = 4\n\tmetadataResponse.ControllerID = leader.BrokerID()\n\tmetadataResponse.AddBroker(leader.Addr(), leader.BrokerID())\n\tmetadataResponse.AddTopic(\"my_topic\", ErrNoError)\n\tmetadataResponse.AddTopicPartition(\"my_topic\", 0, leader.BrokerID(), nil, nil, nil, ErrNoError)\n\tseedBroker.Returns(metadataResponse)\n\n\tclient, err := NewClient([]string{seedBroker.Addr()}, config)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer safeClose(t, client)\n\n\tfindCoordinatorResponse := new(FindCoordinatorResponse)\n\tfindCoordinatorResponse.Coordinator = client.Brokers()[0]\n\tfindCoordinatorResponse.Version = 1\n\tleader.Returns(findCoordinatorResponse)\n\n\tinitProducerIdResponse := new(InitProducerIDResponse)\n\tleader.Returns(initProducerIdResponse)\n\n\taddPartitionToTxn := new(AddPartitionsToTxnResponse)\n\taddPartitionToTxn.Errors = map[string][]*PartitionError{\n\t\t\"my_topic\": {\n\t\t\t{\n\t\t\t\tPartition: 0,\n\t\t\t},\n\t\t},\n\t}\n\tleader.Returns(addPartitionToTxn)\n\n\tprodSuccess := new(ProduceResponse)\n\tprodSuccess.Version = 3\n\tprodSuccess.AddTopicPartition(\"my_topic\", 0, ErrNoError)\n\tfor i := 0; i < 10; i++ {\n\t\tleader.Returns(prodSuccess)\n\t}\n\n\tendTxnResponse := &EndTxnResponse{}\n\tleader.Returns(endTxnResponse)\n\n\tproducer, err := NewSyncProducerFromClient(client)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif !producer.IsTransactional() {\n\t\tt.Error(\"producer is not transactional\")\n\t}\n\n\terr = producer.BeginTxn()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tif producer.TxnStatus()&ProducerTxnFlagInTransaction == 0 {\n\t\tt.Error(\"transaction must started\")\n\t}\n\n\tfor i := 0; i < 10; i++ {\n\t\tmsg := &ProducerMessage{\n\t\t\tTopic: \"my_topic\",\n\t\t\tValue: StringEncoder(TestMessage),\n\t\t\tMetadata: \"test\",\n\t\t}\n\n\t\tpartition, offset, err := producer.SendMessage(msg)\n\n\t\tif partition != 0 || msg.Partition != partition {\n\t\t\tt.Error(\"Unexpected partition\")\n\t\t}\n\t\tif offset != 0 || msg.Offset != offset {\n\t\t\tt.Error(\"Unexpected offset\")\n\t\t}\n\t\tif str, ok := msg.Metadata.(string); !ok || str != \"test\" {\n\t\t\tt.Error(\"Unexpected metadata\")\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t}\n\t}\n\terr = producer.CommitTxn()\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tsafeClose(t, producer)\n}\n\nfunc TestSyncProducerBatch(t *testing.T) {\n\tseedBroker := NewMockBroker(t, 1)\n\tleader := NewMockBroker(t, 2)\n\n\tmetadataResponse := new(MetadataResponse)\n\tmetadataResponse.AddBroker(leader.Addr(), leader.BrokerID())\n\tmetadataResponse.AddTopicPartition(\"my_topic\", 0, leader.BrokerID(), nil, nil, nil, ErrNoError)\n\tseedBroker.Returns(metadataResponse)\n\n\tprodSuccess := new(ProduceResponse)\n\tprodSuccess.AddTopicPartition(\"my_topic\", 0, ErrNoError)\n\tleader.Returns(prodSuccess)\n\n\tconfig := NewTestConfig()\n\tconfig.Producer.Flush.Messages = 3\n\tconfig.Producer.Return.Successes = true\n\tproducer, err := NewSyncProducer([]string{seedBroker.Addr()}, config)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\terr = producer.SendMessages([]*ProducerMessage{\n\t\t{\n\t\t\tTopic: \"my_topic\",\n\t\t\tValue: StringEncoder(TestMessage),\n\t\t\tMetadata: \"test\",\n\t\t},\n\t\t{\n\t\t\tTopic: \"my_topic\",\n\t\t\tValue: StringEncoder(TestMessage),\n\t\t\tMetadata: \"test\",\n\t\t},\n\t\t{\n\t\t\tTopic: \"my_topic\",\n\t\t\tValue: StringEncoder(TestMessage),\n\t\t\tMetadata: \"test\",\n\t\t},\n\t})\n\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tsafeClose(t, producer)\n\tleader.Close()\n\tseedBroker.Close()\n}\n\nfunc TestConcurrentSyncProducer(t *testing.T) {\n\tseedBroker := NewMockBroker(t, 1)\n\tleader := NewMockBroker(t, 2)\n\n\tmetadataResponse := new(MetadataResponse)\n\tmetadataResponse.AddBroker(leader.Addr(), leader.BrokerID())\n\tmetadataResponse.AddTopicPartition(\"my_topic\", 0, leader.BrokerID(), nil, nil, nil, ErrNoError)\n\tseedBroker.Returns(metadataResponse)\n\n\tprodSuccess := new(ProduceResponse)\n\tprodSuccess.AddTopicPartition(\"my_topic\", 0, ErrNoError)\n\tleader.Returns(prodSuccess)\n\n\tconfig := NewTestConfig()\n\tconfig.Producer.Flush.Messages = 100\n\tconfig.Producer.Return.Successes = true\n\tproducer, err := NewSyncProducer([]string{seedBroker.Addr()}, config)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\twg := sync.WaitGroup{}\n\n\tfor i := 0; i < 100; i++ {\n\t\twg.Add(1)\n\t\tgo func() {\n\t\t\tmsg := &ProducerMessage{Topic: \"my_topic\", Value: StringEncoder(TestMessage)}\n\t\t\tpartition, _, err := producer.SendMessage(msg)\n\t\t\tif partition != 0 {\n\t\t\t\tt.Error(\"Unexpected partition\")\n\t\t\t}\n\t\t\tif err != nil {\n\t\t\t\tt.Error(err)\n\t\t\t}\n\t\t\twg.Done()\n\t\t}()\n\t}\n\twg.Wait()\n\n\tsafeClose(t, producer)\n\tleader.Close()\n\tseedBroker.Close()\n}\n\nfunc TestSyncProducerToNonExistingTopic(t *testing.T) {\n\tbroker := NewMockBroker(t, 1)\n\n\tmetadataResponse := new(MetadataResponse)\n\tmetadataResponse.AddBroker(broker.Addr(), broker.BrokerID())\n\tmetadataResponse.AddTopicPartition(\"my_topic\", 0, broker.BrokerID(), nil, nil, nil, ErrNoError)\n\tbroker.Returns(metadataResponse)\n\n\tconfig := NewTestConfig()\n\tconfig.Metadata.Retry.Max = 0\n\tconfig.Producer.Retry.Max = 0\n\tconfig.Producer.Return.Successes = true\n\n\tproducer, err := NewSyncProducer([]string{broker.Addr()}, config)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tmetadataResponse = new(MetadataResponse)\n\tmetadataResponse.AddBroker(broker.Addr(), broker.BrokerID())\n\tmetadataResponse.AddTopic(\"unknown\", ErrUnknownTopicOrPartition)\n\tbroker.Returns(metadataResponse)\n\n\t_, _, err = producer.SendMessage(&ProducerMessage{Topic: \"unknown\"})\n\tif !errors.Is(err, ErrUnknownTopicOrPartition) {\n\t\tt.Error(\"Uxpected ErrUnknownTopicOrPartition, found:\", err)\n\t}\n\n\tsafeClose(t, producer)\n\tbroker.Close()\n}\n\nfunc TestSyncProducerRecoveryWithRetriesDisabled(t *testing.T) {\n\tseedBroker := NewMockBroker(t, 1)\n\tleader1 := NewMockBroker(t, 2)\n\tleader2 := NewMockBroker(t, 3)\n\n\tmetadataLeader1 := new(MetadataResponse)\n\tmetadataLeader1.AddBroker(leader1.Addr(), leader1.BrokerID())\n\tmetadataLeader1.AddTopicPartition(\"my_topic\", 0, leader1.BrokerID(), nil, nil, nil, ErrNoError)\n\tseedBroker.Returns(metadataLeader1)\n\n\tconfig := NewTestConfig()\n\tconfig.Producer.Retry.Max = 0 // disable!\n\tconfig.Producer.Retry.Backoff = 0\n\tconfig.Producer.Return.Successes = true\n\tproducer, err := NewSyncProducer([]string{seedBroker.Addr()}, config)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tseedBroker.Close()\n\n\tprodNotLeader := new(ProduceResponse)\n\tprodNotLeader.AddTopicPartition(\"my_topic\", 0, ErrNotLeaderForPartition)\n\tleader1.Returns(prodNotLeader)\n\t_, _, err = producer.SendMessage(&ProducerMessage{Topic: \"my_topic\", Value: StringEncoder(TestMessage)})\n\tif !errors.Is(err, ErrNotLeaderForPartition) {\n\t\tt.Fatal(err)\n\t}\n\n\tmetadataLeader2 := new(MetadataResponse)\n\tmetadataLeader2.AddBroker(leader2.Addr(), leader2.BrokerID())\n\tmetadataLeader2.AddTopicPartition(\"my_topic\", 0, leader2.BrokerID(), nil, nil, nil, ErrNoError)\n\tleader1.Returns(metadataLeader2)\n\tprodSuccess := new(ProduceResponse)\n\tprodSuccess.AddTopicPartition(\"my_topic\", 0, ErrNoError)\n\tleader2.Returns(prodSuccess)\n\t_, _, err = producer.SendMessage(&ProducerMessage{Topic: \"my_topic\", Value: StringEncoder(TestMessage)})\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tleader1.Close()\n\tleader2.Close()\n\tsafeClose(t, producer)\n}\n\n// This example shows the basic usage pattern of the SyncProducer.\nfunc ExampleSyncProducer() {\n\tproducer, err := NewSyncProducer([]string{\"localhost:9092\"}, nil)\n\tif err != nil {\n\t\tlog.Fatalln(err)\n\t}\n\tdefer func() {\n\t\tif err := producer.Close(); err != nil {\n\t\t\tlog.Fatalln(err)\n\t\t}\n\t}()\n\n\tmsg := &ProducerMessage{Topic: \"my_topic\", Value: StringEncoder(\"testing 123\")}\n\tpartition, offset, err := producer.SendMessage(msg)\n\tif err != nil {\n\t\tlog.Printf(\"FAILED to send message: %s\\n\", err)\n\t} else {\n\t\tlog.Printf(\"> message sent to partition %d at offset %d\\n\", partition, offset)\n\t}\n}\n"
},
{
"path": "timestamp.go",
"content": "package sarama\n\nimport (\n\t\"fmt\"\n\t\"time\"\n)\n\ntype Timestamp struct {\n\t*time.Time\n}\n\nfunc (t Timestamp) encode(pe packetEncoder) error {\n\ttimestamp := int64(-1)\n\n\tif !t.Before(time.Unix(0, 0)) {\n\t\ttimestamp = t.UnixNano() / int64(time.Millisecond)\n\t} else if !t.IsZero() {\n\t\treturn PacketEncodingError{fmt.Sprintf(\"invalid timestamp (%v)\", t)}\n\t}\n\n\tpe.putInt64(timestamp)\n\treturn nil\n}\n\nfunc (t Timestamp) decode(pd packetDecoder) error {\n\tmillis, err := pd.getInt64()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t// negative timestamps are invalid, in these cases we should return\n\t// a zero time\n\ttimestamp := time.Time{}\n\tif millis >= 0 {\n\t\ttimestamp = time.Unix(millis/1000, (millis%1000)*int64(time.Millisecond))\n\t}\n\n\t*t.Time = timestamp\n\treturn nil\n}\n"
},
{
"path": "tools/README.md",
"content": "# Sarama tools\n\nThis folder contains applications that are useful for exploration of your Kafka cluster, or instrumentation.\nSome of these tools mirror tools that ship with Kafka, but these tools won't require installing the JVM to function.\n\n- [kafka-console-producer](./kafka-console-producer): a command line tool to produce a single message to your Kafka custer.\n- [kafka-console-partitionconsumer](./kafka-console-partitionconsumer): (deprecated) a command line tool to consume a single partition of a topic on your Kafka cluster.\n- [kafka-console-consumer](./kafka-console-consumer): a command line tool to consume arbitrary partitions of a topic on your Kafka cluster.\n- [kafka-producer-performance](./kafka-producer-performance): a command line tool to performance test producers (sync and async) on your Kafka cluster.\n\nTo install all tools, run `go install github.com/IBM/sarama/tools/...@latest`\n"
},
{
"path": "tools/kafka-producer-performance/README.md",
"content": "# kafka-producer-performance\n\nA command line tool to test producer performance.\n\n### Installation\n\n go get github.com/IBM/sarama/tools/kafka-producer-performance\n\n\n### Usage\n\n # Display all command line options\n kafka-producer-performance -help\n\n\t# Minimum invocation\n kafka-producer-performance \\\n\t\t-brokers=kafka:9092 \\\n\t\t-message-load=50000 \\\n\t\t-message-size=100 \\\n\t\t-topic=producer_test\n"
},
{
"path": "tools/kafka-producer-performance/main.go",
"content": "package main\n\nimport (\n\t\"context\"\n\t\"crypto/rand\"\n\t\"crypto/x509\"\n\t\"flag\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n\t\"os\"\n\t\"strings\"\n\tgosync \"sync\"\n\t\"time\"\n\n\t\"github.com/rcrowley/go-metrics\"\n\n\t\"github.com/IBM/sarama\"\n\t\"github.com/IBM/sarama/tools/tls\"\n)\n\nvar (\n\tsync = flag.Bool(\n\t\t\"sync\",\n\t\tfalse,\n\t\t\"Use a synchronous producer.\",\n\t)\n\tmessageLoad = flag.Int(\n\t\t\"message-load\",\n\t\t0,\n\t\t\"REQUIRED: The number of messages to produce to -topic.\",\n\t)\n\tmessageSize = flag.Int(\n\t\t\"message-size\",\n\t\t0,\n\t\t\"REQUIRED: The approximate size (in bytes) of each message to produce to -topic.\",\n\t)\n\tbrokers = flag.String(\n\t\t\"brokers\",\n\t\t\"\",\n\t\t\"REQUIRED: A comma separated list of broker addresses.\",\n\t)\n\tsecurityProtocol = flag.String(\n\t\t\"security-protocol\",\n\t\t\"PLAINTEXT\",\n\t\t\"The name of the security protocol to talk to Kafka (PLAINTEXT, SSL).\",\n\t)\n\ttlsRootCACerts = flag.String(\n\t\t\"tls-ca-certs\",\n\t\t\"\",\n\t\t\"The path to a file that contains a set of root certificate authorities in PEM format \"+\n\t\t\t\"to trust when verifying broker certificates when -security-protocol=SSL \"+\n\t\t\t\"(leave empty to use the host's root CA set).\",\n\t)\n\ttlsClientCert = flag.String(\n\t\t\"tls-client-cert\",\n\t\t\"\",\n\t\t\"The path to a file that contains the client certificate to send to the broker \"+\n\t\t\t\"in PEM format if client authentication is required when -security-protocol=SSL \"+\n\t\t\t\"(leave empty to disable client authentication).\",\n\t)\n\ttlsClientKey = flag.String(\n\t\t\"tls-client-key\",\n\t\t\"\",\n\t\t\"The path to a file that contains the client private key linked to the client certificate \"+\n\t\t\t\"in PEM format when -security-protocol=SSL (REQUIRED if tls-client-cert is provided).\",\n\t)\n\ttopic = flag.String(\n\t\t\"topic\",\n\t\t\"\",\n\t\t\"REQUIRED: The topic to run the performance test on.\",\n\t)\n\tpartition = flag.Int(\n\t\t\"partition\",\n\t\t-1,\n\t\t\"The partition of -topic to run the performance test on.\",\n\t)\n\tthroughput = flag.Int(\n\t\t\"throughput\",\n\t\t0,\n\t\t\"The maximum number of messages to send per second (0 for no limit).\",\n\t)\n\tmaxOpenRequests = flag.Int(\n\t\t\"max-open-requests\",\n\t\t5,\n\t\t\"The maximum number of unacknowledged requests the client will send on a single connection before blocking.\",\n\t)\n\tmaxMessageBytes = flag.Int(\n\t\t\"max-message-bytes\",\n\t\t1000000,\n\t\t\"The max permitted size of a message.\",\n\t)\n\trequiredAcks = flag.Int(\n\t\t\"required-acks\",\n\t\t1,\n\t\t\"The required number of acks needed from the broker (-1: all, 0: none, 1: local).\",\n\t)\n\ttimeout = flag.Duration(\n\t\t\"timeout\",\n\t\t10*time.Second,\n\t\t\"The duration the producer will wait to receive -required-acks.\",\n\t)\n\tpartitioner = flag.String(\n\t\t\"partitioner\",\n\t\t\"roundrobin\",\n\t\t\"The partitioning scheme to use (hash, manual, random, roundrobin).\",\n\t)\n\tcompression = flag.String(\n\t\t\"compression\",\n\t\t\"none\",\n\t\t\"The compression method to use (none, gzip, snappy, lz4).\",\n\t)\n\tflushFrequency = flag.Duration(\n\t\t\"flush-frequency\",\n\t\t0,\n\t\t\"The best-effort frequency of flushes.\",\n\t)\n\tflushBytes = flag.Int(\n\t\t\"flush-bytes\",\n\t\t0,\n\t\t\"The best-effort number of bytes needed to trigger a flush.\",\n\t)\n\tflushMessages = flag.Int(\n\t\t\"flush-messages\",\n\t\t0,\n\t\t\"The best-effort number of messages needed to trigger a flush.\",\n\t)\n\tflushMaxMessages = flag.Int(\n\t\t\"flush-max-messages\",\n\t\t0,\n\t\t\"The maximum number of messages the producer will send in a single request.\",\n\t)\n\tclientID = flag.String(\n\t\t\"client-id\",\n\t\t\"sarama\",\n\t\t\"The client ID sent with every request to the brokers.\",\n\t)\n\tchannelBufferSize = flag.Int(\n\t\t\"channel-buffer-size\",\n\t\t256,\n\t\t\"The number of events to buffer in internal and external channels.\",\n\t)\n\troutines = flag.Int(\n\t\t\"routines\",\n\t\t1,\n\t\t\"The number of routines to send the messages from (-sync only).\",\n\t)\n\tversion = flag.String(\n\t\t\"version\",\n\t\t\"0.8.2.0\",\n\t\t\"The assumed version of Kafka.\",\n\t)\n\tverbose = flag.Bool(\n\t\t\"verbose\",\n\t\tfalse,\n\t\t\"Turn on sarama logging to stderr\",\n\t)\n)\n\nfunc parseCompression(scheme string) sarama.CompressionCodec {\n\tswitch scheme {\n\tcase \"none\":\n\t\treturn sarama.CompressionNone\n\tcase \"gzip\":\n\t\treturn sarama.CompressionGZIP\n\tcase \"snappy\":\n\t\treturn sarama.CompressionSnappy\n\tcase \"lz4\":\n\t\treturn sarama.CompressionLZ4\n\tdefault:\n\t\tprintUsageErrorAndExit(fmt.Sprintf(\"Unknown -compression: %s\", scheme))\n\t}\n\tpanic(\"should not happen\")\n}\n\nfunc parsePartitioner(scheme string, partition int) sarama.PartitionerConstructor {\n\tif partition < 0 && scheme == \"manual\" {\n\t\tprintUsageErrorAndExit(\"-partition must not be -1 for -partitioning=manual\")\n\t}\n\tswitch scheme {\n\tcase \"manual\":\n\t\treturn sarama.NewManualPartitioner\n\tcase \"hash\":\n\t\treturn sarama.NewHashPartitioner\n\tcase \"random\":\n\t\treturn sarama.NewRandomPartitioner\n\tcase \"roundrobin\":\n\t\treturn sarama.NewRoundRobinPartitioner\n\tdefault:\n\t\tprintUsageErrorAndExit(fmt.Sprintf(\"Unknown -partitioning: %s\", scheme))\n\t}\n\tpanic(\"should not happen\")\n}\n\nfunc parseVersion(version string) sarama.KafkaVersion {\n\tresult, err := sarama.ParseKafkaVersion(version)\n\tif err != nil {\n\t\tprintUsageErrorAndExit(fmt.Sprintf(\"unknown -version: %s\", version))\n\t}\n\treturn result\n}\n\nfunc generateMessages(topic string, partition, messageLoad, messageSize int) []*sarama.ProducerMessage {\n\tmessages := make([]*sarama.ProducerMessage, messageLoad)\n\tfor i := 0; i < messageLoad; i++ {\n\t\tpayload := make([]byte, messageSize)\n\t\tif _, err := rand.Read(payload); err != nil {\n\t\t\tprintErrorAndExit(69, \"Failed to generate message payload: %s\", err)\n\t\t}\n\t\tmessages[i] = &sarama.ProducerMessage{\n\t\t\tTopic: topic,\n\t\t\tPartition: int32(partition),\n\t\t\tValue: sarama.ByteEncoder(payload),\n\t\t}\n\t}\n\treturn messages\n}\n\nfunc main() {\n\tflag.Parse()\n\n\tif *brokers == \"\" {\n\t\tprintUsageErrorAndExit(\"-brokers is required\")\n\t}\n\tif *topic == \"\" {\n\t\tprintUsageErrorAndExit(\"-topic is required\")\n\t}\n\tif *messageLoad <= 0 {\n\t\tprintUsageErrorAndExit(\"-message-load must be greater than 0\")\n\t}\n\tif *messageSize <= 0 {\n\t\tprintUsageErrorAndExit(\"-message-size must be greater than 0\")\n\t}\n\tif *routines < 1 || *routines > *messageLoad {\n\t\tprintUsageErrorAndExit(\"-routines must be greater than 0 and less than or equal to -message-load\")\n\t}\n\tif *securityProtocol != \"PLAINTEXT\" && *securityProtocol != \"SSL\" {\n\t\tprintUsageErrorAndExit(fmt.Sprintf(\"-security-protocol %q is not supported\", *securityProtocol))\n\t}\n\tif *verbose {\n\t\tsarama.Logger = log.New(os.Stderr, \"\", log.LstdFlags)\n\t}\n\n\tconfig := sarama.NewConfig()\n\n\tconfig.Net.MaxOpenRequests = *maxOpenRequests\n\tconfig.Producer.MaxMessageBytes = *maxMessageBytes\n\tconfig.Producer.RequiredAcks = sarama.RequiredAcks(*requiredAcks)\n\tconfig.Producer.Timeout = *timeout\n\tconfig.Producer.Partitioner = parsePartitioner(*partitioner, *partition)\n\tconfig.Producer.Compression = parseCompression(*compression)\n\tconfig.Producer.Flush.Frequency = *flushFrequency\n\tconfig.Producer.Flush.Bytes = *flushBytes\n\tconfig.Producer.Flush.Messages = *flushMessages\n\tconfig.Producer.Flush.MaxMessages = *flushMaxMessages\n\tconfig.Producer.Return.Successes = true\n\tconfig.ClientID = *clientID\n\tconfig.ChannelBufferSize = *channelBufferSize\n\tconfig.Version = parseVersion(*version)\n\n\tif *securityProtocol == \"SSL\" {\n\t\ttlsConfig, err := tls.NewConfig(*tlsClientCert, *tlsClientKey)\n\t\tif err != nil {\n\t\t\tprintErrorAndExit(69, \"failed to load client certificate from: %s and private key from: %s: %v\",\n\t\t\t\t*tlsClientCert, *tlsClientKey, err)\n\t\t}\n\n\t\tif *tlsRootCACerts != \"\" {\n\t\t\trootCAsBytes, err := os.ReadFile(*tlsRootCACerts)\n\t\t\tif err != nil {\n\t\t\t\tprintErrorAndExit(69, \"failed to read root CA certificates: %v\", err)\n\t\t\t}\n\t\t\tcertPool := x509.NewCertPool()\n\t\t\tif !certPool.AppendCertsFromPEM(rootCAsBytes) {\n\t\t\t\tprintErrorAndExit(69, \"failed to load root CA certificates from file: %s\", *tlsRootCACerts)\n\t\t\t}\n\t\t\t// Use specific root CA set vs the host's set\n\t\t\ttlsConfig.RootCAs = certPool\n\t\t}\n\n\t\tconfig.Net.TLS.Enable = true\n\t\tconfig.Net.TLS.Config = tlsConfig\n\t}\n\n\tif err := config.Validate(); err != nil {\n\t\tprintErrorAndExit(69, \"Invalid configuration: %s\", err)\n\t}\n\n\t// Print out metrics periodically.\n\tdone := make(chan struct{})\n\tctx, cancel := context.WithCancel(context.Background())\n\tgo func(ctx context.Context) {\n\t\tdefer close(done)\n\t\tt := time.Tick(5 * time.Second)\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-t:\n\t\t\t\tprintMetrics(os.Stdout, config.MetricRegistry)\n\t\t\tcase <-ctx.Done():\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}(ctx)\n\n\tbrokers := strings.Split(*brokers, \",\")\n\tif *sync {\n\t\trunSyncProducer(*topic, *partition, *messageLoad, *messageSize, *routines,\n\t\t\tconfig, brokers, *throughput)\n\t} else {\n\t\trunAsyncProducer(*topic, *partition, *messageLoad, *messageSize,\n\t\t\tconfig, brokers, *throughput)\n\t}\n\n\tcancel()\n\t<-done\n}\n\nfunc runAsyncProducer(topic string, partition, messageLoad, messageSize int,\n\tconfig *sarama.Config, brokers []string, throughput int) {\n\tproducer, err := sarama.NewAsyncProducer(brokers, config)\n\tif err != nil {\n\t\tprintErrorAndExit(69, \"Failed to create producer: %s\", err)\n\t}\n\tdefer func() {\n\t\t// Print final metrics.\n\t\tprintMetrics(os.Stdout, config.MetricRegistry)\n\t\tif err := producer.Close(); err != nil {\n\t\t\tprintErrorAndExit(69, \"Failed to close producer: %s\", err)\n\t\t}\n\t}()\n\n\tmessages := generateMessages(topic, partition, messageLoad, messageSize)\n\n\tmessagesDone := make(chan struct{})\n\tgo func() {\n\t\tfor i := 0; i < messageLoad; i++ {\n\t\t\tselect {\n\t\t\tcase <-producer.Successes():\n\t\t\tcase err = <-producer.Errors():\n\t\t\t\tprintErrorAndExit(69, \"%s\", err)\n\t\t\t}\n\t\t}\n\t\tmessagesDone <- struct{}{}\n\t}()\n\n\tif throughput > 0 {\n\t\tticker := time.NewTicker(time.Second)\n\t\tfor idx, message := range messages {\n\t\t\tproducer.Input() <- message\n\t\t\tif (idx+1)%throughput == 0 {\n\t\t\t\t<-ticker.C\n\t\t\t}\n\t\t}\n\t\tticker.Stop()\n\t} else {\n\t\tfor _, message := range messages {\n\t\t\tproducer.Input() <- message\n\t\t}\n\t}\n\n\t<-messagesDone\n\tclose(messagesDone)\n}\n\nfunc runSyncProducer(topic string, partition, messageLoad, messageSize, routines int,\n\tconfig *sarama.Config, brokers []string, throughput int) {\n\tproducer, err := sarama.NewSyncProducer(brokers, config)\n\tif err != nil {\n\t\tprintErrorAndExit(69, \"Failed to create producer: %s\", err)\n\t}\n\tdefer func() {\n\t\t// Print final metrics.\n\t\tprintMetrics(os.Stdout, config.MetricRegistry)\n\t\tif err := producer.Close(); err != nil {\n\t\t\tprintErrorAndExit(69, \"Failed to close producer: %s\", err)\n\t\t}\n\t}()\n\n\tmessages := make([][]*sarama.ProducerMessage, routines)\n\tfor i := 0; i < routines; i++ {\n\t\tif i == routines-1 {\n\t\t\tmessages[i] = generateMessages(topic, partition, messageLoad/routines+messageLoad%routines, messageSize)\n\t\t} else {\n\t\t\tmessages[i] = generateMessages(topic, partition, messageLoad/routines, messageSize)\n\t\t}\n\t}\n\n\tvar wg gosync.WaitGroup\n\tif throughput > 0 {\n\t\tfor _, messages := range messages {\n\t\t\twg.Add(1)\n\t\t\tgo func() {\n\t\t\t\tticker := time.NewTicker(time.Second)\n\t\t\t\tfor _, message := range messages {\n\t\t\t\t\tfor i := 0; i < throughput; i++ {\n\t\t\t\t\t\t_, _, err = producer.SendMessage(message)\n\t\t\t\t\t\tif err != nil {\n\t\t\t\t\t\t\tprintErrorAndExit(69, \"Failed to send message: %s\", err)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\t<-ticker.C\n\t\t\t\t}\n\t\t\t\tticker.Stop()\n\t\t\t\twg.Done()\n\t\t\t}()\n\t\t}\n\t} else {\n\t\tfor _, messages := range messages {\n\t\t\twg.Add(1)\n\t\t\tgo func() {\n\t\t\t\tfor _, message := range messages {\n\t\t\t\t\t_, _, err = producer.SendMessage(message)\n\t\t\t\t\tif err != nil {\n\t\t\t\t\t\tprintErrorAndExit(69, \"Failed to send message: %s\", err)\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\twg.Done()\n\t\t\t}()\n\t\t}\n\t}\n\twg.Wait()\n}\n\nfunc printMetrics(w io.Writer, r metrics.Registry) {\n\trecordSendRateMetric := r.Get(\"record-send-rate\")\n\trequestLatencyMetric := r.Get(\"request-latency-in-ms\")\n\toutgoingByteRateMetric := r.Get(\"outgoing-byte-rate\")\n\trequestsInFlightMetric := r.Get(\"requests-in-flight\")\n\n\tif recordSendRateMetric == nil || requestLatencyMetric == nil || outgoingByteRateMetric == nil ||\n\t\trequestsInFlightMetric == nil {\n\t\treturn\n\t}\n\trecordSendRate := recordSendRateMetric.(metrics.Meter).Snapshot()\n\trequestLatency := requestLatencyMetric.(metrics.Histogram).Snapshot()\n\trequestLatencyPercentiles := requestLatency.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})\n\toutgoingByteRate := outgoingByteRateMetric.(metrics.Meter).Snapshot()\n\trequestsInFlight := requestsInFlightMetric.(metrics.Counter).Count()\n\tfmt.Fprintf(w, \"%d records sent, %.1f records/sec (%.2f MiB/sec ingress, %.2f MiB/sec egress), \"+\n\t\t\"%.1f ms avg latency, %.1f ms stddev, %.1f ms 50th, %.1f ms 75th, \"+\n\t\t\"%.1f ms 95th, %.1f ms 99th, %.1f ms 99.9th, %d total req. in flight\\n\",\n\t\trecordSendRate.Count(),\n\t\trecordSendRate.RateMean(),\n\t\trecordSendRate.RateMean()*float64(*messageSize)/1024/1024,\n\t\toutgoingByteRate.RateMean()/1024/1024,\n\t\trequestLatency.Mean(),\n\t\trequestLatency.StdDev(),\n\t\trequestLatencyPercentiles[0],\n\t\trequestLatencyPercentiles[1],\n\t\trequestLatencyPercentiles[2],\n\t\trequestLatencyPercentiles[3],\n\t\trequestLatencyPercentiles[4],\n\t\trequestsInFlight,\n\t)\n}\n\nfunc printUsageErrorAndExit(message string) {\n\tfmt.Fprintln(os.Stderr, \"ERROR:\", message)\n\tfmt.Fprintln(os.Stderr)\n\tfmt.Fprintln(os.Stderr, \"Available command line options:\")\n\tflag.PrintDefaults()\n\tos.Exit(64)\n}\n\nfunc printErrorAndExit(code int, format string, values ...interface{}) {\n\tfmt.Fprintf(os.Stderr, \"ERROR: %s\\n\", fmt.Sprintf(format, values...))\n\tfmt.Fprintln(os.Stderr)\n\tos.Exit(code)\n}\n"
},
{
"path": "tools/tls/config.go",
"content": "package tls\n\nimport \"crypto/tls\"\n\nfunc NewConfig(clientCert, clientKey string) (*tls.Config, error) {\n\ttlsConfig := tls.Config{\n\t\tMinVersion: tls.VersionTLS12,\n\t}\n\n\tif clientCert != \"\" && clientKey != \"\" {\n\t\tcert, err := tls.LoadX509KeyPair(clientCert, clientKey)\n\t\tif err != nil {\n\t\t\treturn &tlsConfig, err\n\t\t}\n\t\ttlsConfig.Certificates = []tls.Certificate{cert}\n\t}\n\n\treturn &tlsConfig, nil\n}\n"
},
{
"path": "transaction_manager.go",
"content": "package sarama\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n)\n\n// ProducerTxnStatusFlag mark current transaction status.\ntype ProducerTxnStatusFlag int16\n\nconst (\n\t// ProducerTxnFlagUninitialized when txnmgr is created\n\tProducerTxnFlagUninitialized ProducerTxnStatusFlag = 1 << iota\n\t// ProducerTxnFlagInitializing when txnmgr is initializing\n\tProducerTxnFlagInitializing\n\t// ProducerTxnFlagReady when is ready to receive transaction\n\tProducerTxnFlagReady\n\t// ProducerTxnFlagInTransaction when transaction is started\n\tProducerTxnFlagInTransaction\n\t// ProducerTxnFlagEndTransaction when transaction will be committed\n\tProducerTxnFlagEndTransaction\n\t// ProducerTxnFlagInError when having abortable or fatal error\n\tProducerTxnFlagInError\n\t// ProducerTxnFlagCommittingTransaction when committing txn\n\tProducerTxnFlagCommittingTransaction\n\t// ProducerTxnFlagAbortingTransaction when committing txn\n\tProducerTxnFlagAbortingTransaction\n\t// ProducerTxnFlagAbortableError when producer encounter an abortable error\n\t// Must call AbortTxn in this case.\n\tProducerTxnFlagAbortableError\n\t// ProducerTxnFlagFatalError when producer encounter an fatal error\n\t// Must Close an recreate it.\n\tProducerTxnFlagFatalError\n)\n\nfunc (s ProducerTxnStatusFlag) String() string {\n\tstatus := make([]string, 0)\n\tif s&ProducerTxnFlagUninitialized != 0 {\n\t\tstatus = append(status, \"ProducerTxnStateUninitialized\")\n\t}\n\tif s&ProducerTxnFlagInitializing != 0 {\n\t\tstatus = append(status, \"ProducerTxnStateInitializing\")\n\t}\n\tif s&ProducerTxnFlagReady != 0 {\n\t\tstatus = append(status, \"ProducerTxnStateReady\")\n\t}\n\tif s&ProducerTxnFlagInTransaction != 0 {\n\t\tstatus = append(status, \"ProducerTxnStateInTransaction\")\n\t}\n\tif s&ProducerTxnFlagEndTransaction != 0 {\n\t\tstatus = append(status, \"ProducerTxnStateEndTransaction\")\n\t}\n\tif s&ProducerTxnFlagInError != 0 {\n\t\tstatus = append(status, \"ProducerTxnStateInError\")\n\t}\n\tif s&ProducerTxnFlagCommittingTransaction != 0 {\n\t\tstatus = append(status, \"ProducerTxnStateCommittingTransaction\")\n\t}\n\tif s&ProducerTxnFlagAbortingTransaction != 0 {\n\t\tstatus = append(status, \"ProducerTxnStateAbortingTransaction\")\n\t}\n\tif s&ProducerTxnFlagAbortableError != 0 {\n\t\tstatus = append(status, \"ProducerTxnStateAbortableError\")\n\t}\n\tif s&ProducerTxnFlagFatalError != 0 {\n\t\tstatus = append(status, \"ProducerTxnStateFatalError\")\n\t}\n\treturn strings.Join(status, \"|\")\n}\n\n// transactionManager keeps the state necessary to ensure idempotent production\ntype transactionManager struct {\n\tproducerID int64\n\tproducerEpoch int16\n\tsequenceNumbers map[string]int32\n\tmutex sync.Mutex\n\ttransactionalID string\n\ttransactionTimeout time.Duration\n\tclient Client\n\n\t// when kafka cluster is at least 2.5.0.\n\t// used to recover when producer failed.\n\tcoordinatorSupportsBumpingEpoch bool\n\n\t// When producer need to bump it's epoch.\n\tepochBumpRequired bool\n\t// Record last seen error.\n\tlastError error\n\n\t// Ensure that status is never accessed with a race-condition.\n\tstatusLock sync.RWMutex\n\tstatus ProducerTxnStatusFlag\n\n\t// Ensure that only one goroutine will update partitions in current transaction.\n\tpartitionInTxnLock sync.Mutex\n\tpendingPartitionsInCurrentTxn topicPartitionSet\n\tpartitionsInCurrentTxn topicPartitionSet\n\n\t// Offsets to add to transaction.\n\toffsetsInCurrentTxn map[string]topicPartitionOffsets\n}\n\nconst (\n\tnoProducerID = -1\n\tnoProducerEpoch = -1\n\n\t// see publishTxnPartitions comment.\n\taddPartitionsRetryBackoff = 20 * time.Millisecond\n)\n\n// txnmngr allowed transitions.\nvar producerTxnTransitions = map[ProducerTxnStatusFlag][]ProducerTxnStatusFlag{\n\tProducerTxnFlagUninitialized: {\n\t\tProducerTxnFlagReady,\n\t\tProducerTxnFlagInError,\n\t},\n\t// When we need are initializing\n\tProducerTxnFlagInitializing: {\n\t\tProducerTxnFlagInitializing,\n\t\tProducerTxnFlagReady,\n\t\tProducerTxnFlagInError,\n\t},\n\t// When we have initialized transactional producer\n\tProducerTxnFlagReady: {\n\t\tProducerTxnFlagInTransaction,\n\t},\n\t// When beginTxn has been called\n\tProducerTxnFlagInTransaction: {\n\t\t// When calling commit or abort\n\t\tProducerTxnFlagEndTransaction,\n\t\t// When got an error\n\t\tProducerTxnFlagInError,\n\t},\n\tProducerTxnFlagEndTransaction: {\n\t\t// When epoch bump\n\t\tProducerTxnFlagInitializing,\n\t\t// When commit is good\n\t\tProducerTxnFlagReady,\n\t\t// When got an error\n\t\tProducerTxnFlagInError,\n\t},\n\t// Need to abort transaction\n\tProducerTxnFlagAbortableError: {\n\t\t// Call AbortTxn\n\t\tProducerTxnFlagAbortingTransaction,\n\t\t// When got an error\n\t\tProducerTxnFlagInError,\n\t},\n\t// Need to close producer\n\tProducerTxnFlagFatalError: {\n\t\tProducerTxnFlagFatalError,\n\t},\n}\n\ntype topicPartition struct {\n\ttopic string\n\tpartition int32\n}\n\n// to ensure that we don't do a full scan every time a partition or an offset is added.\ntype (\n\ttopicPartitionSet map[topicPartition]struct{}\n\ttopicPartitionOffsets map[topicPartition]*PartitionOffsetMetadata\n)\n\nfunc (s topicPartitionSet) mapToRequest() map[string][]int32 {\n\tresult := make(map[string][]int32, len(s))\n\tfor tp := range s {\n\t\tresult[tp.topic] = append(result[tp.topic], tp.partition)\n\t}\n\treturn result\n}\n\nfunc (s topicPartitionOffsets) mapToRequest() map[string][]*PartitionOffsetMetadata {\n\tresult := make(map[string][]*PartitionOffsetMetadata, len(s))\n\tfor tp, offset := range s {\n\t\tresult[tp.topic] = append(result[tp.topic], offset)\n\t}\n\treturn result\n}\n\n// Return true if current transition is allowed.\nfunc (t *transactionManager) isTransitionValid(target ProducerTxnStatusFlag) bool {\n\tfor status, allowedTransitions := range producerTxnTransitions {\n\t\tif status&t.status != 0 {\n\t\t\tfor _, allowedTransition := range allowedTransitions {\n\t\t\t\tif allowedTransition&target != 0 {\n\t\t\t\t\treturn true\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n\n// Get current transaction status.\nfunc (t *transactionManager) currentTxnStatus() ProducerTxnStatusFlag {\n\tt.statusLock.RLock()\n\tdefer t.statusLock.RUnlock()\n\n\treturn t.status\n}\n\n// Try to transition to a valid status and return an error otherwise.\nfunc (t *transactionManager) transitionTo(target ProducerTxnStatusFlag, err error) error {\n\tt.statusLock.Lock()\n\tdefer t.statusLock.Unlock()\n\n\tif !t.isTransitionValid(target) {\n\t\treturn ErrTransitionNotAllowed\n\t}\n\n\tif target&ProducerTxnFlagInError != 0 {\n\t\tif err == nil {\n\t\t\treturn ErrCannotTransitionNilError\n\t\t}\n\t\tt.lastError = err\n\t} else {\n\t\tt.lastError = nil\n\t}\n\n\tDebugLogger.Printf(\"txnmgr/transition [%s] transition from %s to %s\\n\", t.transactionalID, t.status, target)\n\n\tt.status = target\n\treturn err\n}\n\nfunc (t *transactionManager) getAndIncrementSequenceNumber(topic string, partition int32) (int32, int16) {\n\tkey := fmt.Sprintf(\"%s-%d\", topic, partition)\n\tt.mutex.Lock()\n\tdefer t.mutex.Unlock()\n\tsequence := t.sequenceNumbers[key]\n\tt.sequenceNumbers[key] = sequence + 1\n\treturn sequence, t.producerEpoch\n}\n\nfunc (t *transactionManager) bumpEpoch() {\n\tt.mutex.Lock()\n\tdefer t.mutex.Unlock()\n\tt.producerEpoch++\n\tfor k := range t.sequenceNumbers {\n\t\tt.sequenceNumbers[k] = 0\n\t}\n}\n\nfunc (t *transactionManager) getProducerID() (int64, int16) {\n\tt.mutex.Lock()\n\tdefer t.mutex.Unlock()\n\treturn t.producerID, t.producerEpoch\n}\n\n// Compute retry backoff considered current attempts.\nfunc (t *transactionManager) computeBackoff(attemptsRemaining int) time.Duration {\n\tif t.client.Config().Producer.Transaction.Retry.BackoffFunc != nil {\n\t\tmaxRetries := t.client.Config().Producer.Transaction.Retry.Max\n\t\tretries := maxRetries - attemptsRemaining\n\t\treturn t.client.Config().Producer.Transaction.Retry.BackoffFunc(retries, maxRetries)\n\t}\n\treturn t.client.Config().Producer.Transaction.Retry.Backoff\n}\n\n// return true is txnmngr is transactinal.\nfunc (t *transactionManager) isTransactional() bool {\n\treturn t.transactionalID != \"\"\n}\n\n// add specified offsets to current transaction.\nfunc (t *transactionManager) addOffsetsToTxn(offsetsToAdd map[string][]*PartitionOffsetMetadata, groupId string) error {\n\tt.mutex.Lock()\n\tdefer t.mutex.Unlock()\n\n\tif t.currentTxnStatus()&ProducerTxnFlagInTransaction == 0 {\n\t\treturn ErrTransactionNotReady\n\t}\n\n\tif t.currentTxnStatus()&ProducerTxnFlagFatalError != 0 {\n\t\treturn t.lastError\n\t}\n\n\tif _, ok := t.offsetsInCurrentTxn[groupId]; !ok {\n\t\tt.offsetsInCurrentTxn[groupId] = topicPartitionOffsets{}\n\t}\n\n\tfor topic, offsets := range offsetsToAdd {\n\t\tfor _, offset := range offsets {\n\t\t\ttp := topicPartition{topic: topic, partition: offset.Partition}\n\t\t\tt.offsetsInCurrentTxn[groupId][tp] = offset\n\t\t}\n\t}\n\treturn nil\n}\n\n// send txnmgnr save offsets to transaction coordinator.\nfunc (t *transactionManager) publishOffsetsToTxn(offsets topicPartitionOffsets, groupId string) (topicPartitionOffsets, error) {\n\t// First AddOffsetsToTxn\n\tattemptsRemaining := t.client.Config().Producer.Transaction.Retry.Max\n\texec := func(run func() (bool, error), err error) error {\n\t\tfor attemptsRemaining >= 0 {\n\t\t\tvar retry bool\n\t\t\tretry, err = run()\n\t\t\tif !retry {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tbackoff := t.computeBackoff(attemptsRemaining)\n\t\t\tLogger.Printf(\"txnmgr/add-offset-to-txn [%s] retrying after %dms... (%d attempts remaining) (%s)\\n\",\n\t\t\t\tt.transactionalID, backoff/time.Millisecond, attemptsRemaining, err)\n\t\t\ttime.Sleep(backoff)\n\t\t\tattemptsRemaining--\n\t\t}\n\t\treturn err\n\t}\n\tlastError := exec(func() (bool, error) {\n\t\tcoordinator, err := t.client.TransactionCoordinator(t.transactionalID)\n\t\tif err != nil {\n\t\t\treturn true, err\n\t\t}\n\t\trequest := &AddOffsetsToTxnRequest{\n\t\t\tTransactionalID: t.transactionalID,\n\t\t\tProducerEpoch: t.producerEpoch,\n\t\t\tProducerID: t.producerID,\n\t\t\tGroupID: groupId,\n\t\t}\n\t\tif t.client.Config().Version.IsAtLeast(V2_7_0_0) {\n\t\t\t// Version 2 adds the support for new error code PRODUCER_FENCED.\n\t\t\trequest.Version = 2\n\t\t} else if t.client.Config().Version.IsAtLeast(V2_0_0_0) {\n\t\t\t// Version 1 is the same as version 0.\n\t\t\trequest.Version = 1\n\t\t}\n\t\tresponse, err := coordinator.AddOffsetsToTxn(request)\n\t\tif err != nil {\n\t\t\t// If an error occurred try to refresh current transaction coordinator.\n\t\t\t_ = coordinator.Close()\n\t\t\t_ = t.client.RefreshTransactionCoordinator(t.transactionalID)\n\t\t\treturn true, err\n\t\t}\n\t\tif response == nil {\n\t\t\t// If no response is returned just retry.\n\t\t\treturn true, ErrTxnUnableToParseResponse\n\t\t}\n\t\tif response.Err == ErrNoError {\n\t\t\tDebugLogger.Printf(\"txnmgr/add-offset-to-txn [%s] successful add-offset-to-txn with group %s %+v\\n\",\n\t\t\t\tt.transactionalID, groupId, response)\n\t\t\t// If no error, just exit.\n\t\t\treturn false, nil\n\t\t}\n\t\tswitch response.Err {\n\t\tcase ErrConsumerCoordinatorNotAvailable:\n\t\t\tfallthrough\n\t\tcase ErrNotCoordinatorForConsumer:\n\t\t\t_ = coordinator.Close()\n\t\t\t_ = t.client.RefreshTransactionCoordinator(t.transactionalID)\n\t\t\tfallthrough\n\t\tcase ErrOffsetsLoadInProgress:\n\t\t\tfallthrough\n\t\tcase ErrConcurrentTransactions:\n\t\t\t// Retry\n\t\tcase ErrUnknownProducerID:\n\t\t\tfallthrough\n\t\tcase ErrInvalidProducerIDMapping:\n\t\t\treturn false, t.abortableErrorIfPossible(response.Err)\n\t\tcase ErrGroupAuthorizationFailed:\n\t\t\treturn false, t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagAbortableError, response.Err)\n\t\tdefault:\n\t\t\t// Others are fatal\n\t\t\treturn false, t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagFatalError, response.Err)\n\t\t}\n\t\treturn true, response.Err\n\t}, nil)\n\n\tif lastError != nil {\n\t\treturn offsets, lastError\n\t}\n\n\tresultOffsets := offsets\n\t// Then TxnOffsetCommit\n\t// note the result is not completed until the TxnOffsetCommit returns\n\tattemptsRemaining = t.client.Config().Producer.Transaction.Retry.Max\n\texecTxnOffsetCommit := func(run func() (topicPartitionOffsets, bool, error), err error) (topicPartitionOffsets, error) {\n\t\tvar r topicPartitionOffsets\n\t\tfor attemptsRemaining >= 0 {\n\t\t\tvar retry bool\n\t\t\tr, retry, err = run()\n\t\t\tif !retry {\n\t\t\t\treturn r, err\n\t\t\t}\n\t\t\tbackoff := t.computeBackoff(attemptsRemaining)\n\t\t\tLogger.Printf(\"txnmgr/txn-offset-commit [%s] retrying after %dms... (%d attempts remaining) (%s)\\n\",\n\t\t\t\tt.transactionalID, backoff/time.Millisecond, attemptsRemaining, err)\n\t\t\ttime.Sleep(backoff)\n\t\t\tattemptsRemaining--\n\t\t}\n\t\treturn r, err\n\t}\n\treturn execTxnOffsetCommit(func() (topicPartitionOffsets, bool, error) {\n\t\tconsumerGroupCoordinator, err := t.client.Coordinator(groupId)\n\t\tif err != nil {\n\t\t\treturn resultOffsets, true, err\n\t\t}\n\t\trequest := &TxnOffsetCommitRequest{\n\t\t\tTransactionalID: t.transactionalID,\n\t\t\tProducerEpoch: t.producerEpoch,\n\t\t\tProducerID: t.producerID,\n\t\t\tGroupID: groupId,\n\t\t\tTopics: offsets.mapToRequest(),\n\t\t}\n\t\tif t.client.Config().Version.IsAtLeast(V2_1_0_0) {\n\t\t\t// Version 2 adds the committed leader epoch.\n\t\t\trequest.Version = 2\n\t\t} else if t.client.Config().Version.IsAtLeast(V2_0_0_0) {\n\t\t\t// Version 1 is the same as version 0.\n\t\t\trequest.Version = 1\n\t\t}\n\t\tresponses, err := consumerGroupCoordinator.TxnOffsetCommit(request)\n\t\tif err != nil {\n\t\t\t_ = consumerGroupCoordinator.Close()\n\t\t\t_ = t.client.RefreshCoordinator(groupId)\n\t\t\treturn resultOffsets, true, err\n\t\t}\n\n\t\tif responses == nil {\n\t\t\treturn resultOffsets, true, ErrTxnUnableToParseResponse\n\t\t}\n\n\t\tvar responseErrors []error\n\t\tfailedTxn := topicPartitionOffsets{}\n\t\tfor topic, partitionErrors := range responses.Topics {\n\t\t\tfor _, partitionError := range partitionErrors {\n\t\t\t\tswitch partitionError.Err {\n\t\t\t\tcase ErrNoError:\n\t\t\t\t\tcontinue\n\t\t\t\t// If the topic is unknown or the coordinator is loading, retry with the current coordinator\n\t\t\t\tcase ErrRequestTimedOut:\n\t\t\t\t\tfallthrough\n\t\t\t\tcase ErrConsumerCoordinatorNotAvailable:\n\t\t\t\t\tfallthrough\n\t\t\t\tcase ErrNotCoordinatorForConsumer:\n\t\t\t\t\t_ = consumerGroupCoordinator.Close()\n\t\t\t\t\t_ = t.client.RefreshCoordinator(groupId)\n\t\t\t\t\tfallthrough\n\t\t\t\tcase ErrUnknownTopicOrPartition:\n\t\t\t\t\tfallthrough\n\t\t\t\tcase ErrOffsetsLoadInProgress:\n\t\t\t\t\t// Do nothing just retry\n\t\t\t\tcase ErrIllegalGeneration:\n\t\t\t\t\tfallthrough\n\t\t\t\tcase ErrUnknownMemberId:\n\t\t\t\t\tfallthrough\n\t\t\t\tcase ErrFencedInstancedId:\n\t\t\t\t\tfallthrough\n\t\t\t\tcase ErrGroupAuthorizationFailed:\n\t\t\t\t\treturn resultOffsets, false, t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagAbortableError, partitionError.Err)\n\t\t\t\tdefault:\n\t\t\t\t\t// Others are fatal\n\t\t\t\t\treturn resultOffsets, false, t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagFatalError, partitionError.Err)\n\t\t\t\t}\n\t\t\t\ttp := topicPartition{topic: topic, partition: partitionError.Partition}\n\t\t\t\tfailedTxn[tp] = offsets[tp]\n\t\t\t\tresponseErrors = append(responseErrors, partitionError.Err)\n\t\t\t}\n\t\t}\n\n\t\tresultOffsets = failedTxn\n\n\t\tif len(resultOffsets) == 0 {\n\t\t\tDebugLogger.Printf(\"txnmgr/txn-offset-commit [%s] successful txn-offset-commit with group %s\\n\",\n\t\t\t\tt.transactionalID, groupId)\n\t\t\treturn resultOffsets, false, nil\n\t\t}\n\t\treturn resultOffsets, true, Wrap(ErrTxnOffsetCommit, responseErrors...)\n\t}, nil)\n}\n\nfunc (t *transactionManager) initProducerId() (int64, int16, error) {\n\tisEpochBump := false\n\n\treq := &InitProducerIDRequest{}\n\tif t.isTransactional() {\n\t\treq.TransactionalID = &t.transactionalID\n\t\treq.TransactionTimeout = t.transactionTimeout\n\t}\n\n\tif t.client.Config().Version.IsAtLeast(V2_5_0_0) {\n\t\tif t.client.Config().Version.IsAtLeast(V2_7_0_0) {\n\t\t\t// Version 4 adds the support for new error code PRODUCER_FENCED.\n\t\t\treq.Version = 4\n\t\t} else {\n\t\t\t// Version 3 adds ProducerId and ProducerEpoch, allowing producers to try\n\t\t\t// to resume after an INVALID_PRODUCER_EPOCH error\n\t\t\treq.Version = 3\n\t\t}\n\t\tisEpochBump = t.producerID != noProducerID && t.producerEpoch != noProducerEpoch\n\t\tt.coordinatorSupportsBumpingEpoch = true\n\t\treq.ProducerID = t.producerID\n\t\treq.ProducerEpoch = t.producerEpoch\n\t} else if t.client.Config().Version.IsAtLeast(V2_4_0_0) {\n\t\t// Version 2 is the first flexible version.\n\t\treq.Version = 2\n\t} else if t.client.Config().Version.IsAtLeast(V2_0_0_0) {\n\t\t// Version 1 is the same as version 0.\n\t\treq.Version = 1\n\t}\n\n\tif isEpochBump {\n\t\terr := t.transitionTo(ProducerTxnFlagInitializing, nil)\n\t\tif err != nil {\n\t\t\treturn -1, -1, err\n\t\t}\n\t\tDebugLogger.Printf(\"txnmgr/init-producer-id [%s] invoking InitProducerId for the first time in order to acquire a producer ID\\n\",\n\t\t\tt.transactionalID)\n\t} else {\n\t\tDebugLogger.Printf(\"txnmgr/init-producer-id [%s] invoking InitProducerId with current producer ID %d and epoch %d in order to bump the epoch\\n\",\n\t\t\tt.transactionalID, t.producerID, t.producerEpoch)\n\t}\n\n\tattemptsRemaining := t.client.Config().Producer.Transaction.Retry.Max\n\texec := func(run func() (int64, int16, bool, error), err error) (int64, int16, error) {\n\t\tpid := int64(-1)\n\t\tpepoch := int16(-1)\n\t\tfor attemptsRemaining >= 0 {\n\t\t\tvar retry bool\n\t\t\tpid, pepoch, retry, err = run()\n\t\t\tif !retry {\n\t\t\t\treturn pid, pepoch, err\n\t\t\t}\n\t\t\tbackoff := t.computeBackoff(attemptsRemaining)\n\t\t\tLogger.Printf(\"txnmgr/init-producer-id [%s] retrying after %dms... (%d attempts remaining) (%s)\\n\",\n\t\t\t\tt.transactionalID, backoff/time.Millisecond, attemptsRemaining, err)\n\t\t\ttime.Sleep(backoff)\n\t\t\tattemptsRemaining--\n\t\t}\n\t\treturn -1, -1, err\n\t}\n\treturn exec(func() (int64, int16, bool, error) {\n\t\tvar err error\n\t\tvar coordinator *Broker\n\t\tif t.isTransactional() {\n\t\t\tcoordinator, err = t.client.TransactionCoordinator(t.transactionalID)\n\t\t} else {\n\t\t\tcoordinator = t.client.LeastLoadedBroker()\n\t\t}\n\t\tif err != nil {\n\t\t\treturn -1, -1, true, err\n\t\t}\n\t\tresponse, err := coordinator.InitProducerID(req)\n\t\tif err != nil {\n\t\t\tif t.isTransactional() {\n\t\t\t\t_ = coordinator.Close()\n\t\t\t\t_ = t.client.RefreshTransactionCoordinator(t.transactionalID)\n\t\t\t}\n\t\t\treturn -1, -1, true, err\n\t\t}\n\t\tif response == nil {\n\t\t\treturn -1, -1, true, ErrTxnUnableToParseResponse\n\t\t}\n\t\tif response.Err == ErrNoError {\n\t\t\tif isEpochBump {\n\t\t\t\tt.sequenceNumbers = make(map[string]int32)\n\t\t\t}\n\t\t\terr := t.transitionTo(ProducerTxnFlagReady, nil)\n\t\t\tif err != nil {\n\t\t\t\treturn -1, -1, true, err\n\t\t\t}\n\t\t\tDebugLogger.Printf(\"txnmgr/init-producer-id [%s] successful init producer id %+v\\n\",\n\t\t\t\tt.transactionalID, response)\n\t\t\treturn response.ProducerID, response.ProducerEpoch, false, nil\n\t\t}\n\t\tswitch response.Err {\n\t\t// Retriable errors\n\t\tcase ErrConsumerCoordinatorNotAvailable, ErrNotCoordinatorForConsumer, ErrOffsetsLoadInProgress:\n\t\t\tif t.isTransactional() {\n\t\t\t\t_ = coordinator.Close()\n\t\t\t\t_ = t.client.RefreshTransactionCoordinator(t.transactionalID)\n\t\t\t}\n\t\t// Fatal errors\n\t\tdefault:\n\t\t\treturn -1, -1, false, t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagFatalError, response.Err)\n\t\t}\n\t\treturn -1, -1, true, response.Err\n\t}, nil)\n}\n\n// if kafka cluster is at least 2.5.0 mark txnmngr to bump epoch else mark it as fatal.\nfunc (t *transactionManager) abortableErrorIfPossible(err error) error {\n\tif t.coordinatorSupportsBumpingEpoch {\n\t\tt.epochBumpRequired = true\n\t\treturn t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagAbortableError, err)\n\t}\n\treturn t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagFatalError, err)\n}\n\n// End current transaction.\nfunc (t *transactionManager) completeTransaction() error {\n\tif t.epochBumpRequired {\n\t\terr := t.transitionTo(ProducerTxnFlagInitializing, nil)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t} else {\n\t\terr := t.transitionTo(ProducerTxnFlagReady, nil)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tt.lastError = nil\n\tt.epochBumpRequired = false\n\tt.partitionsInCurrentTxn = topicPartitionSet{}\n\tt.pendingPartitionsInCurrentTxn = topicPartitionSet{}\n\tt.offsetsInCurrentTxn = map[string]topicPartitionOffsets{}\n\n\treturn nil\n}\n\n// send EndTxn request with commit flag. (true when committing false otherwise)\nfunc (t *transactionManager) endTxn(commit bool) error {\n\tattemptsRemaining := t.client.Config().Producer.Transaction.Retry.Max\n\texec := func(run func() (bool, error), err error) error {\n\t\tfor attemptsRemaining >= 0 {\n\t\t\tvar retry bool\n\t\t\tretry, err = run()\n\t\t\tif !retry {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tbackoff := t.computeBackoff(attemptsRemaining)\n\t\t\tLogger.Printf(\"txnmgr/endtxn [%s] retrying after %dms... (%d attempts remaining) (%s)\\n\",\n\t\t\t\tt.transactionalID, backoff/time.Millisecond, attemptsRemaining, err)\n\t\t\ttime.Sleep(backoff)\n\t\t\tattemptsRemaining--\n\t\t}\n\t\treturn err\n\t}\n\treturn exec(func() (bool, error) {\n\t\tcoordinator, err := t.client.TransactionCoordinator(t.transactionalID)\n\t\tif err != nil {\n\t\t\treturn true, err\n\t\t}\n\t\trequest := &EndTxnRequest{\n\t\t\tTransactionalID: t.transactionalID,\n\t\t\tProducerEpoch: t.producerEpoch,\n\t\t\tProducerID: t.producerID,\n\t\t\tTransactionResult: commit,\n\t\t}\n\t\tif t.client.Config().Version.IsAtLeast(V2_7_0_0) {\n\t\t\t// Version 2 adds the support for new error code PRODUCER_FENCED.\n\t\t\trequest.Version = 2\n\t\t} else if t.client.Config().Version.IsAtLeast(V2_0_0_0) {\n\t\t\t// Version 1 is the same as version 0.\n\t\t\trequest.Version = 1\n\t\t}\n\t\tresponse, err := coordinator.EndTxn(request)\n\t\tif err != nil {\n\t\t\t// Always retry on network error\n\t\t\t_ = coordinator.Close()\n\t\t\t_ = t.client.RefreshTransactionCoordinator(t.transactionalID)\n\t\t\treturn true, err\n\t\t}\n\t\tif response == nil {\n\t\t\treturn true, ErrTxnUnableToParseResponse\n\t\t}\n\t\tif response.Err == ErrNoError {\n\t\t\tDebugLogger.Printf(\"txnmgr/endtxn [%s] successful to end txn %+v\\n\",\n\t\t\t\tt.transactionalID, response)\n\t\t\treturn false, t.completeTransaction()\n\t\t}\n\t\tswitch response.Err {\n\t\t// Need to refresh coordinator\n\t\tcase ErrConsumerCoordinatorNotAvailable:\n\t\t\tfallthrough\n\t\tcase ErrNotCoordinatorForConsumer:\n\t\t\t_ = coordinator.Close()\n\t\t\t_ = t.client.RefreshTransactionCoordinator(t.transactionalID)\n\t\t\tfallthrough\n\t\tcase ErrOffsetsLoadInProgress:\n\t\t\tfallthrough\n\t\tcase ErrConcurrentTransactions:\n\t\t\t// Just retry\n\t\tcase ErrUnknownProducerID:\n\t\t\tfallthrough\n\t\tcase ErrInvalidProducerIDMapping:\n\t\t\treturn false, t.abortableErrorIfPossible(response.Err)\n\t\t// Fatal errors\n\t\tdefault:\n\t\t\treturn false, t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagFatalError, response.Err)\n\t\t}\n\t\treturn true, response.Err\n\t}, nil)\n}\n\n// We will try to publish associated offsets for each groups\n// then send endtxn request to mark transaction as finished.\nfunc (t *transactionManager) finishTransaction(commit bool) error {\n\tt.mutex.Lock()\n\tdefer t.mutex.Unlock()\n\n\t// Ensure no error when committing or aborting\n\tif commit && t.currentTxnStatus()&ProducerTxnFlagInError != 0 {\n\t\treturn t.lastError\n\t} else if !commit && t.currentTxnStatus()&ProducerTxnFlagFatalError != 0 {\n\t\treturn t.lastError\n\t}\n\n\t// if no records has been sent don't do anything.\n\tif len(t.partitionsInCurrentTxn) == 0 {\n\t\treturn t.completeTransaction()\n\t}\n\n\tepochBump := t.epochBumpRequired\n\t// If we're aborting the transaction, so there should be no need to add offsets.\n\tif commit && len(t.offsetsInCurrentTxn) > 0 {\n\t\tfor group, offsets := range t.offsetsInCurrentTxn {\n\t\t\tnewOffsets, err := t.publishOffsetsToTxn(offsets, group)\n\t\t\tif err != nil {\n\t\t\t\tt.offsetsInCurrentTxn[group] = newOffsets\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tdelete(t.offsetsInCurrentTxn, group)\n\t\t}\n\t}\n\n\tif t.currentTxnStatus()&ProducerTxnFlagFatalError != 0 {\n\t\treturn t.lastError\n\t}\n\n\tif !errors.Is(t.lastError, ErrInvalidProducerIDMapping) {\n\t\terr := t.endTxn(commit)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif !epochBump {\n\t\t\treturn nil\n\t\t}\n\t}\n\t// reset pid and epoch if needed.\n\treturn t.initializeTransactions()\n}\n\n// called before sending any transactional record\n// won't do anything if current topic-partition is already added to transaction.\nfunc (t *transactionManager) maybeAddPartitionToCurrentTxn(topic string, partition int32) {\n\tif t.currentTxnStatus()&ProducerTxnFlagInError != 0 {\n\t\treturn\n\t}\n\n\ttp := topicPartition{topic: topic, partition: partition}\n\n\tt.partitionInTxnLock.Lock()\n\tdefer t.partitionInTxnLock.Unlock()\n\tif _, ok := t.partitionsInCurrentTxn[tp]; ok {\n\t\t// partition is already added\n\t\treturn\n\t}\n\n\tt.pendingPartitionsInCurrentTxn[tp] = struct{}{}\n}\n\n// Makes a request to kafka to add a list of partitions ot the current transaction.\nfunc (t *transactionManager) publishTxnPartitions() error {\n\tt.partitionInTxnLock.Lock()\n\tdefer t.partitionInTxnLock.Unlock()\n\n\tif t.currentTxnStatus()&ProducerTxnFlagInError != 0 {\n\t\treturn t.lastError\n\t}\n\n\tif len(t.pendingPartitionsInCurrentTxn) == 0 {\n\t\treturn nil\n\t}\n\n\t// Remove the partitions from the pending set regardless of the result. We use the presence\n\t// of partitions in the pending set to know when it is not safe to send batches. However, if\n\t// the partitions failed to be added and we enter an error state, we expect the batches to be\n\t// aborted anyway. In this case, we must be able to continue sending the batches which are in\n\t// retry for partitions that were successfully added.\n\tremoveAllPartitionsOnFatalOrAbortedError := func() {\n\t\tt.pendingPartitionsInCurrentTxn = topicPartitionSet{}\n\t}\n\n\t// We only want to reduce the backoff when retrying the first AddPartition which errored out due to a\n\t// CONCURRENT_TRANSACTIONS error since this means that the previous transaction is still completing and\n\t// we don't want to wait too long before trying to start the new one.\n\t//\n\t// This is only a temporary fix, the long term solution is being tracked in\n\t// https://issues.apache.org/jira/browse/KAFKA-5482\n\tretryBackoff := t.client.Config().Producer.Transaction.Retry.Backoff\n\tcomputeBackoff := func(attemptsRemaining int) time.Duration {\n\t\tif t.client.Config().Producer.Transaction.Retry.BackoffFunc != nil {\n\t\t\tmaxRetries := t.client.Config().Producer.Transaction.Retry.Max\n\t\t\tretries := maxRetries - attemptsRemaining\n\t\t\treturn t.client.Config().Producer.Transaction.Retry.BackoffFunc(retries, maxRetries)\n\t\t}\n\t\treturn retryBackoff\n\t}\n\tattemptsRemaining := t.client.Config().Producer.Transaction.Retry.Max\n\n\texec := func(run func() (bool, error), err error) error {\n\t\tfor attemptsRemaining >= 0 {\n\t\t\tvar retry bool\n\t\t\tretry, err = run()\n\t\t\tif !retry {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tbackoff := computeBackoff(attemptsRemaining)\n\t\t\tLogger.Printf(\"txnmgr/add-partition-to-txn retrying after %dms... (%d attempts remaining) (%s)\\n\", backoff/time.Millisecond, attemptsRemaining, err)\n\t\t\ttime.Sleep(backoff)\n\t\t\tattemptsRemaining--\n\t\t}\n\t\treturn err\n\t}\n\treturn exec(func() (bool, error) {\n\t\tcoordinator, err := t.client.TransactionCoordinator(t.transactionalID)\n\t\tif err != nil {\n\t\t\treturn true, err\n\t\t}\n\t\trequest := &AddPartitionsToTxnRequest{\n\t\t\tTransactionalID: t.transactionalID,\n\t\t\tProducerID: t.producerID,\n\t\t\tProducerEpoch: t.producerEpoch,\n\t\t\tTopicPartitions: t.pendingPartitionsInCurrentTxn.mapToRequest(),\n\t\t}\n\t\tif t.client.Config().Version.IsAtLeast(V2_7_0_0) {\n\t\t\t// Version 2 adds the support for new error code PRODUCER_FENCED.\n\t\t\trequest.Version = 2\n\t\t} else if t.client.Config().Version.IsAtLeast(V2_0_0_0) {\n\t\t\t// Version 1 is the same as version 0.\n\t\t\trequest.Version = 1\n\t\t}\n\t\taddPartResponse, err := coordinator.AddPartitionsToTxn(request)\n\t\tif err != nil {\n\t\t\t_ = coordinator.Close()\n\t\t\t_ = t.client.RefreshTransactionCoordinator(t.transactionalID)\n\t\t\treturn true, err\n\t\t}\n\n\t\tif addPartResponse == nil {\n\t\t\treturn true, ErrTxnUnableToParseResponse\n\t\t}\n\n\t\t// remove from the list partitions that have been successfully updated\n\t\tvar responseErrors []error\n\t\tfor topic, results := range addPartResponse.Errors {\n\t\t\tfor _, response := range results {\n\t\t\t\ttp := topicPartition{topic: topic, partition: response.Partition}\n\t\t\t\tswitch response.Err {\n\t\t\t\tcase ErrNoError:\n\t\t\t\t\t// Mark partition as added to transaction\n\t\t\t\t\tt.partitionsInCurrentTxn[tp] = struct{}{}\n\t\t\t\t\tdelete(t.pendingPartitionsInCurrentTxn, tp)\n\t\t\t\t\tcontinue\n\t\t\t\tcase ErrConsumerCoordinatorNotAvailable:\n\t\t\t\t\tfallthrough\n\t\t\t\tcase ErrNotCoordinatorForConsumer:\n\t\t\t\t\t_ = coordinator.Close()\n\t\t\t\t\t_ = t.client.RefreshTransactionCoordinator(t.transactionalID)\n\t\t\t\t\tfallthrough\n\t\t\t\tcase ErrUnknownTopicOrPartition:\n\t\t\t\t\tfallthrough\n\t\t\t\tcase ErrOffsetsLoadInProgress:\n\t\t\t\t\t// Retry topicPartition\n\t\t\t\tcase ErrConcurrentTransactions:\n\t\t\t\t\tif len(t.partitionsInCurrentTxn) == 0 && retryBackoff > addPartitionsRetryBackoff {\n\t\t\t\t\t\tretryBackoff = addPartitionsRetryBackoff\n\t\t\t\t\t}\n\t\t\t\tcase ErrOperationNotAttempted:\n\t\t\t\t\tfallthrough\n\t\t\t\tcase ErrTopicAuthorizationFailed:\n\t\t\t\t\tremoveAllPartitionsOnFatalOrAbortedError()\n\t\t\t\t\treturn false, t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagAbortableError, response.Err)\n\t\t\t\tcase ErrUnknownProducerID:\n\t\t\t\t\tfallthrough\n\t\t\t\tcase ErrInvalidProducerIDMapping:\n\t\t\t\t\tremoveAllPartitionsOnFatalOrAbortedError()\n\t\t\t\t\treturn false, t.abortableErrorIfPossible(response.Err)\n\t\t\t\t// Fatal errors\n\t\t\t\tdefault:\n\t\t\t\t\tremoveAllPartitionsOnFatalOrAbortedError()\n\t\t\t\t\treturn false, t.transitionTo(ProducerTxnFlagInError|ProducerTxnFlagFatalError, response.Err)\n\t\t\t\t}\n\t\t\t\tresponseErrors = append(responseErrors, response.Err)\n\t\t\t}\n\t\t}\n\n\t\t// handle end\n\t\tif len(t.pendingPartitionsInCurrentTxn) == 0 {\n\t\t\tDebugLogger.Printf(\"txnmgr/add-partition-to-txn [%s] successful to add partitions txn %+v\\n\",\n\t\t\t\tt.transactionalID, addPartResponse)\n\t\t\treturn false, nil\n\t\t}\n\t\treturn true, Wrap(ErrAddPartitionsToTxn, responseErrors...)\n\t}, nil)\n}\n\n// Build a new transaction manager sharing producer client.\nfunc newTransactionManager(conf *Config, client Client) (*transactionManager, error) {\n\ttxnmgr := &transactionManager{\n\t\tproducerID: noProducerID,\n\t\tproducerEpoch: noProducerEpoch,\n\t\tclient: client,\n\t\tpendingPartitionsInCurrentTxn: topicPartitionSet{},\n\t\tpartitionsInCurrentTxn: topicPartitionSet{},\n\t\toffsetsInCurrentTxn: make(map[string]topicPartitionOffsets),\n\t\tstatus: ProducerTxnFlagUninitialized,\n\t}\n\n\tif conf.Producer.Idempotent {\n\t\ttxnmgr.transactionalID = conf.Producer.Transaction.ID\n\t\ttxnmgr.transactionTimeout = conf.Producer.Transaction.Timeout\n\t\ttxnmgr.sequenceNumbers = make(map[string]int32)\n\t\ttxnmgr.mutex = sync.Mutex{}\n\n\t\tvar err error\n\t\ttxnmgr.producerID, txnmgr.producerEpoch, err = txnmgr.initProducerId()\n\t\tif err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t\tLogger.Printf(\"txnmgr/init-producer-id [%s] obtained a ProducerId: %d and ProducerEpoch: %d\\n\",\n\t\t\ttxnmgr.transactionalID, txnmgr.producerID, txnmgr.producerEpoch)\n\t}\n\n\treturn txnmgr, nil\n}\n\n// re-init producer-id and producer-epoch if needed.\nfunc (t *transactionManager) initializeTransactions() (err error) {\n\tt.producerID, t.producerEpoch, err = t.initProducerId()\n\treturn\n}\n"
},
{
"path": "transaction_manager_test.go",
"content": "//go:build !functional\n\npackage sarama\n\nimport (\n\t\"errors\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestTransitions(t *testing.T) {\n\ttestError := errors.New(\"test\")\n\ttype testCase struct {\n\t\ttransitions []ProducerTxnStatusFlag\n\t\texpectedError error\n\t}\n\ttestCases := []testCase{\n\t\t{\n\t\t\ttransitions: []ProducerTxnStatusFlag{\n\t\t\t\tProducerTxnFlagUninitialized,\n\t\t\t\tProducerTxnFlagReady,\n\t\t\t\tProducerTxnFlagInTransaction,\n\t\t\t\tProducerTxnFlagEndTransaction | ProducerTxnFlagCommittingTransaction,\n\t\t\t\tProducerTxnFlagReady,\n\t\t\t},\n\t\t\texpectedError: nil,\n\t\t},\n\t\t{\n\t\t\ttransitions: []ProducerTxnStatusFlag{\n\t\t\t\tProducerTxnFlagUninitialized,\n\t\t\t\tProducerTxnFlagReady,\n\t\t\t\tProducerTxnFlagInTransaction,\n\t\t\t\tProducerTxnFlagEndTransaction | ProducerTxnFlagAbortingTransaction,\n\t\t\t\tProducerTxnFlagReady,\n\t\t\t},\n\t\t\texpectedError: nil,\n\t\t},\n\t\t{\n\t\t\ttransitions: []ProducerTxnStatusFlag{\n\t\t\t\tProducerTxnFlagUninitialized,\n\t\t\t\tProducerTxnFlagReady,\n\t\t\t\tProducerTxnFlagInTransaction,\n\t\t\t\tProducerTxnFlagEndTransaction,\n\t\t\t\tProducerTxnFlagInError | ProducerTxnFlagAbortableError,\n\t\t\t},\n\t\t\texpectedError: testError,\n\t\t},\n\t\t{\n\t\t\ttransitions: []ProducerTxnStatusFlag{\n\t\t\t\tProducerTxnFlagInError | ProducerTxnFlagAbortableError,\n\t\t\t\tProducerTxnFlagEndTransaction | ProducerTxnFlagAbortingTransaction,\n\t\t\t\tProducerTxnFlagReady,\n\t\t\t},\n\t\t\texpectedError: nil,\n\t\t},\n\t\t{\n\t\t\ttransitions: []ProducerTxnStatusFlag{\n\t\t\t\tProducerTxnFlagInError | ProducerTxnFlagAbortableError,\n\t\t\t\tProducerTxnFlagEndTransaction | ProducerTxnFlagCommittingTransaction,\n\t\t\t},\n\t\t\texpectedError: ErrTransitionNotAllowed,\n\t\t},\n\t\t{\n\t\t\ttransitions: []ProducerTxnStatusFlag{\n\t\t\t\tProducerTxnFlagInError | ProducerTxnFlagFatalError,\n\t\t\t\tProducerTxnFlagEndTransaction | ProducerTxnFlagAbortingTransaction,\n\t\t\t},\n\t\t\texpectedError: ErrTransitionNotAllowed,\n\t\t},\n\t}\n\tfor _, tc := range testCases {\n\t\ttxnmgr := transactionManager{}\n\t\ttxnmgr.status = tc.transitions[0]\n\t\tvar lastError error\n\t\tfor i := 1; i < len(tc.transitions); i++ {\n\t\t\tvar baseErr error\n\t\t\tif tc.transitions[i]&ProducerTxnFlagInError != 0 {\n\t\t\t\tbaseErr = testError\n\t\t\t}\n\t\t\tlastError = txnmgr.transitionTo(tc.transitions[i], baseErr)\n\t\t}\n\t\trequire.Equal(t, tc.expectedError, lastError, tc)\n\t}\n}\n\nfunc TestTxnmgrInitProducerIdTxn(t *testing.T) {\n\tbroker := NewMockBroker(t, 1)\n\tdefer broker.Close()\n\n\tmetadataLeader := new(MetadataResponse)\n\tmetadataLeader.Version = 4\n\tmetadataLeader.ControllerID = broker.brokerID\n\tmetadataLeader.AddBroker(broker.Addr(), broker.BrokerID())\n\tbroker.Returns(metadataLeader)\n\n\tconfig := NewTestConfig()\n\tconfig.Producer.Idempotent = true\n\tconfig.Producer.Transaction.ID = \"test\"\n\tconfig.Version = V0_11_0_0\n\tconfig.Producer.RequiredAcks = WaitForAll\n\tconfig.Net.MaxOpenRequests = 1\n\n\tclient, err := NewClient([]string{broker.Addr()}, config)\n\trequire.NoError(t, err)\n\tdefer client.Close()\n\n\tfindCoordinatorResponse := FindCoordinatorResponse{\n\t\tCoordinator: client.Brokers()[0],\n\t\tErr: ErrNoError,\n\t\tVersion: 1,\n\t}\n\tbroker.Returns(&findCoordinatorResponse)\n\n\tproducerIdResponse := &InitProducerIDResponse{\n\t\tErr: ErrNoError,\n\t\tProducerID: 1,\n\t\tProducerEpoch: 0,\n\t}\n\tbroker.Returns(producerIdResponse)\n\n\ttxmng, err := newTransactionManager(config, client)\n\trequire.NoError(t, err)\n\n\trequire.Equal(t, int64(1), txmng.producerID)\n\trequire.Equal(t, int16(0), txmng.producerEpoch)\n\trequire.Equal(t, ProducerTxnFlagReady, txmng.status)\n}\n\n// TestTxnmgrInitProducerIdTxnCoordinatorLoading ensure we retry initProducerId when either FindCoordinator or InitProducerID returns ErrOffsetsLoadInProgress\nfunc TestTxnmgrInitProducerIdTxnCoordinatorLoading(t *testing.T) {\n\tconfig := NewTestConfig()\n\tconfig.Producer.Idempotent = true\n\tconfig.Producer.Transaction.ID = \"txid-group\"\n\tconfig.Version = V0_11_0_0\n\tconfig.Producer.RequiredAcks = WaitForAll\n\tconfig.Net.MaxOpenRequests = 1\n\n\tbroker := NewMockBroker(t, 1)\n\tdefer broker.Close()\n\n\tbroker.SetHandlerByMap(map[string]MockResponse{\n\t\t\"MetadataRequest\": NewMockMetadataResponse(t).\n\t\t\tSetController(broker.BrokerID()).\n\t\t\tSetBroker(broker.Addr(), broker.BrokerID()),\n\t\t\"FindCoordinatorRequest\": NewMockSequence(\n\t\t\tNewMockFindCoordinatorResponse(t).\n\t\t\t\tSetError(CoordinatorTransaction, \"txid-group\", ErrOffsetsLoadInProgress),\n\t\t\tNewMockFindCoordinatorResponse(t).\n\t\t\t\tSetError(CoordinatorTransaction, \"txid-group\", ErrOffsetsLoadInProgress),\n\t\t\tNewMockFindCoordinatorResponse(t).\n\t\t\t\tSetCoordinator(CoordinatorTransaction, \"txid-group\", broker),\n\t\t),\n\t\t\"InitProducerIDRequest\": NewMockSequence(\n\t\t\tNewMockInitProducerIDResponse(t).\n\t\t\t\tSetError(ErrOffsetsLoadInProgress),\n\t\t\tNewMockInitProducerIDResponse(t).\n\t\t\t\tSetError(ErrOffsetsLoadInProgress),\n\t\t\tNewMockInitProducerIDResponse(t).\n\t\t\t\tSetProducerID(1).\n\t\t\t\tSetProducerEpoch(0),\n\t\t),\n\t})\n\n\tclient, err := NewClient([]string{broker.Addr()}, config)\n\trequire.NoError(t, err)\n\tdefer client.Close()\n\n\ttxmng, err := newTransactionManager(config, client)\n\trequire.NoError(t, err)\n\n\trequire.Equal(t, int64(1), txmng.producerID)\n\trequire.Equal(t, int16(0), txmng.producerEpoch)\n\trequire.Equal(t, ProducerTxnFlagReady, txmng.status)\n}\n\nfunc TestMaybeAddPartitionToCurrentTxn(t *testing.T) {\n\ttype testCase struct {\n\t\tinitialFlags ProducerTxnStatusFlag\n\t\tinitialPartitionsInCurrentTxn topicPartitionSet\n\t\tinitialPendingPartitionsInCurrentTxn topicPartitionSet\n\n\t\ttpToAdd map[string][]int32\n\n\t\texpectedPendingPartitions topicPartitionSet\n\t\texpectedPartitionsInTxn topicPartitionSet\n\t}\n\ttestCases := []testCase{\n\t\t{\n\t\t\tinitialFlags: ProducerTxnFlagInTransaction,\n\t\t\tinitialPartitionsInCurrentTxn: topicPartitionSet{\n\t\t\t\t{topic: \"test-topic\", partition: 0}: struct{}{},\n\t\t\t},\n\t\t\tinitialPendingPartitionsInCurrentTxn: topicPartitionSet{},\n\t\t\ttpToAdd: map[string][]int32{\n\t\t\t\t\"test-topic\": {\n\t\t\t\t\t0,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedPendingPartitions: topicPartitionSet{},\n\t\t\texpectedPartitionsInTxn: topicPartitionSet{\n\t\t\t\t{topic: \"test-topic\", partition: 0}: struct{}{},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinitialFlags: ProducerTxnFlagInTransaction,\n\t\t\tinitialPartitionsInCurrentTxn: topicPartitionSet{},\n\t\t\tinitialPendingPartitionsInCurrentTxn: topicPartitionSet{},\n\t\t\ttpToAdd: map[string][]int32{\n\t\t\t\t\"test-topic\": {\n\t\t\t\t\t0,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedPendingPartitions: topicPartitionSet{\n\t\t\t\t{topic: \"test-topic\", partition: 0}: struct{}{},\n\t\t\t},\n\t\t\texpectedPartitionsInTxn: topicPartitionSet{},\n\t\t},\n\t\t{\n\t\t\tinitialFlags: ProducerTxnFlagInTransaction,\n\t\t\tinitialPartitionsInCurrentTxn: topicPartitionSet{\n\t\t\t\t{topic: \"test-topic\", partition: 0}: struct{}{},\n\t\t\t},\n\t\t\tinitialPendingPartitionsInCurrentTxn: topicPartitionSet{},\n\t\t\ttpToAdd: map[string][]int32{\n\t\t\t\t\"test-topic\": {\n\t\t\t\t\t0,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedPendingPartitions: topicPartitionSet{},\n\t\t\texpectedPartitionsInTxn: topicPartitionSet{\n\t\t\t\t{topic: \"test-topic\", partition: 0}: struct{}{},\n\t\t\t},\n\t\t},\n\t\t{\n\t\t\tinitialFlags: ProducerTxnFlagInTransaction,\n\t\t\tinitialPartitionsInCurrentTxn: topicPartitionSet{},\n\t\t\tinitialPendingPartitionsInCurrentTxn: topicPartitionSet{\n\t\t\t\t{topic: \"test-topic\", partition: 0}: struct{}{},\n\t\t\t},\n\t\t\ttpToAdd: map[string][]int32{\n\t\t\t\t\"test-topic\": {\n\t\t\t\t\t0,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedPendingPartitions: topicPartitionSet{\n\t\t\t\t{topic: \"test-topic\", partition: 0}: struct{}{},\n\t\t\t},\n\t\t\texpectedPartitionsInTxn: topicPartitionSet{},\n\t\t},\n\t\t{\n\t\t\tinitialFlags: ProducerTxnFlagInError,\n\t\t\tinitialPartitionsInCurrentTxn: topicPartitionSet{},\n\t\t\tinitialPendingPartitionsInCurrentTxn: topicPartitionSet{},\n\t\t\ttpToAdd: map[string][]int32{\n\t\t\t\t\"test-topic\": {\n\t\t\t\t\t0,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedPendingPartitions: topicPartitionSet{},\n\t\t\texpectedPartitionsInTxn: topicPartitionSet{},\n\t\t},\n\t}\n\n\tbroker := NewMockBroker(t, 1)\n\tdefer broker.Close()\n\n\tmetadataLeader := new(MetadataResponse)\n\tmetadataLeader.Version = 4\n\tmetadataLeader.ControllerID = broker.brokerID\n\tmetadataLeader.AddBroker(broker.Addr(), broker.BrokerID())\n\tmetadataLeader.AddTopic(\"test-topic\", ErrNoError)\n\tmetadataLeader.AddTopicPartition(\"test-topic\", 0, broker.BrokerID(), nil, nil, nil, ErrNoError)\n\n\tconfig := NewTestConfig()\n\tconfig.Producer.Idempotent = true\n\tconfig.Producer.Transaction.ID = \"test\"\n\tconfig.Version = V0_11_0_0\n\tconfig.Producer.RequiredAcks = WaitForAll\n\tconfig.Net.MaxOpenRequests = 1\n\tconfig.Producer.Transaction.Retry.Max = 0\n\tconfig.Producer.Transaction.Retry.Backoff = 0\n\n\tfor _, tc := range testCases {\n\t\tfunc() {\n\t\t\tbroker.Returns(metadataLeader)\n\n\t\t\tclient, err := NewClient([]string{broker.Addr()}, config)\n\t\t\trequire.NoError(t, err)\n\t\t\tdefer client.Close()\n\n\t\t\tfindCoordinatorResponse := FindCoordinatorResponse{\n\t\t\t\tCoordinator: client.Brokers()[0],\n\t\t\t\tErr: ErrNoError,\n\t\t\t\tVersion: 1,\n\t\t\t}\n\t\t\tbroker.Returns(&findCoordinatorResponse)\n\n\t\t\tproducerIdResponse := &InitProducerIDResponse{\n\t\t\t\tErr: ErrNoError,\n\t\t\t\tProducerID: 1,\n\t\t\t\tProducerEpoch: 0,\n\t\t\t}\n\t\t\tbroker.Returns(producerIdResponse)\n\n\t\t\ttxmng, err := newTransactionManager(config, client)\n\t\t\trequire.NoError(t, err)\n\n\t\t\ttxmng.partitionsInCurrentTxn = tc.initialPartitionsInCurrentTxn\n\t\t\ttxmng.pendingPartitionsInCurrentTxn = tc.initialPendingPartitionsInCurrentTxn\n\t\t\ttxmng.status = tc.initialFlags\n\n\t\t\tfor topic, partitions := range tc.tpToAdd {\n\t\t\t\tfor _, partition := range partitions {\n\t\t\t\t\ttxmng.maybeAddPartitionToCurrentTxn(topic, partition)\n\t\t\t\t}\n\t\t\t}\n\t\t\trequire.Equal(t, tc.expectedPartitionsInTxn, txmng.partitionsInCurrentTxn, tc)\n\t\t\trequire.Equal(t, tc.expectedPendingPartitions, txmng.pendingPartitionsInCurrentTxn, tc)\n\t\t}()\n\t}\n}\n\nfunc TestAddOffsetsToTxn(t *testing.T) {\n\ttype testCase struct {\n\t\tbrokerErr KError\n\t\tinitialFlags ProducerTxnStatusFlag\n\t\texpectedFlags ProducerTxnStatusFlag\n\t\texpectedError error\n\t\tnewOffsets topicPartitionOffsets\n\t}\n\n\toriginalOffsets := topicPartitionOffsets{\n\t\ttopicPartition{topic: \"test-topic\", partition: 0}: {\n\t\t\tPartition: 0,\n\t\t\tOffset: 0,\n\t\t},\n\t}\n\n\ttestCases := []testCase{\n\t\t{\n\t\t\tbrokerErr: ErrNoError,\n\t\t\tinitialFlags: ProducerTxnFlagInTransaction,\n\t\t\texpectedFlags: ProducerTxnFlagInTransaction,\n\t\t\texpectedError: nil,\n\t\t\tnewOffsets: topicPartitionOffsets{},\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrConsumerCoordinatorNotAvailable,\n\t\t\tinitialFlags: ProducerTxnFlagInTransaction,\n\t\t\texpectedFlags: ProducerTxnFlagInTransaction,\n\t\t\texpectedError: ErrConsumerCoordinatorNotAvailable,\n\t\t\tnewOffsets: originalOffsets,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrNotCoordinatorForConsumer,\n\t\t\tinitialFlags: ProducerTxnFlagInTransaction,\n\t\t\texpectedFlags: ProducerTxnFlagInTransaction,\n\t\t\texpectedError: ErrNotCoordinatorForConsumer,\n\t\t\tnewOffsets: originalOffsets,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrOffsetsLoadInProgress,\n\t\t\tinitialFlags: ProducerTxnFlagInTransaction,\n\t\t\texpectedFlags: ProducerTxnFlagInTransaction,\n\t\t\texpectedError: ErrOffsetsLoadInProgress,\n\t\t\tnewOffsets: originalOffsets,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrConcurrentTransactions,\n\t\t\tinitialFlags: ProducerTxnFlagInTransaction,\n\t\t\texpectedFlags: ProducerTxnFlagInTransaction,\n\t\t\texpectedError: ErrConcurrentTransactions,\n\t\t\tnewOffsets: originalOffsets,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrUnknownProducerID,\n\t\t\tinitialFlags: ProducerTxnFlagInTransaction,\n\t\t\texpectedFlags: ProducerTxnFlagFatalError,\n\t\t\texpectedError: ErrUnknownProducerID,\n\t\t\tnewOffsets: originalOffsets,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrInvalidProducerIDMapping,\n\t\t\tinitialFlags: ProducerTxnFlagInTransaction,\n\t\t\texpectedFlags: ProducerTxnFlagFatalError,\n\t\t\texpectedError: ErrInvalidProducerIDMapping,\n\t\t\tnewOffsets: originalOffsets,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrGroupAuthorizationFailed,\n\t\t\tinitialFlags: ProducerTxnFlagInTransaction,\n\t\t\texpectedFlags: ProducerTxnFlagAbortableError,\n\t\t\texpectedError: ErrGroupAuthorizationFailed,\n\t\t\tnewOffsets: originalOffsets,\n\t\t},\n\t}\n\n\tbroker := NewMockBroker(t, 1)\n\tdefer broker.Close()\n\n\tmetadataLeader := new(MetadataResponse)\n\tmetadataLeader.Version = 4\n\tmetadataLeader.ControllerID = broker.brokerID\n\tmetadataLeader.AddBroker(broker.Addr(), broker.BrokerID())\n\tmetadataLeader.AddTopic(\"test-topic\", ErrNoError)\n\tmetadataLeader.AddTopicPartition(\"test-topic\", 0, broker.BrokerID(), nil, nil, nil, ErrNoError)\n\n\tconfig := NewTestConfig()\n\tconfig.Producer.Idempotent = true\n\tconfig.Producer.Transaction.ID = \"test\"\n\tconfig.Version = V0_11_0_0\n\tconfig.Producer.RequiredAcks = WaitForAll\n\tconfig.Net.MaxOpenRequests = 1\n\tconfig.Producer.Transaction.Retry.Max = 0\n\tconfig.Producer.Transaction.Retry.Backoff = 0\n\n\toffsets := topicPartitionOffsets{\n\t\ttopicPartition{topic: \"test-topic\", partition: 0}: {\n\t\t\tPartition: 0,\n\t\t\tOffset: 0,\n\t\t},\n\t}\n\n\tfor _, tc := range testCases {\n\t\tfunc() {\n\t\t\tbroker.Returns(metadataLeader)\n\n\t\t\tclient, err := NewClient([]string{broker.Addr()}, config)\n\t\t\trequire.NoError(t, err)\n\n\t\t\tdefer client.Close()\n\n\t\t\tfindCoordinatorResponse := FindCoordinatorResponse{\n\t\t\t\tCoordinator: client.Brokers()[0],\n\t\t\t\tErr: ErrNoError,\n\t\t\t\tVersion: 1,\n\t\t\t}\n\t\t\tbroker.Returns(&findCoordinatorResponse)\n\n\t\t\tproducerIdResponse := &InitProducerIDResponse{\n\t\t\t\tErr: ErrNoError,\n\t\t\t\tProducerID: 1,\n\t\t\t\tProducerEpoch: 0,\n\t\t\t}\n\t\t\tbroker.Returns(producerIdResponse)\n\n\t\t\ttxmng, err := newTransactionManager(config, client)\n\t\t\trequire.NoError(t, err)\n\n\t\t\ttxmng.status = tc.initialFlags\n\n\t\t\tbroker.Returns(&AddOffsetsToTxnResponse{\n\t\t\t\tErr: tc.brokerErr,\n\t\t\t})\n\n\t\t\tif errors.Is(tc.brokerErr, ErrRequestTimedOut) ||\n\t\t\t\terrors.Is(tc.brokerErr, ErrConsumerCoordinatorNotAvailable) ||\n\t\t\t\terrors.Is(tc.brokerErr, ErrNotCoordinatorForConsumer) {\n\t\t\t\tbroker.Returns(&FindCoordinatorResponse{\n\t\t\t\t\tCoordinator: client.Brokers()[0],\n\t\t\t\t\tErr: ErrNoError,\n\t\t\t\t\tVersion: 1,\n\t\t\t\t})\n\t\t\t}\n\n\t\t\tif tc.brokerErr == ErrNoError {\n\t\t\t\tbroker.Returns(&FindCoordinatorResponse{\n\t\t\t\t\tCoordinator: client.Brokers()[0],\n\t\t\t\t\tErr: ErrNoError,\n\t\t\t\t\tVersion: 1,\n\t\t\t\t})\n\t\t\t\tbroker.Returns(&TxnOffsetCommitResponse{\n\t\t\t\t\tTopics: map[string][]*PartitionError{\n\t\t\t\t\t\t\"test-topic\": {\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\t\t\tErr: ErrNoError,\n\t\t\t\t\t\t\t},\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t})\n\t\t\t}\n\n\t\t\tnewOffsets, err := txmng.publishOffsetsToTxn(offsets, \"test-group\")\n\t\t\tif tc.expectedError != nil {\n\t\t\t\trequire.Equal(t, tc.expectedError.Error(), err.Error())\n\t\t\t} else {\n\t\t\t\trequire.Equal(t, tc.expectedError, err)\n\t\t\t}\n\t\t\trequire.Equal(t, tc.newOffsets, newOffsets)\n\t\t\trequire.True(t, tc.expectedFlags&txmng.status != 0)\n\t\t}()\n\t}\n}\n\nfunc TestTxnOffsetsCommit(t *testing.T) {\n\ttype testCase struct {\n\t\tbrokerErr KError\n\t\tinitialFlags ProducerTxnStatusFlag\n\t\tinitialOffsets topicPartitionOffsets\n\t\texpectedFlags ProducerTxnStatusFlag\n\t\texpectedError error\n\t\texpectedOffsets topicPartitionOffsets\n\t}\n\n\toriginalOffsets := topicPartitionOffsets{\n\t\ttopicPartition{topic: \"test-topic\", partition: 0}: {\n\t\t\tPartition: 0,\n\t\t\tOffset: 0,\n\t\t},\n\t}\n\n\ttestCases := []testCase{\n\t\t{\n\t\t\tbrokerErr: ErrConsumerCoordinatorNotAvailable,\n\t\t\tinitialFlags: ProducerTxnFlagInTransaction,\n\t\t\tinitialOffsets: topicPartitionOffsets{\n\t\t\t\ttopicPartition{topic: \"test-topic\", partition: 0}: {\n\t\t\t\t\tPartition: 0,\n\t\t\t\t\tOffset: 0,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedFlags: ProducerTxnFlagInTransaction,\n\t\t\texpectedError: Wrap(ErrTxnOffsetCommit, ErrConsumerCoordinatorNotAvailable),\n\t\t\texpectedOffsets: originalOffsets,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrNotCoordinatorForConsumer,\n\t\t\tinitialFlags: ProducerTxnFlagInTransaction,\n\t\t\tinitialOffsets: topicPartitionOffsets{\n\t\t\t\ttopicPartition{topic: \"test-topic\", partition: 0}: {\n\t\t\t\t\tPartition: 0,\n\t\t\t\t\tOffset: 0,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedFlags: ProducerTxnFlagInTransaction,\n\t\t\texpectedError: Wrap(ErrTxnOffsetCommit, ErrNotCoordinatorForConsumer),\n\t\t\texpectedOffsets: originalOffsets,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrNoError,\n\t\t\tinitialFlags: ProducerTxnFlagInTransaction,\n\t\t\tinitialOffsets: topicPartitionOffsets{\n\t\t\t\ttopicPartition{topic: \"test-topic\", partition: 0}: {\n\t\t\t\t\tPartition: 0,\n\t\t\t\t\tOffset: 0,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedFlags: ProducerTxnFlagInTransaction,\n\t\t\texpectedError: nil,\n\t\t\texpectedOffsets: topicPartitionOffsets{},\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrUnknownTopicOrPartition,\n\t\t\tinitialFlags: ProducerTxnFlagInTransaction,\n\t\t\tinitialOffsets: topicPartitionOffsets{\n\t\t\t\ttopicPartition{topic: \"test-topic\", partition: 0}: {\n\t\t\t\t\tPartition: 0,\n\t\t\t\t\tOffset: 0,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedFlags: ProducerTxnFlagInTransaction,\n\t\t\texpectedError: Wrap(ErrTxnOffsetCommit, ErrUnknownTopicOrPartition),\n\t\t\texpectedOffsets: originalOffsets,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrOffsetsLoadInProgress,\n\t\t\tinitialFlags: ProducerTxnFlagInTransaction,\n\t\t\tinitialOffsets: topicPartitionOffsets{\n\t\t\t\ttopicPartition{topic: \"test-topic\", partition: 0}: {\n\t\t\t\t\tPartition: 0,\n\t\t\t\t\tOffset: 0,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedFlags: ProducerTxnFlagInTransaction,\n\t\t\texpectedError: Wrap(ErrTxnOffsetCommit, ErrOffsetsLoadInProgress),\n\t\t\texpectedOffsets: originalOffsets,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrIllegalGeneration,\n\t\t\tinitialFlags: ProducerTxnFlagInTransaction,\n\t\t\tinitialOffsets: topicPartitionOffsets{\n\t\t\t\ttopicPartition{topic: \"test-topic\", partition: 0}: {\n\t\t\t\t\tPartition: 0,\n\t\t\t\t\tOffset: 0,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedFlags: ProducerTxnFlagAbortableError,\n\t\t\texpectedError: ErrIllegalGeneration,\n\t\t\texpectedOffsets: originalOffsets,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrUnknownMemberId,\n\t\t\tinitialFlags: ProducerTxnFlagInTransaction,\n\t\t\tinitialOffsets: topicPartitionOffsets{\n\t\t\t\ttopicPartition{topic: \"test-topic\", partition: 0}: {\n\t\t\t\t\tPartition: 0,\n\t\t\t\t\tOffset: 0,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedFlags: ProducerTxnFlagAbortableError,\n\t\t\texpectedError: ErrUnknownMemberId,\n\t\t\texpectedOffsets: originalOffsets,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrFencedInstancedId,\n\t\t\tinitialFlags: ProducerTxnFlagInTransaction,\n\t\t\tinitialOffsets: topicPartitionOffsets{\n\t\t\t\ttopicPartition{topic: \"test-topic\", partition: 0}: {\n\t\t\t\t\tPartition: 0,\n\t\t\t\t\tOffset: 0,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedFlags: ProducerTxnFlagAbortableError,\n\t\t\texpectedError: ErrFencedInstancedId,\n\t\t\texpectedOffsets: originalOffsets,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrGroupAuthorizationFailed,\n\t\t\tinitialFlags: ProducerTxnFlagInTransaction,\n\t\t\tinitialOffsets: topicPartitionOffsets{\n\t\t\t\ttopicPartition{topic: \"test-topic\", partition: 0}: {\n\t\t\t\t\tPartition: 0,\n\t\t\t\t\tOffset: 0,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedFlags: ProducerTxnFlagAbortableError,\n\t\t\texpectedError: ErrGroupAuthorizationFailed,\n\t\t\texpectedOffsets: originalOffsets,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrKafkaStorageError,\n\t\t\tinitialFlags: ProducerTxnFlagInTransaction,\n\t\t\tinitialOffsets: topicPartitionOffsets{\n\t\t\t\ttopicPartition{topic: \"test-topic\", partition: 0}: {\n\t\t\t\t\tPartition: 0,\n\t\t\t\t\tOffset: 0,\n\t\t\t\t},\n\t\t\t},\n\t\t\texpectedFlags: ProducerTxnFlagFatalError,\n\t\t\texpectedError: ErrKafkaStorageError,\n\t\t\texpectedOffsets: originalOffsets,\n\t\t},\n\t}\n\n\tbroker := NewMockBroker(t, 1)\n\tdefer broker.Close()\n\n\tconfig := NewTestConfig()\n\tconfig.Producer.Idempotent = true\n\tconfig.Producer.Transaction.ID = \"test\"\n\tconfig.Version = V0_11_0_0\n\tconfig.Producer.RequiredAcks = WaitForAll\n\tconfig.Net.MaxOpenRequests = 1\n\tconfig.Producer.Transaction.Retry.Max = 0\n\tconfig.Producer.Transaction.Retry.Backoff = 0\n\n\tmetadataLeader := new(MetadataResponse)\n\tmetadataLeader.Version = 4\n\tmetadataLeader.ControllerID = broker.brokerID\n\tmetadataLeader.AddBroker(broker.Addr(), broker.BrokerID())\n\tmetadataLeader.AddTopic(\"test-topic\", ErrNoError)\n\tmetadataLeader.AddTopicPartition(\"test-topic\", 0, broker.BrokerID(), nil, nil, nil, ErrNoError)\n\n\tfor _, tc := range testCases {\n\t\tfunc() {\n\t\t\tbroker.Returns(metadataLeader)\n\n\t\t\tclient, err := NewClient([]string{broker.Addr()}, config)\n\t\t\trequire.NoError(t, err)\n\t\t\tdefer client.Close()\n\n\t\t\tfindCoordinatorResponse := FindCoordinatorResponse{\n\t\t\t\tCoordinator: client.Brokers()[0],\n\t\t\t\tErr: ErrNoError,\n\t\t\t\tVersion: 1,\n\t\t\t}\n\t\t\tbroker.Returns(&findCoordinatorResponse)\n\n\t\t\tproducerIdResponse := &InitProducerIDResponse{\n\t\t\t\tErr: ErrNoError,\n\t\t\t\tProducerID: 1,\n\t\t\t\tProducerEpoch: 0,\n\t\t\t}\n\t\t\tbroker.Returns(producerIdResponse)\n\n\t\t\ttxmng, err := newTransactionManager(config, client)\n\t\t\trequire.NoError(t, err)\n\n\t\t\ttxmng.status = tc.initialFlags\n\n\t\t\tbroker.Returns(&AddOffsetsToTxnResponse{\n\t\t\t\tErr: ErrNoError,\n\t\t\t})\n\n\t\t\tbroker.Returns(&FindCoordinatorResponse{\n\t\t\t\tCoordinator: client.Brokers()[0],\n\t\t\t\tErr: ErrNoError,\n\t\t\t\tVersion: 1,\n\t\t\t})\n\t\t\tbroker.Returns(&TxnOffsetCommitResponse{\n\t\t\t\tTopics: map[string][]*PartitionError{\n\t\t\t\t\t\"test-topic\": {\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\t\tErr: tc.brokerErr,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t})\n\t\t\tif errors.Is(tc.brokerErr, ErrRequestTimedOut) ||\n\t\t\t\terrors.Is(tc.brokerErr, ErrConsumerCoordinatorNotAvailable) ||\n\t\t\t\terrors.Is(tc.brokerErr, ErrNotCoordinatorForConsumer) {\n\t\t\t\tbroker.Returns(&FindCoordinatorResponse{\n\t\t\t\t\tCoordinator: client.Brokers()[0],\n\t\t\t\t\tErr: ErrNoError,\n\t\t\t\t\tVersion: 1,\n\t\t\t\t})\n\t\t\t}\n\n\t\t\tnewOffsets, err := txmng.publishOffsetsToTxn(tc.initialOffsets, \"test-group\")\n\t\t\tif tc.expectedError != nil {\n\t\t\t\trequire.Equal(t, tc.expectedError.Error(), err.Error())\n\t\t\t} else {\n\t\t\t\trequire.Equal(t, tc.expectedError, err)\n\t\t\t}\n\t\t\trequire.Equal(t, tc.expectedOffsets, newOffsets)\n\t\t\trequire.True(t, tc.expectedFlags&txmng.status != 0)\n\t\t}()\n\t}\n}\n\nfunc TestEndTxn(t *testing.T) {\n\ttype testCase struct {\n\t\tbrokerErr KError\n\t\tcommit bool\n\t\texpectedFlags ProducerTxnStatusFlag\n\t\texpectedError error\n\t}\n\n\ttestCases := []testCase{\n\t\t{\n\t\t\tbrokerErr: ErrNoError,\n\t\t\tcommit: true,\n\t\t\texpectedFlags: ProducerTxnFlagReady,\n\t\t\texpectedError: nil,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrConsumerCoordinatorNotAvailable,\n\t\t\tcommit: true,\n\t\t\texpectedFlags: ProducerTxnFlagEndTransaction,\n\t\t\texpectedError: ErrConsumerCoordinatorNotAvailable,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrNotCoordinatorForConsumer,\n\t\t\tcommit: true,\n\t\t\texpectedFlags: ProducerTxnFlagEndTransaction,\n\t\t\texpectedError: ErrNotCoordinatorForConsumer,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrOffsetsLoadInProgress,\n\t\t\tcommit: true,\n\t\t\texpectedFlags: ProducerTxnFlagEndTransaction,\n\t\t\texpectedError: ErrOffsetsLoadInProgress,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrConcurrentTransactions,\n\t\t\tcommit: true,\n\t\t\texpectedFlags: ProducerTxnFlagEndTransaction,\n\t\t\texpectedError: ErrConcurrentTransactions,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrUnknownProducerID,\n\t\t\tcommit: true,\n\t\t\texpectedFlags: ProducerTxnFlagFatalError,\n\t\t\texpectedError: ErrUnknownProducerID,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrInvalidProducerIDMapping,\n\t\t\tcommit: true,\n\t\t\texpectedFlags: ProducerTxnFlagFatalError,\n\t\t\texpectedError: ErrInvalidProducerIDMapping,\n\t\t},\n\t}\n\n\tbroker := NewMockBroker(t, 1)\n\tdefer broker.Close()\n\n\tmetadataLeader := new(MetadataResponse)\n\tmetadataLeader.Version = 4\n\tmetadataLeader.ControllerID = broker.brokerID\n\tmetadataLeader.AddBroker(broker.Addr(), broker.BrokerID())\n\tmetadataLeader.AddTopic(\"test-topic\", ErrNoError)\n\tmetadataLeader.AddTopicPartition(\"test-topic\", 0, broker.BrokerID(), nil, nil, nil, ErrNoError)\n\n\tconfig := NewTestConfig()\n\tconfig.Producer.Idempotent = true\n\tconfig.Producer.Transaction.ID = \"test\"\n\tconfig.Version = V0_11_0_0\n\tconfig.Producer.RequiredAcks = WaitForAll\n\tconfig.Net.MaxOpenRequests = 1\n\tconfig.Producer.Transaction.Retry.Max = 0\n\tconfig.Producer.Transaction.Retry.Backoff = 0\n\n\tfor _, tc := range testCases {\n\t\tfunc() {\n\t\t\tbroker.Returns(metadataLeader)\n\n\t\t\tclient, err := NewClient([]string{broker.Addr()}, config)\n\t\t\trequire.NoError(t, err)\n\t\t\tdefer client.Close()\n\n\t\t\tfindCoordinatorResponse := FindCoordinatorResponse{\n\t\t\t\tCoordinator: client.Brokers()[0],\n\t\t\t\tErr: ErrNoError,\n\t\t\t\tVersion: 1,\n\t\t\t}\n\t\t\tbroker.Returns(&findCoordinatorResponse)\n\n\t\t\tproducerIdResponse := &InitProducerIDResponse{\n\t\t\t\tErr: ErrNoError,\n\t\t\t\tProducerID: 1,\n\t\t\t\tProducerEpoch: 0,\n\t\t\t}\n\t\t\tbroker.Returns(producerIdResponse)\n\n\t\t\ttxmng, err := newTransactionManager(config, client)\n\t\t\trequire.NoError(t, err)\n\n\t\t\ttxmng.status = ProducerTxnFlagEndTransaction\n\n\t\t\tendTxnResponse := &EndTxnResponse{\n\t\t\t\tErr: tc.brokerErr,\n\t\t\t\tThrottleTime: 0,\n\t\t\t}\n\t\t\tbroker.Returns(endTxnResponse)\n\n\t\t\tif errors.Is(tc.brokerErr, ErrRequestTimedOut) ||\n\t\t\t\terrors.Is(tc.brokerErr, ErrConsumerCoordinatorNotAvailable) ||\n\t\t\t\terrors.Is(tc.brokerErr, ErrNotCoordinatorForConsumer) {\n\t\t\t\tbroker.Returns(&FindCoordinatorResponse{\n\t\t\t\t\tCoordinator: client.Brokers()[0],\n\t\t\t\t\tErr: ErrNoError,\n\t\t\t\t\tVersion: 1,\n\t\t\t\t})\n\t\t\t}\n\n\t\t\terr = txmng.endTxn(tc.commit)\n\t\t\trequire.Equal(t, tc.expectedError, err)\n\t\t\trequire.True(t, txmng.currentTxnStatus()&tc.expectedFlags != 0)\n\t\t}()\n\t}\n}\n\nfunc TestPublishPartitionToTxn(t *testing.T) {\n\ttype testCase struct {\n\t\tbrokerErr KError\n\t\texpectedFlags ProducerTxnStatusFlag\n\t\texpectedError error\n\t\texpectedPendingPartitions topicPartitionSet\n\t\texpectedPartitionsInTxn topicPartitionSet\n\t}\n\n\tinitialPendingTopicPartitionSet := topicPartitionSet{\n\t\t{\n\t\t\ttopic: \"test-topic\",\n\t\t\tpartition: 0,\n\t\t}: struct{}{},\n\t}\n\n\ttestCases := []testCase{\n\t\t{\n\t\t\tbrokerErr: ErrNoError,\n\t\t\texpectedFlags: ProducerTxnFlagInTransaction,\n\t\t\texpectedError: nil,\n\t\t\texpectedPendingPartitions: topicPartitionSet{},\n\t\t\texpectedPartitionsInTxn: initialPendingTopicPartitionSet,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrConsumerCoordinatorNotAvailable,\n\t\t\texpectedFlags: ProducerTxnFlagInTransaction,\n\t\t\texpectedError: Wrap(ErrAddPartitionsToTxn, ErrConsumerCoordinatorNotAvailable),\n\t\t\texpectedPartitionsInTxn: topicPartitionSet{},\n\t\t\texpectedPendingPartitions: initialPendingTopicPartitionSet,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrNotCoordinatorForConsumer,\n\t\t\texpectedFlags: ProducerTxnFlagInTransaction,\n\t\t\texpectedError: Wrap(ErrAddPartitionsToTxn, ErrNotCoordinatorForConsumer),\n\t\t\texpectedPartitionsInTxn: topicPartitionSet{},\n\t\t\texpectedPendingPartitions: initialPendingTopicPartitionSet,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrUnknownTopicOrPartition,\n\t\t\texpectedFlags: ProducerTxnFlagInTransaction,\n\t\t\texpectedError: Wrap(ErrAddPartitionsToTxn, ErrUnknownTopicOrPartition),\n\t\t\texpectedPartitionsInTxn: topicPartitionSet{},\n\t\t\texpectedPendingPartitions: initialPendingTopicPartitionSet,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrOffsetsLoadInProgress,\n\t\t\texpectedFlags: ProducerTxnFlagInTransaction,\n\t\t\texpectedError: Wrap(ErrAddPartitionsToTxn, ErrOffsetsLoadInProgress),\n\t\t\texpectedPartitionsInTxn: topicPartitionSet{},\n\t\t\texpectedPendingPartitions: initialPendingTopicPartitionSet,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrConcurrentTransactions,\n\t\t\texpectedFlags: ProducerTxnFlagInTransaction,\n\t\t\texpectedError: Wrap(ErrAddPartitionsToTxn, ErrConcurrentTransactions),\n\t\t\texpectedPartitionsInTxn: topicPartitionSet{},\n\t\t\texpectedPendingPartitions: initialPendingTopicPartitionSet,\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrOperationNotAttempted,\n\t\t\texpectedFlags: ProducerTxnFlagAbortableError,\n\t\t\texpectedError: ErrOperationNotAttempted,\n\t\t\texpectedPartitionsInTxn: topicPartitionSet{},\n\t\t\texpectedPendingPartitions: topicPartitionSet{},\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrTopicAuthorizationFailed,\n\t\t\texpectedFlags: ProducerTxnFlagAbortableError,\n\t\t\texpectedError: ErrTopicAuthorizationFailed,\n\t\t\texpectedPartitionsInTxn: topicPartitionSet{},\n\t\t\texpectedPendingPartitions: topicPartitionSet{},\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrUnknownProducerID,\n\t\t\texpectedFlags: ProducerTxnFlagFatalError,\n\t\t\texpectedError: ErrUnknownProducerID,\n\t\t\texpectedPartitionsInTxn: topicPartitionSet{},\n\t\t\texpectedPendingPartitions: topicPartitionSet{},\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrInvalidProducerIDMapping,\n\t\t\texpectedFlags: ProducerTxnFlagFatalError,\n\t\t\texpectedError: ErrInvalidProducerIDMapping,\n\t\t\texpectedPartitionsInTxn: topicPartitionSet{},\n\t\t\texpectedPendingPartitions: topicPartitionSet{},\n\t\t},\n\t\t{\n\t\t\tbrokerErr: ErrKafkaStorageError,\n\t\t\texpectedFlags: ProducerTxnFlagFatalError,\n\t\t\texpectedError: ErrKafkaStorageError,\n\t\t\texpectedPartitionsInTxn: topicPartitionSet{},\n\t\t\texpectedPendingPartitions: topicPartitionSet{},\n\t\t},\n\t}\n\n\tbroker := NewMockBroker(t, 1)\n\tdefer broker.Close()\n\n\tmetadataLeader := new(MetadataResponse)\n\tmetadataLeader.Version = 4\n\tmetadataLeader.ControllerID = broker.brokerID\n\tmetadataLeader.AddBroker(broker.Addr(), broker.BrokerID())\n\tmetadataLeader.AddTopic(\"test-topic\", ErrNoError)\n\tmetadataLeader.AddTopicPartition(\"test-topic\", 0, broker.BrokerID(), nil, nil, nil, ErrNoError)\n\n\tconfig := NewTestConfig()\n\tconfig.Producer.Idempotent = true\n\tconfig.Producer.Transaction.ID = \"test\"\n\tconfig.Version = V0_11_0_0\n\tconfig.Producer.RequiredAcks = WaitForAll\n\tconfig.Net.MaxOpenRequests = 1\n\tconfig.Producer.Transaction.Retry.Max = 0\n\tconfig.Producer.Transaction.Retry.Backoff = 0\n\n\tfor _, tc := range testCases {\n\t\tfunc() {\n\t\t\tbroker.Returns(metadataLeader)\n\n\t\t\tclient, err := NewClient([]string{broker.Addr()}, config)\n\t\t\trequire.NoError(t, err)\n\t\t\tdefer client.Close()\n\n\t\t\tfindCoordinatorResponse := FindCoordinatorResponse{\n\t\t\t\tCoordinator: client.Brokers()[0],\n\t\t\t\tErr: ErrNoError,\n\t\t\t\tVersion: 1,\n\t\t\t}\n\t\t\tbroker.Returns(&findCoordinatorResponse)\n\n\t\t\tproducerIdResponse := &InitProducerIDResponse{\n\t\t\t\tErr: ErrNoError,\n\t\t\t\tProducerID: 1,\n\t\t\t\tProducerEpoch: 0,\n\t\t\t}\n\t\t\tbroker.Returns(producerIdResponse)\n\n\t\t\ttxmng, err := newTransactionManager(config, client)\n\t\t\trequire.NoError(t, err)\n\n\t\t\ttxmng.status = ProducerTxnFlagInTransaction\n\t\t\ttxmng.pendingPartitionsInCurrentTxn = topicPartitionSet{\n\t\t\t\t{\n\t\t\t\t\ttopic: \"test-topic\",\n\t\t\t\t\tpartition: 0,\n\t\t\t\t}: struct{}{},\n\t\t\t}\n\t\t\tbroker.Returns(&AddPartitionsToTxnResponse{\n\t\t\t\tErrors: map[string][]*PartitionError{\n\t\t\t\t\t\"test-topic\": {\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tPartition: 0,\n\t\t\t\t\t\t\tErr: tc.brokerErr,\n\t\t\t\t\t\t},\n\t\t\t\t\t},\n\t\t\t\t},\n\t\t\t})\n\t\t\tif errors.Is(tc.brokerErr, ErrRequestTimedOut) ||\n\t\t\t\terrors.Is(tc.brokerErr, ErrConsumerCoordinatorNotAvailable) ||\n\t\t\t\terrors.Is(tc.brokerErr, ErrNotCoordinatorForConsumer) {\n\t\t\t\tbroker.Returns(&FindCoordinatorResponse{\n\t\t\t\t\tCoordinator: client.Brokers()[0],\n\t\t\t\t\tErr: ErrNoError,\n\t\t\t\t\tVersion: 1,\n\t\t\t\t})\n\t\t\t}\n\t\t\terr = txmng.publishTxnPartitions()\n\t\t\tif tc.expectedError != nil {\n\t\t\t\trequire.Equal(t, tc.expectedError.Error(), err.Error(), tc)\n\t\t\t} else {\n\t\t\t\trequire.Equal(t, tc.expectedError, err, tc)\n\t\t\t}\n\n\t\t\trequire.True(t, txmng.status&tc.expectedFlags != 0, tc)\n\t\t\trequire.Equal(t, tc.expectedPartitionsInTxn, txmng.partitionsInCurrentTxn, tc)\n\t\t\trequire.Equal(t, tc.expectedPendingPartitions, txmng.pendingPartitionsInCurrentTxn, tc)\n\t\t}()\n\t}\n}\n"
},
{
"path": "txn_offset_commit_request.go",
"content": "package sarama\n\ntype TxnOffsetCommitRequest struct {\n\tVersion int16\n\tTransactionalID string\n\tGroupID string\n\tProducerID int64\n\tProducerEpoch int16\n\tTopics map[string][]*PartitionOffsetMetadata\n}\n\nfunc (t *TxnOffsetCommitRequest) setVersion(v int16) {\n\tt.Version = v\n}\n\nfunc (t *TxnOffsetCommitRequest) encode(pe packetEncoder) error {\n\tif err := pe.putString(t.TransactionalID); err != nil {\n\t\treturn err\n\t}\n\tif err := pe.putString(t.GroupID); err != nil {\n\t\treturn err\n\t}\n\tpe.putInt64(t.ProducerID)\n\tpe.putInt16(t.ProducerEpoch)\n\n\tif err := pe.putArrayLength(len(t.Topics)); err != nil {\n\t\treturn err\n\t}\n\tfor topic, partitions := range t.Topics {\n\t\tif err := pe.putString(topic); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := pe.putArrayLength(len(partitions)); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfor _, partition := range partitions {\n\t\t\tif err := partition.encode(pe, t.Version); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (t *TxnOffsetCommitRequest) decode(pd packetDecoder, version int16) (err error) {\n\tt.Version = version\n\tif t.TransactionalID, err = pd.getString(); err != nil {\n\t\treturn err\n\t}\n\tif t.GroupID, err = pd.getString(); err != nil {\n\t\treturn err\n\t}\n\tif t.ProducerID, err = pd.getInt64(); err != nil {\n\t\treturn err\n\t}\n\tif t.ProducerEpoch, err = pd.getInt16(); err != nil {\n\t\treturn err\n\t}\n\n\tn, err := pd.getArrayLength()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tt.Topics = make(map[string][]*PartitionOffsetMetadata)\n\tfor i := 0; i < n; i++ {\n\t\ttopic, err := pd.getString()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tm, err := pd.getArrayLength()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tt.Topics[topic] = make([]*PartitionOffsetMetadata, m)\n\n\t\tfor j := 0; j < m; j++ {\n\t\t\tpartitionOffsetMetadata := new(PartitionOffsetMetadata)\n\t\t\tif err := partitionOffsetMetadata.decode(pd, version); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t\tt.Topics[topic][j] = partitionOffsetMetadata\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (a *TxnOffsetCommitRequest) key() int16 {\n\treturn apiKeyTxnOffsetCommit\n}\n\nfunc (a *TxnOffsetCommitRequest) version() int16 {\n\treturn a.Version\n}\n\nfunc (a *TxnOffsetCommitRequest) headerVersion() int16 {\n\treturn 1\n}\n\nfunc (a *TxnOffsetCommitRequest) isValidVersion() bool {\n\treturn a.Version >= 0 && a.Version <= 2\n}\n\nfunc (a *TxnOffsetCommitRequest) requiredVersion() KafkaVersion {\n\tswitch a.Version {\n\tcase 2:\n\t\treturn V2_1_0_0\n\tcase 1:\n\t\treturn V2_0_0_0\n\tcase 0:\n\t\treturn V0_11_0_0\n\tdefault:\n\t\treturn V2_1_0_0\n\t}\n}\n\ntype PartitionOffsetMetadata struct {\n\t// Partition contains the index of the partition within the topic.\n\tPartition int32\n\t// Offset contains the message offset to be committed.\n\tOffset int64\n\t// LeaderEpoch contains the leader epoch of the last consumed record.\n\tLeaderEpoch int32\n\t// Metadata contains any associated metadata the client wants to keep.\n\tMetadata *string\n}\n\nfunc (p *PartitionOffsetMetadata) encode(pe packetEncoder, version int16) error {\n\tpe.putInt32(p.Partition)\n\tpe.putInt64(p.Offset)\n\n\tif version >= 2 {\n\t\tpe.putInt32(p.LeaderEpoch)\n\t}\n\n\tif err := pe.putNullableString(p.Metadata); err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n\nfunc (p *PartitionOffsetMetadata) decode(pd packetDecoder, version int16) (err error) {\n\tif p.Partition, err = pd.getInt32(); err != nil {\n\t\treturn err\n\t}\n\tif p.Offset, err = pd.getInt64(); err != nil {\n\t\treturn err\n\t}\n\n\tif version >= 2 {\n\t\tif p.LeaderEpoch, err = pd.getInt32(); err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\tif p.Metadata, err = pd.getNullableString(); err != nil {\n\t\treturn err\n\t}\n\n\treturn nil\n}\n"
},
{
"path": "txn_offset_commit_request_test.go",
"content": "//go:build !functional\n\npackage sarama\n\nimport \"testing\"\n\nvar (\n\ttxnOffsetCommitRequest = []byte{\n\t\t0, 3, 't', 'x', 'n',\n\t\t0, 7, 'g', 'r', 'o', 'u', 'p', 'i', 'd',\n\t\t0, 0, 0, 0, 0, 0, 31, 64, // producer ID\n\t\t0, 1, // producer epoch\n\t\t0, 0, 0, 1, // 1 topic\n\t\t0, 5, 't', 'o', 'p', 'i', 'c',\n\t\t0, 0, 0, 1, // 1 partition\n\t\t0, 0, 0, 2, // partition no 2\n\t\t0, 0, 0, 0, 0, 0, 0, 123,\n\t\t255, 255, // no meta data\n\t}\n\n\ttxnOffsetCommitRequestV2 = []byte{\n\t\t0, 3, 't', 'x', 'n',\n\t\t0, 7, 'g', 'r', 'o', 'u', 'p', 'i', 'd',\n\t\t0, 0, 0, 0, 0, 0, 31, 64, // producer ID\n\t\t0, 1, // producer epoch\n\t\t0, 0, 0, 1, // 1 topic\n\t\t0, 5, 't', 'o', 'p', 'i', 'c',\n\t\t0, 0, 0, 1, // 1 partition\n\t\t0, 0, 0, 2, // partition no 2\n\t\t0, 0, 0, 0, 0, 0, 0, 123,\n\t\t0, 0, 0, 9, // leader epoch\n\t\t255, 255, // no meta data\n\t}\n)\n\nfunc TestTxnOffsetCommitRequest(t *testing.T) {\n\treq := &TxnOffsetCommitRequest{\n\t\tTransactionalID: \"txn\",\n\t\tGroupID: \"groupid\",\n\t\tProducerID: 8000,\n\t\tProducerEpoch: 1,\n\t\tTopics: map[string][]*PartitionOffsetMetadata{\n\t\t\t\"topic\": {{\n\t\t\t\tOffset: 123,\n\t\t\t\tPartition: 2,\n\t\t\t}},\n\t\t},\n\t}\n\n\ttestRequest(t, \"V0\", req, txnOffsetCommitRequest)\n}\n\nfunc TestTxnOffsetCommitRequestV2(t *testing.T) {\n\treq := &TxnOffsetCommitRequest{\n\t\tVersion: 2,\n\t\tTransactionalID: \"txn\",\n\t\tGroupID: \"groupid\",\n\t\tProducerID: 8000,\n\t\tProducerEpoch: 1,\n\t\tTopics: map[string][]*PartitionOffsetMetadata{\n\t\t\t\"topic\": {{\n\t\t\t\tOffset: 123,\n\t\t\t\tPartition: 2,\n\t\t\t\tLeaderEpoch: 9,\n\t\t\t}},\n\t\t},\n\t}\n\n\ttestRequest(t, \"V2\", req, txnOffsetCommitRequestV2)\n}\n"
},
{
"path": "txn_offset_commit_response.go",
"content": "package sarama\n\nimport (\n\t\"time\"\n)\n\ntype TxnOffsetCommitResponse struct {\n\tVersion int16\n\tThrottleTime time.Duration\n\tTopics map[string][]*PartitionError\n}\n\nfunc (t *TxnOffsetCommitResponse) setVersion(v int16) {\n\tt.Version = v\n}\n\nfunc (t *TxnOffsetCommitResponse) encode(pe packetEncoder) error {\n\tpe.putDurationMs(t.ThrottleTime)\n\tif err := pe.putArrayLength(len(t.Topics)); err != nil {\n\t\treturn err\n\t}\n\n\tfor topic, e := range t.Topics {\n\t\tif err := pe.putString(topic); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tif err := pe.putArrayLength(len(e)); err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfor _, partitionError := range e {\n\t\t\tif err := partitionError.encode(pe); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (t *TxnOffsetCommitResponse) decode(pd packetDecoder, version int16) (err error) {\n\tt.Version = version\n\tthrottleTime, err := pd.getInt32()\n\tif err != nil {\n\t\treturn err\n\t}\n\tt.ThrottleTime = time.Duration(throttleTime) * time.Millisecond\n\n\tn, err := pd.getArrayLength()\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tt.Topics = make(map[string][]*PartitionError)\n\n\tfor i := 0; i < n; i++ {\n\t\ttopic, err := pd.getString()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tm, err := pd.getArrayLength()\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tt.Topics[topic] = make([]*PartitionError, m)\n\n\t\tfor j := 0; j < m; j++ {\n\t\t\tt.Topics[topic][j] = new(PartitionError)\n\t\t\tif err := t.Topics[topic][j].decode(pd, version); err != nil {\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (a *TxnOffsetCommitResponse) key() int16 {\n\treturn apiKeyTxnOffsetCommit\n}\n\nfunc (a *TxnOffsetCommitResponse) version() int16 {\n\treturn a.Version\n}\n\nfunc (a *TxnOffsetCommitResponse) headerVersion() int16 {\n\treturn 0\n}\n\nfunc (a *TxnOffsetCommitResponse) isValidVersion() bool {\n\treturn a.Version >= 0 && a.Version <= 2\n}\n\nfunc (a *TxnOffsetCommitResponse) requiredVersion() KafkaVersion {\n\tswitch a.Version {\n\tcase 2:\n\t\treturn V2_1_0_0\n\tcase 1:\n\t\treturn V2_0_0_0\n\tcase 0:\n\t\treturn V0_11_0_0\n\tdefault:\n\t\treturn V2_1_0_0\n\t}\n}\n\nfunc (r *TxnOffsetCommitResponse) throttleTime() time.Duration {\n\treturn r.ThrottleTime\n}\n"
},
{
"path": "txn_offset_commit_response_test.go",
"content": "//go:build !functional\n\npackage sarama\n\nimport (\n\t\"testing\"\n\t\"time\"\n)\n\nvar txnOffsetCommitResponse = []byte{\n\t0, 0, 0, 100,\n\t0, 0, 0, 1, // 1 topic\n\t0, 5, 't', 'o', 'p', 'i', 'c',\n\t0, 0, 0, 1, // 1 partition response\n\t0, 0, 0, 2, // partition number 2\n\t0, 47, // err\n}\n\nfunc TestTxnOffsetCommitResponse(t *testing.T) {\n\tresp := &TxnOffsetCommitResponse{\n\t\tThrottleTime: 100 * time.Millisecond,\n\t\tTopics: map[string][]*PartitionError{\n\t\t\t\"topic\": {{\n\t\t\t\tPartition: 2,\n\t\t\t\tErr: ErrInvalidProducerEpoch,\n\t\t\t}},\n\t\t},\n\t}\n\n\ttestResponse(t, \"\", resp, txnOffsetCommitResponse)\n}\n"
},
{
"path": "utils.go",
"content": "package sarama\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"math/rand\"\n\t\"net\"\n\t\"regexp\"\n\t\"time\"\n)\n\nconst (\n\tdefaultRetryBackoff = 100 * time.Millisecond\n\tdefaultRetryMaxBackoff = 1000 * time.Millisecond\n)\n\ntype none struct{}\n\n// make []int32 sortable so we can sort partition numbers\ntype int32Slice []int32\n\nfunc (slice int32Slice) Len() int {\n\treturn len(slice)\n}\n\nfunc (slice int32Slice) Less(i, j int) bool {\n\treturn slice[i] < slice[j]\n}\n\nfunc (slice int32Slice) Swap(i, j int) {\n\tslice[i], slice[j] = slice[j], slice[i]\n}\n\nfunc dupInt32Slice(input []int32) []int32 {\n\tret := make([]int32, 0, len(input))\n\tret = append(ret, input...)\n\treturn ret\n}\n\nfunc withRecover(fn func()) {\n\tdefer func() {\n\t\thandler := PanicHandler\n\t\tif handler != nil {\n\t\t\tif err := recover(); err != nil {\n\t\t\t\thandler(err)\n\t\t\t}\n\t\t}\n\t}()\n\n\tfn()\n}\n\nfunc safeAsyncClose(b *Broker) {\n\tgo withRecover(func() {\n\t\tif connected, _ := b.Connected(); connected {\n\t\t\tif err := b.Close(); err != nil {\n\t\t\t\tLogger.Println(\"Error closing broker\", b.ID(), \":\", err)\n\t\t\t}\n\t\t}\n\t})\n}\n\n// Encoder is a simple interface for any type that can be encoded as an array of bytes\n// in order to be sent as the key or value of a Kafka message. Length() is provided as an\n// optimization, and must return the same as len() on the result of Encode().\ntype Encoder interface {\n\tEncode() ([]byte, error)\n\tLength() int\n}\n\n// make strings and byte slices encodable for convenience so they can be used as keys\n// and/or values in kafka messages\n\n// StringEncoder implements the Encoder interface for Go strings so that they can be used\n// as the Key or Value in a ProducerMessage.\ntype StringEncoder string\n\nfunc (s StringEncoder) Encode() ([]byte, error) {\n\treturn []byte(s), nil\n}\n\nfunc (s StringEncoder) Length() int {\n\treturn len(s)\n}\n\n// ByteEncoder implements the Encoder interface for Go byte slices so that they can be used\n// as the Key or Value in a ProducerMessage.\ntype ByteEncoder []byte\n\nfunc (b ByteEncoder) Encode() ([]byte, error) {\n\treturn b, nil\n}\n\nfunc (b ByteEncoder) Length() int {\n\treturn len(b)\n}\n\n// bufConn wraps a net.Conn with a buffer for reads to reduce the number of\n// reads that trigger syscalls.\ntype bufConn struct {\n\tnet.Conn\n\tbuf *bufio.Reader\n}\n\nfunc newBufConn(conn net.Conn) *bufConn {\n\treturn &bufConn{\n\t\tConn: conn,\n\t\tbuf: bufio.NewReader(conn),\n\t}\n}\n\nfunc (bc *bufConn) Read(b []byte) (n int, err error) {\n\treturn bc.buf.Read(b)\n}\n\n// KafkaVersion instances represent versions of the upstream Kafka broker.\ntype KafkaVersion struct {\n\t// it's a struct rather than just typing the array directly to make it opaque and stop people\n\t// generating their own arbitrary versions\n\tversion [4]uint\n}\n\nfunc newKafkaVersion(major, minor, veryMinor, patch uint) KafkaVersion {\n\treturn KafkaVersion{\n\t\tversion: [4]uint{major, minor, veryMinor, patch},\n\t}\n}\n\n// IsAtLeast return true if and only if the version it is called on is\n// greater than or equal to the version passed in:\n//\n//\tV1.IsAtLeast(V2) // false\n//\tV2.IsAtLeast(V1) // true\nfunc (v KafkaVersion) IsAtLeast(other KafkaVersion) bool {\n\tfor i := range v.version {\n\t\tif v.version[i] > other.version[i] {\n\t\t\treturn true\n\t\t} else if v.version[i] < other.version[i] {\n\t\t\treturn false\n\t\t}\n\t}\n\treturn true\n}\n\n// Effective constants defining the supported kafka versions.\nvar (\n\tV0_8_2_0 = newKafkaVersion(0, 8, 2, 0)\n\tV0_8_2_1 = newKafkaVersion(0, 8, 2, 1)\n\tV0_8_2_2 = newKafkaVersion(0, 8, 2, 2)\n\tV0_9_0_0 = newKafkaVersion(0, 9, 0, 0)\n\tV0_9_0_1 = newKafkaVersion(0, 9, 0, 1)\n\tV0_10_0_0 = newKafkaVersion(0, 10, 0, 0)\n\tV0_10_0_1 = newKafkaVersion(0, 10, 0, 1)\n\tV0_10_1_0 = newKafkaVersion(0, 10, 1, 0)\n\tV0_10_1_1 = newKafkaVersion(0, 10, 1, 1)\n\tV0_10_2_0 = newKafkaVersion(0, 10, 2, 0)\n\tV0_10_2_1 = newKafkaVersion(0, 10, 2, 1)\n\tV0_10_2_2 = newKafkaVersion(0, 10, 2, 2)\n\tV0_11_0_0 = newKafkaVersion(0, 11, 0, 0)\n\tV0_11_0_1 = newKafkaVersion(0, 11, 0, 1)\n\tV0_11_0_2 = newKafkaVersion(0, 11, 0, 2)\n\tV1_0_0_0 = newKafkaVersion(1, 0, 0, 0)\n\tV1_0_1_0 = newKafkaVersion(1, 0, 1, 0)\n\tV1_0_2_0 = newKafkaVersion(1, 0, 2, 0)\n\tV1_1_0_0 = newKafkaVersion(1, 1, 0, 0)\n\tV1_1_1_0 = newKafkaVersion(1, 1, 1, 0)\n\tV2_0_0_0 = newKafkaVersion(2, 0, 0, 0)\n\tV2_0_1_0 = newKafkaVersion(2, 0, 1, 0)\n\tV2_1_0_0 = newKafkaVersion(2, 1, 0, 0)\n\tV2_1_1_0 = newKafkaVersion(2, 1, 1, 0)\n\tV2_2_0_0 = newKafkaVersion(2, 2, 0, 0)\n\tV2_2_1_0 = newKafkaVersion(2, 2, 1, 0)\n\tV2_2_2_0 = newKafkaVersion(2, 2, 2, 0)\n\tV2_3_0_0 = newKafkaVersion(2, 3, 0, 0)\n\tV2_3_1_0 = newKafkaVersion(2, 3, 1, 0)\n\tV2_4_0_0 = newKafkaVersion(2, 4, 0, 0)\n\tV2_4_1_0 = newKafkaVersion(2, 4, 1, 0)\n\tV2_5_0_0 = newKafkaVersion(2, 5, 0, 0)\n\tV2_5_1_0 = newKafkaVersion(2, 5, 1, 0)\n\tV2_6_0_0 = newKafkaVersion(2, 6, 0, 0)\n\tV2_6_1_0 = newKafkaVersion(2, 6, 1, 0)\n\tV2_6_2_0 = newKafkaVersion(2, 6, 2, 0)\n\tV2_6_3_0 = newKafkaVersion(2, 6, 3, 0)\n\tV2_7_0_0 = newKafkaVersion(2, 7, 0, 0)\n\tV2_7_1_0 = newKafkaVersion(2, 7, 1, 0)\n\tV2_7_2_0 = newKafkaVersion(2, 7, 2, 0)\n\tV2_8_0_0 = newKafkaVersion(2, 8, 0, 0)\n\tV2_8_1_0 = newKafkaVersion(2, 8, 1, 0)\n\tV2_8_2_0 = newKafkaVersion(2, 8, 2, 0)\n\tV3_0_0_0 = newKafkaVersion(3, 0, 0, 0)\n\tV3_0_1_0 = newKafkaVersion(3, 0, 1, 0)\n\tV3_0_2_0 = newKafkaVersion(3, 0, 2, 0)\n\tV3_1_0_0 = newKafkaVersion(3, 1, 0, 0)\n\tV3_1_1_0 = newKafkaVersion(3, 1, 1, 0)\n\tV3_1_2_0 = newKafkaVersion(3, 1, 2, 0)\n\tV3_2_0_0 = newKafkaVersion(3, 2, 0, 0)\n\tV3_2_1_0 = newKafkaVersion(3, 2, 1, 0)\n\tV3_2_2_0 = newKafkaVersion(3, 2, 2, 0)\n\tV3_2_3_0 = newKafkaVersion(3, 2, 3, 0)\n\tV3_3_0_0 = newKafkaVersion(3, 3, 0, 0)\n\tV3_3_1_0 = newKafkaVersion(3, 3, 1, 0)\n\tV3_3_2_0 = newKafkaVersion(3, 3, 2, 0)\n\tV3_4_0_0 = newKafkaVersion(3, 4, 0, 0)\n\tV3_4_1_0 = newKafkaVersion(3, 4, 1, 0)\n\tV3_5_0_0 = newKafkaVersion(3, 5, 0, 0)\n\tV3_5_1_0 = newKafkaVersion(3, 5, 1, 0)\n\tV3_5_2_0 = newKafkaVersion(3, 5, 2, 0)\n\tV3_6_0_0 = newKafkaVersion(3, 6, 0, 0)\n\tV3_6_1_0 = newKafkaVersion(3, 6, 1, 0)\n\tV3_6_2_0 = newKafkaVersion(3, 6, 2, 0)\n\tV3_7_0_0 = newKafkaVersion(3, 7, 0, 0)\n\tV3_7_1_0 = newKafkaVersion(3, 7, 1, 0)\n\tV3_7_2_0 = newKafkaVersion(3, 7, 2, 0)\n\tV3_8_0_0 = newKafkaVersion(3, 8, 0, 0)\n\tV3_8_1_0 = newKafkaVersion(3, 8, 1, 0)\n\tV3_9_0_0 = newKafkaVersion(3, 9, 0, 0)\n\tV3_9_1_0 = newKafkaVersion(3, 9, 1, 0)\n\tV3_9_2_0 = newKafkaVersion(3, 9, 2, 0)\n\tV4_0_0_0 = newKafkaVersion(4, 0, 0, 0)\n\tV4_0_1_0 = newKafkaVersion(4, 0, 1, 0)\n\tV4_1_0_0 = newKafkaVersion(4, 1, 0, 0)\n\tV4_1_1_0 = newKafkaVersion(4, 1, 1, 0)\n\tV4_2_0_0 = newKafkaVersion(4, 2, 0, 0)\n\n\tSupportedVersions = []KafkaVersion{\n\t\tV0_8_2_0,\n\t\tV0_8_2_1,\n\t\tV0_8_2_2,\n\t\tV0_9_0_0,\n\t\tV0_9_0_1,\n\t\tV0_10_0_0,\n\t\tV0_10_0_1,\n\t\tV0_10_1_0,\n\t\tV0_10_1_1,\n\t\tV0_10_2_0,\n\t\tV0_10_2_1,\n\t\tV0_10_2_2,\n\t\tV0_11_0_0,\n\t\tV0_11_0_1,\n\t\tV0_11_0_2,\n\t\tV1_0_0_0,\n\t\tV1_0_1_0,\n\t\tV1_0_2_0,\n\t\tV1_1_0_0,\n\t\tV1_1_1_0,\n\t\tV2_0_0_0,\n\t\tV2_0_1_0,\n\t\tV2_1_0_0,\n\t\tV2_1_1_0,\n\t\tV2_2_0_0,\n\t\tV2_2_1_0,\n\t\tV2_2_2_0,\n\t\tV2_3_0_0,\n\t\tV2_3_1_0,\n\t\tV2_4_0_0,\n\t\tV2_4_1_0,\n\t\tV2_5_0_0,\n\t\tV2_5_1_0,\n\t\tV2_6_0_0,\n\t\tV2_6_1_0,\n\t\tV2_6_2_0,\n\t\tV2_6_3_0,\n\t\tV2_7_0_0,\n\t\tV2_7_1_0,\n\t\tV2_7_2_0,\n\t\tV2_8_0_0,\n\t\tV2_8_1_0,\n\t\tV2_8_2_0,\n\t\tV3_0_0_0,\n\t\tV3_0_1_0,\n\t\tV3_0_2_0,\n\t\tV3_1_0_0,\n\t\tV3_1_1_0,\n\t\tV3_1_2_0,\n\t\tV3_2_0_0,\n\t\tV3_2_1_0,\n\t\tV3_2_2_0,\n\t\tV3_2_3_0,\n\t\tV3_3_0_0,\n\t\tV3_3_1_0,\n\t\tV3_3_2_0,\n\t\tV3_4_0_0,\n\t\tV3_4_1_0,\n\t\tV3_5_0_0,\n\t\tV3_5_1_0,\n\t\tV3_5_2_0,\n\t\tV3_6_0_0,\n\t\tV3_6_1_0,\n\t\tV3_6_2_0,\n\t\tV3_7_0_0,\n\t\tV3_7_1_0,\n\t\tV3_7_2_0,\n\t\tV3_8_0_0,\n\t\tV3_8_1_0,\n\t\tV3_9_0_0,\n\t\tV3_9_1_0,\n\t\tV3_9_2_0,\n\t\tV4_0_0_0,\n\t\tV4_0_1_0,\n\t\tV4_1_0_0,\n\t\tV4_1_1_0,\n\t\tV4_2_0_0,\n\t}\n\tMinVersion = V0_8_2_0\n\tMaxVersion = V4_2_0_0\n\tDefaultVersion = V2_1_0_0\n\n\t// reduced set of protocol versions to matrix test\n\tfvtRangeVersions = []KafkaVersion{\n\t\tV0_8_2_2,\n\t\tV0_10_2_2,\n\t\tV1_0_2_0,\n\t\tV1_1_1_0,\n\t\tV2_0_1_0,\n\t\tV2_2_2_0,\n\t\tV2_4_1_0,\n\t\tV2_6_3_0,\n\t\tV2_8_2_0,\n\t\tV3_1_2_0,\n\t\tV3_3_2_0,\n\t\tV3_6_2_0,\n\t}\n)\n\nvar (\n\t// This regex validates that a string complies with the pre kafka 1.0.0 format for version strings, for example 0.11.0.3\n\tvalidPreKafka1Version = regexp.MustCompile(`^0\\.\\d+\\.\\d+\\.\\d+$`)\n\n\t// This regex validates that a string complies with the post Kafka 1.0.0 format, for example 1.0.0\n\tvalidPostKafka1Version = regexp.MustCompile(`^\\d+\\.\\d+\\.\\d+$`)\n)\n\n// ParseKafkaVersion parses and returns kafka version or error from a string\nfunc ParseKafkaVersion(s string) (KafkaVersion, error) {\n\tif len(s) < 5 {\n\t\treturn DefaultVersion, fmt.Errorf(\"invalid version `%s`\", s)\n\t}\n\tvar major, minor, veryMinor, patch uint\n\tvar err error\n\tif s[0] == '0' {\n\t\terr = scanKafkaVersion(s, validPreKafka1Version, \"0.%d.%d.%d\", [3]*uint{&minor, &veryMinor, &patch})\n\t} else {\n\t\terr = scanKafkaVersion(s, validPostKafka1Version, \"%d.%d.%d\", [3]*uint{&major, &minor, &veryMinor})\n\t}\n\tif err != nil {\n\t\treturn DefaultVersion, err\n\t}\n\treturn newKafkaVersion(major, minor, veryMinor, patch), nil\n}\n\nfunc scanKafkaVersion(s string, pattern *regexp.Regexp, format string, v [3]*uint) error {\n\tif !pattern.MatchString(s) {\n\t\treturn fmt.Errorf(\"invalid version `%s`\", s)\n\t}\n\t_, err := fmt.Sscanf(s, format, v[0], v[1], v[2])\n\treturn err\n}\n\nfunc (v KafkaVersion) String() string {\n\tif v.version[0] == 0 {\n\t\treturn fmt.Sprintf(\"0.%d.%d.%d\", v.version[1], v.version[2], v.version[3])\n\t}\n\n\treturn fmt.Sprintf(\"%d.%d.%d\", v.version[0], v.version[1], v.version[2])\n}\n\n// NewExponentialBackoff returns a function that implements an exponential backoff strategy with jitter.\n// It follows KIP-580, implementing the formula:\n// MIN(retry.backoff.max.ms, (retry.backoff.ms * 2**(failures - 1)) * random(0.8, 1.2))\n// This ensures retries start with `backoff` and exponentially increase until `maxBackoff`, with added jitter.\n// The behavior when `failures = 0` is not explicitly defined in KIP-580 and is left to implementation discretion.\n//\n// Example usage:\n//\n//\tbackoffFunc := sarama.NewExponentialBackoff(config.Producer.Retry.Backoff, 2*time.Second)\n//\tconfig.Producer.Retry.BackoffFunc = backoffFunc\nfunc NewExponentialBackoff(backoff time.Duration, maxBackoff time.Duration) func(retries, maxRetries int) time.Duration {\n\tif backoff <= 0 {\n\t\tbackoff = defaultRetryBackoff\n\t}\n\tif maxBackoff <= 0 {\n\t\tmaxBackoff = defaultRetryMaxBackoff\n\t}\n\n\tif backoff > maxBackoff {\n\t\tLogger.Println(\"Warning: backoff is greater than maxBackoff, using maxBackoff instead.\")\n\t\tbackoff = maxBackoff\n\t}\n\n\treturn func(retries, maxRetries int) time.Duration {\n\t\tif retries <= 0 {\n\t\t\treturn backoff\n\t\t}\n\n\t\tcalculatedBackoff := backoff * time.Duration(1<<(retries-1))\n\t\tjitter := 0.8 + 0.4*rand.Float64()\n\t\tcalculatedBackoff = time.Duration(float64(calculatedBackoff) * jitter)\n\n\t\treturn min(calculatedBackoff, maxBackoff)\n\t}\n}\n"
},
{
"path": "utils_test.go",
"content": "//go:build !functional\n\npackage sarama\n\nimport (\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestVersionCompare(t *testing.T) {\n\tif V0_8_2_0.IsAtLeast(V0_8_2_1) {\n\t\tt.Error(\"0.8.2.0 >= 0.8.2.1\")\n\t}\n\tif !V0_8_2_1.IsAtLeast(V0_8_2_0) {\n\t\tt.Error(\"! 0.8.2.1 >= 0.8.2.0\")\n\t}\n\tif !V0_8_2_0.IsAtLeast(V0_8_2_0) {\n\t\tt.Error(\"! 0.8.2.0 >= 0.8.2.0\")\n\t}\n\tif !V0_9_0_0.IsAtLeast(V0_8_2_1) {\n\t\tt.Error(\"! 0.9.0.0 >= 0.8.2.1\")\n\t}\n\tif V0_8_2_1.IsAtLeast(V0_10_0_0) {\n\t\tt.Error(\"0.8.2.1 >= 0.10.0.0\")\n\t}\n\tif !V1_0_0_0.IsAtLeast(V0_9_0_0) {\n\t\tt.Error(\"! 1.0.0.0 >= 0.9.0.0\")\n\t}\n\tif V0_9_0_0.IsAtLeast(V1_0_0_0) {\n\t\tt.Error(\"0.9.0.0 >= 1.0.0.0\")\n\t}\n}\n\nfunc TestVersionParsing(t *testing.T) {\n\tvalidVersions := []string{\n\t\t\"0.8.2.0\",\n\t\t\"0.8.2.1\",\n\t\t\"0.8.2.2\",\n\t\t\"0.9.0.0\",\n\t\t\"0.9.0.1\",\n\t\t\"0.10.0.0\",\n\t\t\"0.10.0.1\",\n\t\t\"0.10.1.0\",\n\t\t\"0.10.1.1\",\n\t\t\"0.10.2.0\",\n\t\t\"0.10.2.1\",\n\t\t\"0.10.2.2\",\n\t\t\"0.11.0.0\",\n\t\t\"0.11.0.1\",\n\t\t\"0.11.0.2\",\n\t\t\"1.0.0\",\n\t\t\"1.0.1\",\n\t\t\"1.0.2\",\n\t\t\"1.1.0\",\n\t\t\"1.1.1\",\n\t\t\"2.0.0\",\n\t\t\"2.0.1\",\n\t\t\"2.1.0\",\n\t\t\"2.1.1\",\n\t\t\"2.2.0\",\n\t\t\"2.2.1\",\n\t\t\"2.2.2\",\n\t\t\"2.3.0\",\n\t\t\"2.3.1\",\n\t\t\"2.4.0\",\n\t\t\"2.4.1\",\n\t\t\"2.5.0\",\n\t\t\"2.5.1\",\n\t\t\"2.6.0\",\n\t\t\"2.6.1\",\n\t\t\"2.6.2\",\n\t\t\"2.6.3\",\n\t\t\"2.7.0\",\n\t\t\"2.7.1\",\n\t\t\"2.7.2\",\n\t\t\"2.8.0\",\n\t\t\"2.8.1\",\n\t\t\"3.0.0\",\n\t\t\"3.0.1\",\n\t\t\"3.1.0\",\n\t\t\"3.1.1\",\n\t\t\"3.2.0\",\n\t}\n\tfor _, s := range validVersions {\n\t\tv, err := ParseKafkaVersion(s)\n\t\tif err != nil {\n\t\t\tt.Errorf(\"could not parse valid version %s: %s\", s, err)\n\t\t}\n\t\tif v.String() != s {\n\t\t\tt.Errorf(\"version %s != %s\", v.String(), s)\n\t\t}\n\t}\n\n\tinvalidVersions := []string{\"0.8.2-4\", \"0.8.20\", \"1.19.0.0\", \"1.0.x\"}\n\tfor _, s := range invalidVersions {\n\t\tif _, err := ParseKafkaVersion(s); err == nil {\n\t\t\tt.Errorf(\"invalid version %s parsed without error\", s)\n\t\t}\n\t}\n}\n\nfunc TestExponentialBackoffValidCases(t *testing.T) {\n\ttestCases := []struct {\n\t\tretries int\n\t\tmaxRetries int\n\t\tminBackoff time.Duration\n\t\tmaxBackoffExpected time.Duration\n\t}{\n\t\t{1, 5, 80 * time.Millisecond, 120 * time.Millisecond},\n\t\t{3, 5, 320 * time.Millisecond, 480 * time.Millisecond},\n\t\t{5, 5, 1280 * time.Millisecond, 1920 * time.Millisecond},\n\t}\n\n\tfor _, tc := range testCases {\n\t\tbackoffFunc := NewExponentialBackoff(100*time.Millisecond, 2*time.Second)\n\t\tbackoff := backoffFunc(tc.retries, tc.maxRetries)\n\t\tif backoff < tc.minBackoff || backoff > tc.maxBackoffExpected {\n\t\t\tt.Errorf(\"backoff(%d, %d): expected between %v and %v, got %v\", tc.retries, tc.maxRetries, tc.minBackoff, tc.maxBackoffExpected, backoff)\n\t\t}\n\t}\n}\n\nfunc TestExponentialBackoffDefaults(t *testing.T) {\n\ttestCases := []struct {\n\t\tbackoff time.Duration\n\t\tmaxBackoff time.Duration\n\t}{\n\t\t{-100 * time.Millisecond, 2 * time.Second},\n\t\t{100 * time.Millisecond, -2 * time.Second},\n\t\t{-100 * time.Millisecond, -2 * time.Second},\n\t\t{0 * time.Millisecond, 2 * time.Second},\n\t\t{100 * time.Millisecond, 0 * time.Second},\n\t\t{0 * time.Millisecond, 0 * time.Second},\n\t}\n\n\tfor _, tc := range testCases {\n\t\tbackoffFunc := NewExponentialBackoff(tc.backoff, tc.maxBackoff)\n\t\tbackoff := backoffFunc(2, 5)\n\t\tif backoff < defaultRetryBackoff || backoff > defaultRetryMaxBackoff {\n\t\t\tt.Errorf(\"backoff(%v, %v): expected between %v and %v, got %v\",\n\t\t\t\ttc.backoff, tc.maxBackoff, defaultRetryBackoff, defaultRetryMaxBackoff, backoff)\n\t\t}\n\t}\n}\n"
},
{
"path": "version.go",
"content": "package sarama\n\nimport (\n\t\"reflect\"\n\t\"runtime/debug\"\n\t\"sync\"\n)\n\nvar (\n\tv string\n\tvOnce sync.Once\n)\n\nfunc version() string {\n\tvOnce.Do(func() {\n\t\t// Determine our package name without hardcoding a string\n\t\ttype getPackageName struct{}\n\t\tthisPackagePath := reflect.TypeFor[getPackageName]().PkgPath()\n\n\t\tbi, ok := debug.ReadBuildInfo()\n\t\tif ok {\n\t\t\tfor _, dep := range bi.Deps {\n\t\t\t\tif dep.Path == thisPackagePath {\n\t\t\t\t\tv = dep.Version\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif v == \"\" || v == \"(devel)\" {\n\t\t\t// if we can't read a go module version then they're using a git\n\t\t\t// clone or vendored module so all we can do is report \"dev\" for\n\t\t\t// the version to make a valid ApiVersions request\n\t\t\tv = \"dev\"\n\t\t}\n\t})\n\treturn v\n}\n"
},
{
"path": "zstd.go",
"content": "package sarama\n\nimport (\n\t\"sync\"\n\n\t\"github.com/klauspost/compress/zstd\"\n)\n\n// zstdMaxBufferedEncoders maximum number of not-in-use zstd encoders\n// If the pool of encoders is exhausted then new encoders will be created on the fly\nconst zstdMaxBufferedEncoders = 1\n\ntype ZstdEncoderParams struct {\n\tLevel int\n}\ntype ZstdDecoderParams struct {\n}\n\nvar zstdDecMap sync.Map\n\nvar zstdAvailableEncoders sync.Map\n\nfunc getZstdEncoderChannel(params ZstdEncoderParams) chan *zstd.Encoder {\n\tif c, ok := zstdAvailableEncoders.Load(params); ok {\n\t\treturn c.(chan *zstd.Encoder)\n\t}\n\tc, _ := zstdAvailableEncoders.LoadOrStore(params, make(chan *zstd.Encoder, zstdMaxBufferedEncoders))\n\treturn c.(chan *zstd.Encoder)\n}\n\nfunc getZstdEncoder(params ZstdEncoderParams) *zstd.Encoder {\n\tselect {\n\tcase enc := <-getZstdEncoderChannel(params):\n\t\treturn enc\n\tdefault:\n\t\tencoderLevel := zstd.SpeedDefault\n\t\tif params.Level != CompressionLevelDefault {\n\t\t\tencoderLevel = zstd.EncoderLevelFromZstd(params.Level)\n\t\t}\n\t\tzstdEnc, _ := zstd.NewWriter(nil, zstd.WithZeroFrames(true),\n\t\t\tzstd.WithEncoderLevel(encoderLevel),\n\t\t\tzstd.WithEncoderConcurrency(1))\n\t\treturn zstdEnc\n\t}\n}\n\nfunc releaseEncoder(params ZstdEncoderParams, enc *zstd.Encoder) {\n\tselect {\n\tcase getZstdEncoderChannel(params) <- enc:\n\tdefault:\n\t}\n}\n\nfunc getDecoder(params ZstdDecoderParams) *zstd.Decoder {\n\tif ret, ok := zstdDecMap.Load(params); ok {\n\t\treturn ret.(*zstd.Decoder)\n\t}\n\t// It's possible to race and create multiple new readers.\n\t// Only one will survive GC after use.\n\tzstdDec, _ := zstd.NewReader(nil, zstd.WithDecoderConcurrency(0))\n\tzstdDecMap.Store(params, zstdDec)\n\treturn zstdDec\n}\n\nfunc zstdDecompress(params ZstdDecoderParams, dst, src []byte) ([]byte, error) {\n\treturn getDecoder(params).DecodeAll(src, dst)\n}\n\nfunc zstdCompress(params ZstdEncoderParams, dst, src []byte) ([]byte, error) {\n\tenc := getZstdEncoder(params)\n\tout := enc.EncodeAll(src, dst)\n\treleaseEncoder(params, enc)\n\treturn out, nil\n}\n"
},
{
"path": "zstd_test.go",
"content": "//go:build !functional\n\npackage sarama\n\nimport (\n\t\"runtime\"\n\t\"testing\"\n)\n\nfunc BenchmarkZstdMemoryConsumption(b *testing.B) {\n\tparams := ZstdEncoderParams{Level: 9}\n\tbuf := make([]byte, 1024*1024)\n\tfor i := 0; i < len(buf); i++ {\n\t\tbuf[i] = byte((i / 256) + (i * 257))\n\t}\n\n\tcpus := 96\n\n\tgomaxprocsBackup := runtime.GOMAXPROCS(cpus)\n\tb.ReportAllocs()\n\tfor b.Loop() {\n\t\tfor j := 0; j < 2*cpus; j++ {\n\t\t\t_, _ = zstdCompress(params, nil, buf)\n\t\t}\n\t\t// drain the buffered encoder\n\t\tgetZstdEncoder(params)\n\t\t// previously this would be achieved with\n\t\t// zstdEncMap.Delete(params)\n\t}\n\truntime.GOMAXPROCS(gomaxprocsBackup)\n}\n"
}
]