[
  {
    "path": ".gitignore",
    "content": "/gen\n/vendor\n!/vendor/manifest\n/bin\n/pkg\n/tmp\n/log\n/vms\n/run\n/go\n.hmake\n.hmakerc\n.project\n.idea\n.vscode\n*_mock_test.go\nfilenames\n\n.DS_Store"
  },
  {
    "path": ".gitreview",
    "content": "[gerrit]\nhost=review.ec.eng.vmware.com\nport=29418\nproject=cascade-kinesis-client\ndefaultbranch=develop\n"
  },
  {
    "path": "CONTRIBUTING.md",
    "content": "# Contributing to vmware-go-kcl\n\nThe vmware-go-kcl project team welcomes contributions from the community. Before you start working with vmware-go-kcl, please read our [Developer Certificate of Origin](https://cla.vmware.com/dco). All contributions to this repository must be signed as described on that page. Your signature certifies that you wrote the patch or have the right to pass it on as an open-source patch.\n\n## Contribution Flow\n\nThis is a rough outline of what a contributor's workflow looks like:\n\n- Create a topic branch from where you want to base your work\n- Make commits of logical units\n- Make sure your commit messages are in the proper format (see below)\n- Push your changes to a topic branch in your fork of the repository\n- Submit a pull request\n\nExample:\n\n``` shell\ngit remote add upstream https://github.com/vmware/vmware-go-kcl.git\ngit checkout -b my-new-feature master\ngit commit -a\ngit push origin my-new-feature\n```\n\n### Staying In Sync With Upstream\n\nWhen your branch gets out of sync with the vmware/master branch, use the following to update:\n\n``` shell\ngit checkout my-new-feature\ngit fetch -a\ngit pull --rebase upstream master\ngit push --force-with-lease origin my-new-feature\n```\n\n### Updating pull requests\n\nIf your PR fails to pass CI or needs changes based on code review, you'll most likely want to squash these changes into\nexisting commits.\n\nIf your pull request contains a single commit or your changes are related to the most recent commit, you can simply\namend the commit.\n\n``` shell\ngit add .\ngit commit --amend\ngit push --force-with-lease origin my-new-feature\n```\n\nIf you need to squash changes into an earlier commit, you can use:\n\n``` shell\ngit add .\ngit commit --fixup <commit>\ngit rebase -i --autosquash master\ngit push --force-with-lease origin my-new-feature\n```\n\nBe sure to add a comment to the PR indicating your new changes are ready to review, as GitHub does not generate a\nnotification when you git push.\n\n### Formatting Commit Messages\n\nWe follow the conventions on [How to Write a Git Commit Message](http://chris.beams.io/posts/git-commit/).\n\nBe sure to include any related GitHub issue references in the commit message.  See\n[GFM syntax](https://guides.github.com/features/mastering-markdown/#GitHub-flavored-markdown) for referencing issues\nand commits.\n\n## Reporting Bugs and Creating Issues\n\nWhen opening a new issue, try to roughly follow the commit message format conventions above.\n"
  },
  {
    "path": "HyperMake",
    "content": "---\nformat: hypermake.v0\n\nname: cascade-kinesis-client\ndescription: Kinesis Client in Go\n\ntargets:\n  rebuild-toolchain:\n    description: build toolchain image\n    watches:\n      - support/toolchain/docker\n    build: support/toolchain/docker\n\n  toolchain:\n    description: placeholder for additional toolchain dependencies\n\n  deps:\n    description: download dependencies to local cache\n    after:\n      - toolchain\n    watches:\n      - go.mod\n    cmds:\n      - go mod download\n      - go mod vendor\n      - go mod tidy\n\n  build:\n    description: build source code\n    after:\n      - 'build-*'\n\n  test:\n    description: run unit tests\n    after:\n      - deps\n      - check\n    always: true\n    cmds:\n      - ./support/scripts/test.sh\n\n  ci:\n    description: run CI tests\n    after:\n      - deps\n    cmds:\n      - ./support/scripts/ci.sh\n\n  checkfmt:\n    description: check code format\n    after:\n      - toolchain\n    watches:\n      - support/scripts/check.sh\n    always: true\n    cmds:\n      - ./support/scripts/check.sh fmt\n\n  lint:\n    description: run lint to check code\n    after:\n      - toolchain\n    watches:\n      - support/scripts/check.sh\n    always: true\n    cmds:\n      - ./support/scripts/check.sh lint\n\n  scanast:\n    description: run Go AST security scan\n    after:\n      - toolchain\n    watches:\n      - '**/**/*.go'\n      - './support/scripts/check.sh'\n    cmds:\n      - ./support/scripts/check.sh scanast\n\n  check:\n    description: run all code checks\n    after:\n      - checkfmt\n      - lint\n      - scanast\n\nsettings:\n  default-targets:\n    - test\n  docker:\n    image: 'vmware/go-kcl-toolchain:0.1.4'\n    src-volume: /go/src/github.com/vmware/vmware-go-kcl\n"
  },
  {
    "path": "LICENSE",
    "content": "MIT License\r\n\r\nCopyright (c) 2018 VMware, Inc.\r\n\r\nPermission is hereby granted, free of charge, to any person obtaining a copy\r\nof this software and associated documentation files (the \"Software\"), to deal\r\nin the Software without restriction, including without limitation the rights\r\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\r\ncopies of the Software, and to permit persons to whom the Software is\r\nfurnished to do so, subject to the following conditions:\r\n\r\nThe above copyright notice and this permission notice shall be included in all\r\ncopies or substantial portions of the Software.\r\n\r\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\r\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\r\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\r\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\r\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\r\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\r\nSOFTWARE."
  },
  {
    "path": "README.md",
    "content": "# VMware-Go-KCL\n\n![technology Go](https://img.shields.io/badge/technology-go-blue.svg)\n[![Go Report Card](https://goreportcard.com/badge/github.com/vmware/vmware-go-kcl)](https://goreportcard.com/report/github.com/vmware/vmware-go-kcl)\n[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)\n\n## Overview\n\n[Amazon Kinesis](https://aws.amazon.com/kinesis/data-streams/)  enables real-time processing of streaming data at massive scale. Kinesis Streams is useful for rapidly moving data off data producers and then continuously processing the data, be it to transform the data before emitting to a data store, run real-time metrics and analytics, or derive more complex data streams for further processing.\n\nThe **VMware Kinesis Client Library for GO** (VMware-Go-KCL) enables Go developers to easily consume and process data from [Amazon Kinesis][kinesis].\n\n**VMware-Go-KCL** brings Go/Kubernetes community with Go language native implementation of KCL matching **exactly the same** API and functional spec of original [Java KCL v2.0](https://docs.aws.amazon.com/streams/latest/dev/kcl-migration.html) without the resource overhead of installing Java based MultiLangDaemon.\n\nBesides, [vmware-go-kcl-v2](https://github.com/vmware/vmware-go-kcl-v2) is the v2 version of VMWare KCL for the Go programming language by utilizing [AWS Go SDK V2](https://github.com/aws/aws-sdk-go-v2). \n\n## Try it out\n\n### Prerequisites\n\n- Install [Go](https://golang.org/)\n- Install [docker](https://www.docker.com)\n- Install [HyperMake](https://evo-cloud.github.io/hmake)\n- Config [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html)\n\nMake sure hmake is version 1.3.1 or above and go is version 1.11 or above\n\n```sh\nhmake --version\n1.3.1\n```\n\nMake sure to launch Docker daemon with specified DNS server `--dns DNS-SERVER-IP`\n\nOn Ubuntu, update the file `/etc/default/docker` to put `--dns DNS-SERVER-IP` in `DOCKER_OPTS`.\n\nOn Mac, set DNS in _Docker Preferences_ – _Daemon_ – _Insecure registries_\n\n### Build & Run\n\n```sh\nhmake\n\n# security scan\nhmake scanast\n\n# run test\nhmake check\n\n# run integration test\n# update the worker_test.go to let it point to your Kinesis stream\nhmake test\n```\n\n## Documentation\n\nVMware-Go-KCL matches exactly the same interface and programming model from original Amazon KCL, the best place for getting reference, tutorial is from Amazon itself:\n\n- [Developing Consumers Using the Kinesis Client Library](https://docs.aws.amazon.com/streams/latest/dev/developing-consumers-with-kcl.html)\n- [Troubleshooting](https://docs.aws.amazon.com/streams/latest/dev/troubleshooting-consumers.html)\n- [Advanced Topics](https://docs.aws.amazon.com/streams/latest/dev/advanced-consumers.html)\n\n\n## Contributing\n\nThe vmware-go-kcl project team welcomes contributions from the community. Before you start working with vmware-go-kcl, please read our [Developer Certificate of Origin](https://cla.vmware.com/dco). All contributions to this repository must be signed as described on that page. Your signature certifies that you wrote the patch or have the right to pass it on as an open-source patch. For more detailed information, refer to [CONTRIBUTING.md](CONTRIBUTING.md).\n\n## License\n\nMIT License\n"
  },
  {
    "path": "clientlibrary/checkpoint/checkpointer.go",
    "content": "/*\n * Copyright (c) 2018 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\n// The implementation is derived from https://github.com/patrobinson/gokini\n//\n// Copyright 2018 Patrick robinson\n//\n// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n//\n// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n//\n// THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\npackage checkpoint\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\n\tpar \"github.com/vmware/vmware-go-kcl/clientlibrary/partition\"\n)\n\nconst (\n\tLeaseKeyKey       = \"ShardID\"\n\tLeaseOwnerKey     = \"AssignedTo\"\n\tLeaseTimeoutKey   = \"LeaseTimeout\"\n\tSequenceNumberKey = \"Checkpoint\"\n\tParentShardIdKey  = \"ParentShardId\"\n\tClaimRequestKey   = \"ClaimRequest\"\n\n\t// We've completely processed all records in this shard.\n\tShardEnd = \"SHARD_END\"\n\n\t// ErrShardClaimed is returned when shard is claimed\n\tErrShardClaimed = \"Shard is already claimed by another node\"\n)\n\ntype ErrLeaseNotAcquired struct {\n\tcause string\n}\n\nfunc (e ErrLeaseNotAcquired) Error() string {\n\treturn fmt.Sprintf(\"lease not acquired: %s\", e.cause)\n}\n\n// Checkpointer handles checkpointing when a record has been processed\ntype Checkpointer interface {\n\t// Init initialises the Checkpoint\n\tInit() error\n\n\t// GetLease attempts to gain a lock on the given shard\n\tGetLease(*par.ShardStatus, string) error\n\n\t// CheckpointSequence writes a checkpoint at the designated sequence ID\n\tCheckpointSequence(*par.ShardStatus) error\n\n\t// FetchCheckpoint retrieves the checkpoint for the given shard\n\tFetchCheckpoint(*par.ShardStatus) error\n\n\t// RemoveLeaseInfo to remove lease info for shard entry because the shard no longer exists\n\tRemoveLeaseInfo(string) error\n\n\t// RemoveLeaseOwner to remove lease owner for the shard entry to make the shard available for reassignment\n\tRemoveLeaseOwner(string) error\n\n\t// New Lease Stealing Methods\n\t// ListActiveWorkers returns active workers and their shards\n\tListActiveWorkers(map[string]*par.ShardStatus) (map[string][]*par.ShardStatus, error)\n\n\t// ClaimShard claims a shard for stealing\n\tClaimShard(*par.ShardStatus, string) error\n}\n\n// ErrSequenceIDNotFound is returned by FetchCheckpoint when no SequenceID is found\nvar ErrSequenceIDNotFound = errors.New(\"SequenceIDNotFoundForShard\")\n\n// ErrShardNotAssigned is returned by ListActiveWorkers when no AssignedTo is found\nvar ErrShardNotAssigned = errors.New(\"AssignedToNotFoundForShard\")\n"
  },
  {
    "path": "clientlibrary/checkpoint/dynamodb-checkpointer.go",
    "content": "/*\n * Copyright (c) 2018 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\n// The implementation is derived from https://github.com/patrobinson/gokini\n//\n// Copyright 2018 Patrick robinson\n//\n// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n//\n// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n//\n// THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\npackage checkpoint\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/aws/client\"\n\t\"github.com/aws/aws-sdk-go/aws/session\"\n\t\"github.com/aws/aws-sdk-go/service/dynamodb\"\n\t\"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface\"\n\n\t\"github.com/vmware/vmware-go-kcl/clientlibrary/config\"\n\tpar \"github.com/vmware/vmware-go-kcl/clientlibrary/partition\"\n\t\"github.com/vmware/vmware-go-kcl/clientlibrary/utils\"\n\t\"github.com/vmware/vmware-go-kcl/logger\"\n)\n\nconst (\n\t// ErrInvalidDynamoDBSchema is returned when there are one or more fields missing from the table\n\tErrInvalidDynamoDBSchema = \"The DynamoDB schema is invalid and may need to be re-created\"\n\n\t// NumMaxRetries is the max times of doing retry\n\tNumMaxRetries = 10\n)\n\n// DynamoCheckpoint implements the Checkpoint interface using DynamoDB as a backend\ntype DynamoCheckpoint struct {\n\tlog                     logger.Logger\n\tTableName               string\n\tleaseTableReadCapacity  int64\n\tleaseTableWriteCapacity int64\n\n\tLeaseDuration int\n\tsvc           dynamodbiface.DynamoDBAPI\n\tkclConfig     *config.KinesisClientLibConfiguration\n\tRetries       int\n\tlastLeaseSync time.Time\n}\n\nfunc NewDynamoCheckpoint(kclConfig *config.KinesisClientLibConfiguration) *DynamoCheckpoint {\n\tcheckpointer := &DynamoCheckpoint{\n\t\tlog:                     kclConfig.Logger,\n\t\tTableName:               kclConfig.TableName,\n\t\tleaseTableReadCapacity:  int64(kclConfig.InitialLeaseTableReadCapacity),\n\t\tleaseTableWriteCapacity: int64(kclConfig.InitialLeaseTableWriteCapacity),\n\t\tLeaseDuration:           kclConfig.FailoverTimeMillis,\n\t\tkclConfig:               kclConfig,\n\t\tRetries:                 NumMaxRetries,\n\t}\n\n\treturn checkpointer\n}\n\n// WithDynamoDB is used to provide DynamoDB service\nfunc (checkpointer *DynamoCheckpoint) WithDynamoDB(svc dynamodbiface.DynamoDBAPI) *DynamoCheckpoint {\n\tcheckpointer.svc = svc\n\treturn checkpointer\n}\n\n// Init initialises the DynamoDB Checkpoint\nfunc (checkpointer *DynamoCheckpoint) Init() error {\n\tcheckpointer.log.Infof(\"Creating DynamoDB session\")\n\n\ts, err := session.NewSession(&aws.Config{\n\t\tRegion:      aws.String(checkpointer.kclConfig.RegionName),\n\t\tEndpoint:    aws.String(checkpointer.kclConfig.DynamoDBEndpoint),\n\t\tCredentials: checkpointer.kclConfig.DynamoDBCredentials,\n\t\tRetryer: client.DefaultRetryer{\n\t\t\tNumMaxRetries:    checkpointer.Retries,\n\t\t\tMinRetryDelay:    client.DefaultRetryerMinRetryDelay,\n\t\t\tMinThrottleDelay: client.DefaultRetryerMinThrottleDelay,\n\t\t\tMaxRetryDelay:    client.DefaultRetryerMaxRetryDelay,\n\t\t\tMaxThrottleDelay: client.DefaultRetryerMaxRetryDelay,\n\t\t},\n\t})\n\n\tif err != nil {\n\t\t// no need to move forward\n\t\tcheckpointer.log.Fatalf(\"Failed in getting DynamoDB session for creating Worker: %+v\", err)\n\t}\n\n\tif checkpointer.svc == nil {\n\t\tcheckpointer.svc = dynamodb.New(s)\n\t}\n\n\tif !checkpointer.doesTableExist() {\n\t\treturn checkpointer.createTable()\n\t}\n\treturn nil\n}\n\n// GetLease attempts to gain a lock on the given shard\nfunc (checkpointer *DynamoCheckpoint) GetLease(shard *par.ShardStatus, newAssignTo string) error {\n\tnewLeaseTimeout := time.Now().Add(time.Duration(checkpointer.LeaseDuration) * time.Millisecond).UTC()\n\tnewLeaseTimeoutString := newLeaseTimeout.Format(time.RFC3339)\n\tcurrentCheckpoint, err := checkpointer.getItem(shard.ID)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tisClaimRequestExpired := shard.IsClaimRequestExpired(checkpointer.kclConfig)\n\n\tvar claimRequest string\n\tif checkpointer.kclConfig.EnableLeaseStealing {\n\t\tif currentCheckpointClaimRequest, ok := currentCheckpoint[ClaimRequestKey]; ok && currentCheckpointClaimRequest.S != nil {\n\t\t\tclaimRequest = *currentCheckpointClaimRequest.S\n\t\t\tif newAssignTo != claimRequest && !isClaimRequestExpired {\n\t\t\t\tcheckpointer.log.Debugf(\"another worker: %s has a claim on this shard. Not going to renew the lease\", claimRequest)\n\t\t\t\treturn errors.New(ErrShardClaimed)\n\t\t\t}\n\t\t}\n\t}\n\n\tassignedVar, assignedToOk := currentCheckpoint[LeaseOwnerKey]\n\tleaseVar, leaseTimeoutOk := currentCheckpoint[LeaseTimeoutKey]\n\n\tvar conditionalExpression string\n\tvar expressionAttributeValues map[string]*dynamodb.AttributeValue\n\n\tif !leaseTimeoutOk || !assignedToOk {\n\t\tconditionalExpression = \"attribute_not_exists(AssignedTo)\"\n\t} else {\n\t\tassignedTo := *assignedVar.S\n\t\tleaseTimeout := *leaseVar.S\n\n\t\tcurrentLeaseTimeout, err := time.Parse(time.RFC3339, leaseTimeout)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\tif checkpointer.kclConfig.EnableLeaseStealing {\n\t\t\tif time.Now().UTC().Before(currentLeaseTimeout) && assignedTo != newAssignTo && !isClaimRequestExpired {\n\t\t\t\treturn ErrLeaseNotAcquired{\"current lease timeout not yet expired\"}\n\t\t\t}\n\t\t} else {\n\t\t\tif time.Now().UTC().Before(currentLeaseTimeout) && assignedTo != newAssignTo {\n\t\t\t\treturn ErrLeaseNotAcquired{\"current lease timeout not yet expired\"}\n\t\t\t}\n\t\t}\n\n\t\tcheckpointer.log.Debugf(\"Attempting to get a lock for shard: %s, leaseTimeout: %s, assignedTo: %s, newAssignedTo: %s\", shard.ID, currentLeaseTimeout, assignedTo, newAssignTo)\n\t\tconditionalExpression = \"ShardID = :id AND AssignedTo = :assigned_to AND LeaseTimeout = :lease_timeout\"\n\t\texpressionAttributeValues = map[string]*dynamodb.AttributeValue{\n\t\t\t\":id\": {\n\t\t\t\tS: aws.String(shard.ID),\n\t\t\t},\n\t\t\t\":assigned_to\": {\n\t\t\t\tS: aws.String(assignedTo),\n\t\t\t},\n\t\t\t\":lease_timeout\": {\n\t\t\t\tS: aws.String(leaseTimeout),\n\t\t\t},\n\t\t}\n\t}\n\n\tmarshalledCheckpoint := map[string]*dynamodb.AttributeValue{\n\t\tLeaseKeyKey: {\n\t\t\tS: aws.String(shard.ID),\n\t\t},\n\t\tLeaseOwnerKey: {\n\t\t\tS: aws.String(newAssignTo),\n\t\t},\n\t\tLeaseTimeoutKey: {\n\t\t\tS: aws.String(newLeaseTimeoutString),\n\t\t},\n\t}\n\n\tif len(shard.ParentShardId) > 0 {\n\t\tmarshalledCheckpoint[ParentShardIdKey] = &dynamodb.AttributeValue{S: aws.String(shard.ParentShardId)}\n\t}\n\n\tif checkpoint := shard.GetCheckpoint(); checkpoint != \"\" {\n\t\tmarshalledCheckpoint[SequenceNumberKey] = &dynamodb.AttributeValue{\n\t\t\tS: aws.String(checkpoint),\n\t\t}\n\t}\n\n\tif checkpointer.kclConfig.EnableLeaseStealing {\n\t\tif claimRequest != \"\" && claimRequest == newAssignTo && !isClaimRequestExpired {\n\t\t\tif expressionAttributeValues == nil {\n\t\t\t\texpressionAttributeValues = make(map[string]*dynamodb.AttributeValue)\n\t\t\t}\n\t\t\tconditionalExpression = conditionalExpression + \" AND ClaimRequest = :claim_request\"\n\t\t\texpressionAttributeValues[\":claim_request\"] = &dynamodb.AttributeValue{\n\t\t\t\tS: &claimRequest,\n\t\t\t}\n\t\t}\n\t}\n\n\terr = checkpointer.conditionalUpdate(conditionalExpression, expressionAttributeValues, marshalledCheckpoint)\n\tif err != nil {\n\t\tif utils.AWSErrCode(err) == dynamodb.ErrCodeConditionalCheckFailedException {\n\t\t\treturn ErrLeaseNotAcquired{dynamodb.ErrCodeConditionalCheckFailedException}\n\t\t}\n\t\treturn err\n\t}\n\n\tshard.Mux.Lock()\n\tshard.AssignedTo = newAssignTo\n\tshard.LeaseTimeout = newLeaseTimeout\n\tshard.Mux.Unlock()\n\n\treturn nil\n}\n\n// CheckpointSequence writes a checkpoint at the designated sequence ID\nfunc (checkpointer *DynamoCheckpoint) CheckpointSequence(shard *par.ShardStatus) error {\n\tleaseTimeout := shard.GetLeaseTimeout().UTC().Format(time.RFC3339)\n\tmarshalledCheckpoint := map[string]*dynamodb.AttributeValue{\n\t\tLeaseKeyKey: {\n\t\t\tS: aws.String(shard.ID),\n\t\t},\n\t\tSequenceNumberKey: {\n\t\t\tS: aws.String(shard.GetCheckpoint()),\n\t\t},\n\t\tLeaseOwnerKey: {\n\t\t\tS: aws.String(shard.GetLeaseOwner()),\n\t\t},\n\t\tLeaseTimeoutKey: {\n\t\t\tS: aws.String(leaseTimeout),\n\t\t},\n\t}\n\n\tif len(shard.ParentShardId) > 0 {\n\t\tmarshalledCheckpoint[ParentShardIdKey] = &dynamodb.AttributeValue{S: &shard.ParentShardId}\n\t}\n\n\treturn checkpointer.saveItem(marshalledCheckpoint)\n}\n\n// FetchCheckpoint retrieves the checkpoint for the given shard\nfunc (checkpointer *DynamoCheckpoint) FetchCheckpoint(shard *par.ShardStatus) error {\n\tcheckpoint, err := checkpointer.getItem(shard.ID)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tsequenceID, ok := checkpoint[SequenceNumberKey]\n\tif !ok {\n\t\treturn ErrSequenceIDNotFound\n\t}\n\tcheckpointer.log.Debugf(\"Retrieved Shard Iterator %s\", *sequenceID.S)\n\tshard.SetCheckpoint(aws.StringValue(sequenceID.S))\n\n\tif assignedTo, ok := checkpoint[LeaseOwnerKey]; ok {\n\t\tshard.SetLeaseOwner(aws.StringValue(assignedTo.S))\n\t}\n\n\t// Use up-to-date leaseTimeout to avoid ConditionalCheckFailedException when claiming\n\tif leaseTimeout, ok := checkpoint[LeaseTimeoutKey]; ok && leaseTimeout.S != nil {\n\t\tcurrentLeaseTimeout, err := time.Parse(time.RFC3339, aws.StringValue(leaseTimeout.S))\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tshard.LeaseTimeout = currentLeaseTimeout\n\t}\n\n\treturn nil\n}\n\n// RemoveLeaseInfo to remove lease info for shard entry in dynamoDB because the shard no longer exists in Kinesis\nfunc (checkpointer *DynamoCheckpoint) RemoveLeaseInfo(shardID string) error {\n\terr := checkpointer.removeItem(shardID)\n\n\tif err != nil {\n\t\tcheckpointer.log.Errorf(\"Error in removing lease info for shard: %s, Error: %+v\", shardID, err)\n\t} else {\n\t\tcheckpointer.log.Infof(\"Lease info for shard: %s has been removed.\", shardID)\n\t}\n\n\treturn err\n}\n\n// RemoveLeaseOwner to remove lease owner for the shard entry\nfunc (checkpointer *DynamoCheckpoint) RemoveLeaseOwner(shardID string) error {\n\tinput := &dynamodb.UpdateItemInput{\n\t\tTableName: aws.String(checkpointer.TableName),\n\t\tKey: map[string]*dynamodb.AttributeValue{\n\t\t\tLeaseKeyKey: {\n\t\t\t\tS: aws.String(shardID),\n\t\t\t},\n\t\t},\n\t\tUpdateExpression: aws.String(\"remove \" + LeaseOwnerKey),\n\t\tExpressionAttributeValues: map[string]*dynamodb.AttributeValue{\n\t\t\t\":assigned_to\": {\n\t\t\t\tS: aws.String(checkpointer.kclConfig.WorkerID),\n\t\t\t},\n\t\t},\n\t\tConditionExpression: aws.String(\"AssignedTo = :assigned_to\"),\n\t}\n\n\t_, err := checkpointer.svc.UpdateItem(input)\n\n\treturn err\n}\n\n// ListActiveWorkers returns a map of workers and their shards\nfunc (checkpointer *DynamoCheckpoint) ListActiveWorkers(shardStatus map[string]*par.ShardStatus) (map[string][]*par.ShardStatus, error) {\n\terr := checkpointer.syncLeases(shardStatus)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tworkers := map[string][]*par.ShardStatus{}\n\tfor _, shard := range shardStatus {\n\t\tif shard.GetCheckpoint() == ShardEnd {\n\t\t\tcontinue\n\t\t}\n\n\t\tleaseOwner := shard.GetLeaseOwner()\n\t\tif leaseOwner == \"\" {\n\t\t\tcheckpointer.log.Debugf(\"Shard Not Assigned Error. ShardID: %s, WorkerID: %s\", shard.ID, checkpointer.kclConfig.WorkerID)\n\t\t\treturn nil, ErrShardNotAssigned\n\t\t}\n\t\tif w, ok := workers[leaseOwner]; ok {\n\t\t\tworkers[leaseOwner] = append(w, shard)\n\t\t} else {\n\t\t\tworkers[leaseOwner] = []*par.ShardStatus{shard}\n\t\t}\n\t}\n\treturn workers, nil\n}\n\n// ClaimShard places a claim request on a shard to signal a steal attempt\nfunc (checkpointer *DynamoCheckpoint) ClaimShard(shard *par.ShardStatus, claimID string) error {\n\terr := checkpointer.FetchCheckpoint(shard)\n\tif err != nil && err != ErrSequenceIDNotFound {\n\t\treturn err\n\t}\n\tleaseTimeoutString := shard.GetLeaseTimeout().Format(time.RFC3339)\n\n\tconditionalExpression := `ShardID = :id AND LeaseTimeout = :lease_timeout AND attribute_not_exists(ClaimRequest)`\n\texpressionAttributeValues := map[string]*dynamodb.AttributeValue{\n\t\t\":id\": {\n\t\t\tS: aws.String(shard.ID),\n\t\t},\n\t\t\":lease_timeout\": {\n\t\t\tS: aws.String(leaseTimeoutString),\n\t\t},\n\t}\n\n\tmarshalledCheckpoint := map[string]*dynamodb.AttributeValue{\n\t\tLeaseKeyKey: {\n\t\t\tS: &shard.ID,\n\t\t},\n\t\tLeaseTimeoutKey: {\n\t\t\tS: &leaseTimeoutString,\n\t\t},\n\t\tSequenceNumberKey: {\n\t\t\tS: &shard.Checkpoint,\n\t\t},\n\t\tClaimRequestKey: {\n\t\t\tS: &claimID,\n\t\t},\n\t}\n\n\tif leaseOwner := shard.GetLeaseOwner(); leaseOwner == \"\" {\n\t\tconditionalExpression += \" AND attribute_not_exists(AssignedTo)\"\n\t} else {\n\t\tmarshalledCheckpoint[LeaseOwnerKey] = &dynamodb.AttributeValue{S: &leaseOwner}\n\t\tconditionalExpression += \"AND AssignedTo = :assigned_to\"\n\t\texpressionAttributeValues[\":assigned_to\"] = &dynamodb.AttributeValue{S: &leaseOwner}\n\t}\n\n\tif checkpoint := shard.GetCheckpoint(); checkpoint == \"\" {\n\t\tconditionalExpression += \" AND attribute_not_exists(Checkpoint)\"\n\t} else if checkpoint == ShardEnd {\n\t\tconditionalExpression += \" AND Checkpoint <> :checkpoint\"\n\t\texpressionAttributeValues[\":checkpoint\"] = &dynamodb.AttributeValue{S: aws.String(ShardEnd)}\n\t} else {\n\t\tconditionalExpression += \" AND Checkpoint = :checkpoint\"\n\t\texpressionAttributeValues[\":checkpoint\"] = &dynamodb.AttributeValue{S: &checkpoint}\n\t}\n\n\tif shard.ParentShardId == \"\" {\n\t\tconditionalExpression += \" AND attribute_not_exists(ParentShardId)\"\n\t} else {\n\t\tmarshalledCheckpoint[ParentShardIdKey] = &dynamodb.AttributeValue{S: aws.String(shard.ParentShardId)}\n\t\tconditionalExpression += \" AND ParentShardId = :parent_shard\"\n\t\texpressionAttributeValues[\":parent_shard\"] = &dynamodb.AttributeValue{S: &shard.ParentShardId}\n\t}\n\n\treturn checkpointer.conditionalUpdate(conditionalExpression, expressionAttributeValues, marshalledCheckpoint)\n}\n\nfunc (checkpointer *DynamoCheckpoint) syncLeases(shardStatus map[string]*par.ShardStatus) error {\n\tlog := checkpointer.kclConfig.Logger\n\n\tif (checkpointer.lastLeaseSync.Add(time.Duration(checkpointer.kclConfig.LeaseSyncingTimeIntervalMillis) * time.Millisecond)).After(time.Now()) {\n\t\treturn nil\n\t}\n\n\tcheckpointer.lastLeaseSync = time.Now()\n\tinput := &dynamodb.ScanInput{\n\t\tProjectionExpression: aws.String(fmt.Sprintf(\"%s,%s,%s\", LeaseKeyKey, LeaseOwnerKey, SequenceNumberKey)),\n\t\tSelect:               aws.String(\"SPECIFIC_ATTRIBUTES\"),\n\t\tTableName:            aws.String(checkpointer.kclConfig.TableName),\n\t}\n\n\terr := checkpointer.svc.ScanPages(input,\n\t\tfunc(pages *dynamodb.ScanOutput, lastPage bool) bool {\n\t\t\tresults := pages.Items\n\t\t\tfor _, result := range results {\n\t\t\t\tshardId, foundShardId := result[LeaseKeyKey]\n\t\t\t\tassignedTo, foundAssignedTo := result[LeaseOwnerKey]\n\t\t\t\tcheckpoint, foundCheckpoint := result[SequenceNumberKey]\n\t\t\t\tif !foundShardId || !foundAssignedTo || !foundCheckpoint {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t\tif shard, ok := shardStatus[aws.StringValue(shardId.S)]; ok {\n\t\t\t\t\tshard.SetLeaseOwner(aws.StringValue(assignedTo.S))\n\t\t\t\t\tshard.SetCheckpoint(aws.StringValue(checkpoint.S))\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn !lastPage\n\t\t})\n\n\tif err != nil {\n\t\tlog.Debugf(\"Error performing SyncLeases. Error: %+v \", err)\n\t\treturn err\n\t}\n\tlog.Debugf(\"Lease sync completed. Next lease sync will occur in %s\", time.Duration(checkpointer.kclConfig.LeaseSyncingTimeIntervalMillis)*time.Millisecond)\n\treturn nil\n}\n\nfunc (checkpointer *DynamoCheckpoint) createTable() error {\n\tinput := &dynamodb.CreateTableInput{\n\t\tAttributeDefinitions: []*dynamodb.AttributeDefinition{\n\t\t\t{\n\t\t\t\tAttributeName: aws.String(LeaseKeyKey),\n\t\t\t\tAttributeType: aws.String(\"S\"),\n\t\t\t},\n\t\t},\n\t\tKeySchema: []*dynamodb.KeySchemaElement{\n\t\t\t{\n\t\t\t\tAttributeName: aws.String(LeaseKeyKey),\n\t\t\t\tKeyType:       aws.String(\"HASH\"),\n\t\t\t},\n\t\t},\n\t\tProvisionedThroughput: &dynamodb.ProvisionedThroughput{\n\t\t\tReadCapacityUnits:  aws.Int64(checkpointer.leaseTableReadCapacity),\n\t\t\tWriteCapacityUnits: aws.Int64(checkpointer.leaseTableWriteCapacity),\n\t\t},\n\t\tTableName: aws.String(checkpointer.TableName),\n\t}\n\t_, err := checkpointer.svc.CreateTable(input)\n\treturn err\n}\n\nfunc (checkpointer *DynamoCheckpoint) doesTableExist() bool {\n\tinput := &dynamodb.DescribeTableInput{\n\t\tTableName: aws.String(checkpointer.TableName),\n\t}\n\t_, err := checkpointer.svc.DescribeTable(input)\n\treturn err == nil\n}\n\nfunc (checkpointer *DynamoCheckpoint) saveItem(item map[string]*dynamodb.AttributeValue) error {\n\treturn checkpointer.putItem(&dynamodb.PutItemInput{\n\t\tTableName: aws.String(checkpointer.TableName),\n\t\tItem:      item,\n\t})\n}\n\nfunc (checkpointer *DynamoCheckpoint) conditionalUpdate(conditionExpression string, expressionAttributeValues map[string]*dynamodb.AttributeValue, item map[string]*dynamodb.AttributeValue) error {\n\treturn checkpointer.putItem(&dynamodb.PutItemInput{\n\t\tConditionExpression:       aws.String(conditionExpression),\n\t\tTableName:                 aws.String(checkpointer.TableName),\n\t\tItem:                      item,\n\t\tExpressionAttributeValues: expressionAttributeValues,\n\t})\n}\n\nfunc (checkpointer *DynamoCheckpoint) putItem(input *dynamodb.PutItemInput) error {\n\t_, err := checkpointer.svc.PutItem(input)\n\treturn err\n}\n\nfunc (checkpointer *DynamoCheckpoint) getItem(shardID string) (map[string]*dynamodb.AttributeValue, error) {\n\titem, err := checkpointer.svc.GetItem(&dynamodb.GetItemInput{\n\t\tTableName: aws.String(checkpointer.TableName),\n\t\tKey: map[string]*dynamodb.AttributeValue{\n\t\t\tLeaseKeyKey: {\n\t\t\t\tS: aws.String(shardID),\n\t\t\t},\n\t\t},\n\t})\n\treturn item.Item, err\n}\n\nfunc (checkpointer *DynamoCheckpoint) removeItem(shardID string) error {\n\t_, err := checkpointer.svc.DeleteItem(&dynamodb.DeleteItemInput{\n\t\tTableName: aws.String(checkpointer.TableName),\n\t\tKey: map[string]*dynamodb.AttributeValue{\n\t\t\tLeaseKeyKey: {\n\t\t\t\tS: aws.String(shardID),\n\t\t\t},\n\t\t},\n\t})\n\treturn err\n}\n"
  },
  {
    "path": "clientlibrary/checkpoint/dynamodb-checkpointer_test.go",
    "content": "/*\n * Copyright (c) 2019 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\n// The implementation is derived from https://github.com/patrobinson/gokini\n//\n// Copyright 2018 Patrick robinson\n//\n// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n//\n// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n//\n// THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\npackage checkpoint\n\nimport (\n\t\"errors\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/aws/awserr\"\n\t\"github.com/aws/aws-sdk-go/service/dynamodb\"\n\t\"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface\"\n\t\"github.com/stretchr/testify/assert\"\n\n\tcfg \"github.com/vmware/vmware-go-kcl/clientlibrary/config\"\n\tpar \"github.com/vmware/vmware-go-kcl/clientlibrary/partition\"\n)\n\nfunc TestDoesTableExist(t *testing.T) {\n\tsvc := &mockDynamoDB{tableExist: true, item: map[string]*dynamodb.AttributeValue{}}\n\tcheckpoint := &DynamoCheckpoint{\n\t\tTableName: \"TableName\",\n\t\tsvc:       svc,\n\t}\n\tif !checkpoint.doesTableExist() {\n\t\tt.Error(\"Table exists but returned false\")\n\t}\n\n\tsvc = &mockDynamoDB{tableExist: false}\n\tcheckpoint.svc = svc\n\tif checkpoint.doesTableExist() {\n\t\tt.Error(\"Table does not exist but returned true\")\n\t}\n}\n\nfunc TestGetLeaseNotAquired(t *testing.T) {\n\tsvc := &mockDynamoDB{tableExist: true, item: map[string]*dynamodb.AttributeValue{}}\n\tkclConfig := cfg.NewKinesisClientLibConfig(\"appName\", \"test\", \"us-west-2\", \"abc\").\n\t\tWithInitialPositionInStream(cfg.LATEST).\n\t\tWithMaxRecords(10).\n\t\tWithMaxLeasesForWorker(1).\n\t\tWithShardSyncIntervalMillis(5000).\n\t\tWithFailoverTimeMillis(300000)\n\n\tcheckpoint := NewDynamoCheckpoint(kclConfig).WithDynamoDB(svc)\n\tcheckpoint.Init()\n\terr := checkpoint.GetLease(&par.ShardStatus{\n\t\tID:         \"0001\",\n\t\tCheckpoint: \"\",\n\t\tMux:        &sync.RWMutex{},\n\t}, \"abcd-efgh\")\n\tif err != nil {\n\t\tt.Errorf(\"Error getting lease %s\", err)\n\t}\n\n\terr = checkpoint.GetLease(&par.ShardStatus{\n\t\tID:         \"0001\",\n\t\tCheckpoint: \"\",\n\t\tMux:        &sync.RWMutex{},\n\t}, \"ijkl-mnop\")\n\n\tif err == nil || !errors.As(err, &ErrLeaseNotAcquired{}) {\n\t\tt.Errorf(\"Got a lease when it was already held by abcd-efgh: %s\", err)\n\t}\n}\n\nfunc TestGetLeaseAquired(t *testing.T) {\n\tsvc := &mockDynamoDB{tableExist: true, item: map[string]*dynamodb.AttributeValue{}}\n\tkclConfig := cfg.NewKinesisClientLibConfig(\"appName\", \"test\", \"us-west-2\", \"abc\").\n\t\tWithInitialPositionInStream(cfg.LATEST).\n\t\tWithMaxRecords(10).\n\t\tWithMaxLeasesForWorker(1).\n\t\tWithShardSyncIntervalMillis(5000).\n\t\tWithFailoverTimeMillis(300000)\n\n\tcheckpoint := NewDynamoCheckpoint(kclConfig).WithDynamoDB(svc)\n\tcheckpoint.Init()\n\tmarshalledCheckpoint := map[string]*dynamodb.AttributeValue{\n\t\tLeaseKeyKey: {\n\t\t\tS: aws.String(\"0001\"),\n\t\t},\n\t\tLeaseOwnerKey: {\n\t\t\tS: aws.String(\"abcd-efgh\"),\n\t\t},\n\t\tLeaseTimeoutKey: {\n\t\t\tS: aws.String(time.Now().AddDate(0, -1, 0).UTC().Format(time.RFC3339)),\n\t\t},\n\t\tSequenceNumberKey: {\n\t\t\tS: aws.String(\"deadbeef\"),\n\t\t},\n\t}\n\tinput := &dynamodb.PutItemInput{\n\t\tTableName: aws.String(\"TableName\"),\n\t\tItem:      marshalledCheckpoint,\n\t}\n\tcheckpoint.svc.PutItem(input)\n\tshard := &par.ShardStatus{\n\t\tID:         \"0001\",\n\t\tCheckpoint: \"deadbeef\",\n\t\tMux:        &sync.RWMutex{},\n\t}\n\terr := checkpoint.GetLease(shard, \"ijkl-mnop\")\n\n\tif err != nil {\n\t\tt.Errorf(\"Lease not aquired after timeout %s\", err)\n\t}\n\n\tid, ok := svc.item[SequenceNumberKey]\n\tif !ok {\n\t\tt.Error(\"Expected checkpoint to be set by GetLease\")\n\t} else if *id.S != \"deadbeef\" {\n\t\tt.Errorf(\"Expected checkpoint to be deadbeef. Got '%s'\", *id.S)\n\t}\n\n\t// release owner info\n\terr = checkpoint.RemoveLeaseOwner(shard.ID)\n\tassert.Nil(t, err)\n\n\tstatus := &par.ShardStatus{\n\t\tID:  shard.ID,\n\t\tMux: &sync.RWMutex{},\n\t}\n\tcheckpoint.FetchCheckpoint(status)\n\n\t// checkpointer and parent shard id should be the same\n\tassert.Equal(t, shard.Checkpoint, status.Checkpoint)\n\tassert.Equal(t, shard.ParentShardId, status.ParentShardId)\n\n\t// Only the lease owner has been wiped out\n\tassert.Equal(t, \"\", status.GetLeaseOwner())\n}\n\nfunc TestGetLeaseShardClaimed(t *testing.T) {\n\tleaseTimeout := time.Now().Add(-100 * time.Second).UTC()\n\tsvc := &mockDynamoDB{\n\t\ttableExist: true,\n\t\titem: map[string]*dynamodb.AttributeValue{\n\t\t\tClaimRequestKey: {S: aws.String(\"ijkl-mnop\")},\n\t\t\tLeaseTimeoutKey: {S: aws.String(leaseTimeout.Format(time.RFC3339))},\n\t\t},\n\t}\n\tkclConfig := cfg.NewKinesisClientLibConfig(\"appName\", \"test\", \"us-west-2\", \"abc\").\n\t\tWithInitialPositionInStream(cfg.LATEST).\n\t\tWithMaxRecords(10).\n\t\tWithMaxLeasesForWorker(1).\n\t\tWithShardSyncIntervalMillis(5000).\n\t\tWithFailoverTimeMillis(300000).\n\t\tWithLeaseStealing(true)\n\n\tcheckpoint := NewDynamoCheckpoint(kclConfig).WithDynamoDB(svc)\n\tcheckpoint.Init()\n\terr := checkpoint.GetLease(&par.ShardStatus{\n\t\tID:           \"0001\",\n\t\tCheckpoint:   \"\",\n\t\tLeaseTimeout: leaseTimeout,\n\t\tMux:          &sync.RWMutex{},\n\t}, \"abcd-efgh\")\n\tif err == nil || err.Error() != ErrShardClaimed {\n\t\tt.Errorf(\"Got a lease when it was already claimed by by ijkl-mnop: %s\", err)\n\t}\n\n\terr = checkpoint.GetLease(&par.ShardStatus{\n\t\tID:           \"0001\",\n\t\tCheckpoint:   \"\",\n\t\tLeaseTimeout: leaseTimeout,\n\t\tMux:          &sync.RWMutex{},\n\t}, \"ijkl-mnop\")\n\tif err != nil {\n\t\tt.Errorf(\"Error getting lease %s\", err)\n\t}\n}\n\nfunc TestGetLeaseClaimRequestExpiredOwner(t *testing.T) {\n\tkclConfig := cfg.NewKinesisClientLibConfig(\"appName\", \"test\", \"us-west-2\", \"abc\").\n\t\tWithInitialPositionInStream(cfg.LATEST).\n\t\tWithMaxRecords(10).\n\t\tWithMaxLeasesForWorker(1).\n\t\tWithShardSyncIntervalMillis(5000).\n\t\tWithFailoverTimeMillis(300000).\n\t\tWithLeaseStealing(true)\n\n\t// Not expired\n\tleaseTimeout := time.Now().\n\t\tAdd(-time.Duration(kclConfig.LeaseStealingClaimTimeoutMillis) * time.Millisecond).\n\t\tAdd(1 * time.Second).\n\t\tUTC()\n\n\tsvc := &mockDynamoDB{\n\t\ttableExist: true,\n\t\titem: map[string]*dynamodb.AttributeValue{\n\t\t\tLeaseOwnerKey:   {S: aws.String(\"abcd-efgh\")},\n\t\t\tClaimRequestKey: {S: aws.String(\"ijkl-mnop\")},\n\t\t\tLeaseTimeoutKey: {S: aws.String(leaseTimeout.Format(time.RFC3339))},\n\t\t},\n\t}\n\n\tcheckpoint := NewDynamoCheckpoint(kclConfig).WithDynamoDB(svc)\n\tcheckpoint.Init()\n\terr := checkpoint.GetLease(&par.ShardStatus{\n\t\tID:           \"0001\",\n\t\tCheckpoint:   \"\",\n\t\tLeaseTimeout: leaseTimeout,\n\t\tMux:          &sync.RWMutex{},\n\t}, \"abcd-efgh\")\n\tif err == nil || err.Error() != ErrShardClaimed {\n\t\tt.Errorf(\"Got a lease when it was already claimed by ijkl-mnop: %s\", err)\n\t}\n}\n\nfunc TestGetLeaseClaimRequestExpiredClaimer(t *testing.T) {\n\tkclConfig := cfg.NewKinesisClientLibConfig(\"appName\", \"test\", \"us-west-2\", \"abc\").\n\t\tWithInitialPositionInStream(cfg.LATEST).\n\t\tWithMaxRecords(10).\n\t\tWithMaxLeasesForWorker(1).\n\t\tWithShardSyncIntervalMillis(5000).\n\t\tWithFailoverTimeMillis(300000).\n\t\tWithLeaseStealing(true)\n\n\t// Not expired\n\tleaseTimeout := time.Now().\n\t\tAdd(-time.Duration(kclConfig.LeaseStealingClaimTimeoutMillis) * time.Millisecond).\n\t\tAdd(121 * time.Second).\n\t\tUTC()\n\n\tsvc := &mockDynamoDB{\n\t\ttableExist: true,\n\t\titem: map[string]*dynamodb.AttributeValue{\n\t\t\tLeaseOwnerKey:   {S: aws.String(\"abcd-efgh\")},\n\t\t\tClaimRequestKey: {S: aws.String(\"ijkl-mnop\")},\n\t\t\tLeaseTimeoutKey: {S: aws.String(leaseTimeout.Format(time.RFC3339))},\n\t\t},\n\t}\n\n\tcheckpoint := NewDynamoCheckpoint(kclConfig).WithDynamoDB(svc)\n\tcheckpoint.Init()\n\terr := checkpoint.GetLease(&par.ShardStatus{\n\t\tID:           \"0001\",\n\t\tCheckpoint:   \"\",\n\t\tLeaseTimeout: leaseTimeout,\n\t\tMux:          &sync.RWMutex{},\n\t}, \"ijkl-mnop\")\n\tif err == nil || !errors.As(err, &ErrLeaseNotAcquired{}) {\n\t\tt.Errorf(\"Got a lease when it was already claimed by ijkl-mnop: %s\", err)\n\t}\n}\n\nfunc TestFetchCheckpointWithStealing(t *testing.T) {\n\tfuture := time.Now().AddDate(0, 1, 0)\n\n\tsvc := &mockDynamoDB{\n\t\ttableExist: true,\n\t\titem: map[string]*dynamodb.AttributeValue{\n\t\t\tSequenceNumberKey: {S: aws.String(\"deadbeef\")},\n\t\t\tLeaseOwnerKey:     {S: aws.String(\"abcd-efgh\")},\n\t\t\tLeaseTimeoutKey: {\n\t\t\t\tS: aws.String(future.Format(time.RFC3339)),\n\t\t\t},\n\t\t},\n\t}\n\n\tkclConfig := cfg.NewKinesisClientLibConfig(\"appName\", \"test\", \"us-west-2\", \"abc\").\n\t\tWithInitialPositionInStream(cfg.LATEST).\n\t\tWithMaxRecords(10).\n\t\tWithMaxLeasesForWorker(1).\n\t\tWithShardSyncIntervalMillis(5000).\n\t\tWithFailoverTimeMillis(300000).\n\t\tWithLeaseStealing(true)\n\n\tcheckpoint := NewDynamoCheckpoint(kclConfig).WithDynamoDB(svc)\n\tcheckpoint.Init()\n\n\tstatus := &par.ShardStatus{\n\t\tID:           \"0001\",\n\t\tCheckpoint:   \"\",\n\t\tLeaseTimeout: time.Now(),\n\t\tMux:          &sync.RWMutex{},\n\t}\n\n\tcheckpoint.FetchCheckpoint(status)\n\n\tleaseTimeout, _ := time.Parse(time.RFC3339, *svc.item[LeaseTimeoutKey].S)\n\tassert.Equal(t, leaseTimeout, status.LeaseTimeout)\n}\n\nfunc TestGetLeaseConditional(t *testing.T) {\n\tsvc := &mockDynamoDB{tableExist: true, item: map[string]*dynamodb.AttributeValue{}}\n\tkclConfig := cfg.NewKinesisClientLibConfig(\"appName\", \"test\", \"us-west-2\", \"abc\").\n\t\tWithInitialPositionInStream(cfg.LATEST).\n\t\tWithMaxRecords(10).\n\t\tWithMaxLeasesForWorker(1).\n\t\tWithShardSyncIntervalMillis(5000).\n\t\tWithFailoverTimeMillis(300000).\n\t\tWithLeaseStealing(true)\n\n\tcheckpoint := NewDynamoCheckpoint(kclConfig).WithDynamoDB(svc)\n\tcheckpoint.Init()\n\tmarshalledCheckpoint := map[string]*dynamodb.AttributeValue{\n\t\tLeaseKeyKey: {\n\t\t\tS: aws.String(\"0001\"),\n\t\t},\n\t\tLeaseOwnerKey: {\n\t\t\tS: aws.String(\"abcd-efgh\"),\n\t\t},\n\t\tLeaseTimeoutKey: {\n\t\t\tS: aws.String(time.Now().Add(-1 * time.Second).UTC().Format(time.RFC3339)),\n\t\t},\n\t\tSequenceNumberKey: {\n\t\t\tS: aws.String(\"deadbeef\"),\n\t\t},\n\t\tClaimRequestKey: {\n\t\t\tS: aws.String(\"ijkl-mnop\"),\n\t\t},\n\t}\n\tinput := &dynamodb.PutItemInput{\n\t\tTableName: aws.String(\"TableName\"),\n\t\tItem:      marshalledCheckpoint,\n\t}\n\tcheckpoint.svc.PutItem(input)\n\tshard := &par.ShardStatus{\n\t\tID:           \"0001\",\n\t\tCheckpoint:   \"deadbeef\",\n\t\tClaimRequest: \"ijkl-mnop\",\n\t\tMux:          &sync.RWMutex{},\n\t}\n\terr := checkpoint.FetchCheckpoint(shard)\n\tif err != nil {\n\t\tt.Errorf(\"Could not fetch checkpoint %s\", err)\n\t}\n\n\terr = checkpoint.GetLease(shard, \"ijkl-mnop\")\n\tif err != nil {\n\t\tt.Errorf(\"Lease not aquired after timeout %s\", err)\n\t}\n\tassert.Equal(t, *svc.expressionAttributeValues[\":claim_request\"].S, \"ijkl-mnop\")\n\tassert.Contains(t, svc.conditionalExpression, \" AND ClaimRequest = :claim_request\")\n}\n\ntype mockDynamoDB struct {\n\tdynamodbiface.DynamoDBAPI\n\ttableExist                bool\n\titem                      map[string]*dynamodb.AttributeValue\n\tconditionalExpression     string\n\texpressionAttributeValues map[string]*dynamodb.AttributeValue\n}\n\nfunc (m *mockDynamoDB) ScanPages(*dynamodb.ScanInput, func(*dynamodb.ScanOutput, bool) bool) error {\n\treturn nil\n}\n\nfunc (m *mockDynamoDB) DescribeTable(*dynamodb.DescribeTableInput) (*dynamodb.DescribeTableOutput, error) {\n\tif !m.tableExist {\n\t\treturn &dynamodb.DescribeTableOutput{}, awserr.New(dynamodb.ErrCodeResourceNotFoundException, \"doesNotExist\", errors.New(\"\"))\n\t}\n\treturn &dynamodb.DescribeTableOutput{}, nil\n}\n\nfunc (m *mockDynamoDB) PutItem(input *dynamodb.PutItemInput) (*dynamodb.PutItemOutput, error) {\n\titem := input.Item\n\n\tif shardID, ok := item[LeaseKeyKey]; ok {\n\t\tm.item[LeaseKeyKey] = shardID\n\t}\n\n\tif owner, ok := item[LeaseOwnerKey]; ok {\n\t\tm.item[LeaseOwnerKey] = owner\n\t}\n\n\tif timeout, ok := item[LeaseTimeoutKey]; ok {\n\t\tm.item[LeaseTimeoutKey] = timeout\n\t}\n\n\tif checkpoint, ok := item[SequenceNumberKey]; ok {\n\t\tm.item[SequenceNumberKey] = checkpoint\n\t}\n\n\tif parent, ok := item[ParentShardIdKey]; ok {\n\t\tm.item[ParentShardIdKey] = parent\n\t}\n\n\tif claimRequest, ok := item[ClaimRequestKey]; ok {\n\t\tm.item[ClaimRequestKey] = claimRequest\n\t}\n\n\tif input.ConditionExpression != nil {\n\t\tm.conditionalExpression = *input.ConditionExpression\n\t}\n\n\tm.expressionAttributeValues = input.ExpressionAttributeValues\n\n\treturn nil, nil\n}\n\nfunc (m *mockDynamoDB) GetItem(input *dynamodb.GetItemInput) (*dynamodb.GetItemOutput, error) {\n\treturn &dynamodb.GetItemOutput{\n\t\tItem: m.item,\n\t}, nil\n}\n\nfunc (m *mockDynamoDB) UpdateItem(input *dynamodb.UpdateItemInput) (*dynamodb.UpdateItemOutput, error) {\n\texp := input.UpdateExpression\n\n\tif aws.StringValue(exp) == \"remove \"+LeaseOwnerKey {\n\t\tdelete(m.item, LeaseOwnerKey)\n\t}\n\n\treturn nil, nil\n}\n\nfunc (m *mockDynamoDB) CreateTable(input *dynamodb.CreateTableInput) (*dynamodb.CreateTableOutput, error) {\n\treturn &dynamodb.CreateTableOutput{}, nil\n}\n\nfunc TestListActiveWorkers(t *testing.T) {\n\tsvc := &mockDynamoDB{tableExist: true, item: map[string]*dynamodb.AttributeValue{}}\n\tkclConfig := cfg.NewKinesisClientLibConfig(\"appName\", \"test\", \"us-west-2\", \"abc\").\n\t\tWithLeaseStealing(true)\n\n\tcheckpoint := NewDynamoCheckpoint(kclConfig).WithDynamoDB(svc)\n\terr := checkpoint.Init()\n\tif err != nil {\n\t\tt.Errorf(\"Checkpoint initialization failed: %+v\", err)\n\t}\n\n\tshardStatus := map[string]*par.ShardStatus{\n\t\t\"0000\": {ID: \"0000\", AssignedTo: \"worker_1\", Checkpoint: \"\", Mux: &sync.RWMutex{}},\n\t\t\"0001\": {ID: \"0001\", AssignedTo: \"worker_2\", Checkpoint: \"\", Mux: &sync.RWMutex{}},\n\t\t\"0002\": {ID: \"0002\", AssignedTo: \"worker_4\", Checkpoint: \"\", Mux: &sync.RWMutex{}},\n\t\t\"0003\": {ID: \"0003\", AssignedTo: \"worker_0\", Checkpoint: \"\", Mux: &sync.RWMutex{}},\n\t\t\"0004\": {ID: \"0004\", AssignedTo: \"worker_1\", Checkpoint: \"\", Mux: &sync.RWMutex{}},\n\t\t\"0005\": {ID: \"0005\", AssignedTo: \"worker_3\", Checkpoint: \"\", Mux: &sync.RWMutex{}},\n\t\t\"0006\": {ID: \"0006\", AssignedTo: \"worker_3\", Checkpoint: \"\", Mux: &sync.RWMutex{}},\n\t\t\"0007\": {ID: \"0007\", AssignedTo: \"worker_0\", Checkpoint: \"\", Mux: &sync.RWMutex{}},\n\t\t\"0008\": {ID: \"0008\", AssignedTo: \"worker_4\", Checkpoint: \"\", Mux: &sync.RWMutex{}},\n\t\t\"0009\": {ID: \"0009\", AssignedTo: \"worker_2\", Checkpoint: \"\", Mux: &sync.RWMutex{}},\n\t\t\"0010\": {ID: \"0010\", AssignedTo: \"worker_0\", Checkpoint: ShardEnd, Mux: &sync.RWMutex{}},\n\t}\n\n\tworkers, err := checkpoint.ListActiveWorkers(shardStatus)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n\n\tfor workerID, shards := range workers {\n\t\tassert.Equal(t, 2, len(shards))\n\t\tfor _, shard := range shards {\n\t\t\tassert.Equal(t, workerID, shard.AssignedTo)\n\t\t}\n\t}\n}\n\nfunc TestListActiveWorkersErrShardNotAssigned(t *testing.T) {\n\tsvc := &mockDynamoDB{tableExist: true, item: map[string]*dynamodb.AttributeValue{}}\n\tkclConfig := cfg.NewKinesisClientLibConfig(\"appName\", \"test\", \"us-west-2\", \"abc\").\n\t\tWithLeaseStealing(true)\n\n\tcheckpoint := NewDynamoCheckpoint(kclConfig).WithDynamoDB(svc)\n\terr := checkpoint.Init()\n\tif err != nil {\n\t\tt.Errorf(\"Checkpoint initialization failed: %+v\", err)\n\t}\n\n\tshardStatus := map[string]*par.ShardStatus{\n\t\t\"0000\": {ID: \"0000\", Mux: &sync.RWMutex{}},\n\t}\n\n\t_, err = checkpoint.ListActiveWorkers(shardStatus)\n\tif err != ErrShardNotAssigned {\n\t\tt.Error(\"Expected ErrShardNotAssigned when shard is missing AssignedTo value\")\n\t}\n}\n\nfunc TestClaimShard(t *testing.T) {\n\tsvc := &mockDynamoDB{tableExist: true, item: map[string]*dynamodb.AttributeValue{}}\n\tkclConfig := cfg.NewKinesisClientLibConfig(\"appName\", \"test\", \"us-west-2\", \"abc\").\n\t\tWithInitialPositionInStream(cfg.LATEST).\n\t\tWithMaxRecords(10).\n\t\tWithMaxLeasesForWorker(1).\n\t\tWithShardSyncIntervalMillis(5000).\n\t\tWithFailoverTimeMillis(300000).\n\t\tWithLeaseStealing(true)\n\n\tcheckpoint := NewDynamoCheckpoint(kclConfig).WithDynamoDB(svc)\n\tcheckpoint.Init()\n\n\tmarshalledCheckpoint := map[string]*dynamodb.AttributeValue{\n\t\t\"ShardID\": {\n\t\t\tS: aws.String(\"0001\"),\n\t\t},\n\t\t\"AssignedTo\": {\n\t\t\tS: aws.String(\"abcd-efgh\"),\n\t\t},\n\t\t\"LeaseTimeout\": {\n\t\t\tS: aws.String(time.Now().AddDate(0, -1, 0).UTC().Format(time.RFC3339)),\n\t\t},\n\t\t\"Checkpoint\": {\n\t\t\tS: aws.String(\"deadbeef\"),\n\t\t},\n\t}\n\tinput := &dynamodb.PutItemInput{\n\t\tTableName: aws.String(\"TableName\"),\n\t\tItem:      marshalledCheckpoint,\n\t}\n\tcheckpoint.svc.PutItem(input)\n\tshard := &par.ShardStatus{\n\t\tID:         \"0001\",\n\t\tCheckpoint: \"deadbeef\",\n\t\tMux:        &sync.RWMutex{},\n\t}\n\n\terr := checkpoint.ClaimShard(shard, \"ijkl-mnop\")\n\tif err != nil {\n\t\tt.Errorf(\"Shard not claimed %s\", err)\n\t}\n\n\tclaimRequest, ok := svc.item[ClaimRequestKey]\n\tif !ok {\n\t\tt.Error(\"Expected claimRequest to be set by ClaimShard\")\n\t} else if *claimRequest.S != \"ijkl-mnop\" {\n\t\tt.Errorf(\"Expected checkpoint to be ijkl-mnop. Got '%s'\", *claimRequest.S)\n\t}\n\n\tstatus := &par.ShardStatus{\n\t\tID:  shard.ID,\n\t\tMux: &sync.RWMutex{},\n\t}\n\tcheckpoint.FetchCheckpoint(status)\n\n\t// asiggnedTo, checkpointer, and parent shard id should be the same\n\tassert.Equal(t, shard.AssignedTo, status.AssignedTo)\n\tassert.Equal(t, shard.Checkpoint, status.Checkpoint)\n\tassert.Equal(t, shard.ParentShardId, status.ParentShardId)\n}\n"
  },
  {
    "path": "clientlibrary/config/config.go",
    "content": "/*\n * Copyright (c) 2018 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\n// The implementation is derived from https://github.com/awslabs/amazon-kinesis-client\n/*\n * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.\n *\n * Licensed under the Amazon Software License (the \"License\").\n * You may not use this file except in compliance with the License.\n * A copy of the License is located at\n *\n * http://aws.amazon.com/asl/\n *\n * or in the \"license\" file accompanying this file. This file is distributed\n * on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either\n * express or implied. See the License for the specific language governing\n * permissions and limitations under the License.\n */\npackage config\n\nimport (\n\t\"log\"\n\t\"math\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\tcreds \"github.com/aws/aws-sdk-go/aws/credentials\"\n\n\t\"github.com/vmware/vmware-go-kcl/clientlibrary/metrics\"\n\t\"github.com/vmware/vmware-go-kcl/logger\"\n)\n\nconst (\n\t// LATEST start after the most recent data record (fetch new data).\n\tLATEST InitialPositionInStream = iota + 1\n\t// TRIM_HORIZON start from the oldest available data record\n\tTRIM_HORIZON\n\t// AT_TIMESTAMP start from the record at or after the specified server-side Timestamp.\n\tAT_TIMESTAMP\n\n\t// The location in the shard from which the KinesisClientLibrary will start fetching records from\n\t// when the application starts for the first time and there is no checkpoint for the shard.\n\tDefaultInitialPositionInStream = LATEST\n\n\t// Fail over time in milliseconds. A worker which does not renew it's lease within this time interval\n\t// will be regarded as having problems and it's shards will be assigned to other workers.\n\t// For applications that have a large number of shards, this may be set to a higher number to reduce\n\t// the number of DynamoDB IOPS required for tracking leases.\n\tDefaultFailoverTimeMillis = 10000\n\n\t// Period before the end of lease during which a lease is refreshed by the owner.\n\tDefaultLeaseRefreshPeriodMillis = 5000\n\n\t// Max records to fetch from Kinesis in a single GetRecords call.\n\tDefaultMaxRecords = 10000\n\n\t// The default value for how long the {@link ShardConsumer} should sleep if no records are returned\n\t// from the call to\n\tDefaultIdletimeBetweenReadsMillis = 1000\n\n\t// Don't call processRecords() on the record processor for empty record lists.\n\tDefaultDontCallProcessRecordsForEmptyRecordList = false\n\n\t// Interval in milliseconds between polling to check for parent shard completion.\n\t// Polling frequently will take up more DynamoDB IOPS (when there are leases for shards waiting on\n\t// completion of parent shards).\n\tDefaultParentShardPollIntervalMillis = 10000\n\n\t// Shard sync interval in milliseconds - e.g. wait for this long between shard sync tasks.\n\tDefaultShardSyncIntervalMillis = 60000\n\n\t// Cleanup leases upon shards completion (don't wait until they expire in Kinesis).\n\t// Keeping leases takes some tracking/resources (e.g. they need to be renewed, assigned), so by\n\t// default we try to delete the ones we don't need any longer.\n\tDefaultCleanupLeasesUponShardsCompletion = true\n\n\t// Backoff time in milliseconds for Amazon Kinesis Client Library tasks (in the event of failures).\n\tDefaultTaskBackoffTimeMillis = 500\n\n\t// KCL will validate client provided sequence numbers with a call to Amazon Kinesis before\n\t// checkpointing for calls to {@link RecordProcessorCheckpointer#checkpoint(String)} by default.\n\tDefaultValidateSequenceNumberBeforeCheckpointing = true\n\n\t// The max number of leases (shards) this worker should process.\n\t// This can be useful to avoid overloading (and thrashing) a worker when a host has resource constraints\n\t// or during deployment.\n\t// NOTE: Setting this to a low value can cause data loss if workers are not able to pick up all shards in the\n\t// stream due to the max limit.\n\tDefaultMaxLeasesForWorker = math.MaxInt16\n\n\t// Max leases to steal from another worker at one time (for load balancing).\n\t// Setting this to a higher number can allow for faster load convergence (e.g. during deployments, cold starts),\n\t// but can cause higher churn in the system.\n\tDefaultMaxLeasesToStealAtOneTime = 1\n\n\t// The Amazon DynamoDB table used for tracking leases will be provisioned with this read capacity.\n\tDefaultInitialLeaseTableReadCapacity = 10\n\n\t// The Amazon DynamoDB table used for tracking leases will be provisioned with this write capacity.\n\tDefaultInitialLeaseTableWriteCapacity = 10\n\n\t// The Worker will skip shard sync during initialization if there are one or more leases in the lease table. This\n\t// assumes that the shards and leases are in-sync. This enables customers to choose faster startup times (e.g.\n\t// during incremental deployments of an application).\n\tDefaultSkipShardSyncAtStartupIfLeasesExist = false\n\n\t// The amount of milliseconds to wait before graceful shutdown forcefully terminates.\n\tDefaultShutdownGraceMillis = 5000\n\n\t// Lease stealing defaults to false for backwards compatibility.\n\tDefaultEnableLeaseStealing = false\n\n\t// Interval between rebalance tasks defaults to 5 seconds.\n\tDefaultLeaseStealingIntervalMillis = 5000\n\n\t// Number of milliseconds to wait before another worker can aquire a claimed shard\n\tDefaultLeaseStealingClaimTimeoutMillis = 120000\n\n\t// Number of milliseconds to wait before syncing with lease table (dynamodDB)\n\tDefaultLeaseSyncingIntervalMillis = 60000\n)\n\ntype (\n\t// InitialPositionInStream Used to specify the Position in the stream where a new application should start from\n\t// This is used during initial application bootstrap (when a checkpoint doesn't exist for a shard or its parents)\n\tInitialPositionInStream int\n\n\t// Class that houses the entities needed to specify the Position in the stream from where a new application should\n\t// start.\n\tInitialPositionInStreamExtended struct {\n\t\tPosition InitialPositionInStream\n\n\t\t// The time stamp of the data record from which to start reading. Used with\n\t\t// shard iterator type AT_TIMESTAMP. A time stamp is the Unix epoch date with\n\t\t// precision in milliseconds. For example, 2016-04-04T19:58:46.480-00:00 or\n\t\t// 1459799926.480. If a record with this exact time stamp does not exist, the\n\t\t// iterator returned is for the next (later) record. If the time stamp is older\n\t\t// than the current trim horizon, the iterator returned is for the oldest untrimmed\n\t\t// data record (TRIM_HORIZON).\n\t\tTimestamp *time.Time `type:\"Timestamp\" timestampFormat:\"unix\"`\n\t}\n\n\t// Configuration for the Kinesis Client Library.\n\t// Note: There is no need to configure credential provider. Credential can be get from InstanceProfile.\n\tKinesisClientLibConfiguration struct {\n\t\t// ApplicationName is name of application. Kinesis allows multiple applications to consume the same stream.\n\t\tApplicationName string\n\n\t\t// DynamoDBEndpoint is an optional endpoint URL that overrides the default generated endpoint for a DynamoDB client.\n\t\t// If this is empty, the default generated endpoint will be used.\n\t\tDynamoDBEndpoint string\n\n\t\t// KinesisEndpoint is an optional endpoint URL that overrides the default generated endpoint for a Kinesis client.\n\t\t// If this is empty, the default generated endpoint will be used.\n\t\tKinesisEndpoint string\n\n\t\t// KinesisCredentials is used to access Kinesis\n\t\tKinesisCredentials *creds.Credentials\n\n\t\t// DynamoDBCredentials is used to access DynamoDB\n\t\tDynamoDBCredentials *creds.Credentials\n\n\t\t// TableName is name of the dynamo db table for managing kinesis stream default to ApplicationName\n\t\tTableName string\n\n\t\t// StreamName is the name of Kinesis stream\n\t\tStreamName string\n\n\t\t// EnableEnhancedFanOutConsumer enables enhanced fan-out consumer\n\t\t// See: https://docs.aws.amazon.com/streams/latest/dev/enhanced-consumers.html\n\t\t// Either consumer name or consumer ARN must be specified when Enhanced Fan-Out is enabled.\n\t\tEnableEnhancedFanOutConsumer bool\n\n\t\t// EnhancedFanOutConsumerName is the name of the enhanced fan-out consumer to create. If this isn't set the ApplicationName will be used.\n\t\tEnhancedFanOutConsumerName string\n\n\t\t// EnhancedFanOutConsumerARN is the ARN of an already created enhanced fan-out consumer, if this is set no automatic consumer creation will be attempted\n\t\tEnhancedFanOutConsumerARN string\n\n\t\t// WorkerID used to distinguish different workers/processes of a Kinesis application\n\t\tWorkerID string\n\n\t\t// InitialPositionInStream specifies the Position in the stream where a new application should start from\n\t\tInitialPositionInStream InitialPositionInStream\n\n\t\t// InitialPositionInStreamExtended provides actual AT_TIMESTAMP value\n\t\tInitialPositionInStreamExtended InitialPositionInStreamExtended\n\n\t\t// credentials to access Kinesis/Dynamo: https://docs.aws.amazon.com/sdk-for-go/api/aws/credentials/\n\t\t// Note: No need to configure here. Use NewEnvCredentials for testing and EC2RoleProvider for production\n\n\t\t// FailoverTimeMillis Lease duration (leases not renewed within this period will be claimed by others)\n\t\tFailoverTimeMillis int\n\n\t\t// LeaseRefreshPeriodMillis is the period before the end of lease during which a lease is refreshed by the owner.\n\t\tLeaseRefreshPeriodMillis int\n\n\t\t// MaxRecords Max records to read per Kinesis getRecords() call\n\t\tMaxRecords int\n\n\t\t// IdleTimeBetweenReadsInMillis Idle time between calls to fetch data from Kinesis\n\t\tIdleTimeBetweenReadsInMillis int\n\n\t\t// CallProcessRecordsEvenForEmptyRecordList Call the IRecordProcessor::processRecords() API even if\n\t\t// GetRecords returned an empty record list.\n\t\tCallProcessRecordsEvenForEmptyRecordList bool\n\n\t\t// ParentShardPollIntervalMillis Wait for this long between polls to check if parent shards are done\n\t\tParentShardPollIntervalMillis int\n\n\t\t// ShardSyncIntervalMillis Time between tasks to sync leases and Kinesis shards\n\t\tShardSyncIntervalMillis int\n\n\t\t// CleanupTerminatedShardsBeforeExpiry Clean up shards we've finished processing (don't wait for expiration)\n\t\tCleanupTerminatedShardsBeforeExpiry bool\n\n\t\t// kinesisClientConfig Client Configuration used by Kinesis client\n\t\t// dynamoDBClientConfig Client Configuration used by DynamoDB client\n\t\t// Note: we will use default client provided by AWS SDK\n\n\t\t// TaskBackoffTimeMillis Backoff period when tasks encounter an exception\n\t\tTaskBackoffTimeMillis int\n\n\t\t// ValidateSequenceNumberBeforeCheckpointing whether KCL should validate client provided sequence numbers\n\t\tValidateSequenceNumberBeforeCheckpointing bool\n\n\t\t// RegionName The region name for the service\n\t\tRegionName string\n\n\t\t// ShutdownGraceMillis The number of milliseconds before graceful shutdown terminates forcefully\n\t\tShutdownGraceMillis int\n\n\t\t// Operation parameters\n\n\t\t// Max leases this Worker can handle at a time\n\t\tMaxLeasesForWorker int\n\n\t\t// Max leases to steal at one time (for load balancing)\n\t\tMaxLeasesToStealAtOneTime int\n\n\t\t// Read capacity to provision when creating the lease table (dynamoDB).\n\t\tInitialLeaseTableReadCapacity int\n\n\t\t// Write capacity to provision when creating the lease table.\n\t\tInitialLeaseTableWriteCapacity int\n\n\t\t// Worker should skip syncing shards and leases at startup if leases are present\n\t\t// This is useful for optimizing deployments to large fleets working on a stable stream.\n\t\tSkipShardSyncAtWorkerInitializationIfLeasesExist bool\n\n\t\t// Logger used to log message.\n\t\tLogger logger.Logger\n\n\t\t// MonitoringService publishes per worker-scoped metrics.\n\t\tMonitoringService metrics.MonitoringService\n\n\t\t// EnableLeaseStealing turns on lease stealing\n\t\tEnableLeaseStealing bool\n\n\t\t// LeaseStealingIntervalMillis The number of milliseconds between rebalance tasks\n\t\tLeaseStealingIntervalMillis int\n\n\t\t// LeaseStealingClaimTimeoutMillis The number of milliseconds to wait before another worker can aquire a claimed shard\n\t\tLeaseStealingClaimTimeoutMillis int\n\n\t\t// LeaseSyncingTimeInterval The number of milliseconds to wait before syncing with lease table (dynamoDB)\n\t\tLeaseSyncingTimeIntervalMillis int\n\t}\n)\n\nvar positionMap = map[InitialPositionInStream]*string{\n\tLATEST:       aws.String(\"LATEST\"),\n\tTRIM_HORIZON: aws.String(\"TRIM_HORIZON\"),\n\tAT_TIMESTAMP: aws.String(\"AT_TIMESTAMP\"),\n}\n\nfunc InitalPositionInStreamToShardIteratorType(pos InitialPositionInStream) *string {\n\treturn positionMap[pos]\n}\n\nfunc empty(s string) bool {\n\treturn len(strings.TrimSpace(s)) == 0\n}\n\n// checkIsValueNotEmpty makes sure the value is not empty.\nfunc checkIsValueNotEmpty(key string, value string) {\n\tif empty(value) {\n\t\t// There is no point to continue for incorrect configuration. Fail fast!\n\t\tlog.Panicf(\"Non-empty value expected for %v, actual: %v\", key, value)\n\t}\n}\n\n// checkIsValuePositive makes sure the value is possitive.\nfunc checkIsValuePositive(key string, value int) {\n\tif value <= 0 {\n\t\t// There is no point to continue for incorrect configuration. Fail fast!\n\t\tlog.Panicf(\"Positive value expected for %v, actual: %v\", key, value)\n\t}\n}\n"
  },
  {
    "path": "clientlibrary/config/config_test.go",
    "content": "/*\n * Copyright (c) 2018 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\npackage config\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\n\t\"github.com/vmware/vmware-go-kcl/logger\"\n)\n\nfunc TestConfig(t *testing.T) {\n\tkclConfig := NewKinesisClientLibConfig(\"appName\", \"StreamName\", \"us-west-2\", \"workerId\").\n\t\tWithFailoverTimeMillis(500).\n\t\tWithMaxRecords(100).\n\t\tWithInitialPositionInStream(TRIM_HORIZON).\n\t\tWithIdleTimeBetweenReadsInMillis(20).\n\t\tWithCallProcessRecordsEvenForEmptyRecordList(true).\n\t\tWithTaskBackoffTimeMillis(10).\n\t\tWithEnhancedFanOutConsumerName(\"fan-out-consumer\")\n\n\tassert.Equal(t, \"appName\", kclConfig.ApplicationName)\n\tassert.Equal(t, 500, kclConfig.FailoverTimeMillis)\n\tassert.Equal(t, 10, kclConfig.TaskBackoffTimeMillis)\n\n\tassert.True(t, kclConfig.EnableEnhancedFanOutConsumer)\n\tassert.Equal(t, \"fan-out-consumer\", kclConfig.EnhancedFanOutConsumerName)\n\n\tassert.Equal(t, false, kclConfig.EnableLeaseStealing)\n\tassert.Equal(t, 5000, kclConfig.LeaseStealingIntervalMillis)\n\n\tcontextLogger := kclConfig.Logger.WithFields(logger.Fields{\"key1\": \"value1\"})\n\tcontextLogger.Debugf(\"Starting with default logger\")\n\tcontextLogger.Infof(\"Default logger is awesome\")\n}\n\nfunc TestConfigLeaseStealing(t *testing.T) {\n\tkclConfig := NewKinesisClientLibConfig(\"appName\", \"StreamName\", \"us-west-2\", \"workerId\").\n\t\tWithFailoverTimeMillis(500).\n\t\tWithMaxRecords(100).\n\t\tWithInitialPositionInStream(TRIM_HORIZON).\n\t\tWithIdleTimeBetweenReadsInMillis(20).\n\t\tWithCallProcessRecordsEvenForEmptyRecordList(true).\n\t\tWithTaskBackoffTimeMillis(10).\n\t\tWithLeaseStealing(true).\n\t\tWithLeaseStealingIntervalMillis(10000)\n\n\tassert.Equal(t, \"appName\", kclConfig.ApplicationName)\n\tassert.Equal(t, 500, kclConfig.FailoverTimeMillis)\n\tassert.Equal(t, 10, kclConfig.TaskBackoffTimeMillis)\n\tassert.Equal(t, true, kclConfig.EnableLeaseStealing)\n\tassert.Equal(t, 10000, kclConfig.LeaseStealingIntervalMillis)\n\n\tcontextLogger := kclConfig.Logger.WithFields(logger.Fields{\"key1\": \"value1\"})\n\tcontextLogger.Debugf(\"Starting with default logger\")\n\tcontextLogger.Infof(\"Default logger is awesome\")\n}\n\nfunc TestConfigDefaultEnhancedFanOutConsumerName(t *testing.T) {\n\tkclConfig := NewKinesisClientLibConfig(\"appName\", \"StreamName\", \"us-west-2\", \"workerId\")\n\n\tassert.Equal(t, \"appName\", kclConfig.ApplicationName)\n\tassert.False(t, kclConfig.EnableEnhancedFanOutConsumer)\n\tassert.Equal(t, \"appName\", kclConfig.EnhancedFanOutConsumerName)\n}\n\nfunc TestEmptyEnhancedFanOutConsumerName(t *testing.T) {\n\tassert.PanicsWithValue(t, \"Non-empty value expected for EnhancedFanOutConsumerName, actual: \", func() {\n\t\tNewKinesisClientLibConfig(\"app\", \"stream\", \"us-west-2\", \"worker\").WithEnhancedFanOutConsumerName(\"\")\n\t})\n}\n\nfunc TestConfigWithEnhancedFanOutConsumerARN(t *testing.T) {\n\tkclConfig := NewKinesisClientLibConfig(\"app\", \"stream\", \"us-west-2\", \"worker\").\n\t\tWithEnhancedFanOutConsumerARN(\"consumer:arn\")\n\n\tassert.True(t, kclConfig.EnableEnhancedFanOutConsumer)\n\tassert.Equal(t, \"consumer:arn\", kclConfig.EnhancedFanOutConsumerARN)\n}\n\nfunc TestEmptyEnhancedFanOutConsumerARN(t *testing.T) {\n\tassert.PanicsWithValue(t, \"Non-empty value expected for EnhancedFanOutConsumerARN, actual: \", func() {\n\t\tNewKinesisClientLibConfig(\"app\", \"stream\", \"us-west-2\", \"worker\").WithEnhancedFanOutConsumerARN(\"\")\n\t})\n}\n"
  },
  {
    "path": "clientlibrary/config/initial-stream-pos.go",
    "content": "/*\n * Copyright (c) 2018 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\n// The implementation is derived from https://github.com/awslabs/amazon-kinesis-client\n/*\n * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.\n *\n * Licensed under the Amazon Software License (the \"License\").\n * You may not use this file except in compliance with the License.\n * A copy of the License is located at\n *\n * http://aws.amazon.com/asl/\n *\n * or in the \"license\" file accompanying this file. This file is distributed\n * on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either\n * express or implied. See the License for the specific language governing\n * permissions and limitations under the License.\n */\npackage config\n\nimport (\n\t\"time\"\n)\n\nfunc newInitialPositionAtTimestamp(timestamp *time.Time) *InitialPositionInStreamExtended {\n\treturn &InitialPositionInStreamExtended{Position: AT_TIMESTAMP, Timestamp: timestamp}\n}\n\nfunc newInitialPosition(position InitialPositionInStream) *InitialPositionInStreamExtended {\n\treturn &InitialPositionInStreamExtended{Position: position, Timestamp: nil}\n}\n"
  },
  {
    "path": "clientlibrary/config/kcl-config.go",
    "content": "/*\n * Copyright (c) 2018 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\n// The implementation is derived from https://github.com/awslabs/amazon-kinesis-client\n/*\n * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.\n *\n * Licensed under the Amazon Software License (the \"License\").\n * You may not use this file except in compliance with the License.\n * A copy of the License is located at\n *\n * http://aws.amazon.com/asl/\n *\n * or in the \"license\" file accompanying this file. This file is distributed\n * on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either\n * express or implied. See the License for the specific language governing\n * permissions and limitations under the License.\n */\npackage config\n\nimport (\n\t\"log\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aws/credentials\"\n\n\t\"github.com/vmware/vmware-go-kcl/clientlibrary/metrics\"\n\t\"github.com/vmware/vmware-go-kcl/clientlibrary/utils\"\n\t\"github.com/vmware/vmware-go-kcl/logger\"\n)\n\n// NewKinesisClientLibConfig creates a default KinesisClientLibConfiguration based on the required fields.\nfunc NewKinesisClientLibConfig(applicationName, streamName, regionName, workerID string) *KinesisClientLibConfiguration {\n\treturn NewKinesisClientLibConfigWithCredentials(applicationName, streamName, regionName, workerID,\n\t\tnil, nil)\n}\n\n// NewKinesisClientLibConfigWithCredential creates a default KinesisClientLibConfiguration based on the required fields and unique credentials.\nfunc NewKinesisClientLibConfigWithCredential(applicationName, streamName, regionName, workerID string,\n\tcreds *credentials.Credentials) *KinesisClientLibConfiguration {\n\treturn NewKinesisClientLibConfigWithCredentials(applicationName, streamName, regionName, workerID, creds, creds)\n}\n\n// NewKinesisClientLibConfigWithCredentials creates a default KinesisClientLibConfiguration based on the required fields and specific credentials for each service.\nfunc NewKinesisClientLibConfigWithCredentials(applicationName, streamName, regionName, workerID string,\n\tkiniesisCreds, dynamodbCreds *credentials.Credentials) *KinesisClientLibConfiguration {\n\tcheckIsValueNotEmpty(\"ApplicationName\", applicationName)\n\tcheckIsValueNotEmpty(\"StreamName\", streamName)\n\tcheckIsValueNotEmpty(\"RegionName\", regionName)\n\n\tif empty(workerID) {\n\t\tworkerID = utils.MustNewUUID()\n\t}\n\n\t// populate the KCL configuration with default values\n\treturn &KinesisClientLibConfiguration{\n\t\tApplicationName:                                  applicationName,\n\t\tKinesisCredentials:                               kiniesisCreds,\n\t\tDynamoDBCredentials:                              dynamodbCreds,\n\t\tTableName:                                        applicationName,\n\t\tEnhancedFanOutConsumerName:                       applicationName,\n\t\tStreamName:                                       streamName,\n\t\tRegionName:                                       regionName,\n\t\tWorkerID:                                         workerID,\n\t\tInitialPositionInStream:                          DefaultInitialPositionInStream,\n\t\tInitialPositionInStreamExtended:                  *newInitialPosition(DefaultInitialPositionInStream),\n\t\tFailoverTimeMillis:                               DefaultFailoverTimeMillis,\n\t\tLeaseRefreshPeriodMillis:                         DefaultLeaseRefreshPeriodMillis,\n\t\tMaxRecords:                                       DefaultMaxRecords,\n\t\tIdleTimeBetweenReadsInMillis:                     DefaultIdletimeBetweenReadsMillis,\n\t\tCallProcessRecordsEvenForEmptyRecordList:         DefaultDontCallProcessRecordsForEmptyRecordList,\n\t\tParentShardPollIntervalMillis:                    DefaultParentShardPollIntervalMillis,\n\t\tShardSyncIntervalMillis:                          DefaultShardSyncIntervalMillis,\n\t\tCleanupTerminatedShardsBeforeExpiry:              DefaultCleanupLeasesUponShardsCompletion,\n\t\tTaskBackoffTimeMillis:                            DefaultTaskBackoffTimeMillis,\n\t\tValidateSequenceNumberBeforeCheckpointing:        DefaultValidateSequenceNumberBeforeCheckpointing,\n\t\tShutdownGraceMillis:                              DefaultShutdownGraceMillis,\n\t\tMaxLeasesForWorker:                               DefaultMaxLeasesForWorker,\n\t\tMaxLeasesToStealAtOneTime:                        DefaultMaxLeasesToStealAtOneTime,\n\t\tInitialLeaseTableReadCapacity:                    DefaultInitialLeaseTableReadCapacity,\n\t\tInitialLeaseTableWriteCapacity:                   DefaultInitialLeaseTableWriteCapacity,\n\t\tSkipShardSyncAtWorkerInitializationIfLeasesExist: DefaultSkipShardSyncAtStartupIfLeasesExist,\n\t\tEnableLeaseStealing:                              DefaultEnableLeaseStealing,\n\t\tLeaseStealingIntervalMillis:                      DefaultLeaseStealingIntervalMillis,\n\t\tLeaseStealingClaimTimeoutMillis:                  DefaultLeaseStealingClaimTimeoutMillis,\n\t\tLeaseSyncingTimeIntervalMillis:                   DefaultLeaseSyncingIntervalMillis,\n\t\tLogger:                                           logger.GetDefaultLogger(),\n\t}\n}\n\n// WithKinesisEndpoint is used to provide an alternative Kinesis endpoint\nfunc (c *KinesisClientLibConfiguration) WithKinesisEndpoint(kinesisEndpoint string) *KinesisClientLibConfiguration {\n\tc.KinesisEndpoint = kinesisEndpoint\n\treturn c\n}\n\n// WithDynamoDBEndpoint is used to provide an alternative DynamoDB endpoint\nfunc (c *KinesisClientLibConfiguration) WithDynamoDBEndpoint(dynamoDBEndpoint string) *KinesisClientLibConfiguration {\n\tc.DynamoDBEndpoint = dynamoDBEndpoint\n\treturn c\n}\n\n// WithTableName to provide alternative lease table in DynamoDB\nfunc (c *KinesisClientLibConfiguration) WithTableName(tableName string) *KinesisClientLibConfiguration {\n\tc.TableName = tableName\n\treturn c\n}\n\nfunc (c *KinesisClientLibConfiguration) WithInitialPositionInStream(initialPositionInStream InitialPositionInStream) *KinesisClientLibConfiguration {\n\tc.InitialPositionInStream = initialPositionInStream\n\tc.InitialPositionInStreamExtended = *newInitialPosition(initialPositionInStream)\n\treturn c\n}\n\nfunc (c *KinesisClientLibConfiguration) WithTimestampAtInitialPositionInStream(timestamp *time.Time) *KinesisClientLibConfiguration {\n\tc.InitialPositionInStream = AT_TIMESTAMP\n\tc.InitialPositionInStreamExtended = *newInitialPositionAtTimestamp(timestamp)\n\treturn c\n}\n\nfunc (c *KinesisClientLibConfiguration) WithFailoverTimeMillis(failoverTimeMillis int) *KinesisClientLibConfiguration {\n\tcheckIsValuePositive(\"FailoverTimeMillis\", failoverTimeMillis)\n\tc.FailoverTimeMillis = failoverTimeMillis\n\treturn c\n}\n\nfunc (c *KinesisClientLibConfiguration) WithLeaseRefreshPeriodMillis(leaseRefreshPeriodMillis int) *KinesisClientLibConfiguration {\n\tcheckIsValuePositive(\"LeaseRefreshPeriodMillis\", leaseRefreshPeriodMillis)\n\tc.LeaseRefreshPeriodMillis = leaseRefreshPeriodMillis\n\treturn c\n}\n\nfunc (c *KinesisClientLibConfiguration) WithShardSyncIntervalMillis(shardSyncIntervalMillis int) *KinesisClientLibConfiguration {\n\tcheckIsValuePositive(\"ShardSyncIntervalMillis\", shardSyncIntervalMillis)\n\tc.ShardSyncIntervalMillis = shardSyncIntervalMillis\n\treturn c\n}\n\nfunc (c *KinesisClientLibConfiguration) WithMaxRecords(maxRecords int) *KinesisClientLibConfiguration {\n\tcheckIsValuePositive(\"MaxRecords\", maxRecords)\n\tc.MaxRecords = maxRecords\n\treturn c\n}\n\n// WithMaxLeasesForWorker configures maximum lease this worker can handles. It determines how maximun number of shards\n// this worker can handle.\nfunc (c *KinesisClientLibConfiguration) WithMaxLeasesForWorker(n int) *KinesisClientLibConfiguration {\n\tcheckIsValuePositive(\"MaxLeasesForWorker\", n)\n\tc.MaxLeasesForWorker = n\n\treturn c\n}\n\n/**\n * Controls how long the KCL will sleep if no records are returned from Kinesis\n *\n * <p>\n * This value is only used when no records are returned; if records are returned, the {@link com.amazonaws.services.kinesis.clientlibrary.lib.worker.ProcessTask} will\n * immediately retrieve the next set of records after the call to\n * {@link com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessor#processRecords(ProcessRecordsInput)}\n * has returned. Setting this value to high may result in the KCL being unable to catch up. If you are changing this\n * value it's recommended that you enable {@link #withCallProcessRecordsEvenForEmptyRecordList(boolean)}, and\n * monitor how far behind the records retrieved are by inspecting\n * {@link com.amazonaws.services.kinesis.clientlibrary.types.ProcessRecordsInput#getMillisBehindLatest()}, and the\n * <a href=\n * \"http://docs.aws.amazon.com/streams/latest/dev/monitoring-with-cloudwatch.html#kinesis-metrics-stream\">CloudWatch\n * Metric: GetRecords.MillisBehindLatest</a>\n * </p>\n *\n * @param IdleTimeBetweenReadsInMillis\n *            how long to sleep between GetRecords calls when no records are returned.\n * @return KinesisClientLibConfiguration\n */\nfunc (c *KinesisClientLibConfiguration) WithIdleTimeBetweenReadsInMillis(idleTimeBetweenReadsInMillis int) *KinesisClientLibConfiguration {\n\tcheckIsValuePositive(\"IdleTimeBetweenReadsInMillis\", idleTimeBetweenReadsInMillis)\n\tc.IdleTimeBetweenReadsInMillis = idleTimeBetweenReadsInMillis\n\treturn c\n}\n\nfunc (c *KinesisClientLibConfiguration) WithCallProcessRecordsEvenForEmptyRecordList(callProcessRecordsEvenForEmptyRecordList bool) *KinesisClientLibConfiguration {\n\tc.CallProcessRecordsEvenForEmptyRecordList = callProcessRecordsEvenForEmptyRecordList\n\treturn c\n}\n\nfunc (c *KinesisClientLibConfiguration) WithTaskBackoffTimeMillis(taskBackoffTimeMillis int) *KinesisClientLibConfiguration {\n\tcheckIsValuePositive(\"TaskBackoffTimeMillis\", taskBackoffTimeMillis)\n\tc.TaskBackoffTimeMillis = taskBackoffTimeMillis\n\treturn c\n}\n\nfunc (c *KinesisClientLibConfiguration) WithLogger(logger logger.Logger) *KinesisClientLibConfiguration {\n\tif logger == nil {\n\t\tlog.Panic(\"Logger cannot be null\")\n\t}\n\tc.Logger = logger\n\treturn c\n}\n\n// WithMonitoringService sets the monitoring service to use to publish metrics.\nfunc (c *KinesisClientLibConfiguration) WithMonitoringService(mService metrics.MonitoringService) *KinesisClientLibConfiguration {\n\t// Nil case is handled downward (at worker creation) so no need to do it here.\n\t// Plus the user might want to be explicit about passing a nil monitoring service here.\n\tc.MonitoringService = mService\n\treturn c\n}\n\n// WithEnhancedFanOutConsumer sets EnableEnhancedFanOutConsumer. If enhanced fan-out is enabled and ConsumerName is not specified ApplicationName is used as ConsumerName.\n// For more info see: https://docs.aws.amazon.com/streams/latest/dev/enhanced-consumers.html\n// Note: You can register up to twenty consumers per stream to use enhanced fan-out.\nfunc (c *KinesisClientLibConfiguration) WithEnhancedFanOutConsumer(enable bool) *KinesisClientLibConfiguration {\n\tc.EnableEnhancedFanOutConsumer = enable\n\treturn c\n}\n\n// WithEnhancedFanOutConsumerName enables enhanced fan-out consumer with the specified name\n// For more info see: https://docs.aws.amazon.com/streams/latest/dev/enhanced-consumers.html\n// Note: You can register up to twenty consumers per stream to use enhanced fan-out.\nfunc (c *KinesisClientLibConfiguration) WithEnhancedFanOutConsumerName(consumerName string) *KinesisClientLibConfiguration {\n\tcheckIsValueNotEmpty(\"EnhancedFanOutConsumerName\", consumerName)\n\tc.EnhancedFanOutConsumerName = consumerName\n\tc.EnableEnhancedFanOutConsumer = true\n\treturn c\n}\n\n// WithEnhancedFanOutConsumerARN enables enhanced fan-out consumer with the specified consumer ARN\n// For more info see: https://docs.aws.amazon.com/streams/latest/dev/enhanced-consumers.html\n// Note: You can register up to twenty consumers per stream to use enhanced fan-out.\nfunc (c *KinesisClientLibConfiguration) WithEnhancedFanOutConsumerARN(consumerARN string) *KinesisClientLibConfiguration {\n\tcheckIsValueNotEmpty(\"EnhancedFanOutConsumerARN\", consumerARN)\n\tc.EnhancedFanOutConsumerARN = consumerARN\n\tc.EnableEnhancedFanOutConsumer = true\n\treturn c\n}\n\nfunc (c *KinesisClientLibConfiguration) WithLeaseStealing(enableLeaseStealing bool) *KinesisClientLibConfiguration {\n\tc.EnableLeaseStealing = enableLeaseStealing\n\treturn c\n}\n\nfunc (c *KinesisClientLibConfiguration) WithLeaseStealingIntervalMillis(leaseStealingIntervalMillis int) *KinesisClientLibConfiguration {\n\tc.LeaseStealingIntervalMillis = leaseStealingIntervalMillis\n\treturn c\n}\n\nfunc (c *KinesisClientLibConfiguration) WithLeaseSyncingIntervalMillis(leaseSyncingIntervalMillis int) *KinesisClientLibConfiguration {\n\tc.LeaseSyncingTimeIntervalMillis = leaseSyncingIntervalMillis\n\treturn c\n}\n"
  },
  {
    "path": "clientlibrary/interfaces/inputs.go",
    "content": "/*\n * Copyright (c) 2020 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\n// The implementation is derived from https://github.com/awslabs/amazon-kinesis-client\n/*\n * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.\n *\n * Licensed under the Amazon Software License (the \"License\").\n * You may not use this file except in compliance with the License.\n * A copy of the License is located at\n *\n * http://aws.amazon.com/asl/\n *\n * or in the \"license\" file accompanying this file. This file is distributed\n * on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either\n * express or implied. See the License for the specific language governing\n * permissions and limitations under the License.\n */\npackage interfaces\n\nimport (\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\tks \"github.com/aws/aws-sdk-go/service/kinesis\"\n)\n\nconst (\n\t/**\n\t * Indicates that the entire application is being shutdown, and if desired the record processor will be given a\n\t * final chance to checkpoint. This state will not trigger a direct call to\n\t * {@link com.amazonaws.services.kinesis.clientlibrary.interfaces.v2.IRecordProcessor#shutdown(ShutdownInput)}, but\n\t * instead depend on a different interface for backward compatibility.\n\t */\n\tREQUESTED ShutdownReason = iota + 1\n\n\t/**\n\t * Terminate processing for this RecordProcessor (resharding use case).\n\t * Indicates that the shard is closed and all records from the shard have been delivered to the application.\n\t * Applications SHOULD checkpoint their progress to indicate that they have successfully processed all records\n\t * from this shard and processing of child shards can be started.\n\t */\n\tTERMINATE\n\n\t/**\n\t * Processing will be moved to a different record processor (fail over, load balancing use cases).\n\t * Applications SHOULD NOT checkpoint their progress (as another record processor may have already started\n\t * processing data).\n\t */\n\tZOMBIE\n)\n\n// Containers for the parameters to the IRecordProcessor\ntype (\n\t/**\n\t * Reason the RecordProcessor is being shutdown.\n\t * Used to distinguish between a fail-over vs. a termination (shard is closed and all records have been delivered).\n\t * In case of a fail over, applications should NOT checkpoint as part of shutdown,\n\t * since another record processor may have already started processing records for that shard.\n\t * In case of termination (resharding use case), applications SHOULD checkpoint their progress to indicate\n\t * that they have successfully processed all the records (processing of child shards can then begin).\n\t */\n\tShutdownReason int\n\n\tInitializationInput struct {\n\t\t// The shardId that the record processor is being initialized for.\n\t\tShardId string\n\n\t\t// The last extended sequence number that was successfully checkpointed by the previous record processor.\n\t\tExtendedSequenceNumber *ExtendedSequenceNumber\n\t}\n\n\tProcessRecordsInput struct {\n\t\t// The time that this batch of records was received by the KCL.\n\t\tCacheEntryTime *time.Time\n\n\t\t// The time that this batch of records was prepared to be provided to the RecordProcessor.\n\t\tCacheExitTime *time.Time\n\n\t\t// The records received from Kinesis. These records may have been de-aggregated if they were published by the KPL.\n\t\tRecords []*ks.Record\n\n\t\t// A checkpointer that the RecordProcessor can use to checkpoint its progress.\n\t\tCheckpointer IRecordProcessorCheckpointer\n\n\t\t// How far behind this batch of records was when received from Kinesis.\n\t\tMillisBehindLatest int64\n\t}\n\n\tShutdownInput struct {\n\t\t// ShutdownReason shows why RecordProcessor is going to be shutdown.\n\t\tShutdownReason ShutdownReason\n\n\t\t// Checkpointer is used to record the current progress.\n\t\tCheckpointer IRecordProcessorCheckpointer\n\t}\n)\n\nvar shutdownReasonMap = map[ShutdownReason]*string{\n\tREQUESTED: aws.String(\"REQUESTED\"),\n\tTERMINATE: aws.String(\"TERMINATE\"),\n\tZOMBIE:    aws.String(\"ZOMBIE\"),\n}\n\nfunc ShutdownReasonMessage(reason ShutdownReason) *string {\n\treturn shutdownReasonMap[reason]\n}\n"
  },
  {
    "path": "clientlibrary/interfaces/record-processor-checkpointer.go",
    "content": "/*\n * Copyright (c) 2018 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\n// The implementation is derived from https://github.com/awslabs/amazon-kinesis-client\n/*\n * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.\n *\n * Licensed under the Amazon Software License (the \"License\").\n * You may not use this file except in compliance with the License.\n * A copy of the License is located at\n *\n * http://aws.amazon.com/asl/\n *\n * or in the \"license\" file accompanying this file. This file is distributed\n * on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either\n * express or implied. See the License for the specific language governing\n * permissions and limitations under the License.\n */\npackage interfaces\n\ntype (\n\tIPreparedCheckpointer interface {\n\t\tGetPendingCheckpoint() *ExtendedSequenceNumber\n\n\t\t/**\n\t\t * This method will record a pending checkpoint.\n\t\t *\n\t\t * @error ThrottlingError Can't store checkpoint. Can be caused by checkpointing too frequently.\n\t\t *         Consider increasing the throughput/capacity of the checkpoint store or reducing checkpoint frequency.\n\t\t * @error ShutdownError The record processor instance has been shutdown. Another instance may have\n\t\t *         started processing some of these records already.\n\t\t *         The application should abort processing via this RecordProcessor instance.\n\t\t * @error InvalidStateError Can't store checkpoint.\n\t\t *         Unable to store the checkpoint in the DynamoDB table (e.g. table doesn't exist).\n\t\t * @error KinesisClientLibDependencyError Encountered an issue when storing the checkpoint. The application can\n\t\t *         backoff and retry.\n\t\t * @error IllegalArgumentError The sequence number being checkpointed is invalid because it is out of range,\n\t\t *         i.e. it is smaller than the last check point value (prepared or committed), or larger than the greatest\n\t\t *         sequence number seen by the associated record processor.\n\t\t */\n\t\tCheckpoint() error\n\t}\n\n\t/**\n\t * Used by RecordProcessors when they want to checkpoint their progress.\n\t * The Kinesis Client Library will pass an object implementing this interface to RecordProcessors, so they can\n\t * checkpoint their progress.\n\t */\n\tIRecordProcessorCheckpointer interface {\n\t\t/**\n\t\t * This method will checkpoint the progress at the provided sequenceNumber. This method is analogous to\n\t\t * {@link #checkpoint()} but provides the ability to specify the sequence number at which to\n\t\t * checkpoint.\n\t\t *\n\t\t * @param sequenceNumber A sequence number at which to checkpoint in this shard. Upon failover,\n\t\t *        the Kinesis Client Library will start fetching records after this sequence number.\n\t\t * @error ThrottlingError Can't store checkpoint. Can be caused by checkpointing too frequently.\n\t\t *         Consider increasing the throughput/capacity of the checkpoint store or reducing checkpoint frequency.\n\t\t * @error ShutdownError The record processor instance has been shutdown. Another instance may have\n\t\t *         started processing some of these records already.\n\t\t *         The application should abort processing via this RecordProcessor instance.\n\t\t * @error InvalidStateError Can't store checkpoint.\n\t\t *         Unable to store the checkpoint in the DynamoDB table (e.g. table doesn't exist).\n\t\t * @error KinesisClientLibDependencyError Encountered an issue when storing the checkpoint. The application can\n\t\t *         backoff and retry.\n\t\t * @error IllegalArgumentError The sequence number is invalid for one of the following reasons:\n\t\t *         1.) It appears to be out of range, i.e. it is smaller than the last check point value, or larger than the\n\t\t *         greatest sequence number seen by the associated record processor.\n\t\t *         2.) It is not a valid sequence number for a record in this shard.\n\t\t */\n\t\tCheckpoint(sequenceNumber *string) error\n\n\t\t/**\n\t\t * This method will record a pending checkpoint at the provided sequenceNumber.\n\t\t *\n\t\t * @param sequenceNumber A sequence number at which to prepare checkpoint in this shard.\n\n\t\t * @return an IPreparedCheckpointer object that can be called later to persist the checkpoint.\n\t\t *\n\t\t * @error ThrottlingError Can't store pending checkpoint. Can be caused by checkpointing too frequently.\n\t\t *         Consider increasing the throughput/capacity of the checkpoint store or reducing checkpoint frequency.\n\t\t * @error ShutdownError The record processor instance has been shutdown. Another instance may have\n\t\t *         started processing some of these records already.\n\t\t *         The application should abort processing via this RecordProcessor instance.\n\t\t * @error InvalidStateError Can't store pending checkpoint.\n\t\t *         Unable to store the checkpoint in the DynamoDB table (e.g. table doesn't exist).\n\t\t * @error KinesisClientLibDependencyError Encountered an issue when storing the pending checkpoint. The\n\t\t *         application can backoff and retry.\n\t\t * @error IllegalArgumentError The sequence number is invalid for one of the following reasons:\n\t\t *         1.) It appears to be out of range, i.e. it is smaller than the last check point value, or larger than the\n\t\t *         greatest sequence number seen by the associated record processor.\n\t\t *         2.) It is not a valid sequence number for a record in this shard.\n\t\t */\n\t\tPrepareCheckpoint(sequenceNumber *string) (IPreparedCheckpointer, error)\n\t}\n)\n"
  },
  {
    "path": "clientlibrary/interfaces/record-processor.go",
    "content": "/*\n * Copyright (c) 2018 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\n// The implementation is derived from https://github.com/awslabs/amazon-kinesis-client\n/*\n * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.\n *\n * Licensed under the Amazon Software License (the \"License\").\n * You may not use this file except in compliance with the License.\n * A copy of the License is located at\n *\n * http://aws.amazon.com/asl/\n *\n * or in the \"license\" file accompanying this file. This file is distributed\n * on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either\n * express or implied. See the License for the specific language governing\n * permissions and limitations under the License.\n */\npackage interfaces\n\ntype (\n\t// IRecordProcessor is the interface for some callback functions invoked by KCL will\n\t// The main task of using KCL is to provide implementation on IRecordProcessor interface.\n\t// Note: This is exactly the same interface as Amazon KCL IRecordProcessor v2\n\tIRecordProcessor interface {\n\t\t/**\n\t\t * Invoked by the Amazon Kinesis Client Library before data records are delivered to the RecordProcessor instance\n\t\t * (via processRecords).\n\t\t *\n\t\t * @param initializationInput Provides information related to initialization\n\t\t */\n\t\tInitialize(initializationInput *InitializationInput)\n\n\t\t/**\n\t\t * Process data records. The Amazon Kinesis Client Library will invoke this method to deliver data records to the\n\t\t * application.\n\t\t * Upon fail over, the new instance will get records with sequence number > checkpoint position\n\t\t * for each partition key.\n\t\t *\n\t\t * @param processRecordsInput Provides the records to be processed as well as information and capabilities related\n\t\t *        to them (eg checkpointing).\n\t\t */\n\t\tProcessRecords(processRecordsInput *ProcessRecordsInput)\n\n\t\t/**\n\t\t * Invoked by the Amazon Kinesis Client Library to indicate it will no longer send data records to this\n\t\t * RecordProcessor instance.\n\t\t *\n\t\t * <h2><b>Warning</b></h2>\n\t\t *\n\t\t * When the value of {@link ShutdownInput#getShutdownReason()} is\n\t\t * {@link com.amazonaws.services.kinesis.clientlibrary.lib.worker.ShutdownReason#TERMINATE} it is required that you\n\t\t * checkpoint. Failure to do so will result in an IllegalArgumentException, and the KCL no longer making progress.\n\t\t *\n\t\t * @param shutdownInput\n\t\t *            Provides information and capabilities (eg checkpointing) related to shutdown of this record processor.\n\t\t */\n\t\tShutdown(shutdownInput *ShutdownInput)\n\t}\n\n\t// IRecordProcessorFactory is interface for creating IRecordProcessor. Each Worker can have multiple threads\n\t// for processing shard. Client can choose either creating one processor per shard or sharing them.\n\tIRecordProcessorFactory interface {\n\n\t\t/**\n\t\t * Returns a record processor to be used for processing data records for a (assigned) shard.\n\t\t *\n\t\t * @return Returns a processor object.\n\t\t */\n\t\tCreateProcessor() IRecordProcessor\n\t}\n)\n"
  },
  {
    "path": "clientlibrary/interfaces/sequence-number.go",
    "content": "/*\n * Copyright (c) 2018 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\n// The implementation is derived from https://github.com/awslabs/amazon-kinesis-client\n/*\n * Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.\n *\n * Licensed under the Amazon Software License (the \"License\").\n * You may not use this file except in compliance with the License.\n * A copy of the License is located at\n *\n * http://aws.amazon.com/asl/\n *\n * or in the \"license\" file accompanying this file. This file is distributed\n * on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either\n * express or implied. See the License for the specific language governing\n * permissions and limitations under the License.\n */\npackage interfaces\n\n// ExtendedSequenceNumber represents a two-part sequence number for records aggregated by the Kinesis Producer Library.\n//\n// The KPL combines multiple user records into a single Kinesis record. Each user record therefore has an integer\n// sub-sequence number, in addition to the regular sequence number of the Kinesis record. The sub-sequence number\n// is used to checkpoint within an aggregated record.\ntype ExtendedSequenceNumber struct {\n\tSequenceNumber    *string\n\tSubSequenceNumber int64\n}\n"
  },
  {
    "path": "clientlibrary/metrics/cloudwatch/cloudwatch.go",
    "content": "/*\n * Copyright (c) 2018 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\n// The implementation is derived from https://github.com/patrobinson/gokini\n//\n// Copyright 2018 Patrick robinson\n//\n// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n//\n// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n//\n// THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\npackage cloudwatch\n\nimport (\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/aws/credentials\"\n\t\"github.com/aws/aws-sdk-go/aws/session\"\n\tcwatch \"github.com/aws/aws-sdk-go/service/cloudwatch\"\n\t\"github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface\"\n\n\t\"github.com/vmware/vmware-go-kcl/logger\"\n)\n\n// Buffer metrics for at most this long before publishing to CloudWatch.\nconst DEFAULT_CLOUDWATCH_METRICS_BUFFER_DURATION = 10 * time.Second\n\ntype MonitoringService struct {\n\tappName     string\n\tstreamName  string\n\tworkerID    string\n\tregion      string\n\tcredentials *credentials.Credentials\n\tlogger      logger.Logger\n\n\t// control how often to publish to CloudWatch\n\tbufferDuration time.Duration\n\n\tstop         *chan struct{}\n\twaitGroup    *sync.WaitGroup\n\tsvc          cloudwatchiface.CloudWatchAPI\n\tshardMetrics *sync.Map\n}\n\ntype cloudWatchMetrics struct {\n\tsync.Mutex\n\n\tprocessedRecords   int64\n\tprocessedBytes     int64\n\tbehindLatestMillis []float64\n\tleasesHeld         int64\n\tleaseRenewals      int64\n\tgetRecordsTime     []float64\n\tprocessRecordsTime []float64\n}\n\n// NewMonitoringService returns a Monitoring service publishing metrics to CloudWatch.\nfunc NewMonitoringService(region string, creds *credentials.Credentials) *MonitoringService {\n\treturn NewMonitoringServiceWithOptions(region, creds, logger.GetDefaultLogger(), DEFAULT_CLOUDWATCH_METRICS_BUFFER_DURATION)\n}\n\n// NewMonitoringServiceWithOptions returns a Monitoring service publishing metrics to\n// CloudWatch with the provided credentials, buffering duration and logger.\nfunc NewMonitoringServiceWithOptions(region string, creds *credentials.Credentials, logger logger.Logger, bufferDur time.Duration) *MonitoringService {\n\treturn &MonitoringService{\n\t\tregion:         region,\n\t\tcredentials:    creds,\n\t\tlogger:         logger,\n\t\tbufferDuration: bufferDur,\n\t}\n}\n\nfunc (cw *MonitoringService) Init(appName, streamName, workerID string) error {\n\tcw.appName = appName\n\tcw.streamName = streamName\n\tcw.workerID = workerID\n\n\tcfg := &aws.Config{Region: aws.String(cw.region)}\n\tcfg.Credentials = cw.credentials\n\ts, err := session.NewSession(cfg)\n\tif err != nil {\n\t\tcw.logger.Errorf(\"Error in creating session for cloudwatch. %+v\", err)\n\t\treturn err\n\t}\n\tcw.svc = cwatch.New(s)\n\tcw.shardMetrics = new(sync.Map)\n\n\tstopChan := make(chan struct{})\n\tcw.stop = &stopChan\n\twg := sync.WaitGroup{}\n\tcw.waitGroup = &wg\n\n\treturn nil\n}\n\nfunc (cw *MonitoringService) Start() error {\n\tcw.waitGroup.Add(1)\n\t// entering eventloop for sending metrics to CloudWatch\n\tgo cw.eventloop()\n\treturn nil\n}\n\nfunc (cw *MonitoringService) Shutdown() {\n\tcw.logger.Infof(\"Shutting down cloudwatch metrics system...\")\n\tclose(*cw.stop)\n\tcw.waitGroup.Wait()\n\tcw.logger.Infof(\"Cloudwatch metrics system has been shutdown.\")\n}\n\n// Start daemon to flush metrics periodically\nfunc (cw *MonitoringService) eventloop() {\n\tdefer cw.waitGroup.Done()\n\n\tfor {\n\t\tif err := cw.flush(); err != nil {\n\t\t\tcw.logger.Errorf(\"Error sending metrics to CloudWatch. %+v\", err)\n\t\t}\n\n\t\tselect {\n\t\tcase <-*cw.stop:\n\t\t\tcw.logger.Infof(\"Shutting down monitoring system\")\n\t\t\tif err := cw.flush(); err != nil {\n\t\t\t\tcw.logger.Errorf(\"Error sending metrics to CloudWatch. %+v\", err)\n\t\t\t}\n\t\t\treturn\n\t\tcase <-time.After(cw.bufferDuration):\n\t\t}\n\t}\n}\n\nfunc (cw *MonitoringService) flushShard(shard string, metric *cloudWatchMetrics) bool {\n\tmetric.Lock()\n\tdefaultDimensions := []*cwatch.Dimension{\n\t\t{\n\t\t\tName:  aws.String(\"Shard\"),\n\t\t\tValue: &shard,\n\t\t},\n\t\t{\n\t\t\tName:  aws.String(\"KinesisStreamName\"),\n\t\t\tValue: &cw.streamName,\n\t\t},\n\t}\n\n\tleaseDimensions := []*cwatch.Dimension{\n\t\t{\n\t\t\tName:  aws.String(\"Shard\"),\n\t\t\tValue: &shard,\n\t\t},\n\t\t{\n\t\t\tName:  aws.String(\"KinesisStreamName\"),\n\t\t\tValue: &cw.streamName,\n\t\t},\n\t\t{\n\t\t\tName:  aws.String(\"WorkerID\"),\n\t\t\tValue: &cw.workerID,\n\t\t},\n\t}\n\tmetricTimestamp := time.Now()\n\n\tdata := []*cwatch.MetricDatum{\n\t\t{\n\t\t\tDimensions: defaultDimensions,\n\t\t\tMetricName: aws.String(\"RecordsProcessed\"),\n\t\t\tUnit:       aws.String(\"Count\"),\n\t\t\tTimestamp:  &metricTimestamp,\n\t\t\tValue:      aws.Float64(float64(metric.processedRecords)),\n\t\t},\n\t\t{\n\t\t\tDimensions: defaultDimensions,\n\t\t\tMetricName: aws.String(\"DataBytesProcessed\"),\n\t\t\tUnit:       aws.String(\"Bytes\"),\n\t\t\tTimestamp:  &metricTimestamp,\n\t\t\tValue:      aws.Float64(float64(metric.processedBytes)),\n\t\t},\n\t\t{\n\t\t\tDimensions: leaseDimensions,\n\t\t\tMetricName: aws.String(\"RenewLease.Success\"),\n\t\t\tUnit:       aws.String(\"Count\"),\n\t\t\tTimestamp:  &metricTimestamp,\n\t\t\tValue:      aws.Float64(float64(metric.leaseRenewals)),\n\t\t},\n\t\t{\n\t\t\tDimensions: leaseDimensions,\n\t\t\tMetricName: aws.String(\"CurrentLeases\"),\n\t\t\tUnit:       aws.String(\"Count\"),\n\t\t\tTimestamp:  &metricTimestamp,\n\t\t\tValue:      aws.Float64(float64(metric.leasesHeld)),\n\t\t},\n\t}\n\n\tif len(metric.behindLatestMillis) > 0 {\n\t\tdata = append(data, &cwatch.MetricDatum{\n\t\t\tDimensions: defaultDimensions,\n\t\t\tMetricName: aws.String(\"MillisBehindLatest\"),\n\t\t\tUnit:       aws.String(\"Milliseconds\"),\n\t\t\tTimestamp:  &metricTimestamp,\n\t\t\tStatisticValues: &cwatch.StatisticSet{\n\t\t\t\tSampleCount: aws.Float64(float64(len(metric.behindLatestMillis))),\n\t\t\t\tSum:         sumFloat64(metric.behindLatestMillis),\n\t\t\t\tMaximum:     maxFloat64(metric.behindLatestMillis),\n\t\t\t\tMinimum:     minFloat64(metric.behindLatestMillis),\n\t\t\t}})\n\t}\n\n\tif len(metric.getRecordsTime) > 0 {\n\t\tdata = append(data, &cwatch.MetricDatum{\n\t\t\tDimensions: defaultDimensions,\n\t\t\tMetricName: aws.String(\"KinesisDataFetcher.getRecords.Time\"),\n\t\t\tUnit:       aws.String(\"Milliseconds\"),\n\t\t\tTimestamp:  &metricTimestamp,\n\t\t\tStatisticValues: &cwatch.StatisticSet{\n\t\t\t\tSampleCount: aws.Float64(float64(len(metric.getRecordsTime))),\n\t\t\t\tSum:         sumFloat64(metric.getRecordsTime),\n\t\t\t\tMaximum:     maxFloat64(metric.getRecordsTime),\n\t\t\t\tMinimum:     minFloat64(metric.getRecordsTime),\n\t\t\t}})\n\t}\n\n\tif len(metric.processRecordsTime) > 0 {\n\t\tdata = append(data, &cwatch.MetricDatum{\n\t\t\tDimensions: defaultDimensions,\n\t\t\tMetricName: aws.String(\"RecordProcessor.processRecords.Time\"),\n\t\t\tUnit:       aws.String(\"Milliseconds\"),\n\t\t\tTimestamp:  &metricTimestamp,\n\t\t\tStatisticValues: &cwatch.StatisticSet{\n\t\t\t\tSampleCount: aws.Float64(float64(len(metric.processRecordsTime))),\n\t\t\t\tSum:         sumFloat64(metric.processRecordsTime),\n\t\t\t\tMaximum:     maxFloat64(metric.processRecordsTime),\n\t\t\t\tMinimum:     minFloat64(metric.processRecordsTime),\n\t\t\t}})\n\t}\n\n\t// Publish metrics data to cloud watch\n\t_, err := cw.svc.PutMetricData(&cwatch.PutMetricDataInput{\n\t\tNamespace:  aws.String(cw.appName),\n\t\tMetricData: data,\n\t})\n\n\tif err == nil {\n\t\tmetric.processedRecords = 0\n\t\tmetric.processedBytes = 0\n\t\tmetric.behindLatestMillis = []float64{}\n\t\tmetric.leaseRenewals = 0\n\t\tmetric.getRecordsTime = []float64{}\n\t\tmetric.processRecordsTime = []float64{}\n\t} else {\n\t\tcw.logger.Errorf(\"Error in publishing cloudwatch metrics. Error: %+v\", err)\n\t}\n\n\tmetric.Unlock()\n\treturn true\n}\n\nfunc (cw *MonitoringService) flush() error {\n\tcw.logger.Debugf(\"Flushing metrics data. Stream: %s, Worker: %s\", cw.streamName, cw.workerID)\n\t// publish per shard metrics\n\tcw.shardMetrics.Range(func(k, v interface{}) bool {\n\t\tshard, metric := k.(string), v.(*cloudWatchMetrics)\n\t\treturn cw.flushShard(shard, metric)\n\t})\n\n\treturn nil\n}\n\nfunc (cw *MonitoringService) IncrRecordsProcessed(shard string, count int) {\n\tm := cw.getOrCreatePerShardMetrics(shard)\n\tm.Lock()\n\tdefer m.Unlock()\n\tm.processedRecords += int64(count)\n}\n\nfunc (cw *MonitoringService) IncrBytesProcessed(shard string, count int64) {\n\tm := cw.getOrCreatePerShardMetrics(shard)\n\tm.Lock()\n\tdefer m.Unlock()\n\tm.processedBytes += count\n}\n\nfunc (cw *MonitoringService) MillisBehindLatest(shard string, millSeconds float64) {\n\tm := cw.getOrCreatePerShardMetrics(shard)\n\tm.Lock()\n\tdefer m.Unlock()\n\tm.behindLatestMillis = append(m.behindLatestMillis, millSeconds)\n}\n\nfunc (cw *MonitoringService) LeaseGained(shard string) {\n\tm := cw.getOrCreatePerShardMetrics(shard)\n\tm.Lock()\n\tdefer m.Unlock()\n\tm.leasesHeld++\n}\n\nfunc (cw *MonitoringService) LeaseLost(shard string) {\n\tm := cw.getOrCreatePerShardMetrics(shard)\n\tm.Lock()\n\tdefer m.Unlock()\n\tm.leasesHeld--\n}\n\nfunc (cw *MonitoringService) LeaseRenewed(shard string) {\n\tm := cw.getOrCreatePerShardMetrics(shard)\n\tm.Lock()\n\tdefer m.Unlock()\n\tm.leaseRenewals++\n}\n\nfunc (cw *MonitoringService) RecordGetRecordsTime(shard string, time float64) {\n\tm := cw.getOrCreatePerShardMetrics(shard)\n\tm.Lock()\n\tdefer m.Unlock()\n\tm.getRecordsTime = append(m.getRecordsTime, time)\n}\nfunc (cw *MonitoringService) RecordProcessRecordsTime(shard string, time float64) {\n\tm := cw.getOrCreatePerShardMetrics(shard)\n\tm.Lock()\n\tdefer m.Unlock()\n\tm.processRecordsTime = append(m.processRecordsTime, time)\n}\n\nfunc (cw *MonitoringService) getOrCreatePerShardMetrics(shard string) *cloudWatchMetrics {\n\tvar i interface{}\n\tvar ok bool\n\tif i, ok = cw.shardMetrics.Load(shard); !ok {\n\t\tm := &cloudWatchMetrics{}\n\t\tcw.shardMetrics.Store(shard, m)\n\t\treturn m\n\t}\n\n\treturn i.(*cloudWatchMetrics)\n}\n\nfunc sumFloat64(slice []float64) *float64 {\n\tsum := float64(0)\n\tfor _, num := range slice {\n\t\tsum += num\n\t}\n\treturn &sum\n}\n\nfunc maxFloat64(slice []float64) *float64 {\n\tif len(slice) < 1 {\n\t\treturn aws.Float64(0)\n\t}\n\tmax := slice[0]\n\tfor _, num := range slice {\n\t\tif num > max {\n\t\t\tmax = num\n\t\t}\n\t}\n\treturn &max\n}\n\nfunc minFloat64(slice []float64) *float64 {\n\tif len(slice) < 1 {\n\t\treturn aws.Float64(0)\n\t}\n\tmin := slice[0]\n\tfor _, num := range slice {\n\t\tif num < min {\n\t\t\tmin = num\n\t\t}\n\t}\n\treturn &min\n}\n"
  },
  {
    "path": "clientlibrary/metrics/interfaces.go",
    "content": "/*\n * Copyright (c) 2018 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\n// The implementation is derived from https://github.com/patrobinson/gokini\n//\n// Copyright 2018 Patrick robinson\n//\n// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n//\n// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n//\n// THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\npackage metrics\n\ntype MonitoringService interface {\n\tInit(appName, streamName, workerID string) error\n\tStart() error\n\tIncrRecordsProcessed(string, int)\n\tIncrBytesProcessed(string, int64)\n\tMillisBehindLatest(string, float64)\n\tLeaseGained(string)\n\tLeaseLost(string)\n\tLeaseRenewed(string)\n\tRecordGetRecordsTime(string, float64)\n\tRecordProcessRecordsTime(string, float64)\n\tShutdown()\n}\n\n// NoopMonitoringService implements MonitoringService by does nothing.\ntype NoopMonitoringService struct{}\n\nfunc (NoopMonitoringService) Init(appName, streamName, workerID string) error { return nil }\nfunc (NoopMonitoringService) Start() error                                    { return nil }\nfunc (NoopMonitoringService) Shutdown()                                       {}\n\nfunc (NoopMonitoringService) IncrRecordsProcessed(shard string, count int)         {}\nfunc (NoopMonitoringService) IncrBytesProcessed(shard string, count int64)         {}\nfunc (NoopMonitoringService) MillisBehindLatest(shard string, millSeconds float64) {}\nfunc (NoopMonitoringService) LeaseGained(shard string)                             {}\nfunc (NoopMonitoringService) LeaseLost(shard string)                               {}\nfunc (NoopMonitoringService) LeaseRenewed(shard string)                            {}\nfunc (NoopMonitoringService) RecordGetRecordsTime(shard string, time float64)      {}\nfunc (NoopMonitoringService) RecordProcessRecordsTime(shard string, time float64)  {}\n"
  },
  {
    "path": "clientlibrary/metrics/prometheus/prometheus.go",
    "content": "/*\n * Copyright (c) 2018 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\n// The implementation is derived from https://github.com/patrobinson/gokini\n//\n// Copyright 2018 Patrick robinson\n//\n// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n//\n// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n//\n// THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\npackage prometheus\n\nimport (\n\t\"net/http\"\n\n\tprom \"github.com/prometheus/client_golang/prometheus\"\n\t\"github.com/prometheus/client_golang/prometheus/promhttp\"\n\n\t\"github.com/vmware/vmware-go-kcl/logger\"\n)\n\n// MonitoringService publishes kcl metrics to Prometheus.\n// It might be trick if the service onboarding with KCL already uses Prometheus.\ntype MonitoringService struct {\n\tlistenAddress string\n\tnamespace     string\n\tstreamName    string\n\tworkerID      string\n\tregion        string\n\tlogger        logger.Logger\n\n\tprocessedRecords   *prom.CounterVec\n\tprocessedBytes     *prom.CounterVec\n\tbehindLatestMillis *prom.GaugeVec\n\tleasesHeld         *prom.GaugeVec\n\tleaseRenewals      *prom.CounterVec\n\tgetRecordsTime     *prom.HistogramVec\n\tprocessRecordsTime *prom.HistogramVec\n}\n\n// NewMonitoringService returns a Monitoring service publishing metrics to Prometheus.\nfunc NewMonitoringService(listenAddress, region string, logger logger.Logger) *MonitoringService {\n\treturn &MonitoringService{\n\t\tlistenAddress: listenAddress,\n\t\tregion:        region,\n\t\tlogger:        logger,\n\t}\n}\n\nfunc (p *MonitoringService) Init(appName, streamName, workerID string) error {\n\tp.namespace = appName\n\tp.streamName = streamName\n\tp.workerID = workerID\n\n\tp.processedBytes = prom.NewCounterVec(prom.CounterOpts{\n\t\tName: p.namespace + `_processed_bytes`,\n\t\tHelp: \"Number of bytes processed\",\n\t}, []string{\"kinesisStream\", \"shard\"})\n\tp.processedRecords = prom.NewCounterVec(prom.CounterOpts{\n\t\tName: p.namespace + `_processed_records`,\n\t\tHelp: \"Number of records processed\",\n\t}, []string{\"kinesisStream\", \"shard\"})\n\tp.behindLatestMillis = prom.NewGaugeVec(prom.GaugeOpts{\n\t\tName: p.namespace + `_behind_latest_millis`,\n\t\tHelp: \"The amount of milliseconds processing is behind\",\n\t}, []string{\"kinesisStream\", \"shard\"})\n\tp.leasesHeld = prom.NewGaugeVec(prom.GaugeOpts{\n\t\tName: p.namespace + `_leases_held`,\n\t\tHelp: \"The number of leases held by the worker\",\n\t}, []string{\"kinesisStream\", \"shard\", \"workerID\"})\n\tp.leaseRenewals = prom.NewCounterVec(prom.CounterOpts{\n\t\tName: p.namespace + `_lease_renewals`,\n\t\tHelp: \"The number of successful lease renewals\",\n\t}, []string{\"kinesisStream\", \"shard\", \"workerID\"})\n\tp.getRecordsTime = prom.NewHistogramVec(prom.HistogramOpts{\n\t\tName: p.namespace + `_get_records_duration_milliseconds`,\n\t\tHelp: \"The time taken to fetch records and process them\",\n\t}, []string{\"kinesisStream\", \"shard\"})\n\tp.processRecordsTime = prom.NewHistogramVec(prom.HistogramOpts{\n\t\tName: p.namespace + `_process_records_duration_milliseconds`,\n\t\tHelp: \"The time taken to process records\",\n\t}, []string{\"kinesisStream\", \"shard\"})\n\n\tmetrics := []prom.Collector{\n\t\tp.processedBytes,\n\t\tp.processedRecords,\n\t\tp.behindLatestMillis,\n\t\tp.leasesHeld,\n\t\tp.leaseRenewals,\n\t\tp.getRecordsTime,\n\t\tp.processRecordsTime,\n\t}\n\tfor _, metric := range metrics {\n\t\terr := prom.Register(metric)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc (p *MonitoringService) Start() error {\n\thttp.Handle(\"/metrics\", promhttp.Handler())\n\tgo func() {\n\t\tp.logger.Infof(\"Starting Prometheus listener on %s\", p.listenAddress)\n\t\terr := http.ListenAndServe(p.listenAddress, nil)\n\t\tif err != nil {\n\t\t\tp.logger.Errorf(\"Error starting Prometheus metrics endpoint. %+v\", err)\n\t\t}\n\t\tp.logger.Infof(\"Stopped metrics server\")\n\t}()\n\n\treturn nil\n}\n\nfunc (p *MonitoringService) Shutdown() {}\n\nfunc (p *MonitoringService) IncrRecordsProcessed(shard string, count int) {\n\tp.processedRecords.With(prom.Labels{\"shard\": shard, \"kinesisStream\": p.streamName}).Add(float64(count))\n}\n\nfunc (p *MonitoringService) IncrBytesProcessed(shard string, count int64) {\n\tp.processedBytes.With(prom.Labels{\"shard\": shard, \"kinesisStream\": p.streamName}).Add(float64(count))\n}\n\nfunc (p *MonitoringService) MillisBehindLatest(shard string, millSeconds float64) {\n\tp.behindLatestMillis.With(prom.Labels{\"shard\": shard, \"kinesisStream\": p.streamName}).Set(millSeconds)\n}\n\nfunc (p *MonitoringService) LeaseGained(shard string) {\n\tp.leasesHeld.With(prom.Labels{\"shard\": shard, \"kinesisStream\": p.streamName, \"workerID\": p.workerID}).Inc()\n}\n\nfunc (p *MonitoringService) LeaseLost(shard string) {\n\tp.leasesHeld.With(prom.Labels{\"shard\": shard, \"kinesisStream\": p.streamName, \"workerID\": p.workerID}).Dec()\n}\n\nfunc (p *MonitoringService) LeaseRenewed(shard string) {\n\tp.leaseRenewals.With(prom.Labels{\"shard\": shard, \"kinesisStream\": p.streamName, \"workerID\": p.workerID}).Inc()\n}\n\nfunc (p *MonitoringService) RecordGetRecordsTime(shard string, time float64) {\n\tp.getRecordsTime.With(prom.Labels{\"shard\": shard, \"kinesisStream\": p.streamName}).Observe(time)\n}\n\nfunc (p *MonitoringService) RecordProcessRecordsTime(shard string, time float64) {\n\tp.processRecordsTime.With(prom.Labels{\"shard\": shard, \"kinesisStream\": p.streamName}).Observe(time)\n}\n"
  },
  {
    "path": "clientlibrary/partition/partition.go",
    "content": "/*\n * Copyright (c) 2018 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\n// The implementation is derived from https://github.com/patrobinson/gokini\n//\n// Copyright 2018 Patrick robinson\n//\n// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n//\n// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n//\n// THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\npackage worker\n\nimport (\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/vmware/vmware-go-kcl/clientlibrary/config\"\n)\n\ntype ShardStatus struct {\n\tID            string\n\tParentShardId string\n\tCheckpoint    string\n\tAssignedTo    string\n\tMux           *sync.RWMutex\n\tLeaseTimeout  time.Time\n\t// Shard Range\n\tStartingSequenceNumber string\n\t// child shard doesn't have end sequence number\n\tEndingSequenceNumber string\n\tClaimRequest         string\n}\n\nfunc (ss *ShardStatus) GetLeaseOwner() string {\n\tss.Mux.RLock()\n\tdefer ss.Mux.RUnlock()\n\treturn ss.AssignedTo\n}\n\nfunc (ss *ShardStatus) SetLeaseOwner(owner string) {\n\tss.Mux.Lock()\n\tdefer ss.Mux.Unlock()\n\tss.AssignedTo = owner\n}\n\nfunc (ss *ShardStatus) GetCheckpoint() string {\n\tss.Mux.RLock()\n\tdefer ss.Mux.RUnlock()\n\treturn ss.Checkpoint\n}\n\nfunc (ss *ShardStatus) SetCheckpoint(c string) {\n\tss.Mux.Lock()\n\tdefer ss.Mux.Unlock()\n\tss.Checkpoint = c\n}\n\nfunc (ss *ShardStatus) GetLeaseTimeout() time.Time {\n\tss.Mux.Lock()\n\tdefer ss.Mux.Unlock()\n\treturn ss.LeaseTimeout\n}\n\nfunc (ss *ShardStatus) SetLeaseTimeout(timeout time.Time) {\n\tss.Mux.Lock()\n\tdefer ss.Mux.Unlock()\n\tss.LeaseTimeout = timeout\n}\n\nfunc (ss *ShardStatus) IsClaimRequestExpired(kclConfig *config.KinesisClientLibConfiguration) bool {\n\tif leaseTimeout := ss.GetLeaseTimeout(); leaseTimeout.IsZero() {\n\t\treturn false\n\t} else {\n\t\treturn leaseTimeout.\n\t\t\tBefore(time.Now().UTC().Add(time.Duration(-kclConfig.LeaseStealingClaimTimeoutMillis) * time.Millisecond))\n\t}\n}\n"
  },
  {
    "path": "clientlibrary/utils/awserr.go",
    "content": "/*\n * Copyright (c) 2021 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\npackage utils\n\nimport (\n\t\"github.com/aws/aws-sdk-go/aws/awserr\"\n)\n\nfunc AWSErrCode(err error) string {\n\tawsErr, _ := err.(awserr.Error)\n\tif awsErr != nil {\n\t\treturn awsErr.Code()\n\t}\n\treturn \"\"\n}\n"
  },
  {
    "path": "clientlibrary/utils/random.go",
    "content": "/*\n * Copyright (c) 2018 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\n\n// Package utils\npackage utils\n\nimport (\n\t\"crypto/rand\"\n\t\"math/big\"\n\t\"time\"\n)\n\nconst letterBytes = \"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\"\nconst (\n\tletterIdxBits = 6                    // 6 bits to represent a letter index\n\tletterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits\n\tletterIdxMax  = 63 / letterIdxBits   // # of letter indices fitting in 63 bits\n)\n\nfunc RandStringBytesMaskImpr(n int) string {\n\tb := make([]byte, n)\n\tseed := time.Now().UTC().UnixNano()\n\trnd, _ := rand.Int(rand.Reader, big.NewInt(seed))\n\t// A rand.Int64() generates 64 random bits, enough for letterIdxMax letters!\n\tfor i, cache, remain := n-1, rnd.Int64(), letterIdxMax; i >= 0; {\n\t\tif remain == 0 {\n\t\t\trnd, _ = rand.Int(rand.Reader, big.NewInt(seed))\n\t\t\tcache, remain = rnd.Int64(), letterIdxMax\n\t\t}\n\t\tif idx := int(cache & letterIdxMask); idx < len(letterBytes) {\n\t\t\tb[i] = letterBytes[idx]\n\t\t\ti--\n\t\t}\n\t\tcache >>= letterIdxBits\n\t\tremain--\n\t}\n\n\treturn string(b)\n}\n"
  },
  {
    "path": "clientlibrary/utils/random_test.go",
    "content": "/*\n * Copyright (c) 2018 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\npackage utils\n\nimport (\n\t\"fmt\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestRandom(t *testing.T) {\n\tfor i := 0; i < 10; i++ {\n\t\ts1 := RandStringBytesMaskImpr(10)\n\t\ts2 := RandStringBytesMaskImpr(10)\n\t\tif s1 == s2 {\n\t\t\tt.Fatalf(\"failed in generating random string. s1: %s, s2: %s\", s1, s2)\n\t\t}\n\t\tfmt.Println(s1)\n\t\tfmt.Println(s2)\n\t}\n}\n\nfunc TestRandomNum(t *testing.T) {\n\tfor i := 0; i < 10; i++ {\n\t\tseed := time.Now().UTC().Second()\n\t\ts1 := RandStringBytesMaskImpr(seed)\n\t\ts2 := RandStringBytesMaskImpr(seed)\n\t\tif s1 == s2 {\n\t\t\tt.Fatalf(\"failed in generating random string. s1: %s, s2: %s\", s1, s2)\n\t\t}\n\t\tfmt.Println(s1)\n\t\tfmt.Println(s2)\n\t}\n}\n"
  },
  {
    "path": "clientlibrary/utils/uuid.go",
    "content": "/*\n * Copyright (c) 2018 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\npackage utils\n\nimport (\n\tguuid \"github.com/google/uuid\"\n)\n\n// MustNewUUID generates a new UUID and panics if failed\nfunc MustNewUUID() string {\n\tid, err := guuid.NewUUID()\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\treturn id.String()\n}\n"
  },
  {
    "path": "clientlibrary/worker/common-shard-consumer.go",
    "content": "/*\n * Copyright (c) 2021 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\npackage worker\n\nimport (\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/service/kinesis\"\n\t\"github.com/aws/aws-sdk-go/service/kinesis/kinesisiface\"\n\tdeagg \"github.com/awslabs/kinesis-aggregation/go/deaggregator\"\n\n\tchk \"github.com/vmware/vmware-go-kcl/clientlibrary/checkpoint\"\n\t\"github.com/vmware/vmware-go-kcl/clientlibrary/config\"\n\tkcl \"github.com/vmware/vmware-go-kcl/clientlibrary/interfaces\"\n\t\"github.com/vmware/vmware-go-kcl/clientlibrary/metrics\"\n\tpar \"github.com/vmware/vmware-go-kcl/clientlibrary/partition\"\n)\n\ntype shardConsumer interface {\n\tgetRecords() error\n}\n\n// commonShardConsumer implements common functionality for regular and enhanced fan-out consumers\ntype commonShardConsumer struct {\n\tshard           *par.ShardStatus\n\tkc              kinesisiface.KinesisAPI\n\tcheckpointer    chk.Checkpointer\n\trecordProcessor kcl.IRecordProcessor\n\tkclConfig       *config.KinesisClientLibConfiguration\n\tmService        metrics.MonitoringService\n}\n\n// Cleanup the internal lease cache\nfunc (sc *commonShardConsumer) releaseLease() {\n\tlog := sc.kclConfig.Logger\n\tlog.Infof(\"Release lease for shard %s\", sc.shard.ID)\n\tsc.shard.SetLeaseOwner(\"\")\n\n\t// Release the lease by wiping out the lease owner for the shard\n\t// Note: we don't need to do anything in case of error here and shard lease will eventually be expired.\n\tif err := sc.checkpointer.RemoveLeaseOwner(sc.shard.ID); err != nil {\n\t\tlog.Errorf(\"Failed to release shard lease or shard: %s Error: %+v\", sc.shard.ID, err)\n\t}\n\n\t// reporting lease lose metrics\n\tsc.mService.LeaseLost(sc.shard.ID)\n}\n\n// getStartingPosition gets kinesis stating position.\n// First try to fetch checkpoint. If checkpoint is not found use InitialPositionInStream\nfunc (sc *commonShardConsumer) getStartingPosition() (*kinesis.StartingPosition, error) {\n\terr := sc.checkpointer.FetchCheckpoint(sc.shard)\n\tif err != nil && err != chk.ErrSequenceIDNotFound {\n\t\treturn nil, err\n\t}\n\n\tcheckpoint := sc.shard.GetCheckpoint()\n\tif checkpoint != \"\" {\n\t\tsc.kclConfig.Logger.Debugf(\"Start shard: %v at checkpoint: %v\", sc.shard.ID, checkpoint)\n\t\treturn &kinesis.StartingPosition{\n\t\t\tType:           aws.String(\"AFTER_SEQUENCE_NUMBER\"),\n\t\t\tSequenceNumber: &checkpoint,\n\t\t}, nil\n\t}\n\n\tshardIteratorType := config.InitalPositionInStreamToShardIteratorType(sc.kclConfig.InitialPositionInStream)\n\tsc.kclConfig.Logger.Debugf(\"No checkpoint recorded for shard: %v, starting with: %v\", sc.shard.ID, aws.StringValue(shardIteratorType))\n\n\tif sc.kclConfig.InitialPositionInStream == config.AT_TIMESTAMP {\n\t\treturn &kinesis.StartingPosition{\n\t\t\tType:      shardIteratorType,\n\t\t\tTimestamp: sc.kclConfig.InitialPositionInStreamExtended.Timestamp,\n\t\t}, nil\n\t}\n\n\treturn &kinesis.StartingPosition{\n\t\tType: shardIteratorType,\n\t}, nil\n}\n\n// Need to wait until the parent shard finished\nfunc (sc *commonShardConsumer) waitOnParentShard() error {\n\tif len(sc.shard.ParentShardId) == 0 {\n\t\treturn nil\n\t}\n\n\tpshard := &par.ShardStatus{\n\t\tID:  sc.shard.ParentShardId,\n\t\tMux: &sync.RWMutex{},\n\t}\n\n\tfor {\n\t\tif err := sc.checkpointer.FetchCheckpoint(pshard); err != nil {\n\t\t\treturn err\n\t\t}\n\n\t\t// Parent shard is finished.\n\t\tif pshard.GetCheckpoint() == chk.ShardEnd {\n\t\t\treturn nil\n\t\t}\n\n\t\ttime.Sleep(time.Duration(sc.kclConfig.ParentShardPollIntervalMillis) * time.Millisecond)\n\t}\n}\n\nfunc (sc *commonShardConsumer) processRecords(getRecordsStartTime time.Time, records []*kinesis.Record, millisBehindLatest *int64, recordCheckpointer kcl.IRecordProcessorCheckpointer) {\n\tlog := sc.kclConfig.Logger\n\n\tgetRecordsTime := time.Since(getRecordsStartTime).Milliseconds()\n\tsc.mService.RecordGetRecordsTime(sc.shard.ID, float64(getRecordsTime))\n\n\tlog.Debugf(\"Received %d original records.\", len(records))\n\n\t// De-aggregate the records if they were published by the KPL.\n\tdars, err := deagg.DeaggregateRecords(records)\n\tif err != nil {\n\t\t// The error is caused by bad KPL publisher and just skip the bad records\n\t\t// instead of being stuck here.\n\t\tlog.Errorf(\"Error in de-aggregating KPL records: %+v\", err)\n\t}\n\n\tinput := &kcl.ProcessRecordsInput{\n\t\tRecords:            dars,\n\t\tMillisBehindLatest: aws.Int64Value(millisBehindLatest),\n\t\tCheckpointer:       recordCheckpointer,\n\t}\n\n\trecordLength := len(input.Records)\n\trecordBytes := int64(0)\n\tlog.Debugf(\"Received %d de-aggregated records, MillisBehindLatest: %v\", recordLength, input.MillisBehindLatest)\n\n\tfor _, r := range input.Records {\n\t\trecordBytes += int64(len(r.Data))\n\t}\n\n\tif recordLength > 0 || sc.kclConfig.CallProcessRecordsEvenForEmptyRecordList {\n\t\tprocessRecordsStartTime := time.Now()\n\n\t\t// Delivery the events to the record processor\n\t\tinput.CacheEntryTime = &getRecordsStartTime\n\t\tinput.CacheExitTime = &processRecordsStartTime\n\t\tsc.recordProcessor.ProcessRecords(input)\n\n\t\tprocessedRecordsTiming := time.Since(processRecordsStartTime).Milliseconds()\n\t\tsc.mService.RecordProcessRecordsTime(sc.shard.ID, float64(processedRecordsTiming))\n\t}\n\n\tsc.mService.IncrRecordsProcessed(sc.shard.ID, recordLength)\n\tsc.mService.IncrBytesProcessed(sc.shard.ID, recordBytes)\n\tsc.mService.MillisBehindLatest(sc.shard.ID, float64(*millisBehindLatest))\n}\n"
  },
  {
    "path": "clientlibrary/worker/fan-out-shard-consumer.go",
    "content": "/*\n * Copyright (c) 2021 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\npackage worker\n\nimport (\n\t\"errors\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/service/kinesis\"\n\n\tchk \"github.com/vmware/vmware-go-kcl/clientlibrary/checkpoint\"\n\tkcl \"github.com/vmware/vmware-go-kcl/clientlibrary/interfaces\"\n)\n\n// FanOutShardConsumer is  responsible for consuming data records of a (specified) shard.\n// Note: FanOutShardConsumer only deal with one shard.\n// For more info see: https://docs.aws.amazon.com/streams/latest/dev/enhanced-consumers.html\ntype FanOutShardConsumer struct {\n\tcommonShardConsumer\n\tconsumerARN string\n\tconsumerID  string\n\tstop        *chan struct{}\n}\n\n// getRecords subscribes to a shard and reads events from it.\n// Precondition: it currently has the lease on the shard.\nfunc (sc *FanOutShardConsumer) getRecords() error {\n\tdefer sc.releaseLease()\n\n\tlog := sc.kclConfig.Logger\n\n\t// If the shard is child shard, need to wait until the parent finished.\n\tif err := sc.waitOnParentShard(); err != nil {\n\t\t// If parent shard has been deleted by Kinesis system already, just ignore the error.\n\t\tif err != chk.ErrSequenceIDNotFound {\n\t\t\tlog.Errorf(\"Error in waiting for parent shard: %v to finish. Error: %+v\", sc.shard.ParentShardId, err)\n\t\t\treturn err\n\t\t}\n\t}\n\n\tshardSub, err := sc.subscribeToShard()\n\tif err != nil {\n\t\tlog.Errorf(\"Unable to subscribe to shard %s: %v\", sc.shard.ID, err)\n\t\treturn err\n\t}\n\tdefer func() {\n\t\tif shardSub == nil || shardSub.EventStream == nil {\n\t\t\tlog.Debugf(\"Nothing to close, EventStream is nil\")\n\t\t\treturn\n\t\t}\n\t\terr = shardSub.EventStream.Close()\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Unable to close event stream for %s: %v\", sc.shard.ID, err)\n\t\t}\n\t}()\n\n\tinput := &kcl.InitializationInput{\n\t\tShardId:                sc.shard.ID,\n\t\tExtendedSequenceNumber: &kcl.ExtendedSequenceNumber{SequenceNumber: aws.String(sc.shard.GetCheckpoint())},\n\t}\n\tsc.recordProcessor.Initialize(input)\n\trecordCheckpointer := NewRecordProcessorCheckpoint(sc.shard, sc.checkpointer)\n\n\tvar continuationSequenceNumber *string\n\trefreshLeaseTimer := time.After(time.Until(sc.shard.LeaseTimeout.Add(-time.Duration(sc.kclConfig.LeaseRefreshPeriodMillis) * time.Millisecond)))\n\tfor {\n\t\tgetRecordsStartTime := time.Now()\n\t\tselect {\n\t\tcase <-*sc.stop:\n\t\t\tshutdownInput := &kcl.ShutdownInput{ShutdownReason: kcl.REQUESTED, Checkpointer: recordCheckpointer}\n\t\t\tsc.recordProcessor.Shutdown(shutdownInput)\n\t\t\treturn nil\n\t\tcase <-refreshLeaseTimer:\n\t\t\tlog.Debugf(\"Refreshing lease on shard: %s for worker: %s\", sc.shard.ID, sc.consumerID)\n\t\t\terr = sc.checkpointer.GetLease(sc.shard, sc.consumerID)\n\t\t\tif err != nil {\n\t\t\t\tif errors.As(err, &chk.ErrLeaseNotAcquired{}) {\n\t\t\t\t\tlog.Warnf(\"Failed in acquiring lease on shard: %s for worker: %s\", sc.shard.ID, sc.consumerID)\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\tlog.Errorf(\"Error in refreshing lease on shard: %s for worker: %s. Error: %+v\", sc.shard.ID, sc.consumerID, err)\n\t\t\t\treturn err\n\t\t\t}\n\t\t\trefreshLeaseTimer = time.After(time.Until(sc.shard.LeaseTimeout.Add(-time.Duration(sc.kclConfig.LeaseRefreshPeriodMillis) * time.Millisecond)))\n\t\tcase event, ok := <-shardSub.EventStream.Events():\n\t\t\tif !ok {\n\t\t\t\t// need to resubscribe to shard\n\t\t\t\tlog.Debugf(\"Event stream ended, refreshing subscription on shard: %s for worker: %s\", sc.shard.ID, sc.consumerID)\n\t\t\t\tif continuationSequenceNumber == nil || *continuationSequenceNumber == \"\" {\n\t\t\t\t\tlog.Debugf(\"No continuation sequence number\")\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\tshardSub, err = sc.resubscribe(shardSub, continuationSequenceNumber)\n\t\t\t\tif err != nil {\n\t\t\t\t\treturn err\n\t\t\t\t}\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tsubEvent, ok := event.(*kinesis.SubscribeToShardEvent)\n\t\t\tif !ok {\n\t\t\t\tlog.Errorf(\"Received unexpected event type: %T\", event)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tcontinuationSequenceNumber = subEvent.ContinuationSequenceNumber\n\t\t\tsc.processRecords(getRecordsStartTime, subEvent.Records, subEvent.MillisBehindLatest, recordCheckpointer)\n\n\t\t\t// The shard has been closed, so no new records can be read from it\n\t\t\tif continuationSequenceNumber == nil {\n\t\t\t\tlog.Infof(\"Shard %s closed\", sc.shard.ID)\n\t\t\t\tshutdownInput := &kcl.ShutdownInput{ShutdownReason: kcl.TERMINATE, Checkpointer: recordCheckpointer}\n\t\t\t\tsc.recordProcessor.Shutdown(shutdownInput)\n\t\t\t\treturn nil\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (sc *FanOutShardConsumer) subscribeToShard() (*kinesis.SubscribeToShardOutput, error) {\n\tstartPosition, err := sc.getStartingPosition()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn sc.kc.SubscribeToShard(&kinesis.SubscribeToShardInput{\n\t\tConsumerARN:      &sc.consumerARN,\n\t\tShardId:          &sc.shard.ID,\n\t\tStartingPosition: startPosition,\n\t})\n}\n\nfunc (sc *FanOutShardConsumer) resubscribe(shardSub *kinesis.SubscribeToShardOutput, continuationSequence *string) (*kinesis.SubscribeToShardOutput, error) {\n\terr := shardSub.EventStream.Close()\n\tif err != nil {\n\t\tsc.kclConfig.Logger.Errorf(\"Unable to close event stream for %s: %v\", sc.shard.ID, err)\n\t\treturn nil, err\n\t}\n\tstartPosition := &kinesis.StartingPosition{\n\t\tType:           aws.String(\"AFTER_SEQUENCE_NUMBER\"),\n\t\tSequenceNumber: continuationSequence,\n\t}\n\tshardSub, err = sc.kc.SubscribeToShard(&kinesis.SubscribeToShardInput{\n\t\tConsumerARN:      &sc.consumerARN,\n\t\tShardId:          &sc.shard.ID,\n\t\tStartingPosition: startPosition,\n\t})\n\tif err != nil {\n\t\tsc.kclConfig.Logger.Errorf(\"Unable to resubscribe to shard %s: %v\", sc.shard.ID, err)\n\t\treturn nil, err\n\t}\n\treturn shardSub, nil\n}\n"
  },
  {
    "path": "clientlibrary/worker/polling-shard-consumer.go",
    "content": "/*\n * Copyright (c) 2018 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\n// The implementation is derived from https://github.com/patrobinson/gokini\n//\n// Copyright 2018 Patrick robinson\n//\n// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n//\n// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n//\n// THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\npackage worker\n\nimport (\n\t\"errors\"\n\t\"math\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/service/kinesis\"\n\n\tchk \"github.com/vmware/vmware-go-kcl/clientlibrary/checkpoint\"\n\tkcl \"github.com/vmware/vmware-go-kcl/clientlibrary/interfaces\"\n\t\"github.com/vmware/vmware-go-kcl/clientlibrary/metrics\"\n\t\"github.com/vmware/vmware-go-kcl/clientlibrary/utils\"\n)\n\n// PollingShardConsumer is responsible for polling data records from a (specified) shard.\n// Note: PollingShardConsumer only deal with one shard.\ntype PollingShardConsumer struct {\n\tcommonShardConsumer\n\tstreamName string\n\tstop       *chan struct{}\n\tconsumerID string\n\tmService   metrics.MonitoringService\n}\n\nfunc (sc *PollingShardConsumer) getShardIterator() (*string, error) {\n\tstartPosition, err := sc.getStartingPosition()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tshardIterArgs := &kinesis.GetShardIteratorInput{\n\t\tShardId:                &sc.shard.ID,\n\t\tShardIteratorType:      startPosition.Type,\n\t\tStartingSequenceNumber: startPosition.SequenceNumber,\n\t\tTimestamp:              startPosition.Timestamp,\n\t\tStreamName:             &sc.streamName,\n\t}\n\titerResp, err := sc.kc.GetShardIterator(shardIterArgs)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn iterResp.ShardIterator, nil\n}\n\n// getRecords continously poll one shard for data record\n// Precondition: it currently has the lease on the shard.\nfunc (sc *PollingShardConsumer) getRecords() error {\n\tdefer sc.releaseLease()\n\n\tlog := sc.kclConfig.Logger\n\n\t// If the shard is child shard, need to wait until the parent finished.\n\tif err := sc.waitOnParentShard(); err != nil {\n\t\t// If parent shard has been deleted by Kinesis system already, just ignore the error.\n\t\tif err != chk.ErrSequenceIDNotFound {\n\t\t\tlog.Errorf(\"Error in waiting for parent shard: %v to finish. Error: %+v\", sc.shard.ParentShardId, err)\n\t\t\treturn err\n\t\t}\n\t}\n\n\tshardIterator, err := sc.getShardIterator()\n\tif err != nil {\n\t\tlog.Errorf(\"Unable to get shard iterator for %s: %v\", sc.shard.ID, err)\n\t\treturn err\n\t}\n\n\t// Start processing events and notify record processor on shard and starting checkpoint\n\tinput := &kcl.InitializationInput{\n\t\tShardId:                sc.shard.ID,\n\t\tExtendedSequenceNumber: &kcl.ExtendedSequenceNumber{SequenceNumber: aws.String(sc.shard.GetCheckpoint())},\n\t}\n\tsc.recordProcessor.Initialize(input)\n\n\trecordCheckpointer := NewRecordProcessorCheckpoint(sc.shard, sc.checkpointer)\n\tretriedErrors := 0\n\n\tfor {\n\t\tif time.Now().UTC().After(sc.shard.GetLeaseTimeout().Add(-time.Duration(sc.kclConfig.LeaseRefreshPeriodMillis) * time.Millisecond)) {\n\t\t\tlog.Debugf(\"Refreshing lease on shard: %s for worker: %s\", sc.shard.ID, sc.consumerID)\n\t\t\terr = sc.checkpointer.GetLease(sc.shard, sc.consumerID)\n\t\t\tif err != nil {\n\t\t\t\tif errors.As(err, &chk.ErrLeaseNotAcquired{}) {\n\t\t\t\t\tlog.Warnf(\"Failed in acquiring lease on shard: %s for worker: %s\", sc.shard.ID, sc.consumerID)\n\t\t\t\t\treturn nil\n\t\t\t\t}\n\t\t\t\t// log and return error\n\t\t\t\tlog.Errorf(\"Error in refreshing lease on shard: %s for worker: %s. Error: %+v\",\n\t\t\t\t\tsc.shard.ID, sc.consumerID, err)\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\n\t\tgetRecordsStartTime := time.Now()\n\n\t\tlog.Debugf(\"Trying to read %d record from iterator: %v\", sc.kclConfig.MaxRecords, aws.StringValue(shardIterator))\n\t\tgetRecordsArgs := &kinesis.GetRecordsInput{\n\t\t\tLimit:         aws.Int64(int64(sc.kclConfig.MaxRecords)),\n\t\t\tShardIterator: shardIterator,\n\t\t}\n\t\t// Get records from stream and retry as needed\n\t\tgetResp, err := sc.kc.GetRecords(getRecordsArgs)\n\t\tif err != nil {\n\t\t\tif utils.AWSErrCode(err) == kinesis.ErrCodeProvisionedThroughputExceededException || utils.AWSErrCode(err) == kinesis.ErrCodeKMSThrottlingException {\n\t\t\t\tlog.Errorf(\"Error getting records from shard %v: %+v\", sc.shard.ID, err)\n\t\t\t\tretriedErrors++\n\t\t\t\t// exponential backoff\n\t\t\t\t// https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Programming.Errors.html#Programming.Errors.RetryAndBackoff\n\t\t\t\ttime.Sleep(time.Duration(math.Exp2(float64(retriedErrors))*100) * time.Millisecond)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tlog.Errorf(\"Error getting records from Kinesis that cannot be retried: %+v Request: %s\", err, getRecordsArgs)\n\t\t\treturn err\n\t\t}\n\t\t// reset the retry count after success\n\t\tretriedErrors = 0\n\n\t\tsc.processRecords(getRecordsStartTime, getResp.Records, getResp.MillisBehindLatest, recordCheckpointer)\n\n\t\t// The shard has been closed, so no new records can be read from it\n\t\tif getResp.NextShardIterator == nil {\n\t\t\tlog.Infof(\"Shard %s closed\", sc.shard.ID)\n\t\t\tshutdownInput := &kcl.ShutdownInput{ShutdownReason: kcl.TERMINATE, Checkpointer: recordCheckpointer}\n\t\t\tsc.recordProcessor.Shutdown(shutdownInput)\n\t\t\treturn nil\n\t\t}\n\t\tshardIterator = getResp.NextShardIterator\n\n\t\t// Idle between each read, the user is responsible for checkpoint the progress\n\t\t// This value is only used when no records are returned; if records are returned, it should immediately\n\t\t// retrieve the next set of records.\n\t\tif len(getResp.Records) == 0 && aws.Int64Value(getResp.MillisBehindLatest) < int64(sc.kclConfig.IdleTimeBetweenReadsInMillis) {\n\t\t\ttime.Sleep(time.Duration(sc.kclConfig.IdleTimeBetweenReadsInMillis) * time.Millisecond)\n\t\t}\n\n\t\tselect {\n\t\tcase <-*sc.stop:\n\t\t\tshutdownInput := &kcl.ShutdownInput{ShutdownReason: kcl.REQUESTED, Checkpointer: recordCheckpointer}\n\t\t\tsc.recordProcessor.Shutdown(shutdownInput)\n\t\t\treturn nil\n\t\tdefault:\n\t\t}\n\t}\n}\n"
  },
  {
    "path": "clientlibrary/worker/record-processor-checkpointer.go",
    "content": "/*\n * Copyright (c) 2018 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\npackage worker\n\nimport (\n\t\"github.com/aws/aws-sdk-go/aws\"\n\n\tchk \"github.com/vmware/vmware-go-kcl/clientlibrary/checkpoint\"\n\tkcl \"github.com/vmware/vmware-go-kcl/clientlibrary/interfaces\"\n\tpar \"github.com/vmware/vmware-go-kcl/clientlibrary/partition\"\n)\n\ntype (\n\n\t/* Objects of this class are prepared to checkpoint at a specific sequence number. They use an\n\t * IRecordProcessorCheckpointer to do the actual checkpointing, so their checkpoint is subject to the same 'didn't go\n\t * backwards' validation as a normal checkpoint.\n\t */\n\tPreparedCheckpointer struct {\n\t\tpendingCheckpointSequenceNumber *kcl.ExtendedSequenceNumber\n\t\tcheckpointer                    kcl.IRecordProcessorCheckpointer\n\t}\n\n\t/**\n\t * This class is used to enable RecordProcessors to checkpoint their progress.\n\t * The Amazon Kinesis Client Library will instantiate an object and provide a reference to the application\n\t * RecordProcessor instance. Amazon Kinesis Client Library will create one instance per shard assignment.\n\t */\n\tRecordProcessorCheckpointer struct {\n\t\tshard      *par.ShardStatus\n\t\tcheckpoint chk.Checkpointer\n\t}\n)\n\nfunc NewRecordProcessorCheckpoint(shard *par.ShardStatus, checkpoint chk.Checkpointer) kcl.IRecordProcessorCheckpointer {\n\treturn &RecordProcessorCheckpointer{\n\t\tshard:      shard,\n\t\tcheckpoint: checkpoint,\n\t}\n}\n\nfunc (pc *PreparedCheckpointer) GetPendingCheckpoint() *kcl.ExtendedSequenceNumber {\n\treturn pc.pendingCheckpointSequenceNumber\n}\n\nfunc (pc *PreparedCheckpointer) Checkpoint() error {\n\treturn pc.checkpointer.Checkpoint(pc.pendingCheckpointSequenceNumber.SequenceNumber)\n}\n\nfunc (rc *RecordProcessorCheckpointer) Checkpoint(sequenceNumber *string) error {\n\t// checkpoint the last sequence of a closed shard\n\tif sequenceNumber == nil {\n\t\trc.shard.SetCheckpoint(chk.ShardEnd)\n\t} else {\n\t\trc.shard.SetCheckpoint(aws.StringValue(sequenceNumber))\n\t}\n\n\treturn rc.checkpoint.CheckpointSequence(rc.shard)\n}\n\nfunc (rc *RecordProcessorCheckpointer) PrepareCheckpoint(sequenceNumber *string) (kcl.IPreparedCheckpointer, error) {\n\treturn &PreparedCheckpointer{}, nil\n\n}\n"
  },
  {
    "path": "clientlibrary/worker/worker-fan-out.go",
    "content": "/*\n * Copyright (c) 2021 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\npackage worker\n\nimport (\n\t\"fmt\"\n\t\"math\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/service/kinesis\"\n\n\t\"github.com/vmware/vmware-go-kcl/clientlibrary/utils\"\n)\n\n// fetchConsumerARNWithRetry tries to fetch consumer ARN. Retries 10 times with exponential backoff in case of an error\nfunc (w *Worker) fetchConsumerARNWithRetry() (string, error) {\n\tfor retry := 0; ; retry++ {\n\t\tconsumerARN, err := w.fetchConsumerARN()\n\t\tif err == nil {\n\t\t\treturn consumerARN, nil\n\t\t}\n\t\tif retry < 10 {\n\t\t\tsleepDuration := time.Duration(math.Exp2(float64(retry))*100) * time.Millisecond\n\t\t\tw.kclConfig.Logger.Errorf(\"Could not get consumer ARN: %v, retrying after: %s\", err, sleepDuration)\n\t\t\ttime.Sleep(sleepDuration)\n\t\t\tcontinue\n\t\t}\n\t\treturn consumerARN, err\n\t}\n}\n\n// fetchConsumerARN gets enhanced fan-out consumerARN.\n// Registers enhanced fan-out consumer if the consumer is not found\nfunc (w *Worker) fetchConsumerARN() (string, error) {\n\tlog := w.kclConfig.Logger\n\tlog.Debugf(\"Fetching stream consumer ARN\")\n\tstreamDescription, err := w.kc.DescribeStream(&kinesis.DescribeStreamInput{\n\t\tStreamName: &w.kclConfig.StreamName,\n\t})\n\tif err != nil {\n\t\tlog.Errorf(\"Could not describe stream: %v\", err)\n\t\treturn \"\", err\n\t}\n\tstreamConsumerDescription, err := w.kc.DescribeStreamConsumer(&kinesis.DescribeStreamConsumerInput{\n\t\tConsumerName: &w.kclConfig.EnhancedFanOutConsumerName,\n\t\tStreamARN:    streamDescription.StreamDescription.StreamARN,\n\t})\n\tif err == nil {\n\t\tlog.Infof(\"Enhanced fan-out consumer found, consumer status: %s\", *streamConsumerDescription.ConsumerDescription.ConsumerStatus)\n\t\tif *streamConsumerDescription.ConsumerDescription.ConsumerStatus != kinesis.ConsumerStatusActive {\n\t\t\treturn \"\", fmt.Errorf(\"consumer is not in active status yet, current status: %s\", *streamConsumerDescription.ConsumerDescription.ConsumerStatus)\n\t\t}\n\t\treturn *streamConsumerDescription.ConsumerDescription.ConsumerARN, nil\n\t}\n\tif utils.AWSErrCode(err) == kinesis.ErrCodeResourceNotFoundException {\n\t\tlog.Infof(\"Enhanced fan-out consumer not found, registering new consumer with name: %s\", w.kclConfig.EnhancedFanOutConsumerName)\n\t\tout, err := w.kc.RegisterStreamConsumer(&kinesis.RegisterStreamConsumerInput{\n\t\t\tConsumerName: &w.kclConfig.EnhancedFanOutConsumerName,\n\t\t\tStreamARN:    streamDescription.StreamDescription.StreamARN,\n\t\t})\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Could not register enhanced fan-out consumer: %v\", err)\n\t\t\treturn \"\", err\n\t\t}\n\t\tif *out.Consumer.ConsumerStatus != kinesis.ConsumerStatusActive {\n\t\t\treturn \"\", fmt.Errorf(\"consumer is not in active status yet, current status: %s\", *out.Consumer.ConsumerStatus)\n\t\t}\n\t\treturn *out.Consumer.ConsumerARN, nil\n\t}\n\tlog.Errorf(\"Could not describe stream consumer: %v\", err)\n\treturn \"\", err\n}\n"
  },
  {
    "path": "clientlibrary/worker/worker.go",
    "content": "/*\n * Copyright (c) 2018 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\n\n// Package worker\n// The implementation is derived from https://github.com/patrobinson/gokini\n//\n// Copyright 2018 Patrick robinson\n//\n// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the \"Software\"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n//\n// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n//\n// THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\npackage worker\n\nimport (\n\t\"crypto/rand\"\n\t\"errors\"\n\t\"math/big\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/aws/session\"\n\t\"github.com/aws/aws-sdk-go/service/kinesis\"\n\t\"github.com/aws/aws-sdk-go/service/kinesis/kinesisiface\"\n\n\tchk \"github.com/vmware/vmware-go-kcl/clientlibrary/checkpoint\"\n\t\"github.com/vmware/vmware-go-kcl/clientlibrary/config\"\n\tkcl \"github.com/vmware/vmware-go-kcl/clientlibrary/interfaces\"\n\t\"github.com/vmware/vmware-go-kcl/clientlibrary/metrics\"\n\tpar \"github.com/vmware/vmware-go-kcl/clientlibrary/partition\"\n)\n\n//Worker is the high level class that Kinesis applications use to start processing data. It initializes and oversees\n//different components (e.g. syncing shard and lease information, tracking shard assignments, and processing data from\n//the shards).\ntype Worker struct {\n\tstreamName  string\n\tregionName  string\n\tworkerID    string\n\tconsumerARN string\n\n\tprocessorFactory kcl.IRecordProcessorFactory\n\tkclConfig        *config.KinesisClientLibConfiguration\n\tkc               kinesisiface.KinesisAPI\n\tcheckpointer     chk.Checkpointer\n\tmService         metrics.MonitoringService\n\n\tstop      *chan struct{}\n\twaitGroup *sync.WaitGroup\n\tdone      bool\n\n\trandomSeed int64\n\n\tshardStatus          map[string]*par.ShardStatus\n\tshardStealInProgress bool\n}\n\n// NewWorker constructs a Worker instance for processing Kinesis stream data.\nfunc NewWorker(factory kcl.IRecordProcessorFactory, kclConfig *config.KinesisClientLibConfiguration) *Worker {\n\tmService := kclConfig.MonitoringService\n\tif mService == nil {\n\t\t// Replaces nil with noop monitor service (not emitting any metrics).\n\t\tmService = metrics.NoopMonitoringService{}\n\t}\n\n\treturn &Worker{\n\t\tstreamName:       kclConfig.StreamName,\n\t\tregionName:       kclConfig.RegionName,\n\t\tworkerID:         kclConfig.WorkerID,\n\t\tprocessorFactory: factory,\n\t\tkclConfig:        kclConfig,\n\t\tmService:         mService,\n\t\tdone:             false,\n\t\trandomSeed:       time.Now().UTC().UnixNano(),\n\t}\n}\n\n// WithKinesis is used to provide Kinesis service for either custom implementation or unit testing.\nfunc (w *Worker) WithKinesis(svc kinesisiface.KinesisAPI) *Worker {\n\tw.kc = svc\n\treturn w\n}\n\n// WithCheckpointer is used to provide a custom checkpointer service for non-dynamodb implementation\n// or unit testing.\nfunc (w *Worker) WithCheckpointer(checker chk.Checkpointer) *Worker {\n\tw.checkpointer = checker\n\treturn w\n}\n\n// Start Run starts consuming data from the stream, and pass it to the application record processors.\nfunc (w *Worker) Start() error {\n\tlog := w.kclConfig.Logger\n\tif err := w.initialize(); err != nil {\n\t\tlog.Errorf(\"Failed to initialize Worker: %+v\", err)\n\t\treturn err\n\t}\n\n\t// Start monitoring service\n\tlog.Infof(\"Starting monitoring service.\")\n\tif err := w.mService.Start(); err != nil {\n\t\tlog.Errorf(\"Failed to start monitoring service: %+v\", err)\n\t\treturn err\n\t}\n\n\tlog.Infof(\"Starting worker event loop.\")\n\tw.waitGroup.Add(1)\n\tgo func() {\n\t\tdefer w.waitGroup.Done()\n\t\t// entering event loop\n\t\tw.eventLoop()\n\t}()\n\treturn nil\n}\n\n// Shutdown signals worker to shut down. Worker will try initiating shutdown of all record processors.\nfunc (w *Worker) Shutdown() {\n\tlog := w.kclConfig.Logger\n\tlog.Infof(\"Worker shutdown in requested.\")\n\n\tif w.done || w.stop == nil {\n\t\treturn\n\t}\n\n\tclose(*w.stop)\n\tw.done = true\n\tw.waitGroup.Wait()\n\n\tw.mService.Shutdown()\n\tlog.Infof(\"Worker loop is complete. Exiting from worker.\")\n}\n\n// initialize\nfunc (w *Worker) initialize() error {\n\tlog := w.kclConfig.Logger\n\tlog.Infof(\"Worker initialization in progress...\")\n\n\t// Create default Kinesis session\n\tif w.kc == nil {\n\t\t// create session for Kinesis\n\t\tlog.Infof(\"Creating Kinesis session\")\n\n\t\ts, err := session.NewSession(&aws.Config{\n\t\t\tRegion:      aws.String(w.regionName),\n\t\t\tEndpoint:    &w.kclConfig.KinesisEndpoint,\n\t\t\tCredentials: w.kclConfig.KinesisCredentials,\n\t\t})\n\n\t\tif err != nil {\n\t\t\t// no need to move forward\n\t\t\tlog.Fatalf(\"Failed in getting Kinesis session for creating Worker: %+v\", err)\n\t\t}\n\t\tw.kc = kinesis.New(s)\n\t} else {\n\t\tlog.Infof(\"Use custom Kinesis service.\")\n\t}\n\n\t// Create default dynamodb based checkpointer implementation\n\tif w.checkpointer == nil {\n\t\tlog.Infof(\"Creating DynamoDB based checkpointer\")\n\t\tw.checkpointer = chk.NewDynamoCheckpoint(w.kclConfig)\n\t} else {\n\t\tlog.Infof(\"Use custom checkpointer implementation.\")\n\t}\n\n\tif w.kclConfig.EnableEnhancedFanOutConsumer {\n\t\tlog.Debugf(\"Enhanced fan-out is enabled\")\n\t\tw.consumerARN = w.kclConfig.EnhancedFanOutConsumerARN\n\t\tif w.consumerARN == \"\" {\n\t\t\tvar err error\n\t\t\tw.consumerARN, err = w.fetchConsumerARNWithRetry()\n\t\t\tif err != nil {\n\t\t\t\tlog.Errorf(\"Failed to fetch consumer ARN for: %s, %v\", w.kclConfig.EnhancedFanOutConsumerName, err)\n\t\t\t\treturn err\n\t\t\t}\n\t\t}\n\t}\n\n\terr := w.mService.Init(w.kclConfig.ApplicationName, w.streamName, w.workerID)\n\tif err != nil {\n\t\tlog.Errorf(\"Failed to start monitoring service: %+v\", err)\n\t}\n\n\tlog.Infof(\"Initializing Checkpointer\")\n\tif err := w.checkpointer.Init(); err != nil {\n\t\tlog.Errorf(\"Failed to start Checkpointer: %+v\", err)\n\t\treturn err\n\t}\n\n\tw.shardStatus = make(map[string]*par.ShardStatus)\n\n\tstopChan := make(chan struct{})\n\tw.stop = &stopChan\n\n\tw.waitGroup = &sync.WaitGroup{}\n\n\tlog.Infof(\"Initialization complete.\")\n\n\treturn nil\n}\n\n// newShardConsumer creates shard consumer for the specified shard\nfunc (w *Worker) newShardConsumer(shard *par.ShardStatus) shardConsumer {\n\tcommon := commonShardConsumer{\n\t\tshard:           shard,\n\t\tkc:              w.kc,\n\t\tcheckpointer:    w.checkpointer,\n\t\trecordProcessor: w.processorFactory.CreateProcessor(),\n\t\tkclConfig:       w.kclConfig,\n\t\tmService:        w.mService,\n\t}\n\tif w.kclConfig.EnableEnhancedFanOutConsumer {\n\t\tw.kclConfig.Logger.Infof(\"Start enhanced fan-out shard consumer for shard: %v\", shard.ID)\n\t\treturn &FanOutShardConsumer{\n\t\t\tcommonShardConsumer: common,\n\t\t\tconsumerARN:         w.consumerARN,\n\t\t\tconsumerID:          w.workerID,\n\t\t\tstop:                w.stop,\n\t\t}\n\t}\n\tw.kclConfig.Logger.Infof(\"Start polling shard consumer for shard: %v\", shard.ID)\n\treturn &PollingShardConsumer{\n\t\tcommonShardConsumer: common,\n\t\tstreamName:          w.streamName,\n\t\tconsumerID:          w.workerID,\n\t\tstop:                w.stop,\n\t\tmService:            w.mService,\n\t}\n}\n\n// eventLoop\nfunc (w *Worker) eventLoop() {\n\tlog := w.kclConfig.Logger\n\n\tvar foundShards int\n\tfor {\n\t\t// Add [-50%, +50%] random jitter to ShardSyncIntervalMillis. When multiple workers\n\t\t// starts at the same time, this decreases the probability of them calling\n\t\t// kinesis.DescribeStream at the same time, and hit the hard-limit on aws API calls.\n\t\t// On average the period remains the same so that doesn't affect behavior.\n\t\trnd, _ := rand.Int(rand.Reader, big.NewInt(int64(w.kclConfig.ShardSyncIntervalMillis)))\n\t\tshardSyncSleep := w.kclConfig.ShardSyncIntervalMillis/2 + int(rnd.Int64())\n\n\t\terr := w.syncShard()\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Error syncing shards: %+v, Retrying in %d ms...\", err, shardSyncSleep)\n\t\t\ttime.Sleep(time.Duration(shardSyncSleep) * time.Millisecond)\n\t\t\tcontinue\n\t\t}\n\n\t\tif foundShards == 0 || foundShards != len(w.shardStatus) {\n\t\t\tfoundShards = len(w.shardStatus)\n\t\t\tlog.Infof(\"Found %d shards\", foundShards)\n\t\t}\n\n\t\t// Count the number of leases held by this worker excluding the processed shard\n\t\tcounter := 0\n\t\tfor _, shard := range w.shardStatus {\n\t\t\tif shard.GetLeaseOwner() == w.workerID && shard.GetCheckpoint() != chk.ShardEnd {\n\t\t\t\tcounter++\n\t\t\t}\n\t\t}\n\n\t\t// max number of lease has not been reached yet\n\t\tif counter < w.kclConfig.MaxLeasesForWorker {\n\t\t\tfor _, shard := range w.shardStatus {\n\t\t\t\t// already owner of the shard\n\t\t\t\tif shard.GetLeaseOwner() == w.workerID {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\terr := w.checkpointer.FetchCheckpoint(shard)\n\t\t\t\tif err != nil {\n\t\t\t\t\t// checkpoint may not exist yet is not an error condition.\n\t\t\t\t\tif err != chk.ErrSequenceIDNotFound {\n\t\t\t\t\t\tlog.Warnf(\"Couldn't fetch checkpoint: %+v\", err)\n\t\t\t\t\t\t// move on to next shard\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\t// The shard is closed and we have processed all records\n\t\t\t\tif shard.GetCheckpoint() == chk.ShardEnd {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\tvar stealShard bool\n\t\t\t\tif w.kclConfig.EnableLeaseStealing && shard.ClaimRequest != \"\" {\n\t\t\t\t\tupcomingStealingInterval := time.Now().UTC().Add(time.Duration(w.kclConfig.LeaseStealingIntervalMillis) * time.Millisecond)\n\t\t\t\t\tif shard.GetLeaseTimeout().Before(upcomingStealingInterval) && !shard.IsClaimRequestExpired(w.kclConfig) {\n\t\t\t\t\t\tif shard.ClaimRequest == w.workerID {\n\t\t\t\t\t\t\tstealShard = true\n\t\t\t\t\t\t\tlog.Debugf(\"Stealing shard: %s\", shard.ID)\n\t\t\t\t\t\t} else {\n\t\t\t\t\t\t\tlog.Debugf(\"Shard being stolen: %s\", shard.ID)\n\t\t\t\t\t\t\tcontinue\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\terr = w.checkpointer.GetLease(shard, w.workerID)\n\t\t\t\tif err != nil {\n\t\t\t\t\t// cannot get lease on the shard\n\t\t\t\t\tif !errors.As(err, &chk.ErrLeaseNotAcquired{}) {\n\t\t\t\t\t\tlog.Errorf(\"Cannot get lease: %+v\", err)\n\t\t\t\t\t}\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\tif stealShard {\n\t\t\t\t\tlog.Debugf(\"Successfully stole shard: %+v\", shard.ID)\n\t\t\t\t\tw.shardStealInProgress = false\n\t\t\t\t}\n\n\t\t\t\t// log metrics on got lease\n\t\t\t\tw.mService.LeaseGained(shard.ID)\n\t\t\t\tw.waitGroup.Add(1)\n\t\t\t\tgo func(shard *par.ShardStatus) {\n\t\t\t\t\tdefer w.waitGroup.Done()\n\t\t\t\t\tif err := w.newShardConsumer(shard).getRecords(); err != nil {\n\t\t\t\t\t\tlog.Errorf(\"Error in getRecords: %+v\", err)\n\t\t\t\t\t}\n\t\t\t\t}(shard)\n\t\t\t\t// exit from for loop and not to grab more shard for now.\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\tif w.kclConfig.EnableLeaseStealing {\n\t\t\terr = w.rebalance()\n\t\t\tif err != nil {\n\t\t\t\tlog.Warnf(\"Error in rebalance: %+v\", err)\n\t\t\t}\n\t\t}\n\n\t\tselect {\n\t\tcase <-*w.stop:\n\t\t\tlog.Infof(\"Shutting down...\")\n\t\t\treturn\n\t\tcase <-time.After(time.Duration(shardSyncSleep) * time.Millisecond):\n\t\t\tlog.Debugf(\"Waited %d ms to sync shards...\", shardSyncSleep)\n\t\t}\n\t}\n}\n\nfunc (w *Worker) rebalance() error {\n\tlog := w.kclConfig.Logger\n\n\tworkers, err := w.checkpointer.ListActiveWorkers(w.shardStatus)\n\tif err != nil {\n\t\tlog.Debugf(\"Error listing workers. workerID: %s. Error: %+v \", w.workerID, err)\n\t\treturn err\n\t}\n\n\t// Only attempt to steal one shard at time, to allow for linear convergence\n\tif w.shardStealInProgress {\n\t\tshardInfo := make(map[string]bool)\n\t\terr := w.getShardIDs(\"\", shardInfo)\n\t\tif err != nil {\n\t\t\treturn err\n\t\t}\n\t\tfor _, shard := range w.shardStatus {\n\t\t\tif shard.ClaimRequest != \"\" && shard.ClaimRequest == w.workerID {\n\t\t\t\tlog.Debugf(\"Steal in progress. workerID: %s\", w.workerID)\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\t// Our shard steal was stomped on by a Checkpoint.\n\t\t\t// We could deal with that, but instead just try again\n\t\t\tw.shardStealInProgress = false\n\t\t}\n\t}\n\n\tvar numShards int\n\tfor _, shards := range workers {\n\t\tnumShards += len(shards)\n\t}\n\n\tnumWorkers := len(workers)\n\n\t// 1:1 shards to workers is optimal, so we cannot possibly rebalance\n\tif numWorkers >= numShards {\n\t\tlog.Debugf(\"Optimal shard allocation, not stealing any shards. workerID: %s, %v > %v. \", w.workerID, numWorkers, numShards)\n\t\treturn nil\n\t}\n\n\tcurrentShards, ok := workers[w.workerID]\n\tvar numCurrentShards int\n\tif !ok {\n\t\tnumCurrentShards = 0\n\t\tnumWorkers++\n\t} else {\n\t\tnumCurrentShards = len(currentShards)\n\t}\n\n\toptimalShards := numShards / numWorkers\n\n\t// We have more than or equal optimal shards, so no rebalancing can take place\n\tif numCurrentShards >= optimalShards || numCurrentShards == w.kclConfig.MaxLeasesForWorker {\n\t\tlog.Debugf(\"We have enough shards, not attempting to steal any. workerID: %s\", w.workerID)\n\t\treturn nil\n\t}\n\n\tvar workerSteal string\n\tfor worker, shards := range workers {\n\t\tif worker != w.workerID && len(shards) > optimalShards {\n\t\t\tworkerSteal = worker\n\t\t\toptimalShards = len(shards)\n\t\t}\n\t}\n\t// Not all shards are allocated so fallback to default shard allocation mechanisms\n\tif workerSteal == \"\" {\n\t\tlog.Infof(\"Not all shards are allocated, not stealing any. workerID: %s\", w.workerID)\n\t\treturn nil\n\t}\n\n\t// Steal a random shard from the worker with the most shards\n\tw.shardStealInProgress = true\n\trnd, _ := rand.Int(rand.Reader, big.NewInt(int64(len(workers[workerSteal]))))\n\trandIndex := int(rnd.Int64())\n\tshardToSteal := workers[workerSteal][randIndex]\n\tlog.Debugf(\"Stealing shard %s from %s\", shardToSteal, workerSteal)\n\n\terr = w.checkpointer.ClaimShard(w.shardStatus[shardToSteal.ID], w.workerID)\n\tif err != nil {\n\t\tw.shardStealInProgress = false\n\t\treturn err\n\t}\n\treturn nil\n}\n\n// List all shards and store them into shardStatus table\n// If shard has been removed, need to exclude it from cached shard status.\nfunc (w *Worker) getShardIDs(nextToken string, shardInfo map[string]bool) error {\n\tlog := w.kclConfig.Logger\n\n\targs := &kinesis.ListShardsInput{}\n\n\t// When you have a nextToken, you can't set the streamName\n\tif nextToken != \"\" {\n\t\targs.NextToken = aws.String(nextToken)\n\t} else {\n\t\targs.StreamName = aws.String(w.streamName)\n\t}\n\n\tlistShards, err := w.kc.ListShards(args)\n\tif err != nil {\n\t\tlog.Errorf(\"Error in ListShards: %s Error: %+v Request: %s\", w.streamName, err, args)\n\t\treturn err\n\t}\n\n\tfor _, s := range listShards.Shards {\n\t\t// record avail shardId from fresh reading from Kinesis\n\t\tshardInfo[*s.ShardId] = true\n\n\t\t// found new shard\n\t\tif _, ok := w.shardStatus[*s.ShardId]; !ok {\n\t\t\tlog.Infof(\"Found new shard with id %s\", *s.ShardId)\n\t\t\tw.shardStatus[*s.ShardId] = &par.ShardStatus{\n\t\t\t\tID:                     *s.ShardId,\n\t\t\t\tParentShardId:          aws.StringValue(s.ParentShardId),\n\t\t\t\tMux:                    &sync.RWMutex{},\n\t\t\t\tStartingSequenceNumber: aws.StringValue(s.SequenceNumberRange.StartingSequenceNumber),\n\t\t\t\tEndingSequenceNumber:   aws.StringValue(s.SequenceNumberRange.EndingSequenceNumber),\n\t\t\t}\n\t\t}\n\t}\n\n\tif listShards.NextToken != nil {\n\t\terr := w.getShardIDs(aws.StringValue(listShards.NextToken), shardInfo)\n\t\tif err != nil {\n\t\t\tlog.Errorf(\"Error in ListShards: %s Error: %+v Request: %s\", w.streamName, err, args)\n\t\t\treturn err\n\t\t}\n\t}\n\n\treturn nil\n}\n\n// syncShard to sync the cached shard info with actual shard info from Kinesis\nfunc (w *Worker) syncShard() error {\n\tlog := w.kclConfig.Logger\n\tshardInfo := make(map[string]bool)\n\terr := w.getShardIDs(\"\", shardInfo)\n\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tfor _, shard := range w.shardStatus {\n\t\t// The cached shard no longer existed, remove it.\n\t\tif _, ok := shardInfo[shard.ID]; !ok {\n\t\t\t// remove the shard from local status cache\n\t\t\tdelete(w.shardStatus, shard.ID)\n\t\t\t// remove the shard entry in dynamoDB as well\n\t\t\t// Note: syncShard runs periodically. we don't need to do anything in case of error here.\n\t\t\tif err := w.checkpointer.RemoveLeaseInfo(shard.ID); err != nil {\n\t\t\t\tlog.Errorf(\"Failed to remove shard lease info: %s Error: %+v\", shard.ID, err)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n"
  },
  {
    "path": "go.mod",
    "content": "module github.com/vmware/vmware-go-kcl\n\ngo 1.17\n\nrequire (\n\tgithub.com/aws/aws-sdk-go v1.41.7\n\tgithub.com/awslabs/kinesis-aggregation/go v0.0.0-20210630091500-54e17340d32f\n\tgithub.com/golang/protobuf v1.5.2\n\tgithub.com/google/uuid v1.3.0\n\tgithub.com/prometheus/client_golang v1.11.1\n\tgithub.com/prometheus/common v0.32.1\n\tgithub.com/rs/zerolog v1.25.0\n\tgithub.com/sirupsen/logrus v1.8.1\n\tgithub.com/stretchr/testify v1.7.0\n\tgo.uber.org/zap v1.19.1\n\tgopkg.in/natefinch/lumberjack.v2 v2.0.0\n)\n\nrequire (\n\tgithub.com/BurntSushi/toml v0.4.1 // indirect\n\tgithub.com/beorn7/perks v1.0.1 // indirect\n\tgithub.com/cespare/xxhash/v2 v2.1.2 // indirect\n\tgithub.com/davecgh/go-spew v1.1.1 // indirect\n\tgithub.com/jmespath/go-jmespath v0.4.0 // indirect\n\tgithub.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect\n\tgithub.com/pmezard/go-difflib v1.0.0 // indirect\n\tgithub.com/prometheus/client_model v0.2.0 // indirect\n\tgithub.com/prometheus/procfs v0.7.3 // indirect\n\tgo.uber.org/atomic v1.9.0 // indirect\n\tgo.uber.org/multierr v1.7.0 // indirect\n\tgolang.org/x/sys v0.1.0 // indirect\n\tgoogle.golang.org/protobuf v1.27.1 // indirect\n\tgopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect\n)\n"
  },
  {
    "path": "go.sum",
    "content": "cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=\ncloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=\ncloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=\ncloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=\ncloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=\ncloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=\ncloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=\ncloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=\ncloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=\ncloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=\ncloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=\ncloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk=\ncloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs=\ncloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc=\ncloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY=\ncloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=\ncloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=\ncloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=\ncloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=\ncloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=\ncloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=\ncloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=\ncloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=\ncloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=\ncloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=\ncloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=\ncloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=\ncloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=\ncloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=\ncloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=\ncloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=\ncloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=\ndmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=\ngithub.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=\ngithub.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw=\ngithub.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=\ngithub.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=\ngithub.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=\ngithub.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=\ngithub.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=\ngithub.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=\ngithub.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=\ngithub.com/aws/aws-sdk-go v1.19.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=\ngithub.com/aws/aws-sdk-go v1.41.7 h1:vlpR8Cky3ZxUVNINgeRZS6N0p6zmFvu/ZqRRwrTI25U=\ngithub.com/aws/aws-sdk-go v1.41.7/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q=\ngithub.com/awslabs/kinesis-aggregation/go v0.0.0-20210630091500-54e17340d32f h1:Pf0BjJDga7C98f0vhw+Ip5EaiE07S3lTKpIYPNS0nMo=\ngithub.com/awslabs/kinesis-aggregation/go v0.0.0-20210630091500-54e17340d32f/go.mod h1:SghidfnxvX7ribW6nHI7T+IBbc9puZ9kk5Tx/88h8P4=\ngithub.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=\ngithub.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=\ngithub.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=\ngithub.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=\ngithub.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=\ngithub.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=\ngithub.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=\ngithub.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=\ngithub.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=\ngithub.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=\ngithub.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=\ngithub.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=\ngithub.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=\ngithub.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=\ngithub.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=\ngithub.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=\ngithub.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=\ngithub.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=\ngithub.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=\ngithub.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=\ngithub.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=\ngithub.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=\ngithub.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=\ngithub.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=\ngithub.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=\ngithub.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=\ngithub.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=\ngithub.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=\ngithub.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=\ngithub.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=\ngithub.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=\ngithub.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=\ngithub.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=\ngithub.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=\ngithub.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=\ngithub.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=\ngithub.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=\ngithub.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=\ngithub.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=\ngithub.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=\ngithub.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=\ngithub.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=\ngithub.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=\ngithub.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=\ngithub.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=\ngithub.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=\ngithub.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=\ngithub.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=\ngithub.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=\ngithub.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=\ngithub.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=\ngithub.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=\ngithub.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=\ngithub.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=\ngithub.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=\ngithub.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=\ngithub.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=\ngithub.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=\ngithub.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=\ngithub.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=\ngithub.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=\ngithub.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=\ngithub.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=\ngithub.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=\ngithub.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=\ngithub.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=\ngithub.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=\ngithub.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=\ngithub.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=\ngithub.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=\ngithub.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=\ngithub.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=\ngithub.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=\ngithub.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=\ngithub.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=\ngithub.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=\ngithub.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=\ngithub.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=\ngithub.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=\ngithub.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=\ngithub.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=\ngithub.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=\ngithub.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=\ngithub.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=\ngithub.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=\ngithub.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=\ngithub.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=\ngithub.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=\ngithub.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=\ngithub.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=\ngithub.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=\ngithub.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=\ngithub.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=\ngithub.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=\ngithub.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=\ngithub.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=\ngithub.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=\ngithub.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=\ngithub.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=\ngithub.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=\ngithub.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=\ngithub.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=\ngithub.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=\ngithub.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=\ngithub.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=\ngithub.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=\ngithub.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=\ngithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=\ngithub.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=\ngithub.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=\ngithub.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=\ngithub.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=\ngithub.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=\ngithub.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=\ngithub.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=\ngithub.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=\ngithub.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=\ngithub.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=\ngithub.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s=\ngithub.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=\ngithub.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=\ngithub.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=\ngithub.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=\ngithub.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=\ngithub.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=\ngithub.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=\ngithub.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=\ngithub.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=\ngithub.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4=\ngithub.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=\ngithub.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=\ngithub.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=\ngithub.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=\ngithub.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=\ngithub.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=\ngithub.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=\ngithub.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=\ngithub.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=\ngithub.com/rs/zerolog v1.25.0 h1:Rj7XygbUHKUlDPcVdoLyR91fJBsduXj5fRxyqIQj/II=\ngithub.com/rs/zerolog v1.25.0/go.mod h1:7KHcEGe0QZPOm2IE4Kpb5rTh6n1h2hIgS5OOnu1rUaI=\ngithub.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=\ngithub.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=\ngithub.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=\ngithub.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=\ngithub.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=\ngithub.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=\ngithub.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=\ngithub.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=\ngithub.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=\ngithub.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngithub.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngithub.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngithub.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=\ngo.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=\ngo.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=\ngo.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=\ngo.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=\ngo.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=\ngo.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=\ngo.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE=\ngo.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=\ngo.uber.org/goleak v1.1.11-0.20210813005559-691160354723 h1:sHOAIxRGBp443oHZIPB+HsUGaksVCXVQENPxwTfQdH4=\ngo.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=\ngo.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=\ngo.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec=\ngo.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=\ngo.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI=\ngo.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI=\ngolang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=\ngolang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=\ngolang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=\ngolang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=\ngolang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=\ngolang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=\ngolang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=\ngolang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=\ngolang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=\ngolang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=\ngolang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=\ngolang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=\ngolang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=\ngolang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=\ngolang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=\ngolang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=\ngolang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=\ngolang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=\ngolang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=\ngolang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=\ngolang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=\ngolang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=\ngolang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=\ngolang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=\ngolang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=\ngolang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=\ngolang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=\ngolang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=\ngolang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=\ngolang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=\ngolang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=\ngolang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=\ngolang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=\ngolang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=\ngolang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=\ngolang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=\ngolang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=\ngolang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=\ngolang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=\ngolang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=\ngolang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=\ngolang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=\ngolang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=\ngolang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=\ngolang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=\ngolang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=\ngolang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=\ngolang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=\ngolang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=\ngolang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=\ngolang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=\ngolang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=\ngolang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=\ngolang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=\ngolang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=\ngolang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=\ngolang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=\ngolang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=\ngolang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=\ngolang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=\ngolang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=\ngolang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8=\ngolang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=\ngolang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=\ngolang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=\ngolang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=\ngolang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=\ngolang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngoogle.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=\ngoogle.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=\ngoogle.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=\ngoogle.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=\ngoogle.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=\ngoogle.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=\ngoogle.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=\ngoogle.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=\ngoogle.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=\ngoogle.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=\ngoogle.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=\ngoogle.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=\ngoogle.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=\ngoogle.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=\ngoogle.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=\ngoogle.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=\ngoogle.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=\ngoogle.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=\ngoogle.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=\ngoogle.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=\ngoogle.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=\ngoogle.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=\ngoogle.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=\ngoogle.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=\ngoogle.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=\ngoogle.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=\ngoogle.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=\ngoogle.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=\ngoogle.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=\ngoogle.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=\ngoogle.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=\ngoogle.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=\ngoogle.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U=\ngoogle.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=\ngoogle.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=\ngoogle.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=\ngoogle.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=\ngoogle.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=\ngoogle.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=\ngoogle.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=\ngoogle.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=\ngoogle.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=\ngoogle.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=\ngoogle.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=\ngoogle.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=\ngoogle.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=\ngoogle.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=\ngoogle.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=\ngoogle.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=\ngoogle.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=\ngoogle.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=\ngoogle.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=\ngoogle.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=\ngoogle.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=\ngoogle.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=\ngoogle.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=\ngoogle.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=\ngoogle.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=\ngoogle.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=\ngoogle.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=\ngoogle.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=\ngopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=\ngopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=\ngopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=\ngopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=\ngopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=\ngopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\ngopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=\ngopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\nhonnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nhonnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nhonnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nhonnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nhonnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=\nhonnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=\nhonnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=\nrsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=\nrsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=\nrsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=\n"
  },
  {
    "path": "logger/logger.go",
    "content": "/*\n * Copyright (c) 2019 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\n// Note: The implementation comes from https://www.mountedthoughts.com/golang-logger-interface/\n// https://github.com/amitrai48/logger\n\npackage logger\n\nimport (\n\t\"github.com/sirupsen/logrus\"\n)\n\n// Fields Type to pass when we want to call WithFields for structured logging\ntype Fields map[string]interface{}\n\nconst (\n\t//Debug has verbose message\n\tDebug = \"debug\"\n\t//Info is default log level\n\tInfo = \"info\"\n\t//Warn is for logging messages about possible issues\n\tWarn = \"warn\"\n\t//Error is for logging errors\n\tError = \"error\"\n\t//Fatal is for logging fatal messages. The sytem shutsdown after logging the message.\n\tFatal = \"fatal\"\n)\n\n// Logger is the common interface for logging.\ntype Logger interface {\n\tDebugf(format string, args ...interface{})\n\n\tInfof(format string, args ...interface{})\n\n\tWarnf(format string, args ...interface{})\n\n\tErrorf(format string, args ...interface{})\n\n\tFatalf(format string, args ...interface{})\n\n\tPanicf(format string, args ...interface{})\n\n\tWithFields(keyValues Fields) Logger\n}\n\n// Configuration stores the config for the logger\n// For some loggers there can only be one level across writers, for such the level of Console is picked by default\ntype Configuration struct {\n\tEnableConsole     bool\n\tConsoleJSONFormat bool\n\tConsoleLevel      string\n\tEnableFile        bool\n\tFileJSONFormat    bool\n\tFileLevel         string\n\n\t// Filename is the file to write logs to.  Backup log files will be retained\n\t// in the same directory.  It uses <processname>-lumberjack.log in\n\t// os.TempDir() if empty.\n\tFilename string\n\n\t// MaxSize is the maximum size in megabytes of the log file before it gets\n\t// rotated. It defaults to 100 megabytes.\n\tMaxSizeMB int\n\n\t// MaxAge is the maximum number of days to retain old log files based on the\n\t// timestamp encoded in their filename.  Note that a day is defined as 24\n\t// hours and may not exactly correspond to calendar days due to daylight\n\t// savings, leap seconds, etc. The default is 7 days.\n\tMaxAgeDays int\n\n\t// MaxBackups is the maximum number of old log files to retain.  The default\n\t// is to retain all old log files (though MaxAge may still cause them to get\n\t// deleted.)\n\tMaxBackups int\n\n\t// LocalTime determines if the time used for formatting the timestamps in\n\t// backup files is the computer's local time.  The default is to use UTC\n\t// time.\n\tLocalTime bool\n}\n\n// GetDefaultLogger creates a default logger.\nfunc GetDefaultLogger() Logger {\n\treturn NewLogrusLogger(logrus.StandardLogger())\n}\n\n// normalizeConfig to enforce default value in configuration.\nfunc normalizeConfig(config *Configuration) {\n\tif config.MaxSizeMB <= 0 {\n\t\tconfig.MaxSizeMB = 100\n\t}\n\n\tif config.MaxAgeDays <= 0 {\n\t\tconfig.MaxAgeDays = 7\n\t}\n\n\tif config.MaxBackups < 0 {\n\t\tconfig.MaxBackups = 0\n\t}\n}\n"
  },
  {
    "path": "logger/logger_test.go",
    "content": "/*\n * Copyright (c) 2019 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\n// Note: The implementation comes from https://www.mountedthoughts.com/golang-logger-interface/\n\npackage logger\n\nimport (\n\t\"testing\"\n\n\t\"github.com/sirupsen/logrus\"\n)\n\nfunc TestLogrusLoggerWithConfig(t *testing.T) {\n\tconfig := Configuration{\n\t\tEnableConsole:     true,\n\t\tConsoleLevel:      Debug,\n\t\tConsoleJSONFormat: false,\n\t\tEnableFile:        false,\n\t\tFileLevel:         Info,\n\t\tFileJSONFormat:    true,\n\t}\n\n\tlog := NewLogrusLoggerWithConfig(config)\n\n\tcontextLogger := log.WithFields(Fields{\"key1\": \"value1\"})\n\tcontextLogger.Debugf(\"Starting with logrus\")\n\tcontextLogger.Infof(\"Logrus is awesome\")\n}\n\nfunc TestLogrusLogger(t *testing.T) {\n\t// adapts to Logger interface\n\tlog := NewLogrusLogger(logrus.StandardLogger())\n\n\tcontextLogger := log.WithFields(Fields{\"key1\": \"value1\"})\n\tcontextLogger.Debugf(\"Starting with logrus\")\n\tcontextLogger.Infof(\"Logrus is awesome\")\n}\n"
  },
  {
    "path": "logger/logrus.go",
    "content": "/*\n * Copyright (c) 2019 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\n// Note: The implementation comes from https://www.mountedthoughts.com/golang-logger-interface/\n// https://github.com/amitrai48/logger\n\npackage logger\n\nimport (\n\t\"io\"\n\t\"os\"\n\n\t\"github.com/sirupsen/logrus\"\n\tlumberjack \"gopkg.in/natefinch/lumberjack.v2\"\n)\n\ntype LogrusLogEntry struct {\n\tentry *logrus.Entry\n}\n\ntype LogrusLogger struct {\n\tlogger logrus.FieldLogger\n}\n\n// NewLogrusLogger adapts existing logrus logger to Logger interface.\n// The call is responsible for configuring logrus logger appropriately.\nfunc NewLogrusLogger(lLogger logrus.FieldLogger) Logger {\n\treturn &LogrusLogger{\n\t\tlogger: lLogger,\n\t}\n}\n\n// NewLogrusLoggerWithConfig creates and configs Logger instance backed by\n// logrus logger.\nfunc NewLogrusLoggerWithConfig(config Configuration) Logger {\n\tlogLevel := config.ConsoleLevel\n\tif logLevel == \"\" {\n\t\tlogLevel = config.FileLevel\n\t}\n\n\tlevel, err := logrus.ParseLevel(logLevel)\n\tif err != nil {\n\t\t// fallback to InfoLevel\n\t\tlevel = logrus.InfoLevel\n\t}\n\n\tnormalizeConfig(&config)\n\n\tstdOutHandler := os.Stdout\n\tfileHandler := &lumberjack.Logger{\n\t\tFilename:   config.Filename,\n\t\tMaxSize:    config.MaxSizeMB,\n\t\tCompress:   true,\n\t\tMaxAge:     config.MaxAgeDays,\n\t\tMaxBackups: config.MaxBackups,\n\t\tLocalTime:  config.LocalTime,\n\t}\n\tlLogger := &logrus.Logger{\n\t\tOut:       stdOutHandler,\n\t\tFormatter: getFormatter(config.ConsoleJSONFormat),\n\t\tHooks:     make(logrus.LevelHooks),\n\t\tLevel:     level,\n\t}\n\n\tif config.EnableConsole && config.EnableFile {\n\t\tlLogger.SetOutput(io.MultiWriter(stdOutHandler, fileHandler))\n\t} else {\n\t\tif config.EnableFile {\n\t\t\tlLogger.SetOutput(fileHandler)\n\t\t\tlLogger.SetFormatter(getFormatter(config.FileJSONFormat))\n\t\t}\n\t}\n\n\treturn &LogrusLogger{\n\t\tlogger: lLogger,\n\t}\n}\n\nfunc (l *LogrusLogger) Debugf(format string, args ...interface{}) {\n\tl.logger.Debugf(format, args...)\n}\n\nfunc (l *LogrusLogger) Infof(format string, args ...interface{}) {\n\tl.logger.Infof(format, args...)\n}\n\nfunc (l *LogrusLogger) Warnf(format string, args ...interface{}) {\n\tl.logger.Warnf(format, args...)\n}\n\nfunc (l *LogrusLogger) Errorf(format string, args ...interface{}) {\n\tl.logger.Errorf(format, args...)\n}\n\nfunc (l *LogrusLogger) Fatalf(format string, args ...interface{}) {\n\tl.logger.Fatalf(format, args...)\n}\n\nfunc (l *LogrusLogger) Panicf(format string, args ...interface{}) {\n\tl.logger.Fatalf(format, args...)\n}\n\nfunc (l *LogrusLogger) WithFields(fields Fields) Logger {\n\treturn &LogrusLogEntry{\n\t\tentry: l.logger.WithFields(convertToLogrusFields(fields)),\n\t}\n}\n\nfunc (l *LogrusLogEntry) Debugf(format string, args ...interface{}) {\n\tl.entry.Debugf(format, args...)\n}\n\nfunc (l *LogrusLogEntry) Infof(format string, args ...interface{}) {\n\tl.entry.Infof(format, args...)\n}\n\nfunc (l *LogrusLogEntry) Warnf(format string, args ...interface{}) {\n\tl.entry.Warnf(format, args...)\n}\n\nfunc (l *LogrusLogEntry) Errorf(format string, args ...interface{}) {\n\tl.entry.Errorf(format, args...)\n}\n\nfunc (l *LogrusLogEntry) Fatalf(format string, args ...interface{}) {\n\tl.entry.Fatalf(format, args...)\n}\n\nfunc (l *LogrusLogEntry) Panicf(format string, args ...interface{}) {\n\tl.entry.Fatalf(format, args...)\n}\n\nfunc (l *LogrusLogEntry) WithFields(fields Fields) Logger {\n\treturn &LogrusLogEntry{\n\t\tentry: l.entry.WithFields(convertToLogrusFields(fields)),\n\t}\n}\n\nfunc getFormatter(isJSON bool) logrus.Formatter {\n\tif isJSON {\n\t\treturn &logrus.JSONFormatter{}\n\t}\n\treturn &logrus.TextFormatter{\n\t\tFullTimestamp:          true,\n\t\tDisableLevelTruncation: true,\n\t}\n}\n\nfunc convertToLogrusFields(fields Fields) logrus.Fields {\n\tlogrusFields := logrus.Fields{}\n\tfor index, val := range fields {\n\t\tlogrusFields[index] = val\n\t}\n\treturn logrusFields\n}\n"
  },
  {
    "path": "logger/zap/zap.go",
    "content": "/*\n * Copyright (c) 2019 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\n// Note: The implementation comes from https://www.mountedthoughts.com/golang-logger-interface/\n// https://github.com/amitrai48/logger\n\npackage zap\n\nimport (\n\t\"os\"\n\n\t\"github.com/vmware/vmware-go-kcl/logger\"\n\tuzap \"go.uber.org/zap\"\n\t\"go.uber.org/zap/zapcore\"\n\tlumberjack \"gopkg.in/natefinch/lumberjack.v2\"\n)\n\ntype ZapLogger struct {\n\tsugaredLogger *uzap.SugaredLogger\n}\n\n// NewZapLogger adapts existing sugared zap logger to Logger interface.\n// The call is responsible for configuring sugard zap logger appropriately.\n//\n// Note: Sugar wraps the Logger to provide a more ergonomic, but slightly slower,\n// API. Sugaring a Logger is quite inexpensive, so it's reasonable for a\n// single application to use both Loggers and SugaredLoggers, converting\n// between them on the boundaries of performance-sensitive code.\n//\n// Base zap logger can be convert to SugaredLogger by calling to add a wrapper:\n// sugaredLogger := log.Sugar()\n//\nfunc NewZapLogger(logger *uzap.SugaredLogger) logger.Logger {\n\treturn &ZapLogger{\n\t\tsugaredLogger: logger,\n\t}\n}\n\n// NewZapLoggerWithConfig creates and configs Logger instance backed by\n// zap Sugared logger.\nfunc NewZapLoggerWithConfig(config logger.Configuration) logger.Logger {\n\tcores := []zapcore.Core{}\n\n\tif config.EnableConsole {\n\t\tlevel := getZapLevel(config.ConsoleLevel)\n\t\twriter := zapcore.Lock(os.Stdout)\n\t\tcore := zapcore.NewCore(getEncoder(config.ConsoleJSONFormat), writer, level)\n\t\tcores = append(cores, core)\n\t}\n\n\tif config.EnableFile {\n\t\tlevel := getZapLevel(config.FileLevel)\n\t\twriter := zapcore.AddSync(&lumberjack.Logger{\n\t\t\tFilename:   config.Filename,\n\t\t\tMaxSize:    config.MaxSizeMB,\n\t\t\tCompress:   true,\n\t\t\tMaxAge:     config.MaxAgeDays,\n\t\t\tMaxBackups: config.MaxBackups,\n\t\t\tLocalTime:  config.LocalTime,\n\t\t})\n\t\tcore := zapcore.NewCore(getEncoder(config.FileJSONFormat), writer, level)\n\t\tcores = append(cores, core)\n\t}\n\n\tcombinedCore := zapcore.NewTee(cores...)\n\n\t// AddCallerSkip skips 2 number of callers, this is important else the file that gets\n\t// logged will always be the wrapped file. In our case zap.go\n\tlogger := uzap.New(combinedCore,\n\t\tuzap.AddCallerSkip(2),\n\t\tuzap.AddCaller(),\n\t).Sugar()\n\n\treturn &ZapLogger{\n\t\tsugaredLogger: logger,\n\t}\n}\n\nfunc (l *ZapLogger) Debugf(format string, args ...interface{}) {\n\tl.sugaredLogger.Debugf(format, args...)\n}\n\nfunc (l *ZapLogger) Infof(format string, args ...interface{}) {\n\tl.sugaredLogger.Infof(format, args...)\n}\n\nfunc (l *ZapLogger) Warnf(format string, args ...interface{}) {\n\tl.sugaredLogger.Warnf(format, args...)\n}\n\nfunc (l *ZapLogger) Errorf(format string, args ...interface{}) {\n\tl.sugaredLogger.Errorf(format, args...)\n}\n\nfunc (l *ZapLogger) Fatalf(format string, args ...interface{}) {\n\tl.sugaredLogger.Fatalf(format, args...)\n}\n\nfunc (l *ZapLogger) Panicf(format string, args ...interface{}) {\n\tl.sugaredLogger.Fatalf(format, args...)\n}\n\nfunc (l *ZapLogger) WithFields(fields logger.Fields) logger.Logger {\n\tvar f = make([]interface{}, 0)\n\tfor k, v := range fields {\n\t\tf = append(f, k)\n\t\tf = append(f, v)\n\t}\n\tnewLogger := l.sugaredLogger.With(f...)\n\treturn &ZapLogger{newLogger}\n}\n\nfunc getEncoder(isJSON bool) zapcore.Encoder {\n\tencoderConfig := uzap.NewProductionEncoderConfig()\n\tencoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder\n\tif isJSON {\n\t\treturn zapcore.NewJSONEncoder(encoderConfig)\n\t}\n\treturn zapcore.NewConsoleEncoder(encoderConfig)\n}\n\nfunc getZapLevel(level string) zapcore.Level {\n\tswitch level {\n\tcase logger.Info:\n\t\treturn zapcore.InfoLevel\n\tcase logger.Warn:\n\t\treturn zapcore.WarnLevel\n\tcase logger.Debug:\n\t\treturn zapcore.DebugLevel\n\tcase logger.Error:\n\t\treturn zapcore.ErrorLevel\n\tcase logger.Fatal:\n\t\treturn zapcore.FatalLevel\n\tdefault:\n\t\treturn zapcore.InfoLevel\n\t}\n}\n"
  },
  {
    "path": "logger/zap/zap_test.go",
    "content": "package zap_test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/vmware/vmware-go-kcl/logger\"\n\t\"github.com/vmware/vmware-go-kcl/logger/zap\"\n\tuzap \"go.uber.org/zap\"\n)\n\nfunc TestZapLoggerWithConfig(t *testing.T) {\n\tconfig := logger.Configuration{\n\t\tEnableConsole:     true,\n\t\tConsoleLevel:      logger.Debug,\n\t\tConsoleJSONFormat: true,\n\t\tEnableFile:        false,\n\t\tFileLevel:         logger.Info,\n\t\tFileJSONFormat:    true,\n\t\tFilename:          \"log.log\",\n\t}\n\n\tlog := zap.NewZapLoggerWithConfig(config)\n\n\tcontextLogger := log.WithFields(logger.Fields{\"key1\": \"value1\"})\n\tcontextLogger.Debugf(\"Starting with zap\")\n\tcontextLogger.Infof(\"Zap is awesome\")\n}\n\nfunc TestZapLogger(t *testing.T) {\n\tzapLogger, err := uzap.NewProduction()\n\tassert.Nil(t, err)\n\n\tlog := zap.NewZapLogger(zapLogger.Sugar())\n\n\tcontextLogger := log.WithFields(logger.Fields{\"key1\": \"value1\"})\n\tcontextLogger.Debugf(\"Starting with zap\")\n\tcontextLogger.Infof(\"Zap is awesome\")\n}\n"
  },
  {
    "path": "logger/zerolog/zerolog.go",
    "content": "/*\n * Copyright (c) 2019 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\n// Note: The implementation comes from https://www.mountedthoughts.com/golang-logger-interface/\n// https://github.com/amitrai48/logger\n\n// Package zerolog implements the KCL logger using RS Zerolog logger\npackage zerolog\n\nimport (\n\t\"github.com/rs/zerolog\"\n\t\"github.com/vmware/vmware-go-kcl/logger\"\n\t\"gopkg.in/natefinch/lumberjack.v2\"\n\t\"os\"\n)\n\ntype zeroLogger struct {\n\tlog zerolog.Logger\n}\n\n// NewZerologLogger creates a new logger.Logger backed by RS Zerolog using a default config\nfunc NewZerologLogger() logger.Logger {\n\treturn NewZerologLoggerWithConfig(logger.Configuration{\n\t\tEnableConsole:     true,\n\t\tConsoleJSONFormat: true,\n\t\tConsoleLevel:      logger.Info,\n\t\tEnableFile:        false,\n\t\tFileJSONFormat:    false,\n\t\tFileLevel:         logger.Info,\n\t\tFilename:          \"\",\n\t\tMaxSizeMB:         0,\n\t\tMaxAgeDays:        0,\n\t\tMaxBackups:        0,\n\t\tLocalTime:         true,\n\t})\n}\n\n// NewZerologLoggerWithConfig creates a new logger.Logger backed by RS Zerolog using the provided config\nfunc NewZerologLoggerWithConfig(config logger.Configuration) logger.Logger {\n\tvar consoleHandler *zerolog.ConsoleWriter\n\tvar fileHandler *lumberjack.Logger\n\tvar finalLogger zerolog.Logger\n\n\tnormalizeConfig(&config)\n\n\tif config.EnableConsole {\n\t\tconsoleHandler = &zerolog.ConsoleWriter{Out: os.Stdout}\n\t}\n\n\tif config.EnableFile {\n\t\tfileHandler = &lumberjack.Logger{\n\t\t\tFilename:   config.Filename,\n\t\t\tMaxSize:    config.MaxSizeMB,\n\t\t\tCompress:   true,\n\t\t\tMaxAge:     config.MaxAgeDays,\n\t\t\tMaxBackups: config.MaxBackups,\n\t\t\tLocalTime:  config.LocalTime,\n\t\t}\n\t}\n\n\tif config.EnableConsole && config.EnableFile {\n\t\tmulti := zerolog.MultiLevelWriter(consoleHandler, fileHandler)\n\t\tfinalLogger = zerolog.New(multi).Level(getZeroLogLevel(config.ConsoleLevel)).With().Timestamp().Logger()\n\t} else if config.EnableFile {\n\t\tfinalLogger = zerolog.New(fileHandler).Level(getZeroLogLevel(config.FileLevel)).With().Timestamp().Logger()\n\t} else {\n\t\tfinalLogger = zerolog.New(consoleHandler).Level(getZeroLogLevel(config.ConsoleLevel)).With().Timestamp().Logger()\n\t}\n\n\treturn &zeroLogger{log: finalLogger}\n}\n\nfunc (z *zeroLogger) Debugf(format string, args ...interface{}) {\n\tz.log.Debug().Msgf(format, args...)\n}\n\nfunc (z *zeroLogger) Infof(format string, args ...interface{}) {\n\tz.log.Info().Msgf(format, args...)\n}\n\nfunc (z *zeroLogger) Warnf(format string, args ...interface{}) {\n\tz.log.Warn().Msgf(format, args...)\n}\n\nfunc (z *zeroLogger) Errorf(format string, args ...interface{}) {\n\tz.log.Error().Msgf(format, args...)\n}\n\nfunc (z *zeroLogger) Fatalf(format string, args ...interface{}) {\n\tz.log.Fatal().Msgf(format, args...)\n}\n\nfunc (z *zeroLogger) Panicf(format string, args ...interface{}) {\n\tz.log.Panic().Msgf(format, args...)\n}\n\nfunc (z *zeroLogger) WithFields(keyValues logger.Fields) logger.Logger {\n\tnewLogger := z.log.With()\n\tfor k, v := range keyValues {\n\t\tnewLogger.Interface(k, v)\n\t}\n\n\treturn &zeroLogger{\n\t\tlog: newLogger.Logger(),\n\t}\n}\n\nfunc getZeroLogLevel(level string) zerolog.Level {\n\tswitch level {\n\tcase logger.Info:\n\t\treturn zerolog.InfoLevel\n\tcase logger.Warn:\n\t\treturn zerolog.WarnLevel\n\tcase logger.Debug:\n\t\treturn zerolog.DebugLevel\n\tcase logger.Error:\n\t\treturn zerolog.ErrorLevel\n\tcase logger.Fatal:\n\t\treturn zerolog.FatalLevel\n\tdefault:\n\t\treturn zerolog.InfoLevel\n\t}\n}\n\nfunc normalizeConfig(config *logger.Configuration) {\n\tif config.MaxSizeMB <= 0 {\n\t\tconfig.MaxSizeMB = 100\n\t}\n\n\tif config.MaxAgeDays <= 0 {\n\t\tconfig.MaxAgeDays = 7\n\t}\n\n\tif config.MaxBackups < 0 {\n\t\tconfig.MaxBackups = 0\n\t}\n}\n"
  },
  {
    "path": "logger/zerolog/zerolog_test.go",
    "content": "package zerolog\n\nimport (\n\t\"github.com/vmware/vmware-go-kcl/logger\"\n\t\"testing\"\n)\n\nfunc TestZeroLogLoggerWithConfig(t *testing.T) {\n\tconfig := logger.Configuration{\n\t\tEnableConsole:     true,\n\t\tConsoleLevel:      logger.Debug,\n\t\tConsoleJSONFormat: true,\n\t\tEnableFile:        true,\n\t\tFileLevel:         logger.Info,\n\t\tFileJSONFormat:    false,\n\t\tFilename:          \"/tmp/kcl-zerolog-log.log\",\n\t}\n\n\tlog := NewZerologLoggerWithConfig(config)\n\n\tcontextLogger := log.WithFields(logger.Fields{\"key1\": \"value1\"})\n\tcontextLogger.Debugf(\"Starting with rs zerolog\")\n\tcontextLogger.Infof(\"Rs zerolog is awesome\")\n}\n\nfunc TestZeroLogLogger(t *testing.T) {\n\tlog := NewZerologLogger()\n\n\tcontextLogger := log.WithFields(logger.Fields{\"key1\": \"value1\"})\n\tcontextLogger.Debugf(\"Starting with zerolog\")\n\tcontextLogger.Infof(\"Zerolog is awesome\")\n}\n"
  },
  {
    "path": "support/scripts/check.sh",
    "content": "#!/usr/bin/env bash\n\n. support/scripts/functions.sh\n\ncheckfmt() {\n    local files=\"$(gofmt -l $(local_go_pkgs))\"\n    if [ -n \"$files\" ]; then\n        echo \"You need to run \\\"gofmt -w ./\\\" to fix your formating.\"\n        echo \"$files\" >&2\n        return 1\n    fi\n}\n\nlint() {\n    golangci-lint run \\\n        --skip-files=_mock.go \\\n        --disable=golint \\\n        --skip-dirs=test \\\n        --fast \\\n        --timeout=600s \\\n        --verbose \\\n        $(local_go_pkgs)\n}\n\nscanast() {\n    set +e\n    gosec version\n    gosec ./... > security.log 2>&1\n    set -e\n\n    local issues=\"$(grep -E 'Severity: MEDIUM' security.log | wc -l)\"\n    if [ -n $issues ] && [ $issues -gt 0 ]; then\n        echo \"\"\n        echo \"Medium Severity Issues:\"\n        grep -E \"Severity: MEDIUM\" -A 1 security.log\n        echo $issues \"medium severity issues found.\"\n    fi\n\n    local issues=\"$(grep -E 'Severity: HIGH' security.log | grep -v vendor)\"\n    local issues_count=\"$(grep -E 'Severity: HIGH' security.log | grep -v vendor | wc -l)\"\n    if [ -n $issues_count ] && [ $issues_count -gt 0 ]; then\n        echo \"\"\n        echo \"High Severity Issues:\"\n        grep -E \"Severity: HIGH\" -A 1 security.log\n        echo $issues_count \"high severity issues found.\"\n        echo $issues\n        echo \"You need to resolve the high severity issues at the least.\"\n        exit 1\n    fi\n\n    local issues=\"$(grep -E 'Errors unhandled' security.log | grep -v vendor | grep -v /src/go/src)\"\n    local issues_count=\"$(grep -E 'Errors unhandled' security.log | grep -v vendor | grep -v /src/go/src | wc -l)\"\n    if [ -n $issues_count ] && [ $issues_count -gt 0 ]; then\n        echo \"\"\n        echo \"Unhandled errors:\"\n        grep -E \"Errors unhandled\" security.log\n        echo $issues_count \"unhandled errors, please indicate with the right comment that this case is ok, or handle the error.\"\n        echo $issues\n        echo \"You need to resolve the all unhandled errors.\"\n        exit 1\n    fi\n    rm security.log\n}\n\nusage() {\n    echo \"check.sh fmt|lint\" >&2\n    exit 2\n}\n\ncase \"$1\" in\n    fmt) checkfmt ;;\n    lint) lint ;;\n    scanast) scanast;;\n    *) usage ;;\nesac\n"
  },
  {
    "path": "support/scripts/ci.sh",
    "content": "#!/bin/bash\n\n# Run only the integration tests\n# go test -race ./test\necho \"Warning: Cannot find a good way to inject AWS credential to hmake container\"\necho \"Don't use hmake ci. Use the following command directly\"\necho \"go test -race ./test\"\n"
  },
  {
    "path": "support/scripts/functions.sh",
    "content": "set -ex\n\n# PROJ_ROOT specifies the project root\nexport PROJ_ROOT=\"$HMAKE_PROJECT_DIR\"\n\n# Add /go in GOPATH because that's the original GOPATH in toolchain\nexport GOPATH=/go:$PROJ_ROOT\n\nlocal_go_pkgs() {\n    find './clientlibrary/' -name '*.go' | \\\n        grep -Fv '/vendor/' | \\\n        grep -Fv '/go/' | \\\n        grep -Fv '/gen/' | \\\n        grep -Fv '/tmp/' | \\\n        grep -Fv '/run/' | \\\n        grep -Fv '/tests/' | \\\n        sed -r 's|(.+)/[^/]+\\.go$|\\1|g' | \\\n        sort -u\n}\n\nversion_suffix() {\n    local suffix=$(git log -1 --format=%h 2>/dev/null || true)\n    if [ -n \"$suffix\" ]; then\n        test -z \"$(git status --porcelain 2>/dev/null || true)\" || suffix=\"${suffix}+\"\n        echo -n \"-g${suffix}\"\n    else\n        echo -n -dev\n    fi\n}\n\ngit_commit_hash() {\n\techo $(git rev-parse --short HEAD)\n}\n\n# Due to Go plugin genhash algorithm simply takes full source path\n# from archive, it generates different plugin hash if source path of\n# shared pkg is different, and causes load failure.\n# as a workaround, lookup shared pkg and place it to fixed path\nFIX_GOPATH=/tmp/go\n\nfix_go_pkg() {\n    local pkg=\"$1\" base\n    for p in ${GOPATH//:/ }; do\n        if [ -d \"$p/src/$pkg\" ]; then\n            base=\"$p\"\n            break\n        fi\n    done\n\n    if [ -z \"$base\" ]; then\n        echo \"Package $pkg not found in GOPATH: $GOPATH\" >&2\n        return 1\n    fi\n\n    local fix_pkg_path=\"$FIX_GOPATH/src/$pkg\"\n    rm -f \"$fix_pkg_path\"\n    mkdir -p \"$(dirname $fix_pkg_path)\"\n    ln -s \"$base/src/$pkg\" \"$fix_pkg_path\"\n}\n"
  },
  {
    "path": "support/scripts/test.sh",
    "content": "#!/bin/bash\n. support/scripts/functions.sh\n\n# Run only the unit tests and not integration tests\ngo test -cover -race $(local_go_pkgs)\n"
  },
  {
    "path": "support/toolchain/HyperMake",
    "content": "---\nformat: hypermake.v0\n\nname: go-kcl\ndescription: VMWare Go-KCL Amazon Kinesis Client Library in Go\n\ntargets:\n  rebuild-toolchain:\n    description: build toolchain image\n    watches:\n      - docker\n    build: docker\n    cache: false\n    tags:\n      - vmware/go-kcl-toolchain:latest\n\n  push-toolchain:\n    description: push toolchain image\n    after:\n      - rebuild-toolchain\n    push:\n      - vmware/go-kcl-toolchain:latest\n\nsettings:\n  default-targets:\n    - rebuild-toolchain\n  docker:\n    image: 'vmware/go-kcl-toolchain:0.1.4'\n"
  },
  {
    "path": "support/toolchain/docker/Dockerfile",
    "content": "FROM golang:1.17\nENV PATH /go/bin:/src/bin:/root/go/bin:/usr/local/go/bin:$PATH\nENV GOPATH /go:/src\nRUN go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.42.1 && \\\n    go install golang.org/x/tools/cmd/...@latest && \\\n    go install github.com/go-delve/delve/cmd/dlv@latest && \\\n    curl -sfL https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.8.1 && \\\n    chmod -R a+rw /go\n"
  },
  {
    "path": "test/lease_stealing_util_test.go",
    "content": "package test\n\nimport (\n\t\"fmt\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/service/dynamodb\"\n\t\"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface\"\n\t\"github.com/aws/aws-sdk-go/service/kinesis/kinesisiface\"\n\t\"github.com/stretchr/testify/assert\"\n\tchk \"github.com/vmware/vmware-go-kcl/clientlibrary/checkpoint\"\n\tcfg \"github.com/vmware/vmware-go-kcl/clientlibrary/config\"\n\twk \"github.com/vmware/vmware-go-kcl/clientlibrary/worker\"\n)\n\ntype LeaseStealingTest struct {\n\tt       *testing.T\n\tconfig  *TestClusterConfig\n\tcluster *TestCluster\n\tkc      kinesisiface.KinesisAPI\n\tdc      dynamodbiface.DynamoDBAPI\n\n\tbackOffSeconds int\n\tmaxRetries     int\n}\n\nfunc NewLeaseStealingTest(t *testing.T, config *TestClusterConfig, workerFactory TestWorkerFactory) *LeaseStealingTest {\n\tcluster := NewTestCluster(t, config, workerFactory)\n\tclientConfig := cluster.workerFactory.CreateKCLConfig(\"test-client\", config)\n\treturn &LeaseStealingTest{\n\t\tt:              t,\n\t\tconfig:         config,\n\t\tcluster:        cluster,\n\t\tkc:             NewKinesisClient(t, config.regionName, clientConfig.KinesisEndpoint, clientConfig.KinesisCredentials),\n\t\tdc:             NewDynamoDBClient(t, config.regionName, clientConfig.DynamoDBEndpoint, clientConfig.KinesisCredentials),\n\t\tbackOffSeconds: 5,\n\t\tmaxRetries:     60,\n\t}\n}\n\nfunc (lst *LeaseStealingTest) WithBackoffSeconds(backoff int) *LeaseStealingTest {\n\tlst.backOffSeconds = backoff\n\treturn lst\n}\n\nfunc (lst *LeaseStealingTest) WithMaxRetries(retries int) *LeaseStealingTest {\n\tlst.maxRetries = retries\n\treturn lst\n}\n\nfunc (lst *LeaseStealingTest) publishSomeData() (stop func()) {\n\tdone := make(chan int)\n\twg := &sync.WaitGroup{}\n\n\twg.Add(1)\n\tgo func() {\n\t\tticker := time.NewTicker(500 * time.Millisecond)\n\t\tdefer wg.Done()\n\t\tdefer ticker.Stop()\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-done:\n\t\t\t\treturn\n\t\t\tcase <-ticker.C:\n\t\t\t\tlst.t.Log(\"Coninuously publishing records\")\n\t\t\t\tpublishSomeData(lst.t, lst.kc)\n\t\t\t}\n\t\t}\n\t}()\n\n\treturn func() {\n\t\tclose(done)\n\t\twg.Wait()\n\t}\n}\n\nfunc (lst *LeaseStealingTest) getShardCountByWorker() map[string]int {\n\tinput := &dynamodb.ScanInput{\n\t\tTableName: aws.String(lst.config.appName),\n\t}\n\n\tshardsByWorker := map[string]map[string]bool{}\n\terr := lst.dc.ScanPages(input, func(out *dynamodb.ScanOutput, lastPage bool) bool {\n\t\tfor _, result := range out.Items {\n\t\t\tif shardID, ok := result[chk.LeaseKeyKey]; !ok {\n\t\t\t\tcontinue\n\t\t\t} else if assignedTo, ok := result[chk.LeaseOwnerKey]; !ok {\n\t\t\t\tcontinue\n\t\t\t} else {\n\t\t\t\tif _, ok := shardsByWorker[*assignedTo.S]; !ok {\n\t\t\t\t\tshardsByWorker[*assignedTo.S] = map[string]bool{}\n\t\t\t\t}\n\t\t\t\tshardsByWorker[*assignedTo.S][*shardID.S] = true\n\t\t\t}\n\t\t}\n\t\treturn !lastPage\n\t})\n\tassert.Nil(lst.t, err)\n\n\tshardCountByWorker := map[string]int{}\n\tfor worker, shards := range shardsByWorker {\n\t\tshardCountByWorker[worker] = len(shards)\n\t}\n\treturn shardCountByWorker\n}\n\ntype LeaseStealingAssertions struct {\n\texpectedLeasesForIntialWorker int\n\texpectedLeasesPerWorker       int\n}\n\nfunc (lst *LeaseStealingTest) Run(assertions LeaseStealingAssertions) {\n\t// Publish records onto stream thoughtout the entire duration of the test\n\tstop := lst.publishSomeData()\n\tdefer stop()\n\n\t// Start worker 1\n\tworker1, _ := lst.cluster.SpawnWorker()\n\n\t// Wait until the above worker has all leases\n\tvar worker1ShardCount int\n\tfor i := 0; i < lst.maxRetries; i++ {\n\t\ttime.Sleep(time.Duration(lst.backOffSeconds) * time.Second)\n\n\t\tshardCountByWorker := lst.getShardCountByWorker()\n\t\tif shardCount, ok := shardCountByWorker[worker1]; ok && shardCount == assertions.expectedLeasesForIntialWorker {\n\t\t\tworker1ShardCount = shardCount\n\t\t\tbreak\n\t\t}\n\t}\n\n\t// Assert correct number of leases\n\tassert.Equal(lst.t, assertions.expectedLeasesForIntialWorker, worker1ShardCount)\n\n\t// Spawn Remaining Wokers\n\tfor i := 0; i < lst.config.numWorkers-1; i++ {\n\t\tlst.cluster.SpawnWorker()\n\t}\n\n\t// Wait For Rebalance\n\tvar shardCountByWorker map[string]int\n\tfor i := 0; i < lst.maxRetries; i++ {\n\t\ttime.Sleep(time.Duration(lst.backOffSeconds) * time.Second)\n\n\t\tshardCountByWorker = lst.getShardCountByWorker()\n\n\t\tcorrectCount := true\n\t\tfor _, count := range shardCountByWorker {\n\t\t\tif count != assertions.expectedLeasesPerWorker {\n\t\t\t\tcorrectCount = false\n\t\t\t}\n\t\t}\n\n\t\tif correctCount {\n\t\t\tbreak\n\t\t}\n\t}\n\n\t// Assert Rebalanced\n\tassert.Greater(lst.t, len(shardCountByWorker), 0)\n\tfor _, count := range shardCountByWorker {\n\t\tassert.Equal(lst.t, assertions.expectedLeasesPerWorker, count)\n\t}\n\n\t// Shutdown Workers\n\ttime.Sleep(10 * time.Second)\n\tlst.cluster.Shutdown()\n}\n\ntype TestWorkerFactory interface {\n\tCreateWorker(workerID string, kclConfig *cfg.KinesisClientLibConfiguration) *wk.Worker\n\tCreateKCLConfig(workerID string, config *TestClusterConfig) *cfg.KinesisClientLibConfiguration\n}\n\ntype TestClusterConfig struct {\n\tnumShards  int\n\tnumWorkers int\n\n\tappName          string\n\tstreamName       string\n\tregionName       string\n\tworkerIDTemplate string\n}\n\ntype TestCluster struct {\n\tt             *testing.T\n\tconfig        *TestClusterConfig\n\tworkerFactory TestWorkerFactory\n\tworkerIDs     []string\n\tworkers       map[string]*wk.Worker\n}\n\nfunc NewTestCluster(t *testing.T, config *TestClusterConfig, workerFactory TestWorkerFactory) *TestCluster {\n\treturn &TestCluster{\n\t\tt:             t,\n\t\tconfig:        config,\n\t\tworkerFactory: workerFactory,\n\t\tworkerIDs:     make([]string, 0),\n\t\tworkers:       make(map[string]*wk.Worker),\n\t}\n}\n\nfunc (tc *TestCluster) addWorker(workerID string, config *cfg.KinesisClientLibConfiguration) *wk.Worker {\n\tworker := tc.workerFactory.CreateWorker(workerID, config)\n\ttc.workerIDs = append(tc.workerIDs, workerID)\n\ttc.workers[workerID] = worker\n\treturn worker\n}\n\nfunc (tc *TestCluster) SpawnWorker() (string, *wk.Worker) {\n\tid := len(tc.workers)\n\tworkerID := fmt.Sprintf(tc.config.workerIDTemplate, id)\n\n\tconfig := tc.workerFactory.CreateKCLConfig(workerID, tc.config)\n\tworker := tc.addWorker(workerID, config)\n\n\terr := worker.Start()\n\tassert.Nil(tc.t, err)\n\treturn workerID, worker\n}\n\nfunc (tc *TestCluster) Shutdown() {\n\tfor workerID, worker := range tc.workers {\n\t\ttc.t.Logf(\"Shutting down worker: %v\", workerID)\n\t\tworker.Shutdown()\n\t}\n}\n"
  },
  {
    "path": "test/logger_test.go",
    "content": "/*\n * Copyright (c) 2019 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\n// Note: The implementation comes from https://www.mountedthoughts.com/golang-logger-interface/\n\npackage test\n\nimport (\n\t\"github.com/stretchr/testify/assert\"\n\n\t\"testing\"\n\n\t\"github.com/sirupsen/logrus\"\n\t\"go.uber.org/zap\"\n\n\t\"github.com/vmware/vmware-go-kcl/logger\"\n\tzaplogger \"github.com/vmware/vmware-go-kcl/logger/zap\"\n)\n\nfunc TestZapLoggerWithConfig(t *testing.T) {\n\tconfig := logger.Configuration{\n\t\tEnableConsole:     true,\n\t\tConsoleLevel:      logger.Debug,\n\t\tConsoleJSONFormat: true,\n\t\tEnableFile:        true,\n\t\tFileLevel:         logger.Info,\n\t\tFileJSONFormat:    true,\n\t\tFilename:          \"log.log\",\n\t}\n\n\tlog := zaplogger.NewZapLoggerWithConfig(config)\n\n\tcontextLogger := log.WithFields(logger.Fields{\"key1\": \"value1\"})\n\tcontextLogger.Debugf(\"Starting with zap\")\n\tcontextLogger.Infof(\"Zap is awesome\")\n}\n\nfunc TestZapLogger(t *testing.T) {\n\tzapLogger, err := zap.NewProduction()\n\tassert.Nil(t, err)\n\n\tlog := zaplogger.NewZapLogger(zapLogger.Sugar())\n\n\tcontextLogger := log.WithFields(logger.Fields{\"key1\": \"value1\"})\n\tcontextLogger.Debugf(\"Starting with zap\")\n\tcontextLogger.Infof(\"Zap is awesome\")\n}\n\nfunc TestLogrusLoggerWithConfig(t *testing.T) {\n\tconfig := logger.Configuration{\n\t\tEnableConsole:     true,\n\t\tConsoleLevel:      logger.Debug,\n\t\tConsoleJSONFormat: false,\n\t\tEnableFile:        true,\n\t\tFileLevel:         logger.Info,\n\t\tFileJSONFormat:    true,\n\t\tFilename:          \"log.log\",\n\t}\n\n\tlog := logger.NewLogrusLoggerWithConfig(config)\n\n\tcontextLogger := log.WithFields(logger.Fields{\"key1\": \"value1\"})\n\tcontextLogger.Debugf(\"Starting with logrus\")\n\tcontextLogger.Infof(\"Logrus is awesome\")\n}\n\nfunc TestLogrusLogger(t *testing.T) {\n\t// adapts to Logger interface from *logrus.Logger\n\tlog := logger.NewLogrusLogger(logrus.StandardLogger())\n\n\tcontextLogger := log.WithFields(logger.Fields{\"key1\": \"value1\"})\n\tcontextLogger.Debugf(\"Starting with logrus\")\n\tcontextLogger.Infof(\"Logrus is awesome\")\n}\n\nfunc TestLogrusLoggerWithFieldsAtInit(t *testing.T) {\n\t// adapts to Logger interface from *logrus.Entry\n\tfieldLogger := logrus.StandardLogger().WithField(\"key0\", \"value0\")\n\tlog := logger.NewLogrusLogger(fieldLogger)\n\n\tcontextLogger := log.WithFields(logger.Fields{\"key1\": \"value1\"})\n\tcontextLogger.Debugf(\"Starting with logrus\")\n\tcontextLogger.Infof(\"Structured logging is awesome\")\n}\n"
  },
  {
    "path": "test/record_processor_test.go",
    "content": "/*\n * Copyright (c) 2020 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\npackage test\n\nimport (\n\t\"testing\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/stretchr/testify/assert\"\n\tkc \"github.com/vmware/vmware-go-kcl/clientlibrary/interfaces\"\n)\n\n// Record processor factory is used to create RecordProcessor\nfunc recordProcessorFactory(t *testing.T) kc.IRecordProcessorFactory {\n\treturn &dumpRecordProcessorFactory{t: t}\n}\n\n// simple record processor and dump everything\ntype dumpRecordProcessorFactory struct {\n\tt *testing.T\n}\n\nfunc (d *dumpRecordProcessorFactory) CreateProcessor() kc.IRecordProcessor {\n\treturn &dumpRecordProcessor{\n\t\tt: d.t,\n\t}\n}\n\n// Create a dump record processor for printing out all data from record.\ntype dumpRecordProcessor struct {\n\tt     *testing.T\n\tcount int\n}\n\nfunc (dd *dumpRecordProcessor) Initialize(input *kc.InitializationInput) {\n\tdd.t.Logf(\"Processing SharId: %v at checkpoint: %v\", input.ShardId, aws.StringValue(input.ExtendedSequenceNumber.SequenceNumber))\n\tshardID = input.ShardId\n\tdd.count = 0\n}\n\nfunc (dd *dumpRecordProcessor) ProcessRecords(input *kc.ProcessRecordsInput) {\n\tdd.t.Log(\"Processing Records...\")\n\n\t// don't process empty record\n\tif len(input.Records) == 0 {\n\t\treturn\n\t}\n\n\tfor _, v := range input.Records {\n\t\tdd.t.Logf(\"Record = %s\", v.Data)\n\t\tassert.Equal(dd.t, specstr, string(v.Data))\n\t\tdd.count++\n\t}\n\n\t// checkpoint it after processing this batch.\n\t// Especially, for processing de-aggregated KPL records, checkpointing has to happen at the end of batch\n\t// because de-aggregated records share the same sequence number.\n\tlastRecordSequenceNumber := input.Records[len(input.Records)-1].SequenceNumber\n\t// Calculate the time taken from polling records and delivering to record processor for a batch.\n\tdiff := input.CacheExitTime.Sub(*input.CacheEntryTime)\n\tdd.t.Logf(\"Checkpoint progress at: %v,  MillisBehindLatest = %v, KCLProcessTime = %v\", lastRecordSequenceNumber, input.MillisBehindLatest, diff)\n\tinput.Checkpointer.Checkpoint(lastRecordSequenceNumber)\n}\n\nfunc (dd *dumpRecordProcessor) Shutdown(input *kc.ShutdownInput) {\n\tdd.t.Logf(\"Shutdown Reason: %v\", aws.StringValue(kc.ShutdownReasonMessage(input.ShutdownReason)))\n\tdd.t.Logf(\"Processed Record Count = %d\", dd.count)\n\n\t// When the value of {@link ShutdownInput#getShutdownReason()} is\n\t// {@link com.amazonaws.services.kinesis.clientlibrary.lib.worker.ShutdownReason#TERMINATE} it is required that you\n\t// checkpoint. Failure to do so will result in an IllegalArgumentException, and the KCL no longer making progress.\n\tif input.ShutdownReason == kc.TERMINATE {\n\t\tinput.Checkpointer.Checkpoint(nil)\n\t}\n\n\tassert.True(dd.t, dd.count > 0)\n}\n"
  },
  {
    "path": "test/record_publisher_test.go",
    "content": "/*\n * Copyright (c) 2020 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\npackage test\n\nimport (\n\t\"crypto/md5\"\n\t\"fmt\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/aws/credentials\"\n\t\"github.com/aws/aws-sdk-go/aws/session\"\n\t\"github.com/aws/aws-sdk-go/service/dynamodb\"\n\t\"github.com/aws/aws-sdk-go/service/kinesis\"\n\t\"github.com/aws/aws-sdk-go/service/kinesis/kinesisiface\"\n\trec \"github.com/awslabs/kinesis-aggregation/go/records\"\n\t\"github.com/golang/protobuf/proto\"\n\t\"github.com/vmware/vmware-go-kcl/clientlibrary/utils\"\n\n\t\"testing\"\n)\n\nconst specstr = `{\"name\":\"kube-qQyhk\",\"networking\":{\"containerNetworkCidr\":\"10.2.0.0/16\"},\"orgName\":\"BVT-Org-cLQch\",\"projectName\":\"project-tDSJd\",\"serviceLevel\":\"DEVELOPER\",\"size\":{\"count\":1},\"version\":\"1.8.1-4\"}`\n\n// NewKinesisClient to create a Kinesis Client.\nfunc NewKinesisClient(t *testing.T, regionName, endpoint string, credentials *credentials.Credentials) *kinesis.Kinesis {\n\ts, err := session.NewSession(&aws.Config{\n\t\tRegion:      aws.String(regionName),\n\t\tEndpoint:    aws.String(endpoint),\n\t\tCredentials: credentials,\n\t})\n\n\tif err != nil {\n\t\t// no need to move forward\n\t\tt.Fatalf(\"Failed in getting Kinesis session for creating Worker: %+v\", err)\n\t}\n\treturn kinesis.New(s)\n}\n\n// NewDynamoDBClient to create a Kinesis Client.\nfunc NewDynamoDBClient(t *testing.T, regionName, endpoint string, credentials *credentials.Credentials) *dynamodb.DynamoDB {\n\ts, err := session.NewSession(&aws.Config{\n\t\tRegion:      aws.String(regionName),\n\t\tEndpoint:    aws.String(endpoint),\n\t\tCredentials: credentials,\n\t})\n\n\tif err != nil {\n\t\t// no need to move forward\n\t\tt.Fatalf(\"Failed in getting DynamoDB session for creating Worker: %+v\", err)\n\t}\n\treturn dynamodb.New(s)\n}\n\nfunc continuouslyPublishSomeData(t *testing.T, kc kinesisiface.KinesisAPI) func() {\n\tshards := []*kinesis.Shard{}\n\tvar nextToken *string\n\tfor {\n\t\tout, err := kc.ListShards(&kinesis.ListShardsInput{\n\t\t\tStreamName: aws.String(streamName),\n\t\t\tNextToken:  nextToken,\n\t\t})\n\t\tif err != nil {\n\t\t\tt.Errorf(\"Error in ListShards. %+v\", err)\n\t\t}\n\n\t\tshards = append(shards, out.Shards...)\n\t\tif out.NextToken == nil {\n\t\t\tbreak\n\t\t}\n\t\tnextToken = out.NextToken\n\t}\n\n\tdone := make(chan int)\n\twg := &sync.WaitGroup{}\n\n\twg.Add(1)\n\tgo func() {\n\t\tdefer wg.Done()\n\t\tticker := time.NewTicker(500 * time.Millisecond)\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase <-done:\n\t\t\t\treturn\n\t\t\tcase <-ticker.C:\n\t\t\t\tpublishToAllShards(t, kc, shards)\n\t\t\t\tpublishSomeData(t, kc)\n\t\t\t}\n\t\t}\n\t}()\n\n\treturn func() {\n\t\tclose(done)\n\t\twg.Wait()\n\t}\n}\n\nfunc publishToAllShards(t *testing.T, kc kinesisiface.KinesisAPI, shards []*kinesis.Shard) {\n\t// Put records to all shards\n\tfor i := 0; i < 10; i++ {\n\t\tfor _, shard := range shards {\n\t\t\tpublishRecord(t, kc, shard.HashKeyRange.StartingHashKey)\n\t\t}\n\t}\n}\n\n// publishSomeData to put some records into Kinesis stream\nfunc publishSomeData(t *testing.T, kc kinesisiface.KinesisAPI) {\n\t// Put some data into stream.\n\tt.Log(\"Putting data into stream using PutRecord API...\")\n\tfor i := 0; i < 50; i++ {\n\t\tpublishRecord(t, kc, nil)\n\t}\n\tt.Log(\"Done putting data into stream using PutRecord API.\")\n\n\t// Put some data into stream using PutRecords API\n\tt.Log(\"Putting data into stream using PutRecords API...\")\n\tfor i := 0; i < 10; i++ {\n\t\tpublishRecords(t, kc)\n\t}\n\tt.Log(\"Done putting data into stream using PutRecords API.\")\n\n\t// Put some data into stream using KPL Aggregate Record format\n\tt.Log(\"Putting data into stream using KPL Aggregate Record ...\")\n\tfor i := 0; i < 10; i++ {\n\t\tpublishAggregateRecord(t, kc)\n\t}\n\tt.Log(\"Done putting data into stream using KPL Aggregate Record.\")\n}\n\n// publishRecord to put a record into Kinesis stream using PutRecord API.\nfunc publishRecord(t *testing.T, kc kinesisiface.KinesisAPI, hashKey *string) {\n\tinput := &kinesis.PutRecordInput{\n\t\tData:         []byte(specstr),\n\t\tStreamName:   aws.String(streamName),\n\t\tPartitionKey: aws.String(utils.RandStringBytesMaskImpr(10)),\n\t}\n\tif hashKey != nil {\n\t\tinput.ExplicitHashKey = hashKey\n\t}\n\t// Use random string as partition key to ensure even distribution across shards\n\t_, err := kc.PutRecord(input)\n\n\tif err != nil {\n\t\tt.Errorf(\"Error in PutRecord. %+v\", err)\n\t}\n}\n\n// publishRecord to put a record into Kinesis stream using PutRecords API.\nfunc publishRecords(t *testing.T, kc kinesisiface.KinesisAPI) {\n\t// Use random string as partition key to ensure even distribution across shards\n\trecords := make([]*kinesis.PutRecordsRequestEntry, 5)\n\n\tfor i := 0; i < 5; i++ {\n\t\trecord := &kinesis.PutRecordsRequestEntry{\n\t\t\tData:         []byte(specstr),\n\t\t\tPartitionKey: aws.String(utils.RandStringBytesMaskImpr(10)),\n\t\t}\n\t\trecords[i] = record\n\t}\n\n\t_, err := kc.PutRecords(&kinesis.PutRecordsInput{\n\t\tRecords:    records,\n\t\tStreamName: aws.String(streamName),\n\t})\n\n\tif err != nil {\n\t\tt.Errorf(\"Error in PutRecords. %+v\", err)\n\t}\n}\n\n// publishRecord to put a record into Kinesis stream using PutRecord API.\nfunc publishAggregateRecord(t *testing.T, kc kinesisiface.KinesisAPI) {\n\tdata := generateAggregateRecord(5, specstr)\n\t// Use random string as partition key to ensure even distribution across shards\n\t_, err := kc.PutRecord(&kinesis.PutRecordInput{\n\t\tData:         data,\n\t\tStreamName:   aws.String(streamName),\n\t\tPartitionKey: aws.String(utils.RandStringBytesMaskImpr(10)),\n\t})\n\n\tif err != nil {\n\t\tt.Errorf(\"Error in PutRecord. %+v\", err)\n\t}\n}\n\n// generateAggregateRecord generates an aggregate record in the correct AWS-specified format used by KPL.\n// https://github.com/awslabs/amazon-kinesis-producer/blob/master/aggregation-format.md\n// copy from: https://github.com/awslabs/kinesis-aggregation/blob/master/go/deaggregator/deaggregator_test.go\nfunc generateAggregateRecord(numRecords int, content string) []byte {\n\taggr := &rec.AggregatedRecord{}\n\t// Start with the magic header\n\taggRecord := []byte(\"\\xf3\\x89\\x9a\\xc2\")\n\tpartKeyTable := make([]string, 0)\n\n\t// Create proto record with numRecords length\n\tfor i := 0; i < numRecords; i++ {\n\t\tvar partKey uint64\n\t\tvar hashKey uint64\n\t\tpartKey = uint64(i)\n\t\thashKey = uint64(i) * uint64(10)\n\t\tr := &rec.Record{\n\t\t\tPartitionKeyIndex:    &partKey,\n\t\t\tExplicitHashKeyIndex: &hashKey,\n\t\t\tData:                 []byte(content),\n\t\t\tTags:                 make([]*rec.Tag, 0),\n\t\t}\n\n\t\taggr.Records = append(aggr.Records, r)\n\t\tpartKeyVal := fmt.Sprint(i)\n\t\tpartKeyTable = append(partKeyTable, partKeyVal)\n\t}\n\n\taggr.PartitionKeyTable = partKeyTable\n\t// Marshal to protobuf record, create md5 sum from proto record\n\t// and append both to aggRecord with magic header\n\tdata, _ := proto.Marshal(aggr)\n\tmd5Hash := md5.Sum(data)\n\taggRecord = append(aggRecord, data...)\n\taggRecord = append(aggRecord, md5Hash[:]...)\n\treturn aggRecord\n}\n"
  },
  {
    "path": "test/worker_custom_test.go",
    "content": "/*\n * Copyright (c) 2018 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\npackage test\n\nimport (\n\t\"os\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/aws/session\"\n\t\"github.com/aws/aws-sdk-go/service/kinesis\"\n\tlog \"github.com/sirupsen/logrus\"\n\t\"github.com/stretchr/testify/assert\"\n\n\tchk \"github.com/vmware/vmware-go-kcl/clientlibrary/checkpoint\"\n\tcfg \"github.com/vmware/vmware-go-kcl/clientlibrary/config\"\n\tpar \"github.com/vmware/vmware-go-kcl/clientlibrary/partition\"\n\twk \"github.com/vmware/vmware-go-kcl/clientlibrary/worker\"\n)\n\nfunc TestWorkerInjectCheckpointer(t *testing.T) {\n\tkclConfig := cfg.NewKinesisClientLibConfig(appName, streamName, regionName, workerID).\n\t\tWithInitialPositionInStream(cfg.LATEST).\n\t\tWithMaxRecords(10).\n\t\tWithMaxLeasesForWorker(1).\n\t\tWithShardSyncIntervalMillis(5000).\n\t\tWithFailoverTimeMillis(300000)\n\tlog.SetOutput(os.Stdout)\n\tlog.SetLevel(log.DebugLevel)\n\n\tassert.Equal(t, regionName, kclConfig.RegionName)\n\tassert.Equal(t, streamName, kclConfig.StreamName)\n\n\t// configure cloudwatch as metrics system\n\tkclConfig.WithMonitoringService(getMetricsConfig(kclConfig, metricsSystem))\n\n\t// Put some data into stream.\n\tkc := NewKinesisClient(t, regionName, kclConfig.KinesisEndpoint, kclConfig.KinesisCredentials)\n\t// publishSomeData(t, kc)\n\tstop := continuouslyPublishSomeData(t, kc)\n\tdefer stop()\n\n\t// custom checkpointer or a mock checkpointer.\n\tcheckpointer := chk.NewDynamoCheckpoint(kclConfig)\n\n\t// Inject a custom checkpointer into the worker.\n\tworker := wk.NewWorker(recordProcessorFactory(t), kclConfig).\n\t\tWithCheckpointer(checkpointer)\n\n\terr := worker.Start()\n\tassert.Nil(t, err)\n\n\t// wait a few seconds before shutdown processing\n\ttime.Sleep(30 * time.Second)\n\tworker.Shutdown()\n\n\t// verify the checkpointer after graceful shutdown\n\tstatus := &par.ShardStatus{\n\t\tID:  shardID,\n\t\tMux: &sync.RWMutex{},\n\t}\n\tcheckpointer.FetchCheckpoint(status)\n\n\t// checkpointer should be the same\n\tassert.NotEmpty(t, status.Checkpoint)\n\n\t// Only the lease owner has been wiped out\n\tassert.Equal(t, \"\", status.GetLeaseOwner())\n\n}\n\nfunc TestWorkerInjectKinesis(t *testing.T) {\n\tkclConfig := cfg.NewKinesisClientLibConfig(appName, streamName, regionName, workerID).\n\t\tWithInitialPositionInStream(cfg.LATEST).\n\t\tWithMaxRecords(10).\n\t\tWithMaxLeasesForWorker(1).\n\t\tWithShardSyncIntervalMillis(5000).\n\t\tWithFailoverTimeMillis(300000)\n\n\tlog.SetOutput(os.Stdout)\n\tlog.SetLevel(log.DebugLevel)\n\n\tassert.Equal(t, regionName, kclConfig.RegionName)\n\tassert.Equal(t, streamName, kclConfig.StreamName)\n\n\t// configure cloudwatch as metrics system\n\tkclConfig.WithMonitoringService(getMetricsConfig(kclConfig, metricsSystem))\n\n\t// create custom Kinesis\n\ts, err := session.NewSession(&aws.Config{\n\t\tRegion: aws.String(regionName),\n\t})\n\tassert.Nil(t, err)\n\tkc := kinesis.New(s)\n\n\t// Put some data into stream.\n\t// publishSomeData(t, kc)\n\tstop := continuouslyPublishSomeData(t, kc)\n\tdefer stop()\n\n\t// Inject a custom checkpointer into the worker.\n\tworker := wk.NewWorker(recordProcessorFactory(t), kclConfig).\n\t\tWithKinesis(kc)\n\n\terr = worker.Start()\n\tassert.Nil(t, err)\n\n\t// wait a few seconds before shutdown processing\n\ttime.Sleep(30 * time.Second)\n\tworker.Shutdown()\n}\n\nfunc TestWorkerInjectKinesisAndCheckpointer(t *testing.T) {\n\tkclConfig := cfg.NewKinesisClientLibConfig(appName, streamName, regionName, workerID).\n\t\tWithInitialPositionInStream(cfg.LATEST).\n\t\tWithMaxRecords(10).\n\t\tWithMaxLeasesForWorker(1).\n\t\tWithShardSyncIntervalMillis(5000).\n\t\tWithFailoverTimeMillis(300000)\n\n\tlog.SetOutput(os.Stdout)\n\tlog.SetLevel(log.DebugLevel)\n\n\tassert.Equal(t, regionName, kclConfig.RegionName)\n\tassert.Equal(t, streamName, kclConfig.StreamName)\n\n\t// configure cloudwatch as metrics system\n\tkclConfig.WithMonitoringService(getMetricsConfig(kclConfig, metricsSystem))\n\n\t// create custom Kinesis\n\ts, err := session.NewSession(&aws.Config{\n\t\tRegion: aws.String(regionName),\n\t})\n\tassert.Nil(t, err)\n\tkc := kinesis.New(s)\n\n\t// Put some data into stream.\n\t// publishSomeData(t, kc)\n\tstop := continuouslyPublishSomeData(t, kc)\n\tdefer stop()\n\n\t// custom checkpointer or a mock checkpointer.\n\tcheckpointer := chk.NewDynamoCheckpoint(kclConfig)\n\n\t// Inject both custom checkpointer and kinesis into the worker.\n\tworker := wk.NewWorker(recordProcessorFactory(t), kclConfig).\n\t\tWithKinesis(kc).\n\t\tWithCheckpointer(checkpointer)\n\n\terr = worker.Start()\n\tassert.Nil(t, err)\n\n\t// wait a few seconds before shutdown processing\n\ttime.Sleep(30 * time.Second)\n\tworker.Shutdown()\n}\n"
  },
  {
    "path": "test/worker_lease_stealing_test.go",
    "content": "package test\n\nimport (\n\t\"testing\"\n\n\tchk \"github.com/vmware/vmware-go-kcl/clientlibrary/checkpoint\"\n\tcfg \"github.com/vmware/vmware-go-kcl/clientlibrary/config\"\n\twk \"github.com/vmware/vmware-go-kcl/clientlibrary/worker\"\n\t\"github.com/vmware/vmware-go-kcl/logger\"\n)\n\nfunc TestLeaseStealing(t *testing.T) {\n\tconfig := &TestClusterConfig{\n\t\tnumShards:        4,\n\t\tnumWorkers:       2,\n\t\tappName:          appName,\n\t\tstreamName:       streamName,\n\t\tregionName:       regionName,\n\t\tworkerIDTemplate: workerID + \"-%v\",\n\t}\n\ttest := NewLeaseStealingTest(t, config, newLeaseStealingWorkerFactory(t))\n\ttest.Run(LeaseStealingAssertions{\n\t\texpectedLeasesForIntialWorker: config.numShards,\n\t\texpectedLeasesPerWorker:       config.numShards / config.numWorkers,\n\t})\n}\n\ntype leaseStealingWorkerFactory struct {\n\tt *testing.T\n}\n\nfunc newLeaseStealingWorkerFactory(t *testing.T) *leaseStealingWorkerFactory {\n\treturn &leaseStealingWorkerFactory{t}\n}\n\nfunc (wf *leaseStealingWorkerFactory) CreateKCLConfig(workerID string, config *TestClusterConfig) *cfg.KinesisClientLibConfiguration {\n\tlog := logger.NewLogrusLoggerWithConfig(logger.Configuration{\n\t\tEnableConsole:     true,\n\t\tConsoleLevel:      logger.Error,\n\t\tConsoleJSONFormat: false,\n\t\tEnableFile:        true,\n\t\tFileLevel:         logger.Info,\n\t\tFileJSONFormat:    true,\n\t\tFilename:          \"log.log\",\n\t})\n\n\tlog.WithFields(logger.Fields{\"worker\": workerID})\n\n\treturn cfg.NewKinesisClientLibConfig(config.appName, config.streamName, config.regionName, workerID).\n\t\tWithInitialPositionInStream(cfg.LATEST).\n\t\tWithMaxRecords(10).\n\t\tWithShardSyncIntervalMillis(5000).\n\t\tWithFailoverTimeMillis(10000).\n\t\tWithLeaseStealing(true).\n\t\tWithLogger(log)\n}\n\nfunc (wf *leaseStealingWorkerFactory) CreateWorker(workerID string, kclConfig *cfg.KinesisClientLibConfiguration) *wk.Worker {\n\tworker := wk.NewWorker(recordProcessorFactory(wf.t), kclConfig)\n\treturn worker\n}\n\nfunc TestLeaseStealingInjectCheckpointer(t *testing.T) {\n\tconfig := &TestClusterConfig{\n\t\tnumShards:        4,\n\t\tnumWorkers:       2,\n\t\tappName:          appName,\n\t\tstreamName:       streamName,\n\t\tregionName:       regionName,\n\t\tworkerIDTemplate: workerID + \"-%v\",\n\t}\n\ttest := NewLeaseStealingTest(t, config, newleaseStealingWorkerFactoryCustomChk(t))\n\ttest.Run(LeaseStealingAssertions{\n\t\texpectedLeasesForIntialWorker: config.numShards,\n\t\texpectedLeasesPerWorker:       config.numShards / config.numWorkers,\n\t})\n}\n\ntype leaseStealingWorkerFactoryCustom struct {\n\t*leaseStealingWorkerFactory\n}\n\nfunc newleaseStealingWorkerFactoryCustomChk(t *testing.T) *leaseStealingWorkerFactoryCustom {\n\treturn &leaseStealingWorkerFactoryCustom{\n\t\tnewLeaseStealingWorkerFactory(t),\n\t}\n}\n\nfunc (wfc *leaseStealingWorkerFactoryCustom) CreateWorker(workerID string, kclConfig *cfg.KinesisClientLibConfiguration) *wk.Worker {\n\tworker := wfc.leaseStealingWorkerFactory.CreateWorker(workerID, kclConfig)\n\tcheckpointer := chk.NewDynamoCheckpoint(kclConfig)\n\treturn worker.WithCheckpointer(checkpointer)\n}\n\nfunc TestLeaseStealingWithMaxLeasesForWorker(t *testing.T) {\n\tconfig := &TestClusterConfig{\n\t\tnumShards:        4,\n\t\tnumWorkers:       2,\n\t\tappName:          appName,\n\t\tstreamName:       streamName,\n\t\tregionName:       regionName,\n\t\tworkerIDTemplate: workerID + \"-%v\",\n\t}\n\ttest := NewLeaseStealingTest(t, config, newleaseStealingWorkerFactoryMaxLeases(t, config.numShards-1))\n\ttest.Run(LeaseStealingAssertions{\n\t\texpectedLeasesForIntialWorker: config.numShards - 1,\n\t\texpectedLeasesPerWorker:       2,\n\t})\n}\n\ntype leaseStealingWorkerFactoryMaxLeases struct {\n\tmaxLeases int\n\t*leaseStealingWorkerFactory\n}\n\nfunc newleaseStealingWorkerFactoryMaxLeases(t *testing.T, maxLeases int) *leaseStealingWorkerFactoryMaxLeases {\n\treturn &leaseStealingWorkerFactoryMaxLeases{\n\t\tmaxLeases,\n\t\tnewLeaseStealingWorkerFactory(t),\n\t}\n}\n\nfunc (wfm *leaseStealingWorkerFactoryMaxLeases) CreateKCLConfig(workerID string, config *TestClusterConfig) *cfg.KinesisClientLibConfiguration {\n\tkclConfig := wfm.leaseStealingWorkerFactory.CreateKCLConfig(workerID, config)\n\tkclConfig.WithMaxLeasesForWorker(wfm.maxLeases)\n\treturn kclConfig\n}\n"
  },
  {
    "path": "test/worker_test.go",
    "content": "/*\n * Copyright (c) 2018 VMware, Inc.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and\n * associated documentation files (the \"Software\"), to deal in the Software without restriction, including\n * without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell\n * copies of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial\n * portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT\n * NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\n * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,\n * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\n * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */\npackage test\n\nimport (\n\t\"net/http\"\n\t\"os\"\n\t\"os/signal\"\n\t\"syscall\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aws/credentials\"\n\t\"github.com/aws/aws-sdk-go/aws/credentials/stscreds\"\n\t\"github.com/aws/aws-sdk-go/aws/session\"\n\t\"github.com/prometheus/common/expfmt\"\n\t\"github.com/stretchr/testify/assert\"\n\n\tcfg \"github.com/vmware/vmware-go-kcl/clientlibrary/config\"\n\t\"github.com/vmware/vmware-go-kcl/clientlibrary/metrics\"\n\t\"github.com/vmware/vmware-go-kcl/clientlibrary/metrics/cloudwatch\"\n\t\"github.com/vmware/vmware-go-kcl/clientlibrary/metrics/prometheus\"\n\twk \"github.com/vmware/vmware-go-kcl/clientlibrary/worker\"\n\t\"github.com/vmware/vmware-go-kcl/logger\"\n\tzaplogger \"github.com/vmware/vmware-go-kcl/logger/zap\"\n)\n\nconst (\n\tappName      = \"appName\"\n\tstreamName   = \"kcl-test\"\n\tregionName   = \"us-west-2\"\n\tworkerID     = \"test-worker\"\n\tconsumerName = \"enhanced-fan-out-consumer\"\n)\n\nconst metricsSystem = \"cloudwatch\"\n\nvar shardID string\n\nfunc TestWorker(t *testing.T) {\n\t// At minimal. use standard logrus logger\n\t// log := logger.NewLogrusLogger(logrus.StandardLogger())\n\t//\n\t// In order to have precise control over logging. Use logger with config\n\tconfig := logger.Configuration{\n\t\tEnableConsole:     true,\n\t\tConsoleLevel:      logger.Error,\n\t\tConsoleJSONFormat: false,\n\t\tEnableFile:        true,\n\t\tFileLevel:         logger.Info,\n\t\tFileJSONFormat:    true,\n\t\tFilename:          \"log.log\",\n\t}\n\t// Use logrus logger\n\tlog := logger.NewLogrusLoggerWithConfig(config)\n\n\tkclConfig := cfg.NewKinesisClientLibConfig(appName, streamName, regionName, workerID).\n\t\tWithInitialPositionInStream(cfg.LATEST).\n\t\tWithMaxRecords(8).\n\t\tWithMaxLeasesForWorker(1).\n\t\tWithShardSyncIntervalMillis(5000).\n\t\tWithFailoverTimeMillis(300000).\n\t\tWithLogger(log)\n\n\trunTest(kclConfig, false, t)\n}\n\nfunc TestWorkerWithTimestamp(t *testing.T) {\n\t// In order to have precise control over logging. Use logger with config\n\tconfig := logger.Configuration{\n\t\tEnableConsole:     true,\n\t\tConsoleLevel:      logger.Debug,\n\t\tConsoleJSONFormat: false,\n\t}\n\t// Use logrus logger\n\tlog := logger.NewLogrusLoggerWithConfig(config)\n\n\tts := time.Now().Add(time.Second * 5)\n\tkclConfig := cfg.NewKinesisClientLibConfig(appName, streamName, regionName, workerID).\n\t\tWithTimestampAtInitialPositionInStream(&ts).\n\t\tWithMaxRecords(10).\n\t\tWithMaxLeasesForWorker(1).\n\t\tWithShardSyncIntervalMillis(5000).\n\t\tWithFailoverTimeMillis(300000).\n\t\tWithLogger(log)\n\n\trunTest(kclConfig, false, t)\n}\n\nfunc TestWorkerWithSigInt(t *testing.T) {\n\t// At miminal. use standard zap logger\n\t//zapLogger, err := zap.NewProduction()\n\t//assert.Nil(t, err)\n\t//log := zaplogger.NewZapLogger(zapLogger.Sugar())\n\t//\n\t// In order to have precise control over logging. Use logger with config.\n\tconfig := logger.Configuration{\n\t\tEnableConsole:     true,\n\t\tConsoleLevel:      logger.Debug,\n\t\tConsoleJSONFormat: true,\n\t\tEnableFile:        true,\n\t\tFileLevel:         logger.Info,\n\t\tFileJSONFormat:    true,\n\t\tFilename:          \"log.log\",\n\t}\n\t// use zap logger\n\tlog := zaplogger.NewZapLoggerWithConfig(config)\n\n\tkclConfig := cfg.NewKinesisClientLibConfig(appName, streamName, regionName, workerID).\n\t\tWithInitialPositionInStream(cfg.LATEST).\n\t\tWithMaxRecords(10).\n\t\tWithMaxLeasesForWorker(1).\n\t\tWithShardSyncIntervalMillis(5000).\n\t\tWithFailoverTimeMillis(300000).\n\t\tWithLogger(log)\n\n\trunTest(kclConfig, true, t)\n}\n\nfunc TestWorkerStatic(t *testing.T) {\n\tt.Skip(\"Need to provide actual credentials\")\n\n\t// Fill in the credentials for accessing Kinesis and DynamoDB.\n\t// Note: use empty string as SessionToken for long-term credentials.\n\tcreds := credentials.NewStaticCredentials(\"AccessKeyId\", \"SecretAccessKey\", \"SessionToken\")\n\n\tkclConfig := cfg.NewKinesisClientLibConfigWithCredential(appName, streamName, regionName, workerID, creds).\n\t\tWithInitialPositionInStream(cfg.LATEST).\n\t\tWithMaxRecords(10).\n\t\tWithMaxLeasesForWorker(1).\n\t\tWithShardSyncIntervalMillis(5000).\n\t\tWithFailoverTimeMillis(300000)\n\n\trunTest(kclConfig, false, t)\n}\n\nfunc TestWorkerAssumeRole(t *testing.T) {\n\tt.Skip(\"Need to provide actual roleARN\")\n\n\t// Initial credentials loaded from SDK's default credential chain. Such as\n\t// the environment, shared credentials (~/.aws/credentials), or EC2 Instance\n\t// Role. These credentials will be used to to make the STS Assume Role API.\n\tsess := session.Must(session.NewSession())\n\n\t// Create the credentials from AssumeRoleProvider to assume the role\n\t// referenced by the \"myRoleARN\" ARN.\n\tcreds := stscreds.NewCredentials(sess, \"arn:aws:iam::*:role/kcl-test-publisher\")\n\n\tkclConfig := cfg.NewKinesisClientLibConfigWithCredential(appName, streamName, regionName, workerID, creds).\n\t\tWithInitialPositionInStream(cfg.LATEST).\n\t\tWithMaxRecords(10).\n\t\tWithMaxLeasesForWorker(1).\n\t\tWithShardSyncIntervalMillis(5000).\n\t\tWithFailoverTimeMillis(300000)\n\n\trunTest(kclConfig, false, t)\n}\n\nfunc TestEnhancedFanOutConsumer(t *testing.T) {\n\t// At miminal, use standard logrus logger\n\t// log := logger.NewLogrusLogger(logrus.StandardLogger())\n\t//\n\t// In order to have precise control over logging. Use logger with config\n\tconfig := logger.Configuration{\n\t\tEnableConsole:     true,\n\t\tConsoleLevel:      logger.Debug,\n\t\tConsoleJSONFormat: false,\n\t\tEnableFile:        true,\n\t\tFileLevel:         logger.Info,\n\t\tFileJSONFormat:    true,\n\t\tFilename:          \"log.log\",\n\t}\n\t// Use logrus logger\n\tlog := logger.NewLogrusLoggerWithConfig(config)\n\n\tkclConfig := cfg.NewKinesisClientLibConfig(appName, streamName, regionName, workerID).\n\t\tWithInitialPositionInStream(cfg.LATEST).\n\t\tWithEnhancedFanOutConsumerName(consumerName).\n\t\tWithMaxRecords(10).\n\t\tWithMaxLeasesForWorker(1).\n\t\tWithShardSyncIntervalMillis(5000).\n\t\tWithFailoverTimeMillis(300000).\n\t\tWithLogger(log)\n\n\trunTest(kclConfig, false, t)\n}\n\nfunc TestEnhancedFanOutConsumerDefaultConsumerName(t *testing.T) {\n\t// At miminal, use standard logrus logger\n\t// log := logger.NewLogrusLogger(logrus.StandardLogger())\n\t//\n\t// In order to have precise control over logging. Use logger with config\n\tconfig := logger.Configuration{\n\t\tEnableConsole:     true,\n\t\tConsoleLevel:      logger.Debug,\n\t\tConsoleJSONFormat: false,\n\t\tEnableFile:        true,\n\t\tFileLevel:         logger.Info,\n\t\tFileJSONFormat:    true,\n\t\tFilename:          \"log.log\",\n\t}\n\t// Use logrus logger\n\tlog := logger.NewLogrusLoggerWithConfig(config)\n\n\tkclConfig := cfg.NewKinesisClientLibConfig(appName, streamName, regionName, workerID).\n\t\tWithInitialPositionInStream(cfg.LATEST).\n\t\tWithEnhancedFanOutConsumer(true).\n\t\tWithMaxRecords(10).\n\t\tWithMaxLeasesForWorker(1).\n\t\tWithShardSyncIntervalMillis(5000).\n\t\tWithFailoverTimeMillis(300000).\n\t\tWithLogger(log)\n\n\trunTest(kclConfig, false, t)\n}\n\nfunc TestEnhancedFanOutConsumerARN(t *testing.T) {\n\tt.Skip(\"Need to provide actual consumerARN\")\n\n\tconsumerARN := \"arn:aws:kinesis:*:stream/kcl-test/consumer/fanout-poc-consumer-test:*\"\n\t// At miminal, use standard logrus logger\n\t// log := logger.NewLogrusLogger(logrus.StandardLogger())\n\t//\n\t// In order to have precise control over logging. Use logger with config\n\tconfig := logger.Configuration{\n\t\tEnableConsole:     true,\n\t\tConsoleLevel:      logger.Debug,\n\t\tConsoleJSONFormat: false,\n\t\tEnableFile:        true,\n\t\tFileLevel:         logger.Info,\n\t\tFileJSONFormat:    true,\n\t\tFilename:          \"log.log\",\n\t}\n\t// Use logrus logger\n\tlog := logger.NewLogrusLoggerWithConfig(config)\n\n\tkclConfig := cfg.NewKinesisClientLibConfig(appName, streamName, regionName, workerID).\n\t\tWithInitialPositionInStream(cfg.LATEST).\n\t\tWithEnhancedFanOutConsumerARN(consumerARN).\n\t\tWithMaxRecords(10).\n\t\tWithMaxLeasesForWorker(1).\n\t\tWithShardSyncIntervalMillis(5000).\n\t\tWithFailoverTimeMillis(300000).\n\t\tWithLogger(log)\n\n\trunTest(kclConfig, false, t)\n}\n\nfunc runTest(kclConfig *cfg.KinesisClientLibConfiguration, triggersig bool, t *testing.T) {\n\tassert.Equal(t, regionName, kclConfig.RegionName)\n\tassert.Equal(t, streamName, kclConfig.StreamName)\n\n\t// configure cloudwatch as metrics system\n\tkclConfig.WithMonitoringService(getMetricsConfig(kclConfig, metricsSystem))\n\n\t// Put some data into stream.\n\tkc := NewKinesisClient(t, regionName, kclConfig.KinesisEndpoint, kclConfig.KinesisCredentials)\n\t// publishSomeData(t, kc)\n\tstop := continuouslyPublishSomeData(t, kc)\n\tdefer stop()\n\n\tworker := wk.NewWorker(recordProcessorFactory(t), kclConfig)\n\terr := worker.Start()\n\tassert.Nil(t, err)\n\n\tsigs := make(chan os.Signal, 1)\n\tsignal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)\n\n\t// Signal processing.\n\tgo func() {\n\t\tsig := <-sigs\n\t\tt.Logf(\"Received signal %s. Exiting\", sig)\n\t\tworker.Shutdown()\n\t\t// some other processing before exit.\n\t\t//os.Exit(0)\n\t}()\n\n\tif triggersig {\n\t\tt.Log(\"Trigger signal SIGINT\")\n\t\tp, _ := os.FindProcess(os.Getpid())\n\t\tp.Signal(os.Interrupt)\n\t}\n\n\t// wait a few seconds before shutdown processing\n\ttime.Sleep(30 * time.Second)\n\n\tif metricsSystem == \"prometheus\" {\n\t\tres, err := http.Get(\"http://localhost:8080/metrics\")\n\t\tif err != nil {\n\t\t\tt.Fatalf(\"Error scraping Prometheus endpoint %s\", err)\n\t\t}\n\n\t\tvar parser expfmt.TextParser\n\t\tparsed, err := parser.TextToMetricFamilies(res.Body)\n\t\tres.Body.Close()\n\t\tif err != nil {\n\t\t\tt.Errorf(\"Error reading monitoring response %s\", err)\n\t\t}\n\t\tt.Logf(\"Prometheus: %+v\", parsed)\n\n\t}\n\n\tt.Log(\"Calling normal shutdown at the end of application.\")\n\tworker.Shutdown()\n}\n\n// configure different metrics system\nfunc getMetricsConfig(kclConfig *cfg.KinesisClientLibConfiguration, service string) metrics.MonitoringService {\n\n\tif service == \"cloudwatch\" {\n\t\treturn cloudwatch.NewMonitoringServiceWithOptions(kclConfig.RegionName,\n\t\t\tkclConfig.KinesisCredentials,\n\t\t\tkclConfig.Logger,\n\t\t\tcloudwatch.DEFAULT_CLOUDWATCH_METRICS_BUFFER_DURATION)\n\t}\n\n\tif service == \"prometheus\" {\n\t\treturn prometheus.NewMonitoringService(\":8080\", regionName, kclConfig.Logger)\n\t}\n\n\treturn nil\n}\n"
  }
]