[ { "path": ".gitignore", "content": "# Windows Ignores\nThumbs.db\nehthumbs.db\nehthumbs_vista.db\n*.stackdump\n[Dd]esktop.ini\n$RECYCLE.BIN/\n*.cab\n*.msi\n*.msix\n*.msm\n*.msp\n*.lnk\n*.rdb\n# macOS Ignores\n.DS_Store\n.AppleDouble\n.LSOverride\nIcon\n._*\n.DocumentRevisions-V100\n.fseventsd\n.Spotlight-V100\n.TemporaryItems\n.Trashes\n.VolumeIcon.icns\n.com.apple.timemachine.donotpresent\n.AppleDB\n.AppleDesktop\nNetwork Trash Folder\nTemporary Items\n.apdisk\n\n# Text Editor Ignores\n*~\n\\#*\\#\n.\\#*\n*.swp\n[._]*.un~\n[._]*.s[a-v][a-z]\n[._]*.sw[a-p]\n[._]s[a-rt-v][a-z]\n[._]ss[a-gi-z]\n[._]sw[a-p]\ntags\nTAGS\n*.sublime-workspace\n*.sublime-project\n\n# IDE Ignores\n/.idea\n*.iml\n.vscode/\n\n# Go Ignores\n.exe\n*.exe~\n*.dll\n*.so\n*.dylib\n*.test\n*.out\n\n# Project Ignores\ndev/\ntmp/\ntemp/\nout/\nbuild/\nvendor/\n.vendor-new/\n\n/**/application.yml\nexpt/\n\ncmd/debug\napi/*.*\n\nweaver.conf.yaml" }, { "path": ".travis.yml", "content": "language: go\n\ngo: 1.11.5\n\nenv:\n global:\n - ETCD_VER=v3.3.0\n - GO111MODULE=on\n - DOCKER_LATEST=latest\n\nmatrix:\n exclude:\n go: 1.11.5\n\nsetup_etcd: &setup_etcd\n before_script:\n - curl -L https://github.com/coreos/etcd/releases/download/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz\n - mkdir -p /tmp/etcd\n - tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /tmp/etcd --strip-components=1\n - /tmp/etcd/etcd --advertise-client-urls http://127.0.0.1:12379 --listen-client-urls http://127.0.0.1:12379 > /dev/null &\n\nbuild_docker_image: &build_docker_image\n before_script:\n - |\n docker-compose build dev_weaver\n [ ! -z $TRAVIS_TAG ] && docker tag weaver_dev_weaver:$DOCKER_LATEST gojektech/weaver:$TRAVIS_TAG\n [ ! -z $TRAVIS_TAG ] && docker tag gojektech/weaver:$TRAVIS_TAG gojektech/weaver:$DOCKER_LATEST\n [ ! -z $DOCKER_PASSWORD ] && echo \"$DOCKER_PASSWORD\" | docker login -u \"$DOCKER_USERNAME\" --password-stdin\n docker images\n\nservices:\n - docker\n\nstages:\n - test\n - name: deploy\n if: (repo == gojektech/weaver) AND (branch == master) AND (tag IS present)\n\n# travis is stupid to do its own custom install and script things\n# This is the only way to skip them from running\ninstall: echo \"Skip global installing...\"\nscript: echo \"Skip global script...\"\n\njobs:\n include:\n - stage: test\n - name: \"Make Spec\"\n <<: *setup_etcd\n script: make test\n - name: \"Docker Spec\"\n <<: *build_docker_image\n script: make docker-spec\n after_script: make docker-clean\n\n - stage: deploy\n - name: \"Release Builds\"\n script: curl -sL https://git.io/goreleaser | bash -s -- --rm-dist --skip-validate\n - name: \"Docker Builds\"\n <<: *build_docker_image\n script: docker push gojektech/weaver:$TRAVIS_TAG && docker push gojektech/weaver:$DOCKER_LATEST\n\nnotifications:\n email: false\n\n" }, { "path": "CONTRIBUTING.md", "content": "# Weaver - Contributing\n\nWeaver `github.com/gojektech/weaver` is an open-source project. \nIt is licensed using the [Apache License 2.0][1]. \nWe appreciate pull requests; here are our guidelines:\n\n1. [File an issue][2] \n (if there isn't one already). If your patch\n is going to be large it might be a good idea to get the\n discussion started early. We are happy to discuss it in a\n new issue beforehand, and you can always email\n about future work.\n\n2. Please use [Effective Go Community Guidelines][3].\n\n3. We ask that you squash all the commits together before\n pushing and that your commit message references the bug.\n\n## Issue Reporting\n- Check that the issue has not already been reported.\n- Be clear, concise and precise in your description of the problem.\n- Open an issue with a descriptive title and a summary in grammatically correct,\n complete sentences.\n- Include any relevant code to the issue summary.\n\n## Pull Requests\n- Please read this [how to GitHub][4] blog post.\n- Use a topic branch to easily amend a pull request later, if necessary.\n- Write [good commit messages][5].\n- Use the same coding conventions as the rest of the project.\n- Open a [pull request][6] that relates to *only* one subject with a clear title\n and description in grammatically correct, complete sentences.\n\nMuch Thanks! ❤❤❤\n\nGO-JEK Tech\n\n[1]: http://www.apache.org/licenses/LICENSE-2.0\n[2]: https://github.com/gojektech/heimdall/issues\n[3]: https://golang.org/doc/effective_go.html\n[4]: http://gun.io/blog/how-to-github-fork-branch-and-pull-request\n[5]: http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html\n[6]: https://help.github.com/articles/using-pull-requests\n" }, { "path": "Dockerfile", "content": "FROM golang:1.11.5-alpine as base\n\nENV GO111MODULE on\n\nRUN apk --no-cache add gcc g++ make ca-certificates git\nRUN mkdir /weaver\nWORKDIR /weaver\n\nADD go.mod .\nADD go.sum .\nRUN go mod download\n\nFROM base AS weaver_base\n\nADD . /weaver\nRUN make setup\nRUN make build\n\nFROM alpine:latest\n\nCOPY --from=weaver_base /weaver/out/weaver-server /usr/local/bin/weaver\n\nENTRYPOINT [\"weaver\", \"start\"]\n" }, { "path": "LICENSE", "content": " Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n" }, { "path": "Makefile", "content": ".PHONY: all\n\nall: build fmt vet lint test coverage\ndefault: build fmt vet lint test\n\nALL_PACKAGES=$(shell go list ./... | grep -v \"vendor\")\nAPP_EXECUTABLE=\"out/weaver-server\"\nCOMMIT_HASH=$(shell git rev-parse --verify head | cut -c-1-8)\nBUILD_DATE=$(shell date +%Y-%m-%dT%H:%M:%S%z)\n\nsetup:\n\tGO111MODULE=on go get -u golang.org/x/lint/golint\n\tGO111MODULE=on go get github.com/mattn/goveralls\n\ncompile:\n\tmkdir -p out/\n\tGO111MODULE=on go build -o $(APP_EXECUTABLE) -ldflags \"-X main.BuildDate=$(BUILD_DATE) -X main.Commit=$(COMMIT_HASH) -s -w\" ./cmd/weaver-server\n\nbuild: deps compile fmt vet lint\n\ndeps:\n\tGO111MODULE=on go mod tidy -v\n\ninstall:\n\tGO111MODULE=on go install ./...\n\nfmt:\n\tGO111MODULE=on go fmt ./...\n\nvet:\n\tGO111MODULE=on go vet ./...\n\nlint:\n\t@if [[ `golint $(All_PACKAGES) | { grep -vwE \"exported (var|function|method|type|const) \\S+ should have comment\" || true; } | wc -l | tr -d ' '` -ne 0 ]]; then \\\n\t\tgolint $(ALL_PACKAGES) | { grep -vwE \"exported (var|function|method|type|const) \\S+ should have comment\" || true; }; \\\n\t\texit 2; \\\n\tfi;\n\ntest: copy-config\n\tGO111MODULE=on go test ./...\n\ntest-cover-html:\n\t@echo \"mode: count\" > coverage-all.out\n\t$(foreach pkg, $(ALL_PACKAGES),\\\n\tgo test -coverprofile=coverage.out -covermode=count $(pkg);\\\n\ttail -n +2 coverage.out >> coverage-all.out;)\n\tGO111MODULE=on go tool cover -html=coverage-all.out -o out/coverage.html\n\ncopy-config:\n\tcp weaver.conf.yaml.sample weaver.conf.yaml\n\nclean:\n\tgo clean && rm -rf ./vendor ./build ./weaver.conf.yaml\n\ndocker-clean:\n\tdocker-compose down\n\ndocker-spec: docker-clean\n\tdocker-compose build\n\tdocker-compose run --entrypoint \"make test\" dev_weaver\n\ndocker-server:\n\tdocker-compose run --entrypoint \"make local-server\" dev_weaver\n\ndocker-up:\n\tdocker-compose up -d\n\nlocal-server: compile\n\t$(APP_EXECUTABLE) start\n\ncoverage:\n\tgoveralls -service=travis-ci\n\n" }, { "path": "README.md", "content": "# Weaver - A modern HTTP Proxy with Advanced features\n\n

\n\n\"Build [![Go Report Card](https://goreportcard.com/badge/github.com/gojekfarm/weaver)](https://goreportcard.com/report/github.com/gojekfarm/weaver)\n \n[![Coverage Status](https://coveralls.io/repos/github/gojektech/weaver/badge.svg?branch=master)](https://coveralls.io/github/gojektech/weaver?branch=master)\n[![GitHub Release](https://img.shields.io/github/release/gojektech/weaver.svg?style=flat)](https://github.com/gojektech/weaver/releases)\n\n* [Description](#description)\n* [Features](#features)\n* [Installation](#installation)\n* [Architecture](#architecture)\n* [Configuration](#configuration)\n* [Contributing](#contributing)\n* [License](#license)\n\n## Description\nWeaver is a Layer-7 Load Balancer with Dynamic Sharding Strategies. \nIt is a modern HTTP reverse proxy with advanced features.\n\n## Features:\n\n- Sharding request based on headers/path/body fields\n- Emits Metrics on requests per route per backend\n- Dynamic configuring of different routes (No restarts!)\n- Is Fast\n- Supports multiple algorithms for sharding requests (consistent hashing, modulo, s2 etc)\n- Packaged as a single self contained binary\n- Logs on failures (Observability)\n\n## Installation\n\n### Build from source\n\n- Clone the repo:\n```\ngit clone git@github.com:gojektech/weaver.git\n```\n\n- Build to create weaver binary\n```\nmake build\n```\n\n### Binaries for various architectures\n\nDownload the binary for a release from: [here](https://github.com/gojekfarm/weaver/releases)\n\n## Architecture\n\n

\n\nWeaver uses `etcd` as a control plane to match the incoming requests against a particular route config and shard the traffic to different backends based on some sharding strategy.\n\nWeaver can be configured for different routes matching different paths with various sharding strategies through a simple route config named ACL.\n\nThe various sharding strategies supported by weaver are:\n\n- Consistent hashing (hashring)\n- Simple lookup based\n- Modulo\n- Prefix lookup\n- S2 based\n\n## Deploying to Kubernetes\n\nCurrently we support deploying to kubernetes officially. You can check the doc [here](deployment/weaver)\n\n## Examples\n\nWe have examples defined to deploy it to kubernetes and using acls. Please checkout out [examples](examples/body_lookup)\n\n## Configuration\n\n### Defining ACL's\n\nDetails on configuring weaver can be found [here](docs/weaver_acls.md)\n\n### Please note\n\nAs the famous saying goes, `All Load balancers are proxies, but not every proxy is a load balancer`, weaver currently does not support load balancing.\n\n## Contributing\nIf you'd like to contribute to the project, refer to the [contributing documentation](https://github.com/gojektech/weaver/blob/master/CONTRIBUTING.md)\n\n## License\n\n```\nCopyright 2018, GO-JEK Tech (http://gojek.tech)\n\nLicensed under the Apache License, Version 2.0 (the \"License\");\nyou may not use this file except in compliance with the License.\nYou may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing, software\ndistributed under the License is distributed on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\nSee the License for the specific language governing permissions and\nlimitations under the License.\n```\n" }, { "path": "acl.go", "content": "package weaver\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n)\n\n// ACL - Connects to an external endpoint\ntype ACL struct {\n\tID string `json:\"id\"`\n\tCriterion string `json:\"criterion\"`\n\tEndpointConfig *EndpointConfig `json:\"endpoint\"`\n\n\tEndpoint *Endpoint\n}\n\n// GenACL - Generates an ACL from JSON\nfunc (acl *ACL) GenACL(val string) error {\n\treturn json.Unmarshal([]byte(val), &acl)\n}\n\nfunc (acl ACL) String() string {\n\treturn fmt.Sprintf(\"ACL(%s, %s)\", acl.ID, acl.Criterion)\n}\n" }, { "path": "acl_test.go", "content": "package weaver\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nconst (\n\tvalidACLDoc = `{\n\t\t\"id\": \"gojek_hello\",\n\t\t\"criterion\" : \"Method('POST') && Path('/gojek/hello-service')\",\n\t\t\"endpoint\" : {\n\t\t\t\"shard_expr\": \".serviceType\",\n\t\t\t\"matcher\": \"body\",\n\t\t\t\"shard_func\": \"lookup\",\n\t\t\t\"shard_config\": {\n\t\t\t\t\"999\": {\n\t\t\t\t\t\"backend_name\": \"hello_backend\",\n\t\t\t\t\t\"backend\":\"http://hello.golabs.io\"\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}`\n)\n\nvar genACLTests = []struct {\n\taclDoc string\n\texpectedErr error\n}{\n\t{\n\t\tvalidACLDoc,\n\t\tnil,\n\t},\n}\n\nfunc TestGenACL(t *testing.T) {\n\tacl := &ACL{}\n\n\tfor _, tt := range genACLTests {\n\t\tactualErr := acl.GenACL(tt.aclDoc)\n\t\tif actualErr != tt.expectedErr {\n\t\t\tassert.Failf(t, \"Unexpected error message\", \"want: %v got: %v\", tt.expectedErr, actualErr)\n\t\t}\n\t}\n}\n" }, { "path": "backend.go", "content": "package weaver\n\nimport (\n\t\"net\"\n\t\"net/http\"\n\t\"net/http/httputil\"\n\t\"net/url\"\n\t\"time\"\n\n\t\"github.com/gojektech/weaver/config\"\n\t\"github.com/pkg/errors\"\n)\n\ntype Backend struct {\n\tHandler http.Handler\n\tServer *url.URL\n\tName string\n}\n\ntype BackendOptions struct {\n\tTimeout time.Duration\n}\n\nfunc NewBackend(name string, serverURL string, options BackendOptions) (*Backend, error) {\n\tserver, err := url.Parse(serverURL)\n\tif err != nil {\n\t\treturn nil, errors.Wrapf(err, \"URL Parsing failed for: %s\", serverURL)\n\t}\n\n\treturn &Backend{\n\t\tName: name,\n\t\tHandler: newWeaverReverseProxy(server, options),\n\t\tServer: server,\n\t}, nil\n}\n\nfunc newWeaverReverseProxy(target *url.URL, options BackendOptions) *httputil.ReverseProxy {\n\tproxyConfig := config.Proxy()\n\n\tproxy := httputil.NewSingleHostReverseProxy(target)\n\tproxy.Transport = &http.Transport{\n\t\tProxy: http.ProxyFromEnvironment,\n\t\tDialContext: (&net.Dialer{\n\t\t\tTimeout: options.Timeout,\n\t\t\tKeepAlive: proxyConfig.ProxyDialerKeepAliveInMS(),\n\t\t\tDualStack: true,\n\t\t}).DialContext,\n\n\t\tMaxIdleConns: proxyConfig.ProxyMaxIdleConns(),\n\t\tIdleConnTimeout: proxyConfig.ProxyIdleConnTimeoutInMS(),\n\t\tDisableKeepAlives: !proxyConfig.KeepAliveEnabled(),\n\t}\n\n\treturn proxy\n}\n" }, { "path": "backend_test.go", "content": "package weaver\n\nimport (\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestNewBackend(t *testing.T) {\n\tserverURL := \"http://localhost\"\n\n\tbackendOptions := BackendOptions{}\n\tbackend, err := NewBackend(\"foobar\", serverURL, backendOptions)\n\trequire.NoError(t, err, \"should not have failed to create new backend\")\n\n\tassert.NotNil(t, backend.Handler)\n\tassert.Equal(t, serverURL, backend.Server.String())\n}\n\nfunc TestNewBackendFailsWhenURLIsInvalid(t *testing.T) {\n\tserverURL := \":\"\n\n\tbackendOptions := BackendOptions{}\n\tbackend, err := NewBackend(\"foobar\", serverURL, backendOptions)\n\trequire.Error(t, err, \"should have failed to create new backend\")\n\n\tassert.Nil(t, backend)\n}\n" }, { "path": "cmd/weaver-server/main.go", "content": "package main\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"log\"\n\t\"os\"\n\t\"os/signal\"\n\t\"syscall\"\n\n\traven \"github.com/getsentry/raven-go\"\n\t\"github.com/gojektech/weaver/config\"\n\t\"github.com/gojektech/weaver/etcd\"\n\t\"github.com/gojektech/weaver/pkg/instrumentation\"\n\t\"github.com/gojektech/weaver/pkg/logger\"\n\t\"github.com/gojektech/weaver/server\"\n\tcli \"gopkg.in/urfave/cli.v1\"\n)\n\nfunc main() {\n\tapp := cli.NewApp()\n\tapp.Name = \"weaver\"\n\tapp.Usage = \"run weaver-server\"\n\tapp.Version = fmt.Sprintf(\"%s built on %s (commit: %s)\", Version, BuildDate, Commit)\n\tapp.Description = \"An Advanced HTTP Reverse Proxy with Dynamic Sharding Strategies\"\n\tapp.Commands = []cli.Command{\n\t\t{\n\t\t\tName: \"start\",\n\t\t\tDescription: \"Start weaver server\",\n\t\t\tAction: startWeaver,\n\t\t},\n\t}\n\n\tapp.Run(os.Args)\n}\n\nfunc startWeaver(_ *cli.Context) error {\n\tsigC := make(chan os.Signal, 1)\n\tsignal.Notify(sigC, syscall.SIGINT, syscall.SIGKILL, syscall.SIGTERM)\n\n\tconfig.Load()\n\n\traven.SetDSN(config.SentryDSN())\n\tlogger.SetupLogger()\n\n\terr := instrumentation.InitiateStatsDMetrics()\n\tif err != nil {\n\t\tlog.Printf(\"StatsD: Error initiating client %s\", err)\n\t}\n\tdefer instrumentation.CloseStatsDClient()\n\n\tinstrumentation.InitNewRelic()\n\tdefer instrumentation.ShutdownNewRelic()\n\n\trouteLoader, err := etcd.NewRouteLoader()\n\tif err != nil {\n\t\tlog.Printf(\"StartServer: failed to initialise etcd route loader: %s\", err)\n\t}\n\n\tctx, cancel := context.WithCancel(context.Background())\n\tgo server.StartServer(ctx, routeLoader)\n\n\tsig := <-sigC\n\tlog.Printf(\"Received %d, shutting down\", sig)\n\n\tdefer cancel()\n\tserver.ShutdownServer(ctx)\n\n\treturn nil\n}\n\n// Build information (will be injected during build)\nvar (\n\tVersion = \"1.0.0\"\n\tCommit = \"n/a\"\n\tBuildDate = \"n/a\"\n)\n" }, { "path": "config/config.go", "content": "package config\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"net/http\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\tetcd \"github.com/coreos/etcd/client\"\n\tnewrelic \"github.com/newrelic/go-agent\"\n\t\"github.com/spf13/viper\"\n)\n\nvar appConfig Config\n\ntype Config struct {\n\tproxyHost string\n\tproxyPort int\n\tetcdKeyPrefix string\n\tloggerLevel string\n\tetcdEndpoints []string\n\tetcdDialTimeout time.Duration\n\tstatsDConfig StatsDConfig\n\tnewRelicConfig newrelic.Config\n\tsentryDSN string\n\n\tserverReadTimeout time.Duration\n\tserverWriteTimeout time.Duration\n\n\tproxyConfig ProxyConfig\n}\n\nfunc Load() {\n\tviper.SetDefault(\"LOGGER_LEVEL\", \"error\")\n\tviper.SetDefault(\"SERVER_PORT\", \"8080\")\n\tviper.SetDefault(\"PROXY_PORT\", \"8081\")\n\n\tviper.SetConfigName(\"weaver.conf\")\n\n\tviper.AddConfigPath(\"./\")\n\tviper.AddConfigPath(\"../\")\n\tviper.AddConfigPath(\"../../\")\n\tviper.SetConfigType(\"yaml\")\n\n\tviper.ReadInConfig()\n\tviper.AutomaticEnv()\n\n\tappConfig = Config{\n\t\tproxyHost: extractStringValue(\"PROXY_HOST\"),\n\t\tproxyPort: extractIntValue(\"PROXY_PORT\"),\n\t\tetcdKeyPrefix: extractStringValue(\"ETCD_KEY_PREFIX\"),\n\t\tloggerLevel: extractStringValue(\"LOGGER_LEVEL\"),\n\t\tetcdEndpoints: strings.Split(extractStringValue(\"ETCD_ENDPOINTS\"), \",\"),\n\t\tetcdDialTimeout: time.Duration(extractIntValue(\"ETCD_DIAL_TIMEOUT\")),\n\t\tstatsDConfig: loadStatsDConfig(),\n\t\tnewRelicConfig: loadNewRelicConfig(),\n\t\tproxyConfig: loadProxyConfig(),\n\t\tsentryDSN: extractStringValue(\"SENTRY_DSN\"),\n\t\tserverReadTimeout: time.Duration(extractIntValue(\"SERVER_READ_TIMEOUT\")),\n\t\tserverWriteTimeout: time.Duration(extractIntValue(\"SERVER_WRITE_TIMEOUT\")),\n\t}\n}\n\nfunc ServerReadTimeoutInMillis() time.Duration {\n\treturn appConfig.serverReadTimeout * time.Millisecond\n}\n\nfunc ServerWriteTimeoutInMillis() time.Duration {\n\treturn appConfig.serverWriteTimeout * time.Millisecond\n}\n\nfunc ProxyServerAddress() string {\n\treturn fmt.Sprintf(\"%s:%d\", appConfig.proxyHost, appConfig.proxyPort)\n}\n\nfunc ETCDKeyPrefix() string {\n\treturn appConfig.etcdKeyPrefix\n}\n\nfunc NewRelicConfig() newrelic.Config {\n\treturn appConfig.newRelicConfig\n}\n\nfunc SentryDSN() string {\n\treturn appConfig.sentryDSN\n}\n\nfunc StatsD() StatsDConfig {\n\treturn appConfig.statsDConfig\n}\n\nfunc Proxy() ProxyConfig {\n\treturn appConfig.proxyConfig\n}\n\nfunc NewETCDClient() (etcd.Client, error) {\n\treturn etcd.New(etcd.Config{\n\t\tEndpoints: appConfig.etcdEndpoints,\n\t\tHeaderTimeoutPerRequest: appConfig.etcdDialTimeout * time.Second,\n\t\tTransport: &http.Transport{\n\t\t\tProxy: http.ProxyFromEnvironment,\n\t\t\tDial: (&net.Dialer{\n\t\t\t\tTimeout: 30 * time.Second,\n\t\t\t\tKeepAlive: 30 * time.Second,\n\t\t\t}).Dial,\n\t\t\tTLSHandshakeTimeout: 10 * time.Second,\n\t\t},\n\t})\n}\n\nfunc LogLevel() string {\n\treturn appConfig.loggerLevel\n}\n\nfunc extractStringValue(key string) string {\n\tcheckPresenceOf(key)\n\treturn viper.GetString(key)\n}\n\nfunc extractBoolValue(key string) bool {\n\tcheckPresenceOf(key)\n\treturn viper.GetBool(key)\n}\n\nfunc extractBoolValueDefaultToFalse(key string) bool {\n\tif !viper.IsSet(key) {\n\t\treturn false\n\t}\n\n\treturn viper.GetBool(key)\n}\n\nfunc extractIntValue(key string) int {\n\tcheckPresenceOf(key)\n\tv, err := strconv.Atoi(viper.GetString(key))\n\tif err != nil {\n\t\tpanic(fmt.Sprintf(\"key %s is not a valid Integer value\", key))\n\t}\n\n\treturn v\n}\n\nfunc checkPresenceOf(key string) {\n\tif !viper.IsSet(key) {\n\t\tpanic(fmt.Sprintf(\"key %s is not set\", key))\n\t}\n}\n" }, { "path": "config/config_test.go", "content": "package config\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestShouldLoadConfigFromFile(t *testing.T) {\n\tLoad()\n\n\tassert.NotEmpty(t, LogLevel())\n\tassert.NotNil(t, loadStatsDConfig().Prefix())\n\tassert.NotNil(t, loadStatsDConfig().FlushPeriodInSeconds())\n\tassert.NotNil(t, loadStatsDConfig().Port())\n\tassert.NotNil(t, loadStatsDConfig().Enabled())\n}\n\nfunc TestShouldLoadFromEnvVars(t *testing.T) {\n\tconfigVars := map[string]string{\n\t\t\"LOGGER_LEVEL\": \"info\",\n\t\t\"NEW_RELIC_APP_NAME\": \"newrelic\",\n\t\t\"NEW_RELIC_LICENSE_KEY\": \"licence\",\n\t\t\"NEW_RELIC_ENABLED\": \"true\",\n\t\t\"STATSD_PREFIX\": \"weaver\",\n\t\t\"STATSD_FLUSH_PERIOD_IN_SECONDS\": \"20\",\n\t\t\"STATSD_HOST\": \"statsd\",\n\t\t\"STATSD_PORT\": \"8125\",\n\t\t\"STATSD_ENABLED\": \"true\",\n\t\t\"ETCD_KEY_PREFIX\": \"weaver\",\n\n\t\t\"PROXY_DIALER_TIMEOUT_IN_MS\": \"10\",\n\t\t\"PROXY_DIALER_KEEP_ALIVE_IN_MS\": \"10\",\n\t\t\"PROXY_MAX_IDLE_CONNS\": \"200\",\n\t\t\"PROXY_IDLE_CONN_TIMEOUT_IN_MS\": \"20\",\n\t\t\"SENTRY_DSN\": \"dsn\",\n\n\t\t\"SERVER_READ_TIMEOUT\": \"100\",\n\t\t\"SERVER_WRITE_TIMEOUT\": \"100\",\n\t}\n\n\tfor k, v := range configVars {\n\t\terr := os.Setenv(k, v)\n\t\trequire.NoError(t, err, fmt.Sprintf(\"failed to set env for %s key\", k))\n\t}\n\n\tLoad()\n\n\texpectedStatsDConfig := StatsDConfig{\n\t\tprefix: \"weaver\",\n\t\tflushPeriodInSeconds: 20,\n\t\thost: \"statsd\",\n\t\tport: 8125,\n\t\tenabled: true,\n\t}\n\n\tassert.Equal(t, \"info\", LogLevel())\n\n\tassert.Equal(t, \"newrelic\", loadNewRelicConfig().AppName)\n\tassert.Equal(t, \"licence\", loadNewRelicConfig().License)\n\tassert.True(t, loadNewRelicConfig().Enabled)\n\n\tassert.Equal(t, expectedStatsDConfig, loadStatsDConfig())\n\tassert.Equal(t, \"weaver\", ETCDKeyPrefix())\n\tassert.Equal(t, \"dsn\", SentryDSN())\n\n\tassert.Equal(t, time.Duration(10)*time.Millisecond, Proxy().ProxyDialerTimeoutInMS())\n\tassert.Equal(t, time.Duration(10)*time.Millisecond, Proxy().ProxyDialerKeepAliveInMS())\n\tassert.Equal(t, 200, Proxy().ProxyMaxIdleConns())\n\tassert.Equal(t, time.Duration(20)*time.Millisecond, Proxy().ProxyIdleConnTimeoutInMS())\n\n\tassert.Equal(t, time.Duration(100)*time.Millisecond, ServerReadTimeoutInMillis())\n\tassert.Equal(t, time.Duration(100)*time.Millisecond, ServerWriteTimeoutInMillis())\n}\n" }, { "path": "config/newrelic.go", "content": "package config\n\nimport (\n\tnewrelic \"github.com/newrelic/go-agent\"\n)\n\nfunc loadNewRelicConfig() newrelic.Config {\n\tconfig := newrelic.NewConfig(extractStringValue(\"NEW_RELIC_APP_NAME\"),\n\t\textractStringValue(\"NEW_RELIC_LICENSE_KEY\"))\n\tconfig.Enabled = extractBoolValue(\"NEW_RELIC_ENABLED\")\n\treturn config\n}\n" }, { "path": "config/proxy.go", "content": "package config\n\nimport \"time\"\n\ntype ProxyConfig struct {\n\tproxyDialerTimeoutInMS int\n\tproxyDialerKeepAliveInMS int\n\tproxyMaxIdleConns int\n\tproxyIdleConnTimeoutInMS int\n\tkeepAliveEnabled bool\n}\n\nfunc loadProxyConfig() ProxyConfig {\n\treturn ProxyConfig{\n\t\tproxyDialerTimeoutInMS: extractIntValue(\"PROXY_DIALER_TIMEOUT_IN_MS\"),\n\t\tproxyDialerKeepAliveInMS: extractIntValue(\"PROXY_DIALER_KEEP_ALIVE_IN_MS\"),\n\t\tproxyMaxIdleConns: extractIntValue(\"PROXY_MAX_IDLE_CONNS\"),\n\t\tproxyIdleConnTimeoutInMS: extractIntValue(\"PROXY_IDLE_CONN_TIMEOUT_IN_MS\"),\n\t\tkeepAliveEnabled: extractBoolValueDefaultToFalse(\"PROXY_KEEP_ALIVE_ENABLED\"),\n\t}\n}\n\nfunc (pc ProxyConfig) ProxyDialerTimeoutInMS() time.Duration {\n\treturn time.Duration(pc.proxyDialerTimeoutInMS) * time.Millisecond\n}\n\nfunc (pc ProxyConfig) ProxyDialerKeepAliveInMS() time.Duration {\n\treturn time.Duration(pc.proxyDialerKeepAliveInMS) * time.Millisecond\n}\n\nfunc (pc ProxyConfig) ProxyMaxIdleConns() int {\n\treturn pc.proxyMaxIdleConns\n}\n\nfunc (pc ProxyConfig) ProxyIdleConnTimeoutInMS() time.Duration {\n\treturn time.Duration(pc.proxyIdleConnTimeoutInMS) * time.Millisecond\n}\n\nfunc (pc ProxyConfig) KeepAliveEnabled() bool {\n\treturn pc.keepAliveEnabled\n}\n" }, { "path": "config/statsd.go", "content": "package config\n\ntype StatsDConfig struct {\n\tprefix string\n\tflushPeriodInSeconds int\n\thost string\n\tport int\n\tenabled bool\n}\n\nfunc loadStatsDConfig() StatsDConfig {\n\treturn StatsDConfig{\n\t\tprefix: extractStringValue(\"STATSD_PREFIX\"),\n\t\tflushPeriodInSeconds: extractIntValue(\"STATSD_FLUSH_PERIOD_IN_SECONDS\"),\n\t\thost: extractStringValue(\"STATSD_HOST\"),\n\t\tport: extractIntValue(\"STATSD_PORT\"),\n\t\tenabled: extractBoolValue(\"STATSD_ENABLED\"),\n\t}\n}\n\nfunc (sdc StatsDConfig) Prefix() string {\n\treturn sdc.prefix\n}\n\nfunc (sdc StatsDConfig) FlushPeriodInSeconds() int {\n\treturn sdc.flushPeriodInSeconds\n}\n\nfunc (sdc StatsDConfig) Host() string {\n\treturn sdc.host\n}\n\nfunc (sdc StatsDConfig) Port() int {\n\treturn sdc.port\n}\n\nfunc (sdc StatsDConfig) Enabled() bool {\n\treturn sdc.enabled\n}\n" }, { "path": "deployment/weaver/.helmignore", "content": "# Patterns to ignore when building packages.\n# This supports shell glob matching, relative path matching, and\n# negation (prefixed with !). Only one pattern per line.\n.DS_Store\n# Common VCS dirs\n.git/\n.gitignore\n.bzr/\n.bzrignore\n.hg/\n.hgignore\n.svn/\n# Common backup files\n*.swp\n*.bak\n*.tmp\n*~\n# Various IDEs\n.project\n.idea/\n*.tmproj\n.vscode/\n" }, { "path": "deployment/weaver/Chart.yaml", "content": "apiVersion: v1\nappVersion: \"1.0\"\ndescription: A Helm chart to deploy weaver along with etcd and statsd (configurable)\nname: weaver\nversion: 0.1.0\nmaintainers:\n - name: Gowtham Sai\n email: dev@gowtham.me\n" }, { "path": "deployment/weaver/README.md", "content": "# Deploying to Kubernetes\n\nYou can deploy to Kubernetes with the helm charts available in this repo.\n\n### Deploying with ETCD\n\nBy default, helm install will deploy weaver with etcd. But you can disable deploying etcd if you want to reuse existing ETCD.\n\n```sh\nhelm upgrade --debug --install proxy-cluster ./deployment/weaver -f ./deployment/weaver/values-env.yaml\n```\n\nThis will deploy weaver with env values specified in the values-env.yaml file. In case if you want to expose weaver to outside kubernetes you can use NodePort to do that. \n\n```sh\nhelm upgrade --debug --install proxy-cluster ./deployment/weaver --set service.type=NodePort -f ./deployment/weaver/values-env.yaml\n```\n\nThis will deploy along with service of type NodePort. So you can access weaver from outside your kube cluster using kube cluster address and NodePort. In production, you might want to consider ingress/load balancer.\n\n\n### Deploying without ETCD\n\nYou can disable deploying ETCD in case if you want to use existing ETCD in your cluster. To do this, all you have to do is to pass `etcd.enabled` value from command line. \n\n```sh\nhelm upgrade --debug --install proxy-cluster ./deployment/weaver --set etcd.enabled=false -f ./deployment/weaver/values-env.yaml\n```\n\nThis will disable deploying etcd to cluster. But you have to pass etcd host env variable `ETCD_ENDPOINTS` to make weaver work.\n\n\n### Bucket List\n\n1. Helm charts here won't support deploying statsd and sentry yet.\n2. NEWRELIC key can be set to anything if you don't want to monitor your app using newrelic.\n3. Similarly statsd and sentry host can be set to any value.\n\n" }, { "path": "deployment/weaver/templates/_helpers.tpl", "content": "{{/* vim: set filetype=mustache: */}}\n{{/*\nExpand the name of the chart.\n*/}}\n{{- define \"weaver.name\" -}}\n{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix \"-\" -}}\n{{- end -}}\n\n{{/*\nCreate a default fully qualified app name.\nWe truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).\nIf release name contains chart name it will be used as a full name.\n*/}}\n{{- define \"weaver.fullname\" -}}\n{{- if .Values.fullnameOverride -}}\n{{- .Values.fullnameOverride | trunc 63 | trimSuffix \"-\" -}}\n{{- else -}}\n{{- $name := default .Chart.Name .Values.nameOverride -}}\n{{- if contains $name .Release.Name -}}\n{{- .Release.Name | trunc 63 | trimSuffix \"-\" -}}\n{{- else -}}\n{{- printf \"%s-%s\" .Release.Name $name | trunc 63 | trimSuffix \"-\" -}}\n{{- end -}}\n{{- end -}}\n{{- end -}}\n\n{{/*\nCreate chart name and version as used by the chart label.\n*/}}\n{{- define \"weaver.chart\" -}}\n{{- printf \"%s-%s\" .Chart.Name .Chart.Version | replace \"+\" \"_\" | trunc 63 | trimSuffix \"-\" -}}\n{{- end -}}\n" }, { "path": "deployment/weaver/templates/deployment.yaml", "content": "apiVersion: extensions/v1beta1\nkind: Deployment\nname: {{ template \"weaver.fullname\" . }}\nmetadata:\n name: {{ template \"weaver.fullname\" . }}\n labels:\n app.kubernetes.io/managed-by: {{ .Release.Service }}\n app.kubernetes.io/instance: {{ .Release.Name }}\n helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}\n app.kubernetes.io/name: {{ template \"weaver.name\" . }}\nspec:\n replicas: {{ .Values.replicaCount }}\n template:\n metadata:\n {{- if .Values.podAnnotations }}\n # Allows custom annotations to be specified\n annotations:\n {{- toYaml .Values.podAnnotations | nindent 8 }}\n {{- end }}\n labels:\n app.kubernetes.io/name: {{ template \"weaver.name\" . }}\n app.kubernetes.io/instance: {{ .Release.Name }}\n spec:\n containers:\n - name: {{ template \"weaver.name\" . }}\n image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n imagePullPolicy: {{ .Values.image.pullPolicy }}\n ports:\n - name: http\n containerPort: {{ .Values.service.targetPort }}\n protocol: TCP\n {{- if .Values.weaver.env }}\n env:\n {{- toYaml .Values.weaver.env | nindent 12 }}\n {{- end }}\n {{- if .Values.resources }}\n resources:\n # Minikube when high resource requests are specified by default.\n {{- toYaml .Values.resources | nindent 12 }}\n {{- end }}\n {{- if .Values.etcd.enabled }}\n initContainers:\n - name: init-etcd-dns\n image: busybox\n command: ['sh', '-c', 'until nslookup etcd; do echo waiting for etcd dns to be resolving; sleep 1; done;']\n - name: init-etcd-health\n image: busybox\n command: ['sh', '-c', 'until wget --spider etcd:2379/health; do echo waiting for etcd to be up; sleep 1; done;']\n {{- end }}\n {{- if .Values.nodeSelector }}\n nodeSelector:\n # Node selectors can be important on mixed Windows/Linux clusters.\n {{- toYaml .Values.nodeSelector | nindent 8 }}\n {{- end }}\n" }, { "path": "deployment/weaver/templates/service.yaml", "content": "apiVersion: v1\nkind: Service\nmetadata:\n {{- if .Values.service.annotations }}\n annotations:\n {{- toYaml .Values.service.annotations | nindent 4 }}\n {{- end }}\n labels:\n app.kubernetes.io/name: {{ template \"weaver.name\" . }}\n helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}\n app.kubernetes.io/managed-by: {{ .Release.Service }}\n app.kubernetes.io/instance: {{ .Release.Name }}\n app.kubernetes.io/component: server\n app.kubernetes.io/part-of: weaver\n name: {{ template \"weaver.fullname\" . }}\nspec:\n# Provides options for the service so chart users have the full choice\n type: \"{{ .Values.service.type }}\"\n clusterIP: \"{{ .Values.service.clusterIP }}\"\n {{- if .Values.service.externalIPs }}\n externalIPs:\n {{- toYaml .Values.service.externalIPs | nindent 4 }}\n {{- end }}\n {{- if .Values.service.loadBalancerIP }}\n loadBalancerIP: \"{{ .Values.service.loadBalancerIP }}\"\n {{- end }}\n {{- if .Values.service.loadBalancerSourceRanges }}\n loadBalancerSourceRanges:\n {{- toYaml .Values.service.loadBalancerSourceRanges | nindent 4 }}\n {{- end }}\n ports:\n - name: http\n port: {{ .Values.service.port }}\n protocol: TCP\n targetPort: {{ .Values.service.targetPort }}\n {{- if (and (eq .Values.service.type \"NodePort\") (not (empty .Values.service.nodePort))) }}\n nodePort: {{ .Values.service.nodePort }}\n {{- end }}\n selector:\n app.kubernetes.io/name: {{ template \"weaver.name\" . }}\n app.kubernetes.io/instance: {{ .Release.Name }}\n\n---\n\n{{- if .Values.etcd.enabled }}\napiVersion: v1\nkind: Service\nmetadata:\n annotations:\n service.alpha.kubernetes.io/tolerate-unready-endpoints: \"true\"\n {{- if .Values.service.annotations }}\n {{- toYaml .Values.service.annotations | nindent 4 }}\n {{- end }}\n labels:\n app.kubernetes.io/name: \"etcd\"\n helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}\n app.kubernetes.io/managed-by: {{ .Release.Service }}\n app.kubernetes.io/instance: {{ .Release.Name }}\n app.kubernetes.io/component: control-pane\n app.kubernetes.io/part-of: weaver\n name: \"etcd\"\nspec:\n ports:\n - port: {{ .Values.etcd.peerPort }}\n name: etcd-server\n - port: {{ .Values.etcd.clientPort }}\n name: etcd-client\n clusterIP: None\n selector:\n app.kubernetes.io/name: \"etcd\"\n app.kubernetes.io/instance: {{ .Release.Name }}\n{{- end }}\n" }, { "path": "deployment/weaver/templates/statefulset.yaml", "content": "{{- if .Values.etcd.enabled }}\napiVersion: apps/v1beta1\nkind: StatefulSet\nmetadata:\n {{- if .Values.service.annotations }}\n annotations:\n {{- toYaml .Values.service.annotations | indent 4 }}\n {{- end }}\n labels:\n app.kubernetes.io/name: \"etcd\"\n helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}\n app.kubernetes.io/managed-by: {{ .Release.Service }}\n app.kubernetes.io/instance: {{ .Release.Name }}\n app.kubernetes.io/component: control-pane\n app.kubernetes.io/part-of: weaver\n name: \"etcd\"\nspec:\n serviceName: \"etcd\"\n replicas: {{ .Values.etcd.replicas }}\n template:\n metadata:\n {{- if .Values.service.annotations }}\n annotations:\n {{- toYaml .Values.service.annotations | indent 4 }}\n {{- end }}\n labels:\n app.kubernetes.io/name: \"etcd\"\n helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}\n app.kubernetes.io/managed-by: {{ .Release.Service }}\n app.kubernetes.io/instance: {{ .Release.Name }}\n app.kubernetes.io/component: control-pane\n app.kubernetes.io/part-of: weaver\n name: \"etcd\"\n spec:\n{{- if .Values.affinity }}\n affinity:\n{{ toYaml .Values.affinity | indent 8 }}\n{{- end }}\n{{- if .Values.nodeSelector }}\n nodeSelector:\n{{ toYaml .Values.nodeSelector | indent 8 }}\n{{- end }}\n{{- if .Values.tolerations }}\n tolerations:\n{{ toYaml .Values.tolerations | indent 8 }}\n{{- end }}\n containers:\n - name: \"etcd\"\n image: \"{{ .Values.etcd.image.repository }}:{{ .Values.etcd.image.tag }}\"\n imagePullPolicy: \"{{ .Values.etcd.image.pullPolicy }}\"\n ports:\n - containerPort: {{ .Values.etcd.peerPort }}\n name: peer\n - containerPort: {{ .Values.etcd.clientPort }}\n name: client\n resources:\n{{ toYaml .Values.resources | indent 10 }} \n env:\n - name: INITIAL_CLUSTER_SIZE\n value: {{ .Values.etcd.replicas | quote }}\n - name: SET_NAME\n value: \"etcd\"\n{{- if .Values.extraEnv }}\n{{ toYaml .Values.extraEnv | indent 8 }}\n{{- end }}\n volumeMounts:\n - name: datadir\n mountPath: /var/run/etcd\n lifecycle:\n preStop:\n exec:\n command:\n - \"/bin/sh\"\n - \"-ec\"\n - |\n EPS=\"\"\n for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do\n EPS=\"${EPS}${EPS:+,}http://${SET_NAME}-${i}.${SET_NAME}:2379\"\n done\n HOSTNAME=$(hostname)\n member_hash() {\n etcdctl member list | grep http://${HOSTNAME}.${SET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1\n }\n SET_ID=${HOSTNAME##*[^0-9]}\n if [ \"${SET_ID}\" -ge ${INITIAL_CLUSTER_SIZE} ]; then\n echo \"Removing ${HOSTNAME} from etcd cluster\"\n ETCDCTL_ENDPOINT=${EPS} etcdctl member remove $(member_hash)\n if [ $? -eq 0 ]; then\n # Remove everything otherwise the cluster will no longer scale-up\n rm -rf /var/run/etcd/*\n fi\n fi\n command:\n - \"/bin/sh\"\n - \"-ec\"\n - |\n HOSTNAME=$(hostname)\n echo $HOSTNAME\n # store member id into PVC for later member replacement\n collect_member() {\n while ! etcdctl member list &>/dev/null; do sleep 1; done\n etcdctl member list | grep http://${HOSTNAME}.${SET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1 > /var/run/etcd/member_id\n exit 0\n }\n eps() {\n EPS=\"\"\n for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do\n EPS=\"${EPS}${EPS:+,}http://${SET_NAME}-${i}.${SET_NAME}:2379\"\n done\n echo ${EPS}\n }\n member_hash() {\n etcdctl member list | grep http://${HOSTNAME}.${SET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1\n }\n # we should wait for other pods to be up before trying to join\n # otherwise we got \"no such host\" errors when trying to resolve other members\n for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do\n while true; do\n echo \"Waiting for ${SET_NAME}-${i}.${SET_NAME} to come up\"\n ping -W 1 -c 1 ${SET_NAME}-${i}.${SET_NAME} > /dev/null && break\n sleep 1s\n done\n done\n # re-joining after failure?\n echo $(eps)\n if [ -e /var/run/etcd/member_id ]; then\n echo \"Re-joining etcd member\"\n member_id=$(cat /var/run/etcd/member_id)\n # re-join member\n ETCDCTL_ENDPOINT=$(eps) etcdctl member update ${member_id} http://${HOSTNAME}.${SET_NAME}:2380 | true\n exec etcd --name ${HOSTNAME} \\\n --listen-peer-urls http://0.0.0.0:2380 \\\n --listen-client-urls http://0.0.0.0:2379\\\n --advertise-client-urls http://${HOSTNAME}.${SET_NAME}:2379 \\\n --data-dir /var/run/etcd/default.etcd\n fi\n # etcd-SET_ID\n SET_ID=${HOSTNAME##*[^0-9]}\n # adding a new member to existing cluster (assuming all initial pods are available)\n if [ \"${SET_ID}\" -ge ${INITIAL_CLUSTER_SIZE} ]; then\n export ETCDCTL_ENDPOINT=$(eps)\n # member already added?\n MEMBER_HASH=$(member_hash)\n if [ -n \"${MEMBER_HASH}\" ]; then\n # the member hash exists but for some reason etcd failed\n # as the datadir has not be created, we can remove the member\n # and retrieve new hash\n etcdctl member remove ${MEMBER_HASH}\n fi\n echo \"Adding new member\"\n etcdctl member add ${HOSTNAME} http://${HOSTNAME}.${SET_NAME}:2380 | grep \"^ETCD_\" > /var/run/etcd/new_member_envs\n if [ $? -ne 0 ]; then\n echo \"Exiting\"\n rm -f /var/run/etcd/new_member_envs\n exit 1\n fi\n cat /var/run/etcd/new_member_envs\n source /var/run/etcd/new_member_envs\n collect_member &\n exec etcd --name ${HOSTNAME} \\\n --listen-peer-urls http://0.0.0.0:2380 \\\n --listen-client-urls http://0.0.0.0:2379 \\\n --advertise-client-urls http://${HOSTNAME}.${SET_NAME}:2379 \\\n --data-dir /var/run/etcd/default.etcd \\\n --initial-advertise-peer-urls http://${HOSTNAME}.${SET_NAME}:2380 \\\n --initial-cluster ${ETCD_INITIAL_CLUSTER} \\\n --initial-cluster-state ${ETCD_INITIAL_CLUSTER_STATE}\n fi\n PEERS=\"\"\n for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do\n PEERS=\"${PEERS}${PEERS:+,}${SET_NAME}-${i}=http://${SET_NAME}-${i}.${SET_NAME}:2380\"\n done\n collect_member &\n # join member\n exec etcd --name ${HOSTNAME} \\\n --initial-advertise-peer-urls http://${HOSTNAME}.${SET_NAME}:2380 \\\n --listen-peer-urls http://0.0.0.0:2380 \\\n --listen-client-urls http://0.0.0.0:2379 \\\n --advertise-client-urls http://${HOSTNAME}.${SET_NAME}:2379 \\\n --initial-cluster-token etcd-cluster-1 \\\n --initial-cluster ${PEERS} \\\n --initial-cluster-state new \\\n --data-dir /var/run/etcd/default.etcd\n\n {{- if .Values.etcd.persistentVolume.enabled }}\n volumeClaimTemplates:\n - metadata:\n name: datadir\n spec:\n accessModes:\n - \"ReadWriteOnce\"\n resources:\n requests:\n # upstream recommended max is 700M\n storage: \"{{ .Values.etcd.persistentVolume.storage }}\"\n {{- if .Values.etcd.persistentVolume.storageClass }}\n {{- if (eq \"-\" .Values.etcd.persistentVolume.storageClass) }}\n storageClassName: \"\"\n {{- else }}\n storageClassName: \"{{ .Values.etcd.persistentVolume.storageClass }}\"\n {{- end }}\n {{- end }}\n {{- else }}\n volumes:\n - name: datadir\n {{- if .Values.etcd.memoryMode }}\n emptyDir:\n medium: Memory\n {{- else }}\n emptyDir: {}\n {{- end }}\n {{- end }}\n{{- end }}\n" }, { "path": "deployment/weaver/values-env.yaml", "content": "weaver:\n env:\n - name: \"PROXY_HOST\"\n value: \"0.0.0.0\"\n - name: \"PROXY_PORT\"\n value: \"8080\"\n - name: \"PROXY_DIALER_TIMEOUT_IN_MS\"\n value: \"1000\"\n - name: \"PROXY_DIALER_KEEP_ALIVE_IN_MS\"\n value: \"100\"\n - name: \"PROXY_IDLE_CONN_TIMEOUT_IN_MS\"\n value : \"100\"\n - name: \"PROXY_MAX_IDLE_CONNS\"\n value: \"100\"\n - name: \"SENTRY_DSN\"\n value: \"sentry\"\n - name: \"SERVER_READ_TIMEOUT\"\n value: \"100\"\n - name: \"SERVER_WRITE_TIMEOUT\"\n value: \"100\"\n - name: \"ETCD_KEY_PREFIX\"\n value: \"weaver\"\n - name: \"ETCD_ENDPOINTS\"\n value: \"http://etcd:2379\"\n - name: \"ETCD_DIAL_TIMEOUT\"\n value: \"5\"\n - name: \"NEW_RELIC_APP_NAME\"\n value: \"weaver\"\n - name: \"NEW_RELIC_LICENSE_KEY\"\n value: \"weaver-temp-license\"\n - name: \"STATSD_ENABLED\"\n value: \"false\"\n - name: \"STATSD_PREFIX\"\n value: \"weaver\"\n - name: \"STATSD_PORT\"\n value: \"8125\"\n - name: \"STATSD_FLUSH_PERIOD_IN_SECONDS\"\n value: \"10\"\n - name: \"STATSD_HOST\"\n value: \"localhost\"\n - name: \"NEW_RELIC_ENABLED\"\n value: \"false\"\n" }, { "path": "deployment/weaver/values.yaml", "content": "# Default values for weaver.\n# This is a YAML-formatted file.\n# Declare variables to be passed into your templates.\n\nreplicaCount: 1\n\nimage:\n repository: gojektech/weaver\n tag: stable\n pullPolicy: IfNotPresent\n\nnameOverride: \"\"\nfullnameOverride: \"\"\n\nservice:\n type: ClusterIP\n port: 80\n targetPort: 8080\n\nresources: {}\n # We usually recommend not to specify default resources and to leave this as a conscious\n # choice for the user. This also increases chances charts run on environments with little\n # resources, such as Minikube. If you do want to specify resources, uncomment the following\n # lines, adjust them as necessary, and remove the curly braces after 'resources:'.\n # limits:\n # cpu: 100m\n # memory: 128Mi\n # requests:\n # cpu: 100m\n # memory: 128Mi\n\npodAnnotations: {}\nnodeSelector: {}\ntolerations: []\naffinity: {}\n\nweaver:\n env: []\n\n# ETCD Specific configurations\netcd:\n enabled: true\n peerPort: 2380\n clientPort: 2379\n component: \"etcd\"\n replicas: 3\n image:\n repository: \"k8s.gcr.io/etcd-amd64\"\n tag: \"2.2.5\"\n pullPolicy: \"IfNotPresent\"\n resources: {}\n persistentVolume:\n enabled: false\n # storage: \"1Gi\"\n ## etcd data Persistent Volume Storage Class\n ## If defined, storageClassName: \n ## If set to \"-\", storageClassName: \"\", which disables dynamic provisioning\n ## If undefined (the default) or set to null, no storageClassName spec is\n ## set, choosing the default provisioner. (gp2 on AWS, standard on\n ## GKE, AWS & OpenStack)\n ##\n # storageClass: \"-\"\n\n ## This is only available when persistentVolume is false:\n ## If persistentVolume is not enabled, one can choose to use memory mode for ETCD by setting memoryMode to \"true\".\n ## The system will create a volume with \"medium: Memory\"\n memoryMode: false\n" }, { "path": "docker-compose.yml", "content": "version: '3.4'\n\nservices:\n dev_statsd:\n image: gojektech/statsd:0.7.2\n hostname: dev-statsd\n ports:\n - \"18124:8124\"\n - \"18125:8125\"\n - \"18126:8126\"\n networks:\n - weaver_net\n\n dev_etcd:\n image: quay.io/coreos/etcd:v2.3.8\n hostname: dev-etcd\n entrypoint: [\"/etcd\"]\n command: [\n \"-name\",\n \"etcd0\",\n \"-advertise-client-urls\",\n \"http://localhost:2379,http://localhost:4001\",\n \"-listen-client-urls\",\n \"http://0.0.0.0:2379,http://0.0.0.0:4001\",\n \"-initial-advertise-peer-urls\",\n \"http://localhost:2380\",\n \"-listen-peer-urls\",\n \"http://0.0.0.0:2380\",\n \"-initial-cluster-token\",\n \"etcd-cluster-1\",\n \"-initial-cluster\",\n \"etcd0=http://localhost:2380\",\n \"-initial-cluster-state\",\n \"new\"\n ]\n ports:\n - \"12379:2379\"\n networks:\n - weaver_net\n\n dev_weaver:\n build:\n context: .\n target: weaver_base\n hostname: dev-weaver\n depends_on:\n - dev_etcd\n - dev_statsd\n environment:\n STATD_PORT: 8125\n STATSD_HOST: dev_statsd\n ETCD_ENDPOINTS: \"http://dev_etcd:2379\"\n stdin_open: true\n tty: true\n networks:\n - weaver_net\n\nnetworks:\n weaver_net:\n" }, { "path": "docs/weaver_acls.md", "content": "## Weaver ACLs\n\nWeaver ACL is a document formatted in JSON used to decide the destination of downstream traffic. An example of an ACL is\nlike the following\n\n``` json\n{\n \"id\": \"gojek_hello\",\n \"criterion\" : \"Method(`POST`) && Path(`/gojek/hello-service`)\",\n \"endpoint\" : {\n \"shard_expr\": \".serviceType\",\n \"matcher\": \"body\",\n \"shard_func\": \"lookup\",\n \"shard_config\": {\n \"999\": {\n \"backend_name\": \"hello_backend\",\n \"backend\":\"http://hello.golabs.io\"\n }\n }\n }\n}\n```\nThe description of each field is as follows\n\n| Field Name | Description |\n|---|---|\n| `id` | The name of the service |\n| `criterion` | The criterion expressed based on [Vulcand Routing](https://godoc.org/github.com/vulcand/route) |\n| `endpoint` | The endpoint description (see below) |\n\nFor endpoints the keys descriptions are as following:\n\n| Field Name | Description |\n|---|---|\n| `matcher` | The value to match can be `body`, `path` , `header` or `param` |\n| `shard_expr` | Shard expression, the expression to evaluate request based on the matcher |\n| `shard_func` | The function of the sharding (See Below) |\n| `shard_config` | The backends for each evaluated value |\n\nFor each `shard_config` value there are the value evaluated as the result of expression of `shard_expr`. We need to \ndescribe backends for each value.\n\n| Field Name | Description |\n|---|---|\n| `backend_name` | unique name for the evaluated value |\n| `backend` | The URI in which the packet will be forwarded |\n---\n## ACL examples:\n\nPossible **`shard_func`** values accepted by weaver are : `none`, `lookup`, `prefix-lookup`, `modulo` , `hashring`, `s2`. \nSample ACLs for each accepted **`shard_func`** are provided below.\n\n\n**`none`**: \n``` json\n{\n \"id\": \"gojek_hello\",\n \"criterion\" : \"Method(`GET`) && Path(`/gojek/hello-service`)\",\n \"endpoint\" : {\n \"shard_func\": \"none\",\n \"shard_config\": {\n \"backend_name\": \"hello_backend\",\n \"backend\":\"http://hello.golabs.io\"\n }\n }\n}\n```\n\nDetails: Just forwards the traffic. HTTP `GET` to `weaver.io/gojek/hello-service` will be forwarded to backend at `http://hello.golabs.io`.\n\n---\n\n**`lookup`**:\n\n``` json\n{\n \"id\": \"gojek_hello\",\n \"criterion\" : \"Method(`POST`) && Path(`/gojek/hello-service`)\",\n \"endpoint\" : {\n \"shard_expr\": \".serviceType\",\n \"matcher\": \"body\",\n \"shard_func\": \"lookup\",\n \"shard_config\": {\n \"999\": {\n \"backend_name\": \"hello_backend\",\n \"backend\":\"http://hello.golabs.io\"\n },\n \"6969\": {\n \"backend_name\": \"bye_backend\",\n \"backend\":\"http://bye.golabs.io\"\n }\n }\n }\n}\n```\n\nDetails: HTTP `POST` to `weaver.io/gojek/hello-service` will be forwarded based on the `shard_expr` field's value within request body. The request body shall be similar to: \n\n``` json \n{\n \"serviceType\": \"999\",\n ...\n}\n```\n\nIn this scenario the value evaluated by `shard_expr` will be `999`. This will forward the request to `http://hello.golabs.io`. However if the value evaluated by the `shard_expr` is not found within `shard_config`, weaver returns `503` error. \n\n---\n\n**`prefix-lookup`**:\n\n``` json \n{\n \"id\": \"gojek_prefix_hello\",\n \"criterion\": \"Method(`PUT`) && Path(`/gojek/hello/world`)\",\n \"endpoint\": {\n \"shard_expr\": \".orderNo\",\n \"matcher\": \"body\",\n \"shard_func\": \"prefix-lookup\",\n \"shard_config\": {\n \"backends\": {\n \"default\": {\n \"backend_name\": \"backend_1\",\n \"backend\": \"http://backend1\"\n },\n \"AB-\": {\n \"backend_name\": \"backend2\",\n \"backend\": \"http://backend2\"\n },\n \"AD-\": {\n \"backend_name\": \"backend3\",\n \"backend\": \"http://backend3\"\n }\n },\n \"prefix_splitter\": \"-\"\n }\n }\n}\n```\n\nDetails: HTTP `PUT` to `weaver.io/gojek/hello/world` will be forwarded based on the `shard_expr` field's value within request body. The request body shall be similar to: \n``` json \n{\n \"orderNo\": \"AD-2132315\",\n ...\n}\n```\nIn this scenario the value evaluated by `shard_expr` will be `AD-2132315`. This value will be split according to `prefix_splitter` in `shard_config.backends` evaluating to `AD-`. This will forward the request to `http://backend3`. However if the value evaluated by the `shard_expr` is not found within `shard_config`, weaver will fallback to the value in the `default` key. If `default` key is not found within `shard_config.backends`, weaver returns `503` error. \n\n---\n\n**`modulo`**:\n``` json\n{\n \"id\": \"drivers-by-driver-id\",\n \"criterion\": \"Method(`GET`) && PathRegexp(`/v2/drivers/\\\\d+`)\",\n \"endpoint\": {\n \"shard_config\": {\n \"0\": {\n \"backend_name\": \"backend1\",\n \"backend\": \"http://backend1\"\n },\n \"1\": {\n \"backend_name\": \"backend2\",\n \"backend\": \"http://backend2\"\n },\n \"2\": {\n \"backend_name\": \"backend3\",\n \"backend\": \"http://backend3\"\n },\n \"3\": {\n \"backend_name\": \"backend4\",\n \"backend\": \"http://backend4\"\n }\n },\n \"shard_expr\": \"/v2/drivers/(\\\\d+)\",\n \"matcher\": \"path\",\n \"shard_func\": \"modulo\"\n }\n}\n```\n\nDetails: HTTP `GET` to `weaver.io/v2/drivers/2156545453242` will be forwarded based on the value captured by regex in `shard_expr` from `/v2/drivers/2156545453242` path. \nThe value must be an integer. The backend is selected based on the modulo operation between extracted value (`2156545453242`) with number of backends in the `shard_config`. In this scenario the result is `2156545453242 % 4 = 2`. This will forward the request to `http://backend3`. \n\n--- \n\n**`hashring`**:\n\n``` json\n{\n \"id\": \"driver-location\",\n \"criterion\": \"Method(`GET`) && PathRegexp(`/gojek/driver/location`)\",\n \"endpoint\": {\n \"shard_config\": {\n \"totalVirtualBackends\": 1000,\n \"backends\": {\n \"0-249\": {\n \"backend_name\": \"backend1\",\n \"backend\": \"http://backend1\"\n },\n \"250-499\": {\n \"backend_name\": \"backend2\",\n \"backend\": \"http://backend2\"\n },\n \"500-749\": {\n \"backend_name\": \"backend3\",\n \"backend\": \"http://backend3\"\n },\n \"750-999\": {\n \"backend_name\": \"backend4\",\n \"backend\": \"http://backend4\"\n }\n }\n },\n \"shard_expr\": \"DriverID\",\n \"matcher\": \"header\",\n \"shard_func\": \"hashring\"\n }\n}\n```\n\nDetails: HTTP `PUT` to `weaver.io/gojek/driver/location` will be forwarded based on the result of hashing function from the here.\nIn this scenario the key by which we select the backend is obtained by using value of DriverID header since matcher is header. For example if request had DriverID: 34345 header, and hashring calculated hash of that values as hash(34345): 555, it will select backend with 500-749 key. This will forward the request to `http://backend3`\n\n---\n\n**`s2`**:\n\n``` json\n{\n \"id\": \"nearby-driver-service-get-nearby\",\n \"criterion\": \"Method(`GET`) && PathRegexp(`/gojek/nearby`)\",\n \"endpoint\": {\n \"shard_config\": {\n \"shard_key_separator\": \",\",\n \"shard_key_position\": -1,\n \"backends\": {\n \"3477275891585777700\": {\n \"backend_name\": \"backend1\",\n \"backend\": \"http://backend1\",\n \"timeout\": 300\n },\n \"3477284687678800000\": {\n \"backend_name\": \"backend2\",\n \"backend\": \"http://backend2\",\n \"timeout\": 300\n },\n \"3477302279864844300\": {\n \"backend_name\": \"backend3\",\n \"backend\": \"http://backend3\",\n \"timeout\": 300\n },\n \"3477290185236939000\": {\n \"backend_name\": \"backend4\",\n \"backend\": \"http://backend4\",\n \"timeout\": 300\n }\n }\n },\n \"shard_expr\": \"X-Location\",\n \"matcher\": \"header\",\n \"shard_func\": \"s2\"\n }\n}\n```\n\nDetails: HTTP `GET` to `weaver.io/gojek/nearby` will be forwarded based on the result of s2id calculation from X-Location header in the form of lat and long separated by , in accordance to shard_key_separator value. e.g -6.2428103,106.7940571. Weaver calculated s2id from lat and long because shard_key_position value is -1.\n\n" }, { "path": "endpoint.go", "content": "package weaver\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"net/http\"\n\n\t\"github.com/gojektech/weaver/pkg/matcher\"\n\t\"github.com/pkg/errors\"\n)\n\n// EndpointConfig - Defines a config for external service\ntype EndpointConfig struct {\n\tMatcher string `json:\"matcher\"`\n\tShardExpr string `json:\"shard_expr\"`\n\tShardFunc string `json:\"shard_func\"`\n\tShardConfig json.RawMessage `json:\"shard_config\"`\n}\n\nfunc (endpointConfig *EndpointConfig) genShardKeyFunc() (shardKeyFunc, error) {\n\tmatcherFunc, found := matcher.New(endpointConfig.Matcher)\n\tif !found {\n\t\treturn nil, errors.WithStack(fmt.Errorf(\"failed to find a matcherMux for: %s\", endpointConfig.Matcher))\n\t}\n\n\treturn func(req *http.Request) (string, error) {\n\t\treturn matcherFunc(req, endpointConfig.ShardExpr)\n\t}, nil\n}\n\ntype Endpoint struct {\n\tsharder Sharder\n\tshardKeyFunc shardKeyFunc\n}\n\nfunc NewEndpoint(endpointConfig *EndpointConfig, sharder Sharder) (*Endpoint, error) {\n\tif sharder == nil {\n\t\treturn nil, errors.New(\"nil sharder passed in\")\n\t}\n\n\tshardKeyFunc, err := endpointConfig.genShardKeyFunc()\n\tif err != nil {\n\t\treturn nil, errors.Wrapf(err, \"failed to generate shardKeyFunc for %s\", endpointConfig.ShardExpr)\n\t}\n\n\treturn &Endpoint{\n\t\tsharder: sharder,\n\t\tshardKeyFunc: shardKeyFunc,\n\t}, nil\n}\n\nfunc (endpoint *Endpoint) Shard(request *http.Request) (*Backend, error) {\n\tshardKey, err := endpoint.shardKeyFunc(request)\n\tif err != nil {\n\t\treturn nil, errors.Wrapf(err, \"failed to find shardKey\")\n\t}\n\n\treturn endpoint.sharder.Shard(shardKey)\n}\n\ntype shardKeyFunc func(*http.Request) (string, error)\n" }, { "path": "endpoint_test.go", "content": "package weaver\n\nimport (\n\t\"encoding/json\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestNewEndpoint(t *testing.T) {\n\tendpointConfig := &EndpointConfig{\n\t\tMatcher: \"path\",\n\t\tShardExpr: \"/.*\",\n\t\tShardFunc: \"lookup\",\n\t\tShardConfig: json.RawMessage(`{}`),\n\t}\n\tsharder := &stubSharder{}\n\n\tendpoint, err := NewEndpoint(endpointConfig, sharder)\n\trequire.NoError(t, err, \"should not fail to create an endpoint from endpointConfig\")\n\tassert.NotNil(t, endpoint, \"should create an endpoint\")\n\tassert.Equal(t, sharder, endpoint.sharder)\n}\n\nfunc TestNewEndpoint_SharderIsNil(t *testing.T) {\n\tendpointConfig := &EndpointConfig{\n\t\tMatcher: \"path\",\n\t\tShardExpr: \"/.*\",\n\t\tShardFunc: \"lookup\",\n\t\tShardConfig: json.RawMessage(`{}`),\n\t}\n\n\tendpoint, err := NewEndpoint(endpointConfig, nil)\n\tassert.Error(t, err, \"should fail to create an endpoint when sharder is nil\")\n\tassert.Nil(t, endpoint)\n}\n\ntype stubSharder struct {\n}\n\nfunc (stub *stubSharder) Shard(key string) (*Backend, error) {\n\treturn nil, nil\n}\n" }, { "path": "etcd/aclkey.go", "content": "package etcd\n\nimport (\n\t\"fmt\"\n\n\t\"github.com/gojektech/weaver\"\n)\n\nconst (\n\t// ACLKeyFormat - Format for a ACL's key in a KV Store\n\tACLKeyFormat = \"/%s/acls/%s/acl\"\n)\n\n// ACLKey - Points to a stored ACL\ntype ACLKey string\n\n// GenACLKey - Generate an ACL Key given etcd's node key\nfunc GenACLKey(key string) ACLKey {\n\treturn ACLKey(fmt.Sprintf(\"%s/acl\", key))\n}\n\nfunc GenKey(acl *weaver.ACL, pfx string) ACLKey {\n\treturn ACLKey(fmt.Sprintf(ACLKeyFormat, pfx, acl.ID))\n}\n" }, { "path": "etcd/routeloader.go", "content": "package etcd\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"sort\"\n\n\t\"github.com/gojektech/weaver\"\n\t\"github.com/gojektech/weaver/pkg/shard\"\n\n\tetcd \"github.com/coreos/etcd/client\"\n\t\"github.com/gojektech/weaver/config\"\n\t\"github.com/gojektech/weaver/pkg/logger\"\n\t\"github.com/gojektech/weaver/server\"\n\t\"github.com/pkg/errors\"\n)\n\nfunc NewRouteLoader() (*RouteLoader, error) {\n\tetcdClient, err := config.NewETCDClient()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn &RouteLoader{\n\t\tetcdClient: etcdClient,\n\t\tnamespace: config.ETCDKeyPrefix(),\n\t}, nil\n}\n\n// RouteLoader - To store and modify proxy configuration\ntype RouteLoader struct {\n\tetcdClient etcd.Client\n\tnamespace string\n}\n\n// PutACL - Upserts a given ACL\nfunc (routeLoader *RouteLoader) PutACL(acl *weaver.ACL) (ACLKey, error) {\n\tkey := GenKey(acl, routeLoader.namespace)\n\tval, err := json.Marshal(acl)\n\tif err != nil {\n\t\treturn \"\", err\n\t}\n\t_, err = etcd.NewKeysAPI(routeLoader.etcdClient).Set(context.Background(), string(key), string(val), nil)\n\tif err != nil {\n\t\treturn \"\", fmt.Errorf(\"fail to PUT %s:%s with %s\", key, acl, err.Error())\n\t}\n\treturn key, nil\n}\n\n// GetACL - Fetches an ACL given an ACLKey\nfunc (routeLoader *RouteLoader) GetACL(key ACLKey) (*weaver.ACL, error) {\n\tres, err := etcd.NewKeysAPI(routeLoader.etcdClient).Get(context.Background(), string(key), nil)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"fail to GET %s with %s\", key, err.Error())\n\t}\n\tacl := &weaver.ACL{}\n\tif err := json.Unmarshal([]byte(res.Node.Value), acl); err != nil {\n\t\treturn nil, err\n\t}\n\n\tsharder, err := shard.New(acl.EndpointConfig.ShardFunc, acl.EndpointConfig.ShardConfig)\n\tif err != nil {\n\t\treturn nil, errors.Wrapf(err, \"failed to initialize sharder '%s'\", acl.EndpointConfig.ShardFunc)\n\t}\n\n\tacl.Endpoint, err = weaver.NewEndpoint(acl.EndpointConfig, sharder)\n\tif err != nil {\n\t\treturn nil, errors.Wrapf(err, \"failed to create a new Endpoint for key: %s\", key)\n\t}\n\n\treturn acl, nil\n}\n\n// DelACL - Deletes an ACL given an ACLKey\nfunc (routeLoader *RouteLoader) DelACL(key ACLKey) error {\n\t_, err := etcd.NewKeysAPI(routeLoader.etcdClient).Delete(context.Background(), string(key), nil)\n\tif err != nil {\n\t\treturn fmt.Errorf(\"fail to DELETE %s with %s\", key, err.Error())\n\t}\n\treturn nil\n}\n\nfunc (routeLoader *RouteLoader) WatchRoutes(ctx context.Context, upsertRouteFunc server.UpsertRouteFunc, deleteRouteFunc server.DeleteRouteFunc) {\n\tetc, key := initEtcd(routeLoader)\n\twatcher := etc.Watcher(key, &etcd.WatcherOptions{Recursive: true})\n\n\tlogger.Infof(\"starting etcd watcher on %s\", key)\n\tfor {\n\t\tres, err := watcher.Next(ctx)\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"stopping etcd watcher on %s: %v\", key, err)\n\t\t\treturn\n\t\t}\n\n\t\tlogger.Debugf(\"registered etcd watcher event on %v with action %s\", res, res.Action)\n\t\tswitch res.Action {\n\t\tcase \"set\":\n\t\t\tfallthrough\n\t\tcase \"update\":\n\t\t\tlogger.Debugf(\"fetching node key %s\", res.Node.Key)\n\t\t\tacl, err := routeLoader.GetACL(ACLKey(res.Node.Key))\n\t\t\tif err != nil {\n\t\t\t\tlogger.Errorf(\"error in fetching %s: %v\", res.Node.Key, err)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tlogger.Infof(\"upserting %v to router\", acl)\n\t\t\terr = upsertRouteFunc(acl)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Errorf(\"error in upserting %v: %v \", acl, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\tcase \"delete\":\n\t\t\tacl := &weaver.ACL{}\n\t\t\terr := acl.GenACL(res.PrevNode.Value)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Errorf(\"error in unmarshalling %s: %v\", res.PrevNode.Value, err)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tlogger.Infof(\"deleteing %v to router\", acl)\n\t\t\terr = deleteRouteFunc(acl)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Errorf(\"error in deleting %v: %v \", acl, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (routeLoader *RouteLoader) BootstrapRoutes(ctx context.Context, upsertRouteFunc server.UpsertRouteFunc) error {\n\t// TODO: Consider error scenarios and return an error when it makes sense\n\tetc, key := initEtcd(routeLoader)\n\tlogger.Infof(\"bootstrapping router using etcd on %s\", key)\n\tres, err := etc.Get(ctx, key, nil)\n\tif err != nil {\n\t\tlogger.Infof(\"creating router namespace on etcd using %s\", key)\n\t\t_, _ = etc.Set(ctx, key, \"\", &etcd.SetOptions{\n\t\t\tDir: true,\n\t\t})\n\t}\n\n\tif res != nil {\n\t\tsort.Sort(res.Node.Nodes)\n\t\tfor _, nd := range res.Node.Nodes {\n\t\t\tlogger.Debugf(\"fetching node key %s\", nd.Key)\n\t\t\tacl, err := routeLoader.GetACL(GenACLKey(nd.Key))\n\t\t\tif err != nil {\n\t\t\t\tlogger.Errorf(\"error in fetching %s: %v\", nd.Key, err)\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tlogger.Infof(\"upserting %v to router\", acl)\n\t\t\terr = upsertRouteFunc(acl)\n\t\t\tif err != nil {\n\t\t\t\tlogger.Errorf(\"error in upserting %v: %v \", acl, err)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc initEtcd(routeLoader *RouteLoader) (etcd.KeysAPI, string) {\n\tkey := fmt.Sprintf(\"/%s/acls/\", routeLoader.namespace)\n\tetc := etcd.NewKeysAPI(routeLoader.etcdClient)\n\n\treturn etc, key\n}\n" }, { "path": "etcd/routeloader_test.go", "content": "package etcd\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"reflect\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/gojektech/weaver\"\n\n\tetcd \"github.com/coreos/etcd/client\"\n\t\"github.com/gojektech/weaver/config\"\n\t\"github.com/gojektech/weaver/pkg/logger\"\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n\t\"github.com/stretchr/testify/suite\"\n)\n\n// Notice: This test uses time.Sleep, TODO: fix it\ntype RouteLoaderSuite struct {\n\tsuite.Suite\n\n\tng *RouteLoader\n}\n\nfunc (es *RouteLoaderSuite) SetupTest() {\n\tconfig.Load()\n\tlogger.SetupLogger()\n\n\tvar err error\n\n\tes.ng, err = NewRouteLoader()\n\tassert.NoError(es.T(), err)\n}\n\nfunc (es *RouteLoaderSuite) TestNewRouteLoader() {\n\tassert.NotNil(es.T(), es.ng)\n}\n\nfunc TestRouteLoaderSuite(tst *testing.T) {\n\tsuite.Run(tst, new(RouteLoaderSuite))\n}\n\nfunc (es *RouteLoaderSuite) TestPutACL() {\n\taclPut := &weaver.ACL{\n\t\tID: \"svc-01\",\n\t\tCriterion: \"Method(`GET`) && Path(`/ping`)\",\n\t\tEndpointConfig: &weaver.EndpointConfig{\n\t\t\tShardFunc: \"lookup\",\n\t\t\tMatcher: \"path\",\n\t\t\tShardExpr: \"*\",\n\t\t\tShardConfig: json.RawMessage(`{\n\t\t\t\t\"GF-\": {\n\t\t\t\t\t\"backend_name\": \"foobar\",\n\t\t\t\t\t\"backend\": \"http://customer-locations-primary\"\n\t\t\t\t},\n\t\t\t\t\"R-\": {\n\t\t\t\t\t\"timeout\": 100.0,\n\t\t\t\t\t\"backend_name\": \"foobar\",\n\t\t\t\t\t\"backend\": \"http://customer-locations-secondary\"\n\t\t\t\t}\n\t\t\t}`),\n\t\t},\n\t}\n\n\tkey, err := es.ng.PutACL(aclPut)\n\tassert.Nil(es.T(), err, \"fail to PUT %s\", aclPut)\n\taclGet, err := es.ng.GetACL(key)\n\tassert.Nil(es.T(), err, \"fail to GET with %s\", key)\n\tassert.Equal(es.T(), aclPut.ID, aclGet.ID, \"PUT %s =/= GET %s\", aclPut, aclGet)\n\tassert.Nil(es.T(), es.ng.DelACL(key), \"fail to DELETE %+v\", aclPut)\n}\n\nfunc (es *RouteLoaderSuite) TestBootstrapRoutes() {\n\taclPut := &weaver.ACL{\n\t\tID: \"svc-01\",\n\t\tCriterion: \"Method(`GET`) && Path(`/ping`)\",\n\t\tEndpointConfig: &weaver.EndpointConfig{\n\t\t\tShardFunc: \"lookup\",\n\t\t\tMatcher: \"path\",\n\t\t\tShardExpr: \"*\",\n\t\t\tShardConfig: json.RawMessage(`{}`),\n\t\t},\n\t}\n\tkey, err := es.ng.PutACL(aclPut)\n\tassert.NoError(es.T(), err, \"failed to PUT %s\", aclPut)\n\n\taclsChan := make(chan *weaver.ACL, 1)\n\tes.ng.BootstrapRoutes(context.Background(), genRouteProcessorMock(aclsChan))\n\n\tdeepEqual(es.T(), aclPut, <-aclsChan)\n\tassert.Nil(es.T(), es.ng.DelACL(key), \"fail to DELETE %+v\", aclPut)\n}\n\nfunc (es *RouteLoaderSuite) TestBootstrapRoutesSucceedWhenARouteUpsertFails() {\n\taclPut := &weaver.ACL{\n\t\tID: \"svc-01\",\n\t\tCriterion: \"Method(`GET`) && Path(`/ping`)\",\n\t\tEndpointConfig: &weaver.EndpointConfig{\n\t\t\tShardFunc: \"lookup\",\n\t\t\tMatcher: \"path\",\n\t\t\tShardExpr: \"*\",\n\t\t\tShardConfig: json.RawMessage(`{\n\t\t\t\t\"GF-\": {\n\t\t\t\t\t\"backend_name\": \"foobar\",\n\t\t\t\t\t\"backend\": \"http://customer-locations-primary\"\n\t\t\t\t},\n\t\t\t\t\"R-\": {\n\t\t\t\t\t\"timeout\": 100.0,\n\t\t\t\t\t\"backend_name\": \"foobar\",\n\t\t\t\t\t\"backend\": \"http://customer-locations-secondary\"\n\t\t\t\t}\n\t\t\t}`),\n\t\t},\n\t}\n\tkey, err := es.ng.PutACL(aclPut)\n\trequire.NoError(es.T(), err, \"failed to PUT %s\", aclPut)\n\n\terr = es.ng.BootstrapRoutes(context.Background(), failingUpsertRouteFunc)\n\trequire.NoError(es.T(), err, \"should not have failed to bootstrap routes\")\n\tassert.Nil(es.T(), es.ng.DelACL(key), \"fail to DELETE %+v\", aclPut)\n}\n\nfunc (es *RouteLoaderSuite) TestBootstrapRoutesSucceedWhenARouteDoesntExist() {\n\terr := es.ng.BootstrapRoutes(context.Background(), successUpsertRouteFunc)\n\trequire.NoError(es.T(), err, \"should not have failed to bootstrap routes\")\n}\n\nfunc (es *RouteLoaderSuite) TestBootstrapRoutesSucceedWhenARouteHasInvalidData() {\n\taclPut := newTestACL(\"path\")\n\n\tvalue := `{ \"blah\": \"a }`\n\tkey := \"abc\"\n\t_, err := etcd.NewKeysAPI(es.ng.etcdClient).Set(context.Background(), key, value, nil)\n\trequire.NoError(es.T(), err, \"failed to PUT %s\", aclPut)\n\n\terr = es.ng.BootstrapRoutes(context.Background(), successUpsertRouteFunc)\n\trequire.NoError(es.T(), err, \"should not have failed to bootstrap routes\")\n\tassert.Nil(es.T(), es.ng.DelACL(ACLKey(key)), \"fail to DELETE %+v\", aclPut)\n}\n\nfunc (es *RouteLoaderSuite) TestWatchRoutesUpsertRoutesWhenRoutesSet() {\n\tnewACL := newTestACL(\"path\")\n\n\taclsUpserted := make(chan *weaver.ACL, 1)\n\n\twatchCtx, cancelWatch := context.WithCancel(context.Background())\n\tdefer cancelWatch()\n\n\tgo es.ng.WatchRoutes(watchCtx, genRouteProcessorMock(aclsUpserted), successUpsertRouteFunc)\n\ttime.Sleep(100 * time.Millisecond)\n\n\tkey, err := es.ng.PutACL(newACL)\n\trequire.NoError(es.T(), err, \"fail to PUT %+v\", newACL)\n\n\tdeepEqual(es.T(), newACL, <-aclsUpserted)\n\tassert.Nil(es.T(), es.ng.DelACL(key), \"fail to DELETE %+v\", newACL)\n}\n\nfunc (es *RouteLoaderSuite) TestWatchRoutesUpsertRoutesWhenRoutesUpdated() {\n\tnewACL := newTestACL(\"path\")\n\tupdatedACL := newTestACL(\"header\")\n\n\t_, err := es.ng.PutACL(newACL)\n\taclsUpserted := make(chan *weaver.ACL, 1)\n\twatchCtx, cancelWatch := context.WithCancel(context.Background())\n\tdefer cancelWatch()\n\n\tgo es.ng.WatchRoutes(watchCtx, genRouteProcessorMock(aclsUpserted), successUpsertRouteFunc)\n\ttime.Sleep(100 * time.Millisecond)\n\n\tkey, err := es.ng.PutACL(updatedACL)\n\trequire.NoError(es.T(), err, \"fail to PUT %+v\", updatedACL)\n\n\tdeepEqual(es.T(), updatedACL, <-aclsUpserted)\n\tassert.Nil(es.T(), es.ng.DelACL(key), \"fail to DELETE %+v\", updatedACL)\n}\n\nfunc (es *RouteLoaderSuite) TestWatchRoutesDeleteRouteWhenARouteIsDeleted() {\n\tnewACL := newTestACL(\"path\")\n\n\tkey, err := es.ng.PutACL(newACL)\n\trequire.NoError(es.T(), err, \"fail to PUT ACL %+v\", newACL)\n\n\taclsDeleted := make(chan *weaver.ACL, 1)\n\n\twatchCtx, cancelWatch := context.WithCancel(context.Background())\n\tdefer cancelWatch()\n\n\tgo es.ng.WatchRoutes(watchCtx, successUpsertRouteFunc, genRouteProcessorMock(aclsDeleted))\n\ttime.Sleep(100 * time.Millisecond)\n\n\terr = es.ng.DelACL(key)\n\trequire.NoError(es.T(), err, \"fail to Delete %+v\", newACL)\n\n\tdeepEqual(es.T(), newACL, <-aclsDeleted)\n}\n\nfunc newTestACL(matcher string) *weaver.ACL {\n\treturn &weaver.ACL{\n\t\tID: \"svc-01\",\n\t\tCriterion: \"Method(`GET`) && Path(`/ping`)\",\n\t\tEndpointConfig: &weaver.EndpointConfig{\n\t\t\tShardFunc: \"lookup\",\n\t\t\tMatcher: matcher,\n\t\t\tShardExpr: \"*\",\n\t\t\tShardConfig: json.RawMessage(`{\n\t\t\t\t\"GF-\": {\n\t\t\t\t\t\"backend_name\": \"foobar\",\n\t\t\t\t\t\"backend\": \"http://customer-locations-primary\"\n\t\t\t\t},\n\t\t\t\t\"R-\": {\n\t\t\t\t\t\"timeout\": 100.0,\n\t\t\t\t\t\"backend_name\": \"foobar\",\n\t\t\t\t\t\"backend\": \"http://customer-locations-secondary\"\n\t\t\t\t}\n\t\t\t}`),\n\t\t},\n\t}\n}\n\nfunc genRouteProcessorMock(c chan *weaver.ACL) func(*weaver.ACL) error {\n\treturn func(acl *weaver.ACL) error {\n\t\tc <- acl\n\t\treturn nil\n\t}\n}\n\nfunc deepEqual(t *testing.T, expected *weaver.ACL, actual *weaver.ACL) {\n\tassert.Equal(t, expected.ID, actual.ID)\n\tassert.Equal(t, expected.Criterion, actual.Criterion)\n\tassertEqualJSON(t, expected.EndpointConfig.ShardConfig, actual.EndpointConfig.ShardConfig)\n\tassert.Equal(t, expected.EndpointConfig.ShardFunc, actual.EndpointConfig.ShardFunc)\n\tassert.Equal(t, expected.EndpointConfig.Matcher, actual.EndpointConfig.Matcher)\n\tassert.Equal(t, expected.EndpointConfig.ShardExpr, actual.EndpointConfig.ShardExpr)\n}\n\nfunc assertEqualJSON(t *testing.T, json1, json2 json.RawMessage) {\n\tvar jsonVal1 interface{}\n\tvar jsonVal2 interface{}\n\n\terr1 := json.Unmarshal(json1, &jsonVal1)\n\terr2 := json.Unmarshal(json2, &jsonVal2)\n\n\tassert.NoError(t, err1, \"failed to parse json string\")\n\tassert.NoError(t, err2, \"failed to parse json string\")\n\tassert.True(t, reflect.DeepEqual(jsonVal1, jsonVal2))\n}\n\nfunc failingUpsertRouteFunc(acl *weaver.ACL) error {\n\treturn errors.New(\"error\")\n}\n\nfunc successUpsertRouteFunc(acl *weaver.ACL) error {\n\treturn nil\n}\n" }, { "path": "examples/body_lookup/Dockerfile", "content": "FROM golang:1.11.5-alpine as base\n\nENV GO111MODULE off\n\nRUN mkdir /estimate\nADD . /estimate\nWORKDIR /estimate\n\nRUN go build main.go\n\nFROM alpine:latest\nCOPY --from=base /estimate/main /usr/local/bin/estimator\nENTRYPOINT [\"estimator\"]\n\n" }, { "path": "examples/body_lookup/README.md", "content": "# Backend Lookup\n\nIn this example, will deploy etcd and weaver to kubernetes and apply a simple etcd to shard between Singapore estimator and Indonesian Estimator based on key body lookup.\n\n### Setup\n```\n# To kubernetes cluster in local and set current context\nminikube start\nminikube status # verify it is up and running\n\n# You can check dashboard by running following command\nminikube dashboard\n\n# Deploying helm components\nhelm init\n```\n\n### Deploying weaver\n\nNow we have running kubernetes cluster in local. Let's deploy weaver and etcd in kubernetes to play with routes.\n\n1. Clone the repo\n2. On root folder of the project, run the following commands\n\n```sh\n# Connect to kubernetes docker image\neval $(minikube docker-env)\n\n# Build docker weaver image\ndocker build . -t weaver:stable\n\n# Deploy weaver to kubernetes\nhelm upgrade --debug --install proxy-cluster ./deployment/weaver --set service.type=NodePort -f ./deployment/weaver/values-env.yaml\n```\n\nWe are setting service type as NodePort so that we can access it from local machine.\n\nWe have deployed weaver successfully to kubernetes under release name , you can check the same in dashboard.\n\n### Deploying simple service\n\nNow we have to deploy simple service to kubernetes and shard request using weaver.\nNavigate to examples/body_lookup/ and run the following commands.\n\n1. Build docker image for estimate service\n2. Deploy docker image to 2 sharded clusters\n\n```\n# Building docker image for estimate\ndocker build . -t estimate:stable\n\n# Deploying it Singapore Cluster\nhelm upgrade --debug --install singapore-cluster ./examples/body_lookup/estimator -f ./examples/body_lookup/estimator/values-sg.yaml\n\n# Deploying it to Indonesian Cluster\nhelm upgrade --debug --install indonesia-cluster ./examples/body_lookup/estimator -f ./examples/body_lookup/estimator/values-id.yaml\n```\n\nWe have a service called estimator which is sharded (Indonesian cluster, and Singapore cluster) which returns an Amount and Currency.\n\n### Deploying Weaver ACLS\n\nLet's deploy acl to etcd and check weaver in action.\n\n1. Copy acls to weaver pod\n2. Load acs to etcd\n\nWe have to apply acls to etcd so that we can lookup for that acl and load it. In order to apply a acl, first will copy to one of the pod\nand deploy using curl request by issuing following commands.\n\n```sh\n# You can get pod name by running this command - kubectx get pods | grep weaver | awk '{print $1}'\nkubectl cp examples/body_lookup/estimate_acl.json proxy-cluster-weaver-79fb49db6f-tng8r:/go/\n\n# Set path in etcd using curl command\ncurl -v etcd:2379/v2/keys/weaver/acls/estimate/acl -XPUT --data-urlencode \"value@estimate_acl.json\"\n```\n\nOnce we set the acl in etcd, as weaver is watching for path changes continuously it just loads the acl and starts sharding requests.\n\n### Weaver in action\n\n\nNow you have wevaer which is exposed using NodePort service type. This mean you can just shard your request based on currency lookup in body as we defined in the estimate_acl.json file.\n\n1. Get Cluster Info\n2. Send request to weaver to see response from estimator\n\n```sh\n# Get cluster ip from cluster-info\nkubectl cluster-info\n\n# Using cluster ip make a curl request to weaver\ncurl -X POST ${CLUSTER_IP}:${NODE_PORT}/estimate -d '{\"currency\": \"SGD\"}' # This is served by singapore shard\n# {\"Amount\": 23.23, \"Currency\": \"SGD\"}\n\n# Getting estimate from Indonesia shard\ncurl -X POST ${CLUSTER_IP}:${NODE_PORT}/estimate -d '{\"currency\": \"IDR\"}' # This is served by singapore shard\n# {\"Amount\": 81223.23, \"Currency\": \"IDR\"}\n```\n" }, { "path": "examples/body_lookup/estimate_acl.json", "content": "{\n \"id\": \"estimator\",\n \"criterion\" : \"Method(`POST`) && Path(`/estimate`)\",\n \"endpoint\" : {\n \"shard_expr\": \".currency\",\n \"matcher\": \"body\",\n \"shard_func\": \"lookup\",\n \"shard_config\": {\n \"IDR\": {\n \"backend_name\": \"indonesia-cluster\",\n \"backend\":\"http://indonesia-cluster-estimator\"\n },\n \"SGD\": {\n \"backend_name\": \"singapore-cluster\",\n \"backend\":\"http://singapore-cluster-estimator\"\n }\n }\n }\n}\n" }, { "path": "examples/body_lookup/estimator/Chart.yaml", "content": "apiVersion: v1\nappVersion: \"1.0\"\ndescription: A Helm chart to deploy estimator\nname: estimator \nversion: 0.1.0\nmaintainers:\n - name: Gowtham Sai\n email: dev@gowtham.me\n" }, { "path": "examples/body_lookup/estimator/templates/_helpers.tpl", "content": "{{/* vim: set filetype=mustache: */}}\n{{/*\nExpand the name of the chart.\n*/}}\n{{- define \"estimator.name\" -}}\n{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix \"-\" -}}\n{{- end -}}\n\n{{/*\nCreate a default fully qualified app name.\nWe truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).\nIf release name contains chart name it will be used as a full name.\n*/}}\n{{- define \"estimator.fullname\" -}}\n{{- if .Values.fullnameOverride -}}\n{{- .Values.fullnameOverride | trunc 63 | trimSuffix \"-\" -}}\n{{- else -}}\n{{- $name := default .Chart.Name .Values.nameOverride -}}\n{{- if contains $name .Release.Name -}}\n{{- .Release.Name | trunc 63 | trimSuffix \"-\" -}}\n{{- else -}}\n{{- printf \"%s-%s\" .Release.Name $name | trunc 63 | trimSuffix \"-\" -}}\n{{- end -}}\n{{- end -}}\n{{- end -}}\n\n{{/*\nCreate chart name and version as used by the chart label.\n*/}}\n{{- define \"estimator.chart\" -}}\n{{- printf \"%s-%s\" .Chart.Name .Chart.Version | replace \"+\" \"_\" | trunc 63 | trimSuffix \"-\" -}}\n{{- end -}}\n" }, { "path": "examples/body_lookup/estimator/templates/deployment.yaml", "content": "apiVersion: extensions/v1beta1\nkind: Deployment\nname: {{ template \"estimator.fullname\" . }}\nmetadata:\n name: {{ template \"estimator.fullname\" . }}\n labels:\n app.kubernetes.io/managed-by: {{ .Release.Service }}\n app.kubernetes.io/instance: {{ .Release.Name }}\n helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}\n app.kubernetes.io/name: {{ template \"estimator.name\" . }}\nspec:\n replicas: {{ .Values.replicaCount }}\n template:\n metadata:\n {{- if .Values.podAnnotations }}\n # Allows custom annotations to be specified\n annotations:\n {{- toYaml .Values.podAnnotations | indent 8 }}\n {{- end }}\n labels:\n app.kubernetes.io/name: {{ template \"estimator.name\" . }}\n app.kubernetes.io/instance: {{ .Release.Name }}\n spec:\n containers:\n - name: {{ template \"estimator.name\" . }}\n image: \"{{ .Values.image.repository }}:{{ .Values.image.tag }}\"\n imagePullPolicy: {{ .Values.image.pullPolicy }}\n ports:\n - name: http\n containerPort: {{ .Values.service.containerPort }}\n protocol: TCP\n env:\n - name: \"MAX_AMOUNT\"\n value: {{ .Values.env.maxAmount | quote }}\n - name: \"MIN_AMOUNT\"\n value: {{ .Values.env.minAmount | quote }}\n - name: \"CURRENCY\"\n value: {{ .Values.env.currency | quote }}\n {{- if .Values.resources }}\n resources:\n # Minikube when high resource requests are specified by default.\n {{- toYaml .Values.resources | indent 12 }}\n {{- end }}\n {{- if .Values.nodeSelector }}\n nodeSelector:\n # Node selectors can be important on mixed Windows/Linux clusters.\n {{- toYaml .Values.nodeSelector | indent 8 }}\n {{- end }}\n" }, { "path": "examples/body_lookup/estimator/templates/service.yaml", "content": "apiVersion: v1\nkind: Service\nmetadata:\n {{- if .Values.service.annotations }}\n annotations:\n {{- toYaml .Values.service.annotations | indent 4 }}\n {{- end }}\n labels:\n app.kubernetes.io/name: {{ template \"estimator.name\" . }}\n helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}\n app.kubernetes.io/managed-by: {{ .Release.Service }}\n app.kubernetes.io/instance: {{ .Release.Name }}\n app.kubernetes.io/component: server\n app.kubernetes.io/part-of: estimator\n name: {{ template \"estimator.fullname\" . }}\nspec:\n# Provides options for the service so chart users have the full choice\n type: \"{{ .Values.service.type }}\"\n clusterIP: \"{{ .Values.service.clusterIP }}\"\n {{- if .Values.service.externalIPs }}\n externalIPs:\n {{- toYaml .Values.service.externalIPs | indent 4 }}\n {{- end }}\n {{- if .Values.service.loadBalancerIP }}\n loadBalancerIP: \"{{ .Values.service.loadBalancerIP }}\"\n {{- end }}\n {{- if .Values.service.loadBalancerSourceRanges }}\n loadBalancerSourceRanges:\n {{- toYaml .Values.service.loadBalancerSourceRanges | indent 4 }}\n {{- end }}\n ports:\n - name: http\n port: {{ .Values.service.port }}\n protocol: TCP\n targetPort: {{ .Values.service.containerPort }}\n {{- if (and (eq .Values.service.type \"NodePort\") (not (empty .Values.service.nodePort))) }}\n nodePort: {{ .Values.service.nodePort }}\n {{- end }}\n selector:\n app.kubernetes.io/name: {{ template \"estimator.name\" . }}\n app.kubernetes.io/instance: {{ .Release.Name }}\n\n\n" }, { "path": "examples/body_lookup/estimator/values-id.yaml", "content": "# Default env values for Indonesia cluster\nenv:\n maxAmount: 100000\n minAmount: 1000\n currency: IDR\n\n" }, { "path": "examples/body_lookup/estimator/values-sg.yaml", "content": "# Default env values for Singapore cluster\nenv:\n maxAmount: 100\n minAmount: 10\n currency: SGD\n" }, { "path": "examples/body_lookup/estimator/values.yaml", "content": "# Default values for weaver.\n# This is a YAML-formatted file.\n# Declare variables to be passed into your templates.\n\nreplicaCount: 2\n\nimage:\n repository: estimator\n tag: stable\n pullPolicy: IfNotPresent\n\nnameOverride: \"\"\nfullnameOverride: \"\"\n\nservice:\n type: ClusterIP\n port: 80\n containerPort: 8080\n\nresources: {}\n # We usually recommend not to specify default resources and to leave this as a conscious\n # choice for the user. This also increases chances charts run on environments with little\n # resources, such as Minikube. If you do want to specify resources, uncomment the following\n # lines, adjust them as necessary, and remove the curly braces after 'resources:'.\n # limits:\n # cpu: 100m\n # memory: 128Mi\n # requests:\n # cpu: 100m\n # memory: 128Mi\n\npodAnnotations: {}\nnodeSelector: {}\ntolerations: []\naffinity: {}\n\n\nenv:\n maxAmount: \"100\"\n minAmount: \"10\"\n currency: \"SGD\"\n" }, { "path": "examples/body_lookup/main.go", "content": "package main\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"log\"\n\t\"math\"\n\t\"math/rand\"\n\t\"net/http\"\n\t\"os\"\n\t\"strconv\"\n)\n\ntype estimationRequest struct {\n\tAmount float64\n\tCurrency string\n}\n\nfunc getEnv(key string, fallback string) string {\n\tif value, ok := os.LookupEnv(key); ok {\n\t\treturn value\n\t}\n\treturn fallback\n}\n\nfunc handler(w http.ResponseWriter, r *http.Request) {\n\tfmt.Fprintf(w, \"Hi there, welcome to weaver!\")\n}\n\nfunc handlePing(w http.ResponseWriter, r *http.Request) {\n\tfmt.Fprintf(w, \"pong\")\n}\n\nfunc getAmount() float64 {\n\tmaxCap, _ := strconv.ParseFloat(getEnv(\"MAX_AMOUNT\", \"100\"), 64)\n\tminCap, _ := strconv.ParseFloat(getEnv(\"MIN_AMOUNT\", \"100\"), 64)\n\trandomValue := minCap + rand.Float64()*(maxCap-minCap)\n\treturn math.Round(randomValue*100) / 100\n}\n\nfunc handleEstimate(w http.ResponseWriter, r *http.Request) {\n\testimatedValue := estimationRequest{Amount: getAmount(), Currency: getEnv(\"CURRENCY\", \"IDR\")}\n\trespEncoder := json.NewEncoder(w)\n\trespEncoder.Encode(estimatedValue)\n}\n\nfunc main() {\n\thttp.HandleFunc(\"/\", handler)\n\thttp.HandleFunc(\"/ping\", handlePing)\n\thttp.HandleFunc(\"/estimate\", handleEstimate)\n\tlog.Fatal(http.ListenAndServe(\":8080\", nil))\n}\n" }, { "path": "go.mod", "content": "module github.com/gojektech/weaver\n\nrequire (\n\tgithub.com/certifi/gocertifi v0.0.0-20170123212243-03be5e6bb987 // indirect\n\tgithub.com/coreos/bbolt v1.3.2 // indirect\n\tgithub.com/coreos/etcd v3.3.0+incompatible\n\tgithub.com/coreos/go-semver v0.2.0 // indirect\n\tgithub.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 // indirect\n\tgithub.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect\n\tgithub.com/davecgh/go-spew v1.1.1 // indirect\n\tgithub.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect\n\tgithub.com/getsentry/raven-go v0.0.0-20161115135411-3f7439d3e74d\n\tgithub.com/ghodss/yaml v1.0.0 // indirect\n\tgithub.com/gogo/protobuf v1.2.0 // indirect\n\tgithub.com/gojekfarm/hashring v0.0.0-20180330151038-7bba2fd52501\n\tgithub.com/golang/geo v0.0.0-20170430223333-5747e9816367\n\tgithub.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect\n\tgithub.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c // indirect\n\tgithub.com/gorilla/websocket v1.4.0 // indirect\n\tgithub.com/gravitational/trace v0.0.0-20171118015604-0bd13642feb8 // indirect\n\tgithub.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect\n\tgithub.com/grpc-ecosystem/grpc-gateway v1.7.0 // indirect\n\tgithub.com/hashicorp/hcl v0.0.0-20170217164738-630949a3c5fa // indirect\n\tgithub.com/jonboulle/clockwork v0.1.0 // indirect\n\tgithub.com/kr/pretty v0.1.0 // indirect\n\tgithub.com/magiconair/properties v0.0.0-20170113111004-b3b15ef068fd // indirect\n\tgithub.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 // indirect\n\tgithub.com/mitchellh/mapstructure v0.0.0-20170125051937-db1efb556f84 // indirect\n\tgithub.com/newrelic/go-agent v1.11.0\n\tgithub.com/onsi/ginkgo v1.7.0 // indirect\n\tgithub.com/onsi/gomega v1.4.3 // indirect\n\tgithub.com/pelletier/go-buffruneio v0.2.0 // indirect\n\tgithub.com/pelletier/go-toml v0.0.0-20170227222904-361678322880 // indirect\n\tgithub.com/philhofer/fwd v1.0.0 // indirect\n\tgithub.com/pkg/errors v0.8.0\n\tgithub.com/pmezard/go-difflib v1.0.0 // indirect\n\tgithub.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7 // indirect\n\tgithub.com/prometheus/client_golang v0.9.2 // indirect\n\tgithub.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 // indirect\n\tgithub.com/savaki/jq v0.0.0-20161209013833-0e6baecebbf8\n\tgithub.com/sirupsen/logrus v1.0.3\n\tgithub.com/soheilhy/cmux v0.1.4 // indirect\n\tgithub.com/spaolacci/murmur3 v0.0.0-20170819071325-9f5d223c6079 // indirect\n\tgithub.com/spf13/afero v0.0.0-20170217164146-9be650865eab // indirect\n\tgithub.com/spf13/cast v0.0.0-20170221152302-f820543c3592 // indirect\n\tgithub.com/spf13/jwalterweatherman v0.0.0-20170109133355-fa7ca7e836cf // indirect\n\tgithub.com/spf13/pflag v1.0.0 // indirect\n\tgithub.com/spf13/viper v1.0.0\n\tgithub.com/stretchr/objx v0.1.1 // indirect\n\tgithub.com/stretchr/testify v1.2.2\n\tgithub.com/tinylib/msgp v1.1.0 // indirect\n\tgithub.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect\n\tgithub.com/ugorji/go v0.0.0-20171019201919-bdcc60b419d1 // indirect\n\tgithub.com/vulcand/predicate v1.0.0 // indirect\n\tgithub.com/vulcand/route v0.0.0-20160805191529-61904570391b\n\tgithub.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect\n\tgo.etcd.io/bbolt v1.3.2 // indirect\n\tgolang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 // indirect\n\tgolang.org/x/time v0.0.0-20181108054448-85acf8d2951c // indirect\n\tgoogle.golang.org/grpc v1.18.0 // indirect\n\tgopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect\n\tgopkg.in/alexcesaro/statsd.v2 v2.0.0\n\tgopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect\n\tgopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect\n\tgopkg.in/urfave/cli.v1 v1.20.0\n)\n" }, { "path": "go.sum", "content": "cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=\ngithub.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=\ngithub.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=\ngithub.com/certifi/gocertifi v0.0.0-20170123212243-03be5e6bb987 h1:GMSZ85uysw01MMLfnHGjTj/QfUdJcGHuDabY6kWKnVk=\ngithub.com/certifi/gocertifi v0.0.0-20170123212243-03be5e6bb987/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=\ngithub.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=\ngithub.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s=\ngithub.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=\ngithub.com/coreos/etcd v3.3.0+incompatible h1:v3H7yHgF+94suF7Xg6V7Haq6Anac3X6WosuKGTTJCGM=\ngithub.com/coreos/etcd v3.3.0+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=\ngithub.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY=\ngithub.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=\ngithub.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142 h1:3jFq2xL4ZajGK4aZY8jz+DAF0FHjI51BXjjSwCzS1Dk=\ngithub.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=\ngithub.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg=\ngithub.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=\ngithub.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=\ngithub.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=\ngithub.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=\ngithub.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=\ngithub.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=\ngithub.com/getsentry/raven-go v0.0.0-20161115135411-3f7439d3e74d h1:l+MZegqjcffeVt3U7OldySISIA+wDlizPTz9Ki2u3k4=\ngithub.com/getsentry/raven-go v0.0.0-20161115135411-3f7439d3e74d/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=\ngithub.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=\ngithub.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=\ngithub.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI=\ngithub.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=\ngithub.com/gojekfarm/hashring v0.0.0-20180330151038-7bba2fd52501 h1:M711duYkMPGeglzA822WpcmfuLES3eafw7zH7ch+JxM=\ngithub.com/gojekfarm/hashring v0.0.0-20180330151038-7bba2fd52501/go.mod h1:OiCsMsLqQGrKJrRQdHIK8PyJXCv7yo73ZeBweRM4u0w=\ngithub.com/golang/geo v0.0.0-20170430223333-5747e9816367 h1:vfvm90sLVQQU3gbQ+EpAF/Y9SFNvjqCxkyy92aXMnK0=\ngithub.com/golang/geo v0.0.0-20170430223333-5747e9816367/go.mod h1:vgWZ7cu0fq0KY3PpEHsocXOWJpRtkcbKemU4IUw0M60=\ngithub.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=\ngithub.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=\ngithub.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk=\ngithub.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=\ngithub.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=\ngithub.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=\ngithub.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw=\ngithub.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=\ngithub.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=\ngithub.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=\ngithub.com/gravitational/trace v0.0.0-20171118015604-0bd13642feb8 h1:beahEEOlfVHRfa7JFDl3oetCjvCga+p7iU+5RN81evY=\ngithub.com/gravitational/trace v0.0.0-20171118015604-0bd13642feb8/go.mod h1:RvdOUHE4SHqR3oXlFFKnGzms8a5dugHygGw1bqDstYI=\ngithub.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=\ngithub.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=\ngithub.com/grpc-ecosystem/grpc-gateway v1.7.0 h1:tPFY/SM+d656aSgLWO2Eckc3ExwpwwybwdN5Ph20h1A=\ngithub.com/grpc-ecosystem/grpc-gateway v1.7.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=\ngithub.com/hashicorp/hcl v0.0.0-20170217164738-630949a3c5fa h1:10wM7X2JKPrmcvtI9Qy2xsoQI1CBA8dd6LqjyGKlD0c=\ngithub.com/hashicorp/hcl v0.0.0-20170217164738-630949a3c5fa/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=\ngithub.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=\ngithub.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=\ngithub.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=\ngithub.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=\ngithub.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=\ngithub.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=\ngithub.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=\ngithub.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=\ngithub.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=\ngithub.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=\ngithub.com/magiconair/properties v0.0.0-20170113111004-b3b15ef068fd h1:iWbe8Xk8p4yQR6ZVjhS21WRnSM79D/l7YN1mOCdM/wQ=\ngithub.com/magiconair/properties v0.0.0-20170113111004-b3b15ef068fd/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=\ngithub.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 h1:2gxZ0XQIU/5z3Z3bUBu+FXuk2pFbkN6tcwi/pjyaDic=\ngithub.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=\ngithub.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=\ngithub.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=\ngithub.com/mitchellh/mapstructure v0.0.0-20170125051937-db1efb556f84 h1:rrg06yhhsqEELubsnYWqadxdi0CYJ97s899oUXDIrkY=\ngithub.com/mitchellh/mapstructure v0.0.0-20170125051937-db1efb556f84/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=\ngithub.com/newrelic/go-agent v1.11.0 h1:jnd8+H6dB+93UTJHFT1wJoij5spKNN/xZ0nkw0kvt7o=\ngithub.com/newrelic/go-agent v1.11.0/go.mod h1:a8Fv1b/fYhFSReoTU6HDkTYIMZeSVNffmoS726Y0LzQ=\ngithub.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=\ngithub.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=\ngithub.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=\ngithub.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=\ngithub.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=\ngithub.com/pelletier/go-buffruneio v0.2.0 h1:U4t4R6YkofJ5xHm3dJzuRpPZ0mr5MMCoAWooScCR7aA=\ngithub.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=\ngithub.com/pelletier/go-toml v0.0.0-20170227222904-361678322880 h1:3UCAtS/p4J0cFiubq1hFsel1jlSxSp6SZEZoAFUREh8=\ngithub.com/pelletier/go-toml v0.0.0-20170227222904-361678322880/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=\ngithub.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=\ngithub.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=\ngithub.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=\ngithub.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=\ngithub.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7 h1:gGBSHPOU7g8YjTbhwn+lvFm2VDEhhA+PwDIlstkgSxE=\ngithub.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M=\ngithub.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=\ngithub.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=\ngithub.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=\ngithub.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=\ngithub.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=\ngithub.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=\ngithub.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=\ngithub.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=\ngithub.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=\ngithub.com/savaki/jq v0.0.0-20161209013833-0e6baecebbf8 h1:ajJQhvqPSQFJJ4aV5mDAMx8F7iFi6Dxfo6y62wymLNs=\ngithub.com/savaki/jq v0.0.0-20161209013833-0e6baecebbf8/go.mod h1:Nw/CCOXNyF5JDd6UpYxBwG5WWZ2FOJ/d5QnXL4KQ6vY=\ngithub.com/sirupsen/logrus v1.0.3 h1:B5C/igNWoiULof20pKfY4VntcIPqKuwEmoLZrabbUrc=\ngithub.com/sirupsen/logrus v1.0.3/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=\ngithub.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E=\ngithub.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=\ngithub.com/spaolacci/murmur3 v0.0.0-20170819071325-9f5d223c6079 h1:lDiM+yMjW7Ork8mhl0YN0qO1Z02qGoe1vwzGc1gP/8U=\ngithub.com/spaolacci/murmur3 v0.0.0-20170819071325-9f5d223c6079/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=\ngithub.com/spf13/afero v0.0.0-20170217164146-9be650865eab h1:IVAbBHQR8rXL2Fc8Zba/lMF7KOnTi70lqdx91UTuAwQ=\ngithub.com/spf13/afero v0.0.0-20170217164146-9be650865eab/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=\ngithub.com/spf13/cast v0.0.0-20170221152302-f820543c3592 h1:xwZ8A+Sp00knVqYp3nzyQJ931wd7IVQGan4Iur2QpX8=\ngithub.com/spf13/cast v0.0.0-20170221152302-f820543c3592/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=\ngithub.com/spf13/jwalterweatherman v0.0.0-20170109133355-fa7ca7e836cf h1:F3R4gmObzwZfjwH3hCs9WIyyTPjL5yC/cfdsW5hORhI=\ngithub.com/spf13/jwalterweatherman v0.0.0-20170109133355-fa7ca7e836cf/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=\ngithub.com/spf13/pflag v1.0.0 h1:oaPbdDe/x0UncahuwiPxW1GYJyilRAdsPnq3e1yaPcI=\ngithub.com/spf13/pflag v1.0.0/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=\ngithub.com/spf13/viper v1.0.0 h1:RUA/ghS2i64rlnn4ydTfblY8Og8QzcPtCcHvgMn+w/I=\ngithub.com/spf13/viper v1.0.0/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=\ngithub.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=\ngithub.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=\ngithub.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=\ngithub.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU=\ngithub.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=\ngithub.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=\ngithub.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=\ngithub.com/ugorji/go v0.0.0-20171019201919-bdcc60b419d1 h1:UvhxfNjNqlZ/x3cDyqxMhoiUpemd3zXkVQApN6bM/lg=\ngithub.com/ugorji/go v0.0.0-20171019201919-bdcc60b419d1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=\ngithub.com/vulcand/predicate v1.0.0 h1:c5lVsC9SKrQjdWNwTTG3RkADPKhSw1SrUZWq6LJL21k=\ngithub.com/vulcand/predicate v1.0.0/go.mod h1:mlccC5IRBoc2cIFmCB8ZM62I3VDb6p2GXESMHa3CnZg=\ngithub.com/vulcand/route v0.0.0-20160805191529-61904570391b h1:eCB3pa/SYYqLuajbklwy+mJAYNU1U8JQLv3M9gwKSeg=\ngithub.com/vulcand/route v0.0.0-20160805191529-61904570391b/go.mod h1:Pn2LM+/AaNyDRnlxKzatwCJiGBR/ZnRILFto79oYeUg=\ngithub.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=\ngithub.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=\ngo.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk=\ngo.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=\ngolang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 h1:MQ/ZZiDsUapFFiMS+vzwXkCTeEKaum+Do5rINYJDmxc=\ngolang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=\ngolang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=\ngolang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20181201002055-351d144fa1fc h1:a3CU5tJYVj92DY2LaA1kUkrsqD5/3mLDhx2NcNqyW+0=\ngolang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=\ngolang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=\ngolang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20180909124046-d0be0721c37e h1:o3PsSEY8E4eXWkXrIP9YJALUkVZqzHJT5DOasTyn8Vs=\ngolang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=\ngolang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=\ngolang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=\ngolang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngoogle.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=\ngoogle.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=\ngoogle.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=\ngoogle.golang.org/grpc v1.18.0 h1:IZl7mfBGfbhYx2p2rKRtYgDFw6SBz+kclmxYrCksPPA=\ngoogle.golang.org/grpc v1.18.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=\ngopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo=\ngopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=\ngopkg.in/alexcesaro/statsd.v2 v2.0.0 h1:FXkZSCZIH17vLCO5sO2UucTHsH9pc+17F6pl3JVCwMc=\ngopkg.in/alexcesaro/statsd.v2 v2.0.0/go.mod h1:i0ubccKGzBVNBpdGV5MocxyA/XlLUJzA7SLonnE4drU=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=\ngopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=\ngopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=\ngopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0=\ngopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=\ngopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=\ngopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=\ngopkg.in/urfave/cli.v1 v1.20.0 h1:NdAVW6RYxDif9DhDHaAortIu956m2c0v+09AZBPTbE0=\ngopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=\ngopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=\ngopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\nhonnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\n" }, { "path": "goreleaser.yml", "content": "builds:\n - main: ./cmd/weaver-server/\n goos:\n - linux\n - darwin\n - windows\n goarch:\n - amd64\n - 386\n\narchive:\n replacements:\n amd64: 64-bit\n 386: 32-bit\n darwin: macOS\n format: zip\n" }, { "path": "pkg/instrumentation/newrelic.go", "content": "package instrumentation\n\nimport (\n\t\"context\"\n\t\"log\"\n\t\"net/http\"\n\t\"time\"\n\n\t\"github.com/gojektech/weaver/config\"\n\tnewrelic \"github.com/newrelic/go-agent\"\n)\n\ntype ctxKey int\n\nconst txKey ctxKey = 0\n\nvar newRelicApp newrelic.Application\n\nfunc InitNewRelic() newrelic.Application {\n\tcfg := config.NewRelicConfig()\n\tif cfg.Enabled {\n\t\tapp, err := newrelic.NewApplication(cfg)\n\t\tif err != nil {\n\t\t\tlog.Fatalf(err.Error())\n\t\t}\n\n\t\tnewRelicApp = app\n\t}\n\treturn newRelicApp\n}\n\nfunc ShutdownNewRelic() {\n\tif config.NewRelicConfig().Enabled {\n\t\tnewRelicApp.Shutdown(time.Second)\n\t}\n}\n\nfunc NewRelicApp() newrelic.Application {\n\treturn newRelicApp\n}\n\nfunc StartRedisSegmentNow(op string, coll string, txn newrelic.Transaction) newrelic.DatastoreSegment {\n\ts := newrelic.DatastoreSegment{\n\t\tProduct: newrelic.DatastoreRedis,\n\t\tCollection: coll,\n\t\tOperation: op,\n\t}\n\n\ts.StartTime = newrelic.StartSegmentNow(txn)\n\treturn s\n}\n\nfunc NewContext(ctx context.Context, w http.ResponseWriter) context.Context {\n\tif config.NewRelicConfig().Enabled {\n\t\ttx, ok := w.(newrelic.Transaction)\n\t\tif !ok {\n\t\t\treturn ctx\n\t\t}\n\t\treturn context.WithValue(ctx, txKey, tx)\n\t}\n\treturn ctx\n}\n\nfunc NewContextWithTransaction(ctx context.Context, tx newrelic.Transaction) context.Context {\n\treturn context.WithValue(ctx, txKey, tx)\n}\n\nfunc GetTx(ctx context.Context) (newrelic.Transaction, bool) {\n\ttx, ok := ctx.Value(txKey).(newrelic.Transaction)\n\treturn tx, ok\n}\n" }, { "path": "pkg/instrumentation/statsd.go", "content": "package instrumentation\n\nimport (\n\t\"fmt\"\n\t\"net/http\"\n\t\"time\"\n\n\t\"github.com/gojektech/weaver/config\"\n\t\"github.com/gojektech/weaver/pkg/logger\"\n\tstatsd \"gopkg.in/alexcesaro/statsd.v2\"\n)\n\nvar statsD *statsd.Client\n\nfunc InitiateStatsDMetrics() error {\n\tstatsDConfig := config.StatsD()\n\n\tif statsDConfig.Enabled() {\n\t\tflushPeriod := time.Duration(statsDConfig.FlushPeriodInSeconds()) * time.Second\n\t\taddress := fmt.Sprintf(\"%s:%d\", statsDConfig.Host(), statsDConfig.Port())\n\n\t\tvar err error\n\t\tstatsD, err = statsd.New(statsd.Address(address),\n\t\t\tstatsd.Prefix(statsDConfig.Prefix()), statsd.FlushPeriod(flushPeriod))\n\n\t\tif err != nil {\n\t\t\tlogger.Errorf(\"StatsD: Error initiating client %s\", err)\n\t\t\treturn err\n\t\t}\n\n\t\tlogger.Infof(\"StatsD: Sending metrics\")\n\t}\n\n\treturn nil\n}\n\nfunc StatsDClient() *statsd.Client {\n\treturn statsD\n}\n\nfunc CloseStatsDClient() {\n\tif statsD != nil {\n\t\tlogger.Infof(\"StatsD: Shutting down\")\n\t\tstatsD.Close()\n\t}\n}\n\nfunc NewTiming() statsd.Timing {\n\tif statsD != nil {\n\t\treturn statsD.NewTiming()\n\t}\n\n\treturn statsd.Timing{}\n}\n\nfunc IncrementTotalRequestCount() {\n\tincrementProbe(\"request.total.count\")\n}\n\nfunc IncrementAPIRequestCount(apiName string) {\n\tincrementProbe(fmt.Sprintf(\"request.api.%s.count\", apiName))\n}\n\nfunc IncrementAPIStatusCount(apiName string, httpStatusCode int) {\n\tincrementProbe(fmt.Sprintf(\"request.api.%s.status.%d.count\", apiName, httpStatusCode))\n}\n\nfunc IncrementAPIBackendRequestCount(apiName, backendName string) {\n\tincrementProbe(fmt.Sprintf(\"request.api.%s.backend.%s.count\", apiName, backendName))\n}\n\nfunc IncrementAPIBackendStatusCount(apiName, backendName string, httpStatusCode int) {\n\tincrementProbe(fmt.Sprintf(\"request.api.%s.backend.%s.status.%d.count\", apiName, backendName, httpStatusCode))\n}\n\nfunc IncrementCrashCount() {\n\tincrementProbe(\"request.internal.crash.count\")\n}\n\nfunc IncrementNotFound() {\n\tincrementProbe(fmt.Sprintf(\"request.internal.%d.count\", http.StatusNotFound))\n}\n\nfunc IncrementInternalAPIStatusCount(aclName string, statusCode int) {\n\tincrementProbe(fmt.Sprintf(\"request.api.%s.internal.status.%d.count\", aclName, statusCode))\n}\n\nfunc TimeTotalLatency(timing statsd.Timing) {\n\tif statsD != nil {\n\t\ttiming.Send(\"request.time.total\")\n\t}\n\n\treturn\n}\n\nfunc TimeAPILatency(apiName string, timing statsd.Timing) {\n\tif statsD != nil {\n\t\ttiming.Send(fmt.Sprintf(\"request.api.%s.time.total\", apiName))\n\t}\n\n\treturn\n}\n\nfunc TimeAPIBackendLatency(apiName, backendName string, timing statsd.Timing) {\n\tif statsD != nil {\n\t\ttiming.Send(fmt.Sprintf(\"request.api.%s.backend.%s.time.total\", apiName, backendName))\n\t}\n\n\treturn\n}\n\nfunc incrementProbe(key string) {\n\tif statsD == nil {\n\t\treturn\n\t}\n\n\tgo statsD.Increment(key)\n}\n" }, { "path": "pkg/logger/logger.go", "content": "package logger\n\nimport (\n\t\"net/http\"\n\t\"os\"\n\n\t\"github.com/gojektech/weaver/config\"\n\t\"github.com/gojektech/weaver/pkg/util\"\n\t\"github.com/sirupsen/logrus\"\n)\n\nvar logger *logrus.Logger\n\nfunc SetupLogger() {\n\tlevel, err := logrus.ParseLevel(config.LogLevel())\n\tif err != nil {\n\t\tlevel = logrus.WarnLevel\n\t}\n\n\tlogger = &logrus.Logger{\n\t\tOut: os.Stdout,\n\t\tHooks: make(logrus.LevelHooks),\n\t\tLevel: level,\n\t\tFormatter: &logrus.JSONFormatter{},\n\t}\n}\n\nfunc AddHook(hook logrus.Hook) {\n\tlogger.Hooks.Add(hook)\n}\n\nfunc Debug(args ...interface{}) {\n\tlogger.Debug(args...)\n}\n\nfunc Debugf(format string, args ...interface{}) {\n\tlogger.Debugf(format, args...)\n}\n\nfunc Debugln(args ...interface{}) {\n\tlogger.Debugln(args...)\n}\n\nfunc Debugrf(r *http.Request, format string, args ...interface{}) {\n\thttpRequestLogEntry(r).Debugf(format, args...)\n}\n\nfunc Error(args ...interface{}) {\n\tlogger.Error(args...)\n}\n\nfunc Errorf(format string, args ...interface{}) {\n\tlogger.Errorf(format, args...)\n}\n\nfunc Errorln(args ...interface{}) {\n\tlogger.Errorln(args...)\n}\n\nfunc Errorrf(r *http.Request, format string, args ...interface{}) {\n\thttpRequestLogEntry(r).Errorf(format, args...)\n}\n\nfunc ErrorWithFieldsf(fields logrus.Fields, format string, args ...interface{}) {\n\tlogger.WithFields(fields).Errorf(format, args...)\n}\n\nfunc Fatal(args ...interface{}) {\n\tlogger.Fatal(args...)\n}\n\nfunc Fatalf(format string, args ...interface{}) {\n\tlogger.Fatalf(format, args...)\n}\n\nfunc Fatalln(args ...interface{}) {\n\tlogger.Fatalln(args...)\n}\n\nfunc Info(args ...interface{}) {\n\tlogger.Info(args...)\n}\n\nfunc Infof(format string, args ...interface{}) {\n\tlogger.Infof(format, args...)\n}\n\nfunc Infoln(args ...interface{}) {\n\tlogger.Infoln(args...)\n}\n\nfunc Inforf(r *http.Request, format string, args ...interface{}) {\n\thttpRequestLogEntry(r).Infof(format, args...)\n}\n\nfunc InfoWithFieldsf(fields logrus.Fields, format string, args ...interface{}) {\n\tlogger.WithFields(fields).Infof(format, args...)\n}\n\nfunc ProxyInfo(aclName string, downstreamHost string, r *http.Request, responseStatus int, rw http.ResponseWriter) {\n\tlogger.WithFields(logrus.Fields{\n\t\t\"type\": \"proxy\",\n\t\t\"downstream_host\": downstreamHost,\n\t\t\"api_name\": aclName,\n\t\t\"request\": httpRequestFields(r),\n\t\t\"response\": httpResponseFields(responseStatus, rw),\n\t}).Info(\"proxy\")\n}\n\nfunc httpRequestFields(r *http.Request) logrus.Fields {\n\trequestHeaders := map[string]string{}\n\tfor k := range r.Header {\n\t\tnormalizedKey := util.ToSnake(k)\n\t\tif normalizedKey == \"authorization\" {\n\t\t\tcontinue\n\t\t}\n\n\t\trequestHeaders[normalizedKey] = r.Header.Get(k)\n\n\t}\n\treturn logrus.Fields{\n\t\t\"uri\": r.URL.String(),\n\t\t\"query\": r.URL.Query(),\n\t\t\"method\": r.Method,\n\t\t\"headers\": requestHeaders,\n\t}\n}\n\nfunc httpResponseFields(responseStatus int, rw http.ResponseWriter) logrus.Fields {\n\tresponseHeaders := map[string]string{}\n\tfor k := range rw.Header() {\n\t\tresponseHeaders[util.ToSnake(k)] = rw.Header().Get(k)\n\n\t}\n\treturn logrus.Fields{\n\t\t\"status\": responseStatus,\n\t\t\"headers\": responseHeaders,\n\t}\n}\n\nfunc Warn(args ...interface{}) {\n\tlogger.Warn(args...)\n}\n\nfunc Warnf(format string, args ...interface{}) {\n\tlogger.Warnf(format, args...)\n}\n\nfunc Warnln(args ...interface{}) {\n\tlogger.Warnln(args...)\n}\n\nfunc WithField(key string, value interface{}) *logrus.Entry {\n\treturn logger.WithField(key, value)\n}\n\nfunc WithFields(fields logrus.Fields) *logrus.Entry {\n\treturn logger.WithFields(fields)\n}\n\nfunc httpRequestLogEntry(r *http.Request) *logrus.Entry {\n\treturn logger.WithFields(logrus.Fields{\n\t\t\"request_method\": r.Method,\n\t\t\"request_host\": r.Host,\n\t\t\"request_url\": r.URL.String(),\n\t})\n}\n" }, { "path": "pkg/matcher/matcher.go", "content": "package matcher\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"io/ioutil\"\n\t\"net/http\"\n\t\"regexp\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/pkg/errors\"\n\t\"github.com/savaki/jq\"\n)\n\nfunc New(matcherName string) (MatcherFunc, bool) {\n\tmf, found := matcherMux[matcherName]\n\treturn mf, found\n}\n\ntype MatcherFunc func(request *http.Request, shardExpr string) (shardKey string, err error)\n\nvar matcherMux = map[string]MatcherFunc{\n\t\"header\": func(req *http.Request, expr string) (string, error) {\n\t\treturn req.Header.Get(expr), nil\n\t},\n\n\t\"multi-headers\": func(req *http.Request, expr string) (string, error) {\n\t\theaders := strings.Split(expr, \",\")\n\t\tvar headerValues strings.Builder\n\n\t\theadersCount := len(headers)\n\t\tif headersCount == 0 {\n\t\t\treturn \"\", nil\n\t\t}\n\n\t\tfor idx, header := range headers {\n\t\t\theaderValue := req.Header.Get(header)\n\n\t\t\theaderValues.Grow(len(headerValue))\n\t\t\theaderValues.WriteString(headerValue)\n\n\t\t\tif (idx + 1) != headersCount {\n\t\t\t\theaderValues.Grow(1)\n\t\t\t\theaderValues.WriteString(\",\")\n\t\t\t}\n\t\t}\n\n\t\treturn headerValues.String(), nil\n\t},\n\n\t\"param\": func(req *http.Request, expr string) (string, error) {\n\t\treturn req.URL.Query().Get(expr), nil\n\t},\n\n\t\"path\": func(req *http.Request, expr string) (string, error) {\n\t\trex := regexp.MustCompile(expr)\n\t\tmatch := rex.FindStringSubmatch(req.URL.Path)\n\t\tif len(match) == 0 {\n\t\t\treturn \"\", fmt.Errorf(\"no match found for expr: %s\", expr)\n\t\t}\n\n\t\treturn match[1], nil\n\t},\n\n\t\"body\": func(req *http.Request, expr string) (string, error) {\n\t\trequestBody, err := ioutil.ReadAll(req.Body)\n\t\tif err != nil {\n\t\t\treturn \"\", errors.Wrapf(err, \"failed to read request body for expr: %s\", expr)\n\t\t}\n\n\t\treq.Body = ioutil.NopCloser(bytes.NewBuffer(requestBody))\n\n\t\tvar bodyKey interface{}\n\t\top, err := jq.Parse(expr)\n\t\tif err != nil {\n\t\t\treturn \"\", errors.Wrapf(err, \"failed to parse shard expr: %s\", expr)\n\t\t}\n\n\t\tkey, err := op.Apply(requestBody)\n\t\tif err != nil {\n\t\t\treturn \"\", errors.Wrapf(err, \"failed to apply parsed shard expr: %s\", expr)\n\t\t}\n\n\t\tif err := json.Unmarshal(key, &bodyKey); err != nil {\n\t\t\treturn \"\", errors.Wrapf(err, \"failed to unmarshal data for shard expr: %s\", expr)\n\t\t}\n\n\t\tswitch v := bodyKey.(type) {\n\t\tcase string:\n\t\t\treturn v, nil\n\t\tcase float64:\n\t\t\treturn strconv.FormatFloat(v, 'f', -1, 64), nil\n\t\tdefault:\n\t\t\treturn \"\", errors.New(\"failed to type assert bodyKey\")\n\t\t}\n\t},\n}\n" }, { "path": "pkg/matcher/matcher_test.go", "content": "package matcher\n\nimport (\n\t\"bytes\"\n\t\"net/http/httptest\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestBodyMatcher(t *testing.T) {\n\tbody := bytes.NewReader([]byte(`{ \"drivers\": { \"id\": \"123\", \"name\": \"hello\"} }`))\n\n\treq := httptest.NewRequest(\"GET\", \"/drivers\", body)\n\texpr := \".drivers.id\"\n\n\tkey, err := matcherMux[\"body\"](req, expr)\n\trequire.NoError(t, err, \"should not have failed to match a key\")\n\n\tassert.Equal(t, \"123\", key)\n}\n\nfunc TestBodyMatcherParseInt(t *testing.T) {\n\tbody := bytes.NewReader([]byte(`{ \"routeRequests\": [{ \"id\": \"123\", \"serviceType\": 1}] }`))\n\n\treq := httptest.NewRequest(\"GET\", \"/drivers\", body)\n\texpr := \".routeRequests.[0].serviceType\"\n\n\tkey, err := matcherMux[\"body\"](req, expr)\n\trequire.NoError(t, err, \"should not have failed to match a key\")\n\n\tassert.Equal(t, \"1\", key)\n}\n\nfunc TestBodyMatcherParseTypeAssertFail(t *testing.T) {\n\tbody := bytes.NewReader([]byte(`{ \"routeRequests\": [{ \"id\": \"123\", \"serviceType\": []}] }`))\n\n\treq := httptest.NewRequest(\"GET\", \"/drivers\", body)\n\texpr := \".routeRequests.[0].serviceType\"\n\n\tkey, err := matcherMux[\"body\"](req, expr)\n\trequire.Error(t, err, \"should have failed to match a key\")\n\trequire.Equal(t, \"\", key)\n\n\tassert.Equal(t, \"failed to type assert bodyKey\", err.Error())\n}\n\nfunc TestBodyMatcherFail(t *testing.T) {\n\tbody := bytes.NewReader([]byte(`{ \"drivers\": { \"id\": \"123\", \"name\": \"hello\"} }`))\n\n\treq := httptest.NewRequest(\"GET\", \"/drivers\", body)\n\texpr := \".drivers.blah\"\n\n\tkey, err := matcherMux[\"body\"](req, expr)\n\trequire.Error(t, err, \"should have failed to match a key\")\n\n\tassert.Equal(t, \"\", key)\n}\n\nfunc TestHeaderMatcher(t *testing.T) {\n\treq := httptest.NewRequest(\"GET\", \"/drivers\", nil)\n\treq.Header.Add(\"Hello\", \"World\")\n\n\texpr := \"Hello\"\n\n\tkey, err := matcherMux[\"header\"](req, expr)\n\trequire.NoError(t, err, \"should not have failed to match a key\")\n\n\tassert.Equal(t, \"World\", key)\n}\n\nfunc TestHeadersCsvMatcherWithSingleHeader(t *testing.T) {\n\treq := httptest.NewRequest(\"GET\", \"/drivers\", nil)\n\treq.Header.Add(\"H1\", \"One\")\n\treq.Header.Add(\"H2\", \"Two\")\n\treq.Header.Add(\"H3\", \"Three\")\n\n\texpr := \"H2\"\n\n\tkey, err := matcherMux[\"multi-headers\"](req, expr)\n\trequire.NoError(t, err, \"should not have failed to extract headers\")\n\n\tassert.Equal(t, \"Two\", key)\n}\n\nfunc TestHeadersCsvMatcherWithSingleHeaderWhenNoneArePresent(t *testing.T) {\n\treq := httptest.NewRequest(\"GET\", \"/drivers\", nil)\n\texpr := \"H1\"\n\n\tkey, err := matcherMux[\"multi-headers\"](req, expr)\n\trequire.NoError(t, err, \"should not have failed to extract headers\")\n\n\tassert.Equal(t, \"\", key)\n}\n\nfunc TestHeadersCsvMatcherWithZeroHeaders(t *testing.T) {\n\treq := httptest.NewRequest(\"GET\", \"/drivers\", nil)\n\treq.Header.Add(\"H1\", \"One\")\n\treq.Header.Add(\"H2\", \"Two\")\n\treq.Header.Add(\"H3\", \"Three\")\n\n\texpr := \"\"\n\n\tkey, err := matcherMux[\"multi-headers\"](req, expr)\n\trequire.NoError(t, err, \"should not have failed to extract headers\")\n\n\tassert.Equal(t, \"\", key)\n}\n\nfunc TestHeadersCsvMatcherWithMultipleHeaders(t *testing.T) {\n\treq := httptest.NewRequest(\"GET\", \"/drivers\", nil)\n\treq.Header.Add(\"H1\", \"One\")\n\treq.Header.Add(\"H2\", \"Two\")\n\treq.Header.Add(\"H3\", \"Three\")\n\n\texpr := \"H1,H3\"\n\n\tkey, err := matcherMux[\"multi-headers\"](req, expr)\n\trequire.NoError(t, err, \"should not have failed to extract headers\")\n\n\tassert.Equal(t, \"One,Three\", key)\n}\n\nfunc TestHeadersCsvMatcherWithMultipleHeadersWhenSomeArePresent(t *testing.T) {\n\treq := httptest.NewRequest(\"GET\", \"/drivers\", nil)\n\treq.Header.Add(\"H3\", \"Three\")\n\n\texpr := \"H1,H3\"\n\n\tkey, err := matcherMux[\"multi-headers\"](req, expr)\n\trequire.NoError(t, err, \"should not have failed to extract headers\")\n\n\tassert.Equal(t, \",Three\", key)\n}\n\nfunc TestHeadersCsvMatcherWithMultipleHeadersWhenNoneArePresent(t *testing.T) {\n\treq := httptest.NewRequest(\"GET\", \"/drivers\", nil)\n\texpr := \"H1,H3\"\n\n\tkey, err := matcherMux[\"multi-headers\"](req, expr)\n\trequire.NoError(t, err, \"should not have failed to extract headers\")\n\n\tassert.Equal(t, \",\", key)\n}\n\nfunc TestHeaderMatcherFail(t *testing.T) {\n\treq := httptest.NewRequest(\"GET\", \"/drivers\", nil)\n\n\texpr := \"Hello\"\n\n\tkey, err := matcherMux[\"header\"](req, expr)\n\trequire.NoError(t, err, \"should not have failed to match a key\")\n\n\tassert.Equal(t, \"\", key)\n}\n\nfunc TestParamMatcher(t *testing.T) {\n\treq := httptest.NewRequest(\"GET\", \"/drivers?url=blah\", nil)\n\n\texpr := \"url\"\n\n\tkey, err := matcherMux[\"param\"](req, expr)\n\trequire.NoError(t, err, \"should not have failed to match a key\")\n\n\tassert.Equal(t, \"blah\", key)\n}\n\nfunc TestParamMatcherFail(t *testing.T) {\n\treq := httptest.NewRequest(\"GET\", \"/drivers?url=blah\", nil)\n\n\texpr := \"hello\"\n\n\tkey, err := matcherMux[\"param\"](req, expr)\n\trequire.NoError(t, err, \"should not have failed to match a key\")\n\n\tassert.Equal(t, \"\", key)\n}\n\nfunc TestPathMatcher(t *testing.T) {\n\treq := httptest.NewRequest(\"GET\", \"/drivers/123\", nil)\n\n\texpr := `/drivers/(\\d+)`\n\n\tkey, err := matcherMux[\"path\"](req, expr)\n\trequire.NoError(t, err, \"should not have failed to match a key\")\n\n\tassert.Equal(t, \"123\", key)\n}\n\nfunc TestPathMatcherFail(t *testing.T) {\n\treq := httptest.NewRequest(\"GET\", \"/drivers/123\", nil)\n\n\texpr := `/drivers/blah`\n\n\tkey, err := matcherMux[\"path\"](req, expr)\n\trequire.Error(t, err, \"should have failed to match a key\")\n\n\tassert.Equal(t, \"\", key)\n}\n" }, { "path": "pkg/shard/domain.go", "content": "package shard\n\nimport (\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/gojektech/weaver\"\n\t\"github.com/gojektech/weaver/config\"\n\t\"github.com/pkg/errors\"\n)\n\ntype CustomError struct {\n\tExitMessage string\n}\n\nfunc (e *CustomError) Error() string {\n\treturn fmt.Sprintf(\"[error] %s\", e.ExitMessage)\n}\n\nfunc Error(msg string) error {\n\treturn &CustomError{msg}\n}\n\ntype BackendDefinition struct {\n\tBackendName string `json:\"backend_name\"`\n\tBackendURL string `json:\"backend\"`\n\tTimeout *float64 `json:\"timeout,omitempty\"`\n}\n\nfunc (bd BackendDefinition) Validate() error {\n\tif bd.BackendName == \"\" {\n\t\treturn errors.WithStack(fmt.Errorf(\"missing backend name in shard config: %+v\", bd))\n\t}\n\n\tif bd.BackendURL == \"\" {\n\t\treturn errors.WithStack(fmt.Errorf(\"missing backend url in shard config: %+v\", bd))\n\t}\n\n\treturn nil\n}\n\nfunc toBackends(shardConfig map[string]BackendDefinition) (map[string]*weaver.Backend, error) {\n\tbackends := map[string]*weaver.Backend{}\n\n\tfor key, backendDefinition := range shardConfig {\n\t\tif err := backendDefinition.Validate(); err != nil {\n\t\t\treturn nil, errors.Wrapf(err, \"failed to validate backend definition\")\n\t\t}\n\n\t\tbackend, err := parseBackend(backendDefinition)\n\t\tif err != nil {\n\t\t\treturn nil, errors.Wrapf(err, \"failed to parseBackends from backendDefinition\")\n\t\t}\n\n\t\tbackends[key] = backend\n\t}\n\n\treturn backends, nil\n}\n\nfunc parseBackend(shardConfig BackendDefinition) (*weaver.Backend, error) {\n\ttimeoutInDuration := config.Proxy().ProxyDialerTimeoutInMS()\n\n\tif shardConfig.Timeout != nil {\n\t\ttimeoutInDuration = time.Duration(*shardConfig.Timeout)\n\t}\n\n\tbackendOptions := weaver.BackendOptions{\n\t\tTimeout: timeoutInDuration * time.Millisecond,\n\t}\n\n\treturn weaver.NewBackend(shardConfig.BackendName, shardConfig.BackendURL, backendOptions)\n}\n" }, { "path": "pkg/shard/hashring.go", "content": "package shard\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"regexp\"\n\t\"strconv\"\n\n\t\"github.com/gojekfarm/hashring\"\n\t\"github.com/gojektech/weaver\"\n\t\"github.com/pkg/errors\"\n)\n\nfunc NewHashRingStrategy(data json.RawMessage) (weaver.Sharder, error) {\n\tcfg := HashRingStrategyConfig{}\n\tif err := json.Unmarshal(data, &cfg); err != nil {\n\t\treturn nil, err\n\t}\n\n\tif err := cfg.Validate(); err != nil {\n\t\treturn nil, err\n\t}\n\n\thashRing, backends, err := hashringBackends(cfg)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &HashRingStrategy{\n\t\thashRing: hashRing,\n\t\tbackends: backends,\n\t}, nil\n}\n\ntype HashRingStrategy struct {\n\thashRing *hashring.HashRingCluster\n\tbackends map[string]*weaver.Backend\n}\n\nfunc (rs HashRingStrategy) Shard(key string) (*weaver.Backend, error) {\n\tserverName := rs.hashRing.GetServer(key)\n\treturn rs.backends[serverName], nil\n}\n\ntype HashRingStrategyConfig struct {\n\tTotalVirtualBackends *int `json:\"totalVirtualBackends\"`\n\tBackends map[string]BackendDefinition `json:\"backends\"`\n}\n\nfunc (hrCfg HashRingStrategyConfig) Validate() error {\n\tif hrCfg.Backends == nil || len(hrCfg.Backends) == 0 {\n\t\treturn fmt.Errorf(\"No Shard Backends Specified Or Specified Incorrectly\")\n\t}\n\n\tfor _, backend := range hrCfg.Backends {\n\t\tif err := backend.Validate(); err != nil {\n\t\t\treturn errors.Wrapf(err, \"failed to validate backendDefinition for backend: %s\", backend.BackendName)\n\t\t}\n\t}\n\n\treturn nil\n}\n\nfunc hashringBackends(cfg HashRingStrategyConfig) (*hashring.HashRingCluster, map[string]*weaver.Backend, error) {\n\tif cfg.TotalVirtualBackends == nil || *cfg.TotalVirtualBackends < 0 {\n\t\tdefaultBackends := 1000\n\t\tcfg.TotalVirtualBackends = &defaultBackends\n\t}\n\n\tbackendDetails := map[string]*weaver.Backend{}\n\thashRingCluster := hashring.NewHashRingCluster(*cfg.TotalVirtualBackends)\n\n\tvirtualNodesFound := map[int]bool{}\n\tmaxValue := 0\n\trangeRegexp := regexp.MustCompile(\"^([\\\\d]+)-([\\\\d]+)$\")\n\tfor k, v := range cfg.Backends {\n\t\tmatches := rangeRegexp.FindStringSubmatch(k)\n\t\tif len(matches) != 3 {\n\t\t\treturn nil, nil, fmt.Errorf(\"Invalid range key format: %s\", k)\n\t\t}\n\t\tend, _ := strconv.Atoi(matches[2])\n\t\tstart, _ := strconv.Atoi(matches[1])\n\n\t\tif end <= start {\n\t\t\treturn nil, nil, fmt.Errorf(\"Invalid range key %d-%d for backends\", start, end)\n\t\t}\n\t\tfor i := start; i <= end; i++ {\n\t\t\tif _, ok := virtualNodesFound[i]; ok {\n\t\t\t\treturn nil, nil, fmt.Errorf(\"Overlap seen in range key %d\", i)\n\t\t\t}\n\t\t\tvirtualNodesFound[i] = true\n\n\t\t\tif maxValue < i {\n\t\t\t\tmaxValue = i\n\t\t\t}\n\t\t}\n\t\tbackend, err := parseBackend(v)\n\t\tif err != nil {\n\t\t\treturn nil, nil, err\n\t\t}\n\t\tbackendDetails[backend.Name] = backend\n\t\thashRingCluster.AddServer(backend.Name, k)\n\t}\n\n\tif maxValue != *cfg.TotalVirtualBackends-1 {\n\t\treturn nil, nil, fmt.Errorf(\"Shard is out of bounds Max %d found %d\", *cfg.TotalVirtualBackends-1, maxValue)\n\t}\n\n\tfor i := 0; i < *cfg.TotalVirtualBackends; i++ {\n\t\tif _, ok := virtualNodesFound[i]; !ok {\n\t\t\treturn nil, nil, fmt.Errorf(\"Shard is missing coverage for %d\", i)\n\t\t}\n\t}\n\n\treturn hashRingCluster, backendDetails, nil\n}\n" }, { "path": "pkg/shard/hashring_test.go", "content": "package shard_test\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"math\"\n\t\"runtime\"\n\t\"strconv\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/gojektech/weaver/pkg/shard\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestNewHashringStrategy(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n \"totalVirtualBackends\": 1000,\n\t\t\"backends\": {\n\t\t\t\"0-250\": { \"timeout\": 100, \"backend_name\": \"foobar1\", \"backend\": \"http://shard00.local\"},\n\t\t\t\"251-500\": { \"backend_name\": \"foobar2\", \"backend\": \"http://shard01.local\"},\n\t\t\t\"501-725\": { \"backend_name\": \"foobar3\", \"backend\": \"http://shard02.local\"},\n \"726-999\": { \"backend_name\": \"foobar4\", \"backend\": \"http://shard03.local\"}\n\t }\n\t}`)\n\thashRingStrategy, err := shard.NewHashRingStrategy(shardConfig)\n\tassert.Nil(t, err)\n\tassert.NotNil(t, hashRingStrategy)\n}\n\nfunc TestShouldFailToCreateWhenWrongBackends(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"totalVirtualBackends\": 500,\n\t\t\"backends\": \"foo\"\n\t}`)\n\texpectedErr := fmt.Errorf(\"json: cannot unmarshal string into Go struct field HashRingStrategyConfig.backends of type map[string]shard.BackendDefinition\")\n\thashRingStrategy, err := shard.NewHashRingStrategy(shardConfig)\n\tassert.Equal(t, expectedErr.Error(), err.Error())\n\tassert.Nil(t, hashRingStrategy)\n}\n\nfunc TestShouldFailToCreateWhenNoBackends(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"totalVirtualBackends\": 500\n\t}`)\n\texpectedErr := fmt.Errorf(\"No Shard Backends Specified Or Specified Incorrectly\")\n\thashRingStrategy, err := shard.NewHashRingStrategy(shardConfig)\n\tassert.Equal(t, expectedErr, err)\n\tassert.Nil(t, hashRingStrategy)\n}\n\nfunc TestShouldFailToCreateWhenNoBackendURL(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n \"totalVirtualBackends\": 1000,\n\t\t\"backends\": {\n\t\t\t\"0-999\": { \"timeout\": 100, \"backend_name\": \"foobar1\", \"backend\": \"ht$tp://shard00.local\"}\n\t }\n\t}`)\n\thashRingStrategy, err := shard.NewHashRingStrategy(shardConfig)\n\tassert.Contains(t, err.Error(), \"first path segment in URL cannot contain colon\")\n\tassert.Nil(t, hashRingStrategy)\n}\n\nfunc TestShouldFailToCreateWhenTotalVirtualBackendsIsIncorrect(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"totalVirtualBackends\": \"foo\",\n\t\t\"backends\": {\n\t\t\t\"0-10\" : { \"backend_name\": \"foo\"}\n\t\t}\n }`)\n\texpectedErr := fmt.Errorf(\"json: cannot unmarshal string into Go struct field HashRingStrategyConfig.totalVirtualBackends of type int\")\n\thashRingStrategy, err := shard.NewHashRingStrategy(shardConfig)\n\tassert.Equal(t, expectedErr.Error(), err.Error())\n\tassert.Nil(t, hashRingStrategy)\n}\n\nfunc TestShouldFailToCreateWhenBackendURLIsMissing(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"backends\": {\n\t\t\t\"foo\" : { \"backend_name\": \"foo\"}\n\t\t}\n }`)\n\thashRingStrategy, err := shard.NewHashRingStrategy(shardConfig)\n\tassert.Contains(t, err.Error(), \"missing backend url in shard config:\")\n\tassert.Nil(t, hashRingStrategy)\n}\n\nfunc TestShouldDefaultTotalVirtualBackendsWhenValueMissing(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"backends\": {\n\t\t\t\"0-999\" : { \"backend_name\": \"foo\", \"backend\": \"http://backend01\"}\n\t\t}\n\t}`)\n\thashRingStrategy, err := shard.NewHashRingStrategy(shardConfig)\n\tassert.Nil(t, err)\n\tassert.NotNil(t, hashRingStrategy)\n}\n\nfunc TestShouldFailToCreateWithIncorrectRange(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"backends\": {\n\t\t\t\"999-0\" : { \"backend_name\": \"foo\", \"backend\": \"http://blah\"}\n\t\t}\n\t}`)\n\thashRingStrategy, err := shard.NewHashRingStrategy(shardConfig)\n\tassert.Nil(t, hashRingStrategy)\n\tassert.Contains(t, err.Error(), \"Invalid range key 999-0 for backends\")\n}\n\nfunc TestShouldFailToCreateWithIncorrectRangeSpec(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"backends\": {\n\t\t\t\"999-999-0\" : { \"backend_name\": \"foo\", \"backend\": \"http://blah\"}\n\t\t}\n\t}`)\n\thashRingStrategy, err := shard.NewHashRingStrategy(shardConfig)\n\tassert.Nil(t, hashRingStrategy)\n\tassert.Contains(t, err.Error(), \"Invalid range key format:\")\n}\n\nfunc TestShouldFailToCreateHashRingOutOfBounds(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"totalVirtualBackends\": 500,\n\t\t\"backends\": {\n\t\t\t\"0-249\": { \"timeout\": 100, \"backend_name\": \"foobar1\", \"backend\": \"http://shard00.local\"},\n\t\t\t\"250-500\": { \"backend_name\": \"foobar2\", \"backend\": \"http://shard01.local\"}\n\t\t}\n\t}`)\n\texpectedErr := fmt.Errorf(\"Shard is out of bounds Max %d found %d\", 499, 500)\n\t_, err := shard.NewHashRingStrategy(shardConfig)\n\tassert.Equal(t, expectedErr, err)\n}\n\nfunc TestShouldFailToCreateHashRingOnOverlap(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"totalVirtualBackends\": 500,\n\t\t\"backends\": {\n\t\t\t\"0-249\": { \"timeout\": 100, \"backend_name\": \"foobar1\", \"backend\": \"http://shard00.local\"},\n\t\t\t\"249-499\": { \"backend_name\": \"foobar2\", \"backend\": \"http://shard01.local\"}\n\t\t}\n\t}`)\n\texpectedErr := fmt.Errorf(\"Overlap seen in range key %d\", 249)\n\t_, err := shard.NewHashRingStrategy(shardConfig)\n\tassert.Equal(t, expectedErr, err)\n}\n\nfunc TestShouldFailToCreateHashRingForMissingValuesInTheRangeInMiddle(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"totalVirtualBackends\": 500,\n\t\t\"backends\": {\n\t\t\t\"0-248\": { \"timeout\": 100, \"backend_name\": \"foobar1\", \"backend\": \"http://shard00.local\"},\n\t\t\t\"250-499\": { \"backend_name\": \"foobar2\", \"backend\": \"http://shard01.local\"}\n\t\t}\n\t}`)\n\texpectedErr := fmt.Errorf(\"Shard is missing coverage for %d\", 249)\n\t_, err := shard.NewHashRingStrategy(shardConfig)\n\tassert.Equal(t, expectedErr, err)\n}\n\nfunc TestShouldFailToCreateHashRingForMissingValuesInTheRangeAtStart(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"totalVirtualBackends\": 500,\n\t\t\"backends\": {\n\t\t\t\"1-249\": { \"timeout\": 100, \"backend_name\": \"foobar1\", \"backend\": \"http://shard00.local\"},\n\t\t\t\"250-499\": { \"backend_name\": \"foobar2\", \"backend\": \"http://shard01.local\"}\n\t\t}\n\t}`)\n\texpectedErr := fmt.Errorf(\"Shard is missing coverage for %d\", 0)\n\t_, err := shard.NewHashRingStrategy(shardConfig)\n\tassert.Equal(t, expectedErr, err)\n}\n\nfunc TestShouldCheckBackendConfiguration(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"totalVirtualBackends\": 10,\n\t\t\"backends\": {\n\t\t\t\"0-4\": \"foo\",\n\t\t\t\"5-9\": { \"backend_name\": \"foobar2\", \"backend\": \"http://shard01.local\"}\n\t\t}\n\t}`)\n\texpectedErr := fmt.Errorf(\"json: cannot unmarshal string into Go struct field HashRingStrategyConfig.backends of type shard.BackendDefinition\")\n\tstrategy, err := shard.NewHashRingStrategy(shardConfig)\n\tassert.Nil(t, strategy)\n\tassert.Equal(t, expectedErr.Error(), err.Error())\n\n}\n\nfunc TestShouldCheckBackendConfigurationForBackendName(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"totalVirtualBackends\": 10,\n\t\t\"backends\": {\n\t\t\t\"0-4\": {\"foo\": \"bar\"},\n\t\t\t\"5-9\": { \"backend_name\": \"foobar2\", \"backend\": \"http://shard01.local\"}\n\t\t}\n\t}`)\n\tstrategy, err := shard.NewHashRingStrategy(shardConfig)\n\tassert.Nil(t, strategy)\n\tassert.Contains(t, err.Error(), \"missing backend name in shard config:\")\n}\n\nfunc TestShouldCheckBackendConfigurationForBackendUrl(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"totalVirtualBackends\": 10,\n\t\t\"backends\": {\n\t\t\t\"0-4\": {\"foo\": \"bar\", \"backend_name\": \"foo\"},\n\t\t\t\"5-9\": { \"backend_name\": \"foobar2\", \"backend\": \"http://shard01.local\"}\n\t\t}\n\t}`)\n\tstrategy, err := shard.NewHashRingStrategy(shardConfig)\n\tassert.Nil(t, strategy)\n\tassert.Contains(t, err.Error(), \"missing backend url in shard config:\")\n}\n\nfunc TestShouldCheckBackendConfigurationForTimeout(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"totalVirtualBackends\": 10,\n\t\t\"backends\": {\n\t\t\t\"0-4\": {\"backend\": \"http://foo\", \"backend_name\": \"foo\", \"timeout\": \"abc\"},\n\t\t\t\"5-9\": { \"backend_name\": \"foobar2\", \"backend\": \"http://shard01.local\"}\n\t\t}\n\t}`)\n\texpectedErr := fmt.Errorf(\"json: cannot unmarshal string into Go struct field BackendDefinition.timeout of type float64\")\n\tstrategy, err := shard.NewHashRingStrategy(shardConfig)\n\tassert.Nil(t, strategy)\n\tassert.Equal(t, expectedErr.Error(), err.Error())\n}\n\nfunc TestShouldShardConsistently(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"totalVirtualBackends\": 10,\n\t\t\"backends\": {\n\t\t\t\"0-4\": { \"timeout\": 100, \"backend_name\": \"foobar1\", \"backend\": \"http://shard00.local\"},\n\t\t\t\"5-9\": { \"backend_name\": \"foobar2\", \"backend\": \"http://shard01.local\"}\n\t\t}\n\t}`)\n\n\tstrategy, _ := shard.NewHashRingStrategy(shardConfig)\n\texpectedBackend := \"foobar2\"\n\tbackend, err := strategy.Shard(\"1\")\n\tassert.Nil(t, err)\n\tassert.NotNil(t, backend)\n\tassert.Equal(t, expectedBackend, backend.Name, \"Should return foobar2 for key 1\")\n\tbackend, err = strategy.Shard(\"1\")\n\tassert.Equal(t, expectedBackend, backend.Name, \"Should return foobar2 for key 1\")\n\n}\n\nfunc TestShouldShardConsistentlyOverALargeRange(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"totalVirtualBackends\": 10,\n\t\t\"backends\": {\n\t\t\t\"0-4\": { \"timeout\": 100, \"backend_name\": \"foobar1\", \"backend\": \"http://shard00.local\"},\n\t\t\t\"5-9\": { \"backend_name\": \"foobar2\", \"backend\": \"http://shard01.local\"}\n\t\t}\n\t}`)\n\n\tstrategy, _ := shard.NewHashRingStrategy(shardConfig)\n\tshardList := []string{}\n\tfor i := 0; i < 10000; i++ {\n\t\tbackend, err := strategy.Shard(strconv.Itoa(i))\n\t\tassert.Nil(t, err, \"Failed to Shard for key %d\", i)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t\treturn\n\t\t}\n\t\tshardList = append(shardList, backend.Name)\n\t}\n\n\tfor i := 0; i < 10000; i++ {\n\t\tbackend, err := strategy.Shard(strconv.Itoa(i))\n\t\tassert.Nil(t, err, \"Failed to Re - Shard for key %d\", i)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t\treturn\n\t\t}\n\t\tassert.Equal(t, shardList[i], backend.Name, \"Sharded inconsistently for key %d %s -> %s\", i, shardList[i], backend.Name)\n\t}\n}\n\nfunc TestShouldShardConsistentlyAcrossRuns(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"totalVirtualBackends\": 1000,\n\t\t\"backends\": {\n\t\t\t\"0-249\": { \"timeout\": 100, \"backend_name\": \"foobar1\", \"backend\": \"http://shard00.local\"},\n\t\t\t\"250-499\": { \"backend_name\": \"foobar2\", \"backend\": \"http://shard01.local\"},\n\t\t\t\"500-749\": { \"backend_name\": \"foobar3\", \"backend\": \"http://shard02.local\"},\n\t\t\t\"750-999\": { \"backend_name\": \"foobar4\", \"backend\": \"http://shard03.local\"}\n\t\t}\n\t}`)\n\n\tstrategy, _ := shard.NewHashRingStrategy(shardConfig)\n\tshardList := []string{}\n\tfor i := 0; i < 10000; i++ {\n\t\tbackend, err := strategy.Shard(strconv.Itoa(i))\n\t\tassert.Nil(t, err, \"Failed to Shard for key %d\", i)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t\treturn\n\t\t}\n\t\tshardList = append(shardList, backend.Name)\n\t}\n\n\tstrategy2, _ := shard.NewHashRingStrategy(shardConfig)\n\tfor i := 0; i < 10000; i++ {\n\t\tbackend, err := strategy2.Shard(strconv.Itoa(i))\n\t\tassert.Nil(t, err, \"Failed to Re - Shard for key %d\", i)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t\treturn\n\t\t}\n\t\tassert.Equal(t, shardList[i], backend.Name, \"Sharded inconsistently for key %d %s -> %s\", i, shardList[i], backend.Name)\n\t}\n}\n\nfunc TestShouldShardUniformally(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"totalVirtualBackends\": 1000,\n\t\t\"backends\": {\n\t\t\t\"0-249\": { \"timeout\": 100, \"backend_name\": \"foobar1\", \"backend\": \"http://shard00.local\"},\n\t\t\t\"250-499\": { \"backend_name\": \"foobar2\", \"backend\": \"http://shard01.local\"},\n\t\t\t\"500-749\": { \"backend_name\": \"foobar3\", \"backend\": \"http://shard02.local\"},\n\t\t\t\"750-999\": { \"backend_name\": \"foobar4\", \"backend\": \"http://shard03.local\"}\n\t\t}\n\t}`)\n\n\tstrategy, _ := shard.NewHashRingStrategy(shardConfig)\n\tshardDistribution := map[string]int{}\n\tnumKeys := 1000000\n\tfor i := 0; i < numKeys; i++ {\n\t\tbackend, err := strategy.Shard(strconv.Itoa(i))\n\t\tassert.Nil(t, err, \"Failed to Shard for key %d\", i)\n\t\tif err != nil {\n\t\t\tt.Log(err)\n\t\t\treturn\n\t\t}\n\t\tshardDistribution[backend.Name] = shardDistribution[backend.Name] + 1\n\t}\n\tmean := float64(0)\n\tfor _, v := range shardDistribution {\n\t\tmean += float64(v)\n\t}\n\tmean = mean / 4\n\n\tsd := float64(0)\n\n\tfor _, v := range shardDistribution {\n\t\tsd += math.Pow(float64(v)-mean, 2)\n\t}\n\n\tsd = (math.Sqrt(sd/4) / float64(numKeys)) * float64(100)\n\tassert.True(t, (sd < float64(2.5)), \"Standard Deviation should be less than 2.5% -> %f\", sd)\n\n\tt.Log(\"Standard Deviation:\", sd)\n}\n\nfunc bToMb(b uint64) uint64 {\n\treturn b / 1024 / 1024\n}\n\nfunc PrintMemUsage() runtime.MemStats {\n\tvar m runtime.MemStats\n\truntime.ReadMemStats(&m)\n\t// For info on each, see: https://golang.org/pkg/runtime/#MemStats\n\tfmt.Printf(\"Alloc = %v MiB\", bToMb(m.Alloc))\n\tfmt.Printf(\"\\tTotalAlloc = %v MiB\", bToMb(m.TotalAlloc))\n\tfmt.Printf(\"\\tHeapAlloc = %v MiB\", bToMb(m.HeapAlloc))\n\tfmt.Printf(\"\\tSys = %v MiB\", bToMb(m.Sys))\n\tfmt.Printf(\"\\tNumGC = %v\\n\", m.NumGC)\n\treturn m\n}\n\nfunc TestShouldShardWithoutLeakingMemory(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"totalVirtualBackends\": 1000,\n\t\t\"backends\": {\n\t\t\t\"0-249\": { \"timeout\": 100, \"backend_name\": \"foobar1\", \"backend\": \"http://shard00.local\"},\n\t\t\t\"250-499\": { \"backend_name\": \"foobar2\", \"backend\": \"http://shard01.local\"},\n\t\t\t\"500-749\": { \"backend_name\": \"foobar3\", \"backend\": \"http://shard02.local\"},\n\t\t\t\"750-999\": { \"backend_name\": \"foobar4\", \"backend\": \"http://shard03.local\"}\n\t\t}\n\t}`)\n\n\tstrategy, _ := shard.NewHashRingStrategy(shardConfig)\n\tnumKeys := 1000\n\tPrintMemUsage()\n\tstrategy.Shard(\"1\")\n\n\tfor j := 0; j < 1000; j++ {\n\t\tfor i := 0; i < numKeys; i++ {\n\t\t\t_, _ = strategy.Shard(strconv.Itoa(i))\n\t\t}\n\t}\n\n\tPrintMemUsage()\n}\n\nfunc TestToMeasureTimeForSharding(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"totalVirtualBackends\": 1000,\n\t\t\"backends\": {\n\t\t\t\"0-249\": { \"timeout\": 100, \"backend_name\": \"foobar1\", \"backend\": \"http://shard00.local\"},\n\t\t\t\"250-499\": { \"backend_name\": \"foobar2\", \"backend\": \"http://shard01.local\"},\n\t\t\t\"500-749\": { \"backend_name\": \"foobar3\", \"backend\": \"http://shard02.local\"},\n\t\t\t\"750-999\": { \"backend_name\": \"foobar4\", \"backend\": \"http://shard03.local\"}\n\t\t}\n\t}`)\n\tstrategy, _ := shard.NewHashRingStrategy(shardConfig)\n\tnumKeys := 1000\n\tstart := time.Now()\n\tfor j := 0; j < 1000; j++ {\n\t\tfor i := 0; i < numKeys; i++ {\n\t\t\t_, _ = strategy.Shard(strconv.Itoa(i))\n\t\t}\n\t}\n\telapsed := time.Since(start)\n\tfmt.Printf(\"Elapsed Time %v\", elapsed)\n}\n" }, { "path": "pkg/shard/lookup.go", "content": "package shard\n\nimport (\n\t\"encoding/json\"\n\n\t\"github.com/gojektech/weaver\"\n)\n\nfunc NewLookupStrategy(data json.RawMessage) (weaver.Sharder, error) {\n\tshardConfig := map[string]BackendDefinition{}\n\tif err := json.Unmarshal(data, &shardConfig); err != nil {\n\t\treturn nil, err\n\t}\n\n\tbackends, err := toBackends(shardConfig)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &LookupStrategy{\n\t\tbackends: backends,\n\t}, nil\n}\n\ntype LookupStrategy struct {\n\tbackends map[string]*weaver.Backend\n}\n\nfunc (ls *LookupStrategy) Shard(key string) (*weaver.Backend, error) {\n\treturn ls.backends[key], nil\n}\n" }, { "path": "pkg/shard/lookup_test.go", "content": "package shard\n\nimport (\n\t\"encoding/json\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestNewLookupStrategy(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"R-\": { \"timeout\": 100, \"backend_name\": \"foobar\", \"backend\": \"http://ride-service\"},\n\t\t\"GK-\": { \"backend_name\": \"foobar\", \"backend\": \"http://go-kilat\"}\n\t}`)\n\n\tlookupStrategy, err := NewLookupStrategy(shardConfig)\n\trequire.NoError(t, err, \"should not have failed to parse the shard config\")\n\n\tbackend, err := lookupStrategy.Shard(\"GK-\")\n\trequire.NoError(t, err, \"should not have failed when finding shard\")\n\n\tassert.Equal(t, \"http://go-kilat\", backend.Server.String())\n\tassert.NotNil(t, backend.Handler)\n}\n\nfunc TestNewLookupStrategyFailWhenTimeoutIsInvalid(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"R-\": { \"timeout\": \"abc\", \"backend\": \"http://ride-service\"},\n\t\t\"GK-\": { \"backend\": \"http://go-kilat\"}\n\t}`)\n\n\tlookupStrategy, err := NewLookupStrategy(shardConfig)\n\trequire.Error(t, err, \"should have failed to parse the shard config\")\n\n\tassert.Nil(t, lookupStrategy)\n}\n\nfunc TestNewLookupStrategyFailWhenNoBackendGiven(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"R-\": { \"timeout\": \"abc\", \"backend_name\": \"hello\"},\n\t\t\"GK-\": { \"backend_name\": \"mello\", \"backend\": \"http://go-kilat\"}\n\t}`)\n\n\tlookupStrategy, err := NewLookupStrategy(shardConfig)\n\trequire.Error(t, err, \"should have failed to parse the shard config\")\n\n\tassert.Nil(t, lookupStrategy)\n}\n\nfunc TestNewLookupStrategyFailWhenBackendIsNotString(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"R-\": { \"backend\": 123 },\n\t\t\"GK-\": { \"backend\": \"http://go-kilat\"}\n\t}`)\n\n\tlookupStrategy, err := NewLookupStrategy(shardConfig)\n\trequire.Error(t, err, \"should have failed to parse the shard config\")\n\n\tassert.Nil(t, lookupStrategy)\n}\n\nfunc TestNewLookupStrategyFailWhenBackendIsNotAValidURL(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"R-\": { \"backend\": \":\"},\n\t\t\"GK-\": { \"backend\": \"http://go-kilat\"}\n\t}`)\n\n\tlookupStrategy, err := NewLookupStrategy(shardConfig)\n\trequire.Error(t, err, \"should have failed to parse the shard config\")\n\n\tassert.Nil(t, lookupStrategy)\n}\n\nfunc TestNewLookupStrategyFailsWhenConfigIsInvalid(t *testing.T) {\n\tshardConfig := json.RawMessage(`[]`)\n\n\tlookupStrategy, err := NewLookupStrategy(shardConfig)\n\trequire.Error(t, err, \"should have failed to parse the shard config\")\n\n\tassert.Equal(t, err.Error(), \"json: cannot unmarshal array into Go value of type map[string]shard.BackendDefinition\")\n\tassert.Nil(t, lookupStrategy, \"should have failed to parse the shard config\")\n}\n\nfunc TestNewLookupStrategyFailsWhenConfigValueIsInvalid(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"foo\": \"hello\",\n\t\t\"hello\": []\n\t}`)\n\n\tlookupStrategy, err := NewLookupStrategy(shardConfig)\n\trequire.Error(t, err, \"should have failed to parse the shard config\")\n\n\tassert.Contains(t, err.Error(), \"cannot unmarshal string into Go value of type shard.BackendDefinition\")\n\tassert.Nil(t, lookupStrategy, \"should have failed to parse the shard config\")\n}\n" }, { "path": "pkg/shard/modulo.go", "content": "package shard\n\nimport (\n\t\"encoding/json\"\n\t\"strconv\"\n\n\t\"github.com/gojektech/weaver\"\n\t\"github.com/pkg/errors\"\n)\n\nfunc NewModuloStrategy(data json.RawMessage) (weaver.Sharder, error) {\n\tshardConfig := map[string]BackendDefinition{}\n\tif err := json.Unmarshal(data, &shardConfig); err != nil {\n\t\treturn nil, err\n\t}\n\n\tbackends, err := toBackends(shardConfig)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn &ModuloStrategy{\n\t\tbackends: backends,\n\t}, nil\n}\n\ntype ModuloStrategy struct {\n\tbackends map[string]*weaver.Backend\n}\n\nfunc (ms ModuloStrategy) Shard(key string) (*weaver.Backend, error) {\n\tid, err := strconv.Atoi(key)\n\tif err != nil {\n\t\treturn nil, errors.Wrapf(err, \"not an integer key: %s\", key)\n\t}\n\n\tmodulo := id % (len(ms.backends))\n\treturn ms.backends[strconv.Itoa(modulo)], nil\n}\n" }, { "path": "pkg/shard/modulo_test.go", "content": "package shard\n\nimport (\n\t\"encoding/json\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestNewModuloShardStrategy(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"0\": { \"timeout\": 100, \"backend_name\": \"foobar\", \"backend\": \"http://shard00.local\"},\n\t\t\"1\": { \"backend_name\": \"foobar1\", \"backend\": \"http://shard01.local\"},\n\t\t\"2\": { \"backend_name\": \"foobar2\", \"backend\": \"http://shard02.local\"}\n\t}`)\n\n\tmoduloStrategy, err := NewModuloStrategy(shardConfig)\n\trequire.NoError(t, err, \"should not have failed to parse the shard config\")\n\n\tbackend, err := moduloStrategy.Shard(\"5678987\")\n\trequire.NoError(t, err, \"should not have failed to find backend\")\n\n\tassert.Equal(t, \"http://shard02.local\", backend.Server.String())\n\tassert.NotNil(t, backend.Handler)\n}\n\nfunc TestNewModuloShardStrategyFailure(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"0\": { \"timeout\": 100, \"backend_name\": \"foobar\", \"backend\": \"http://shard00.local\"},\n\t\t\"1\": { \"backend_name\": \"foobar1\", \"backend\": \"http://shard01.local\"},\n\t\t\"2\": { \"backend_name\": \"foobar2\", \"backend\": \"http://shard02.local\"}\n\t}`)\n\n\tmoduloStrategy, err := NewModuloStrategy(shardConfig)\n\trequire.NoError(t, err, \"should not have failed to parse the shard config\")\n\n\tbackend, err := moduloStrategy.Shard(\"abcd\")\n\trequire.Error(t, err, \"should have failed to find backend\")\n\n\tassert.Nil(t, backend)\n}\n\nfunc TestNewModuloStrategyFailWhenTimeoutIsInvalid(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"0\": { \"backend_name\": \"A\", \"timeout\": \"abc\", \"backend\": \"http://shard00.local\"},\n\t\t\"1\": { \"backend_name\": \"B\", \"backend\": \"http://shard01.local\"}\n\t}`)\n\n\tmoduloStrategy, err := NewModuloStrategy(shardConfig)\n\trequire.Error(t, err, \"should have failed to parse the shard config\")\n\n\tassert.Nil(t, moduloStrategy)\n\tassert.Contains(t, err.Error(), \"cannot unmarshal string into Go struct field BackendDefinition.timeout of type float64\")\n}\n\nfunc TestNewModuloStrategyFailWhenNoBackendGiven(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"0\": { \"backend_name\": \"hello\"},\n\t\t\"1\": { \"backend_name\": \"mello\", \"backend\": \"http://shard01.local\"}\n\t}`)\n\n\tmoduloStrategy, err := NewModuloStrategy(shardConfig)\n\trequire.Error(t, err, \"should have failed to parse the shard config\")\n\n\tassert.Nil(t, moduloStrategy)\n\tassert.Contains(t, err.Error(), \"missing backend url in shard config:\")\n}\n\nfunc TestNewModuloStrategyFailWhenBackendIsNotString(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"0\": { \"backend_name\": \"hello\", \"backend\": 123 },\n\t\t\"1\": { \"backend_name\": \"mello\", \"backend\": \"http://shard01.local\"}\n\t}`)\n\n\tmoduloStrategy, err := NewModuloStrategy(shardConfig)\n\trequire.Error(t, err, \"should have failed to parse the shard config\")\n\n\tassert.Nil(t, moduloStrategy)\n\tassert.Contains(t, err.Error(), \"cannot unmarshal number\")\n}\n\nfunc TestNewModuloStrategyFailWhenBackendIsNotAValidURL(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"0\": { \"backend_name\": \"hello\", \"backend\": \":\"},\n\t\t\"1\": { \"backend_name\": \"mello\", \"backend\": \"http://shard01.local\"}\n\t}`)\n\n\tmoduloStrategy, err := NewModuloStrategy(shardConfig)\n\trequire.Error(t, err, \"should have failed to parse the shard config\")\n\n\tassert.Nil(t, moduloStrategy)\n\tassert.Contains(t, err.Error(), \"URL Parsing failed for\")\n}\n\nfunc TestNewModuloStrategyFailsWhenConfigIsInvalid(t *testing.T) {\n\tshardConfig := json.RawMessage(`[]`)\n\n\tmoduloStrategy, err := NewModuloStrategy(shardConfig)\n\trequire.Error(t, err, \"should have failed to parse the shard config\")\n\n\tassert.Contains(t, err.Error(), \"json: cannot unmarshal array into Go value of type map[string]shard.BackendDefinition\")\n\tassert.Nil(t, moduloStrategy, \"should have failed to parse the shard config\")\n}\n\nfunc TestNewModuloStrategyFailsWhenConfigValueIsInvalid(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"foo\": \"hello\",\n\t\t\"hello\": []\n\t}`)\n\n\tmoduloStrategy, err := NewModuloStrategy(shardConfig)\n\trequire.Error(t, err, \"should have failed to parse the shard config\")\n\n\tassert.Contains(t, err.Error(), \"cannot unmarshal string into Go value of type shard.BackendDefinition\")\n\tassert.Nil(t, moduloStrategy, \"should have failed to parse the shard config\")\n}\n" }, { "path": "pkg/shard/no.go", "content": "package shard\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\n\t\"github.com/gojektech/weaver\"\n\t\"github.com/pkg/errors\"\n)\n\nfunc NewNoStrategy(data json.RawMessage) (weaver.Sharder, error) {\n\tcfg := NoStrategyConfig{}\n\tif err := json.Unmarshal(data, &cfg); err != nil {\n\t\treturn nil, err\n\t}\n\n\tif err := cfg.Validate(); err != nil {\n\t\treturn nil, err\n\t}\n\n\tbackendOptions := weaver.BackendOptions{}\n\tbackend, err := weaver.NewBackend(cfg.BackendName, cfg.BackendURL, backendOptions)\n\tif err != nil {\n\t\treturn nil, errors.WithStack(fmt.Errorf(\"failed to create backend: %s: %+v\", err, cfg))\n\t}\n\n\treturn &NoStrategy{\n\t\tbackend: backend,\n\t}, nil\n}\n\ntype NoStrategy struct {\n\tbackend *weaver.Backend\n}\n\nfunc (ns *NoStrategy) Shard(key string) (*weaver.Backend, error) {\n\treturn ns.backend, nil\n}\n\ntype NoStrategyConfig struct {\n\tBackendDefinition `json:\",inline\"`\n}\n" }, { "path": "pkg/shard/no_test.go", "content": "package shard\n\nimport (\n\t\"encoding/json\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestNewNoStrategy(t *testing.T) {\n\tshardConfig := json.RawMessage(`{ \"backend_name\": \"foobar\", \"backend\": \"http://localhost\" }`)\n\n\tnoStrategy, err := NewNoStrategy(shardConfig)\n\trequire.NoError(t, err, \"should not have failed to parse the shard config\")\n\n\tbackend, err := noStrategy.Shard(\"whatever\")\n\trequire.NoError(t, err, \"should not have failed when finding shard\")\n\n\tassert.Equal(t, \"http://localhost\", backend.Server.String())\n\tassert.NotNil(t, backend.Handler)\n}\n\nfunc TestNewNoStrategyFailsWhenConfigIsInvalid(t *testing.T) {\n\tshardConfig := json.RawMessage(`[]`)\n\n\tnoStrategy, err := NewNoStrategy(shardConfig)\n\trequire.Error(t, err, \"should have failed to parse the shard config\")\n\n\tassert.Equal(t, \"json: cannot unmarshal array into Go value of type shard.NoStrategyConfig\", err.Error())\n\tassert.Nil(t, noStrategy, \"should have failed to parse the shard config\")\n}\n\nfunc TestNewNoStrategyFailsWhenBackendURLInvalid(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"backend_name\": \"foobar\",\n\t\t\"backend\": \"http$://google.com\"\n\t}`)\n\n\tnoStrategy, err := NewNoStrategy(shardConfig)\n\trequire.Error(t, err, \"should have failed to parse the shard config\")\n\n\tassert.Contains(t, err.Error(), \"failed to create backend:\")\n\tassert.Nil(t, noStrategy, \"should have failed to parse the shard config\")\n}\n\nfunc TestNewNoStrategyFailsWhenConfigValueIsInvalid(t *testing.T) {\n\tshardConfig := json.RawMessage(`{ \"backend_name\": \"foobar\", \"backend\": [] }`)\n\n\tnoStrategy, err := NewNoStrategy(shardConfig)\n\trequire.Error(t, err, \"should have failed to parse the shard config\")\n\n\tassert.Contains(t, err.Error(), \"cannot unmarshal array into Go struct field NoStrategyConfig.backend of type string\")\n\tassert.Nil(t, noStrategy, \"should have failed to parse the shard config\")\n}\n\nfunc TestNoStrategyFailWhenBackendIsNotAValidURL(t *testing.T) {\n\tshardConfig := json.RawMessage(`{ \"server\": \":\" }`)\n\n\tnoStrategy, err := NewNoStrategy(shardConfig)\n\trequire.Error(t, err, \"should have failed to parse the shard config\")\n\n\tassert.Nil(t, noStrategy)\n}\n" }, { "path": "pkg/shard/prefix_lookup.go", "content": "package shard\n\nimport (\n\t\"encoding/json\"\n\t\"errors\"\n\t\"strings\"\n\n\t\"github.com/gojektech/weaver\"\n)\n\nconst (\n\tdefaultBackendKey = \"default\"\n\tdefaultPrefixSplitter = \"-\"\n)\n\ntype prefixLookupConfig struct {\n\tPrefixSplitter string `json:\"prefix_splitter\"`\n\tBackends map[string]BackendDefinition `json:\"backends\"`\n}\n\nfunc (plg prefixLookupConfig) Validate() error {\n\tif len(plg.Backends) == 0 {\n\t\treturn errors.New(\"no backends specified\")\n\t}\n\n\treturn nil\n}\n\nfunc NewPrefixLookupStrategy(data json.RawMessage) (weaver.Sharder, error) {\n\tprefixLookupConfig := &prefixLookupConfig{}\n\n\tif err := json.Unmarshal(data, &prefixLookupConfig); err != nil {\n\t\treturn nil, err\n\t}\n\n\tif err := prefixLookupConfig.Validate(); err != nil {\n\t\treturn nil, err\n\t}\n\n\tbackends, err := toBackends(prefixLookupConfig.Backends)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\tprefixSplitter := prefixLookupConfig.PrefixSplitter\n\tif prefixSplitter == \"\" {\n\t\tprefixSplitter = defaultPrefixSplitter\n\t}\n\n\treturn &PrefixLookupStrategy{\n\t\tbackends: backends,\n\t\tprefixSplitter: prefixSplitter,\n\t}, nil\n}\n\ntype PrefixLookupStrategy struct {\n\tbackends map[string]*weaver.Backend\n\tprefixSplitter string\n}\n\nfunc (pls *PrefixLookupStrategy) Shard(key string) (*weaver.Backend, error) {\n\tprefix := strings.SplitAfter(key, pls.prefixSplitter)[0]\n\tif pls.backends[prefix] == nil {\n\t\treturn pls.backends[defaultBackendKey], nil\n\t}\n\n\treturn pls.backends[prefix], nil\n}\n" }, { "path": "pkg/shard/prefix_lookup_test.go", "content": "package shard\n\nimport (\n\t\"encoding/json\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n)\n\nfunc TestNewPrefixLookupStrategy(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t \"backends\": {\n\t\t\"R_\": { \"timeout\": 100, \"backend_name\": \"foobar\", \"backend\": \"http://ride-service\"},\n\t\t\"GK_\": { \"backend_name\": \"foobar\", \"backend\": \"http://go-kilat\"}\n\t },\n\t \"prefix_splitter\": \"_\"\n\t}`)\n\n\tprefixLookupStrategy, err := NewPrefixLookupStrategy(shardConfig)\n\trequire.NoError(t, err, \"should not have failed to parse the shard config\")\n\n\tbackend, err := prefixLookupStrategy.Shard(\"GK_123444\")\n\trequire.NoError(t, err, \"should not have failed when finding shard\")\n\n\tassert.Equal(t, \"http://go-kilat\", backend.Server.String())\n\tassert.NotNil(t, backend.Handler)\n}\n\nfunc TestNewPrefixLookupStrategyWithDefaultPrefixSplitter(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t \"backends\": {\n\t\t\"R-\": { \"timeout\": 100, \"backend_name\": \"foobar\", \"backend\": \"http://ride-service\"},\n\t\t\"GK-\": { \"backend_name\": \"foobar\", \"backend\": \"http://go-kilat\"}\n\t }\n\t}`)\n\n\tprefixLookupStrategy, err := NewPrefixLookupStrategy(shardConfig)\n\trequire.NoError(t, err, \"should not have failed to parse the shard config\")\n\n\tbackend, err := prefixLookupStrategy.Shard(\"GK-123444\")\n\trequire.NoError(t, err, \"should not have failed when finding shard\")\n\n\tassert.Equal(t, \"http://go-kilat\", backend.Server.String())\n\tassert.NotNil(t, backend.Handler)\n}\n\nfunc TestNewPrefixLookupStrategyForNoPrefix(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t \"backends\": {\n\t\t\"R-\": { \"timeout\": 100, \"backend_name\": \"foobar\", \"backend\": \"http://ride-service\"},\n\t\t\"GK-\": { \"backend_name\": \"foobar\", \"backend\": \"http://go-kilat\"},\n\t\t\"default\": { \"backend_name\": \"hello\", \"backend\": \"http://sm\"}\n\t },\n\t \"prefix_splitter\": \"-\"\n\t}`)\n\n\tprefixLookupStrategy, err := NewPrefixLookupStrategy(shardConfig)\n\trequire.NoError(t, err, \"should not have failed to parse the shard config\")\n\n\tbackend, err := prefixLookupStrategy.Shard(\"123444\")\n\trequire.NoError(t, err, \"should not have failed when finding shard\")\n\n\tassert.Equal(t, \"http://sm\", backend.Server.String())\n\tassert.NotNil(t, backend.Handler)\n}\n\nfunc TestNewPrefixLookupStrategyFailWhenTimeoutIsInvalid(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t \"backends\": {\n\t\t\"R-\": { \"timeout\": \"abc\", \"backend\": \"http://ride-service\"},\n\t\t\"GK-\": { \"backend\": \"http://go-kilat\"},\n\t\t\"default\": { \"backend_name\": \"hello\", \"backend\": \"http://sm\"}\n\t },\n\t \"prefix_splitter\": \"-\"\n\t}`)\n\n\tprefixLookupStrategy, err := NewPrefixLookupStrategy(shardConfig)\n\trequire.Error(t, err, \"should have failed to parse the shard config\")\n\n\tassert.Nil(t, prefixLookupStrategy)\n}\n\nfunc TestNewPrefixLookupStrategyFailWhenNoBackendGiven(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t \"backends\": {\n\t\t\"R-\": { \"timeout\": \"abc\", \"backend_name\": \"hello\"},\n\t\t\"GK-\": { \"backend_name\": \"mello\", \"backend\": \"http://go-kilat\"},\n\t\t\"default\": { \"backend_name\": \"hello\", \"backend\": \"http://sm\"}\n\t },\n\t \"prefix_splitter\": \"-\"\n\t}`)\n\n\tprefixLookupStrategy, err := NewPrefixLookupStrategy(shardConfig)\n\trequire.Error(t, err, \"should have failed to parse the shard config\")\n\n\tassert.Nil(t, prefixLookupStrategy)\n}\n\nfunc TestNewPrefixLookupStrategyFailWhenBackendIsNotString(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t \"backends\": {\n\t\t\"R-\": { \"backend\": 123 },\n\t\t\"GK-\": { \"backend\": \"http://go-kilat\"},\n\t\t\"default\": { \"backend_name\": \"hello\", \"backend\": \"http://sm\"}\n\t },\n\t \"prefix_splitter\": \"-\"\n\t}`)\n\n\tprefixLookupStrategy, err := NewPrefixLookupStrategy(shardConfig)\n\trequire.Error(t, err, \"should have failed to parse the shard config\")\n\n\tassert.Nil(t, prefixLookupStrategy)\n}\n\nfunc TestNewPrefixLookupStrategyFailWhenBackendIsNotAValidURL(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t \"backends\": {\n\t\t\"R-\": { \"backend\": \":\"},\n\t\t\"GK-\": { \"backend\": \"http://go-kilat\"},\n\t\t\"default\": { \"backend_name\": \"hello\", \"backend\": \"http://sm\"}\n\t },\n\t \"prefix_splitter\": \"-\"\n\t}`)\n\n\tprefixLookupStrategy, err := NewPrefixLookupStrategy(shardConfig)\n\trequire.Error(t, err, \"should have failed to parse the shard config\")\n\n\tassert.Nil(t, prefixLookupStrategy)\n}\n\nfunc TestNewPrefixLookupStrategyFailsWhenConfigIsInvalid(t *testing.T) {\n\tshardConfig := json.RawMessage(`[]`)\n\n\tprefixLookupStrategy, err := NewPrefixLookupStrategy(shardConfig)\n\trequire.Error(t, err, \"should have failed to parse the shard config\")\n\n\tassert.Equal(t, err.Error(), \"json: cannot unmarshal array into Go value of type shard.prefixLookupConfig\")\n\tassert.Nil(t, prefixLookupStrategy, \"should have failed to parse the shard config\")\n}\n\nfunc TestNewPrefixLookupStrategyFailsWhenConfigValueIsInvalid(t *testing.T) {\n\tshardConfig := json.RawMessage(`{\n\t\t\"foo\": \"hello\",\n\t\t\"hello\": []\n\t}`)\n\n\tprefixLookupStrategy, err := NewPrefixLookupStrategy(shardConfig)\n\trequire.Error(t, err, \"should have failed to parse the shard config\")\n\n\tassert.Contains(t, err.Error(), \"no backends specified\")\n\tassert.Nil(t, prefixLookupStrategy, \"should have failed to parse the shard config\")\n}\n" }, { "path": "pkg/shard/s2.go", "content": "package shard\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/gojektech/weaver\"\n\t\"github.com/gojektech/weaver/pkg/util\"\n\tgeos2 \"github.com/golang/geo/s2\"\n)\n\nvar (\n\tdefaultBackendS2id = \"default\"\n)\n\nfunc NewS2Strategy(data json.RawMessage) (weaver.Sharder, error) {\n\tcfg := S2StrategyConfig{}\n\tif err := json.Unmarshal(data, &cfg); err != nil {\n\t\treturn nil, err\n\t}\n\n\tif err := cfg.Validate(); err != nil {\n\t\treturn nil, err\n\t}\n\n\ts2Backends := make(map[string]*weaver.Backend, len(cfg.Backends))\n\tfor s2id, backend := range cfg.Backends {\n\t\tvar err error\n\t\tif s2Backends[s2id], err = parseBackend(backend); err != nil {\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\tif cfg.ShardKeyPosition == nil {\n\t\tdefaultPos := -1\n\t\tcfg.ShardKeyPosition = &defaultPos\n\t}\n\n\treturn &S2Strategy{\n\t\tbackends: s2Backends,\n\t\tshardKeySeparator: cfg.ShardKeySeparator,\n\t\tshardKeyPosition: *cfg.ShardKeyPosition,\n\t}, nil\n}\n\ntype S2Strategy struct {\n\tbackends map[string]*weaver.Backend\n\tshardKeySeparator string\n\tshardKeyPosition int\n}\n\ntype S2StrategyConfig struct {\n\tShardKeySeparator string `json:\"shard_key_separator\"`\n\tShardKeyPosition *int `json:\"shard_key_position,omitempty\"`\n\tBackends map[string]BackendDefinition `json:\"backends\"`\n}\n\nfunc (s2cfg S2StrategyConfig) Validate() error {\n\tif s2cfg.ShardKeySeparator == \"\" {\n\t\treturn Error(\"missing required config: shard_key_separator\")\n\t}\n\tif err := s2cfg.validateS2IDs(); err != nil {\n\t\treturn err\n\t}\n\treturn nil\n}\n\nfunc (s2cfg S2StrategyConfig) validateS2IDs() error {\n\tbackendCount := len(s2cfg.Backends)\n\ts2IDs := make([]uint64, backendCount)\n\tfor k := range s2cfg.Backends {\n\t\tif k != defaultBackendS2id {\n\t\t\tid, err := strconv.ParseUint(k, 10, 64)\n\t\t\tif err != nil {\n\t\t\t\treturn fmt.Errorf(\"[error] Bad S2 ID found in backends: %s\", k)\n\t\t\t}\n\t\t\ts2IDs = append(s2IDs, id)\n\t\t}\n\t}\n\n\tif util.ContainsOverlappingS2IDs(s2IDs) {\n\t\treturn fmt.Errorf(\"[error] Overlapping S2 IDs found in backends: %v\", s2cfg.Backends)\n\t}\n\treturn nil\n}\n\nfunc s2idFromLatLng(latLng []string) (s2id geos2.CellID, err error) {\n\tif len(latLng) != 2 {\n\t\terr = Error(\"lat lng key is not valid\")\n\t\treturn\n\t}\n\tlat, err := strconv.ParseFloat(latLng[0], 64)\n\tif err != nil {\n\t\terr = Error(\"fail to parse latitude\")\n\t\treturn\n\t}\n\tlng, err := strconv.ParseFloat(latLng[1], 64)\n\tif err != nil {\n\t\terr = Error(\"fail to parse longitude\")\n\t\treturn\n\t}\n\ts2LatLng := geos2.LatLngFromDegrees(lat, lng)\n\tif !s2LatLng.IsValid() {\n\t\terr = Error(\"fail to convert lat-long to geos2 objects\")\n\t\treturn\n\t}\n\ts2id = geos2.CellIDFromLatLng(s2LatLng)\n\treturn\n}\n\nfunc s2idFromSmartID(smartIDComponents []string, pos int) (s2id geos2.CellID, err error) {\n\tif len(smartIDComponents) <= pos {\n\t\terr = Error(\"failed to get location from smart-id\")\n\t\treturn\n\t}\n\ts2idStr := smartIDComponents[pos]\n\ts2idUint, err := strconv.ParseUint(s2idStr, 10, 64)\n\tif err != nil {\n\t\terr = Error(\"failed to parse s2id\")\n\t\treturn\n\t}\n\ts2id = geos2.CellID(s2idUint)\n\treturn\n\n}\n\nfunc (s2s *S2Strategy) s2ID(key string) (uint64, error) {\n\tshardKeyComponents := strings.Split(key, s2s.shardKeySeparator)\n\tvar s2CellID geos2.CellID\n\tvar err error\n\n\tswitch s2s.shardKeyPosition {\n\tcase -1:\n\t\ts2CellID, err = s2idFromLatLng(shardKeyComponents)\n\tdefault:\n\t\ts2CellID, err = s2idFromSmartID(shardKeyComponents, s2s.shardKeyPosition)\n\t}\n\n\treturn uint64(s2CellID), err\n}\n\nfunc (s2s *S2Strategy) Shard(key string) (*weaver.Backend, error) {\n\ts2id, err := s2s.s2ID(key)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\n\ts2CellID := geos2.CellID(s2id)\n\n\tfor s2Str, backendConfig := range s2s.backends {\n\t\tcellInt, err := strconv.ParseUint(s2Str, 10, 64)\n\t\tif err != nil {\n\t\t\tcontinue\n\t\t}\n\t\tshardCellID := geos2.CellID(cellInt)\n\t\tif shardCellID.Contains(s2CellID) {\n\t\t\treturn backendConfig, nil\n\t\t}\n\t}\n\n\tif _, ok := s2s.backends[defaultBackendS2id]; ok {\n\t\treturn s2s.backends[defaultBackendS2id], nil\n\t}\n\n\treturn nil, Error(\"fail to find backend\")\n}\n" }, { "path": "pkg/shard/s2_test.go", "content": "package shard\n\nimport (\n\t\"encoding/json\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nvar (\n\ts2ShardConfigWithoutDefault = json.RawMessage(`{\n\t\t\"backends\": {\n\t\t\t\"3344472479136481280\": { \"backend_name\": \"jkt-a\", \"backend\": \"http://jkt.a.local\"},\n\t\t\t\"3346530764903677952\": { \"backend_name\": \"jkt-b\", \"backend\": \"http://jkt.b.local\"}\n\t\t},\n\t\t\"shard_key_separator\": \",\"\n\t}`)\n\ts2ShardConfig = json.RawMessage(`{\n\t\t\"backends\": {\n\t\t\t\"3344472479136481280\": { \"backend_name\": \"jkt-a\", \"backend\": \"http://jkt.a.local\"},\n\t\t\t\"3346530764903677952\": { \"backend_name\": \"jkt-b\", \"backend\": \"http://jkt.b.local\"},\n\t\t\t\"default\": {\"backend_name\": \"jkt-c\", \"backend\": \"http://jkt.c.local\"}\n\t\t},\n\t\t\"shard_key_separator\": \",\"\n\t}`)\n\ts2ShardConfigBad = json.RawMessage(`{\n\t\t\"backends\": {\n\t\t\t\"123454321\": { \"backend_name\": \"jkt-a\", \"backend\": \"http://jkt.a.local\"},\n\t\t\t\"123459876\": \"bad-data\"\n\t\t},\n\t\t\"shard_key_separator\": \",\"\n\t}`)\n\n\ts2ShardConfigInvalidS2ID = json.RawMessage(`{\n\t\t\"backends\": {\n\t\t\t\"45435hb344hj3b\": { \"backend_name\": \"jkt-a\", \"backend\": \"http://jkt.a.local\"}\n\t\t},\n\t\t\"shard_key_separator\": \",\"\n\t}`)\n\ts2ShardConfigOverlapping = json.RawMessage(`{\n\t\t\"backends\": {\n\t\t\t\"3344474311656055761\": { \"backend_name\": \"jkt-a\", \"backend\": \"http://jkt.a.local\"},\n\t\t\t\"3344474311656055760\": { \"backend_name\": \"jkt-b\", \"backend\": \"http://jkt.b.local\"},\n\t\t\t\"3344474311656055744\": { \"backend_name\": \"jkt-c\", \"backend\": \"http://jkt.c.local\"},\n\t\t\t\"3344474311656055552\": { \"backend_name\": \"jkt-d\", \"backend\": \"http://jkt.d.local\"},\n\t\t\t\"3344474311656055808\": { \"backend_name\": \"jkt-e\", \"backend\": \"http://jkt.e.local\"},\n\t\t\t\"3573054715985026048\": { \"backend_name\": \"surbaya-a\", \"backend\": \"http://surbaya.a.local\"}\n\t\t},\n\t\t\"shard_key_separator\": \",\"\n\t}`)\n\n\ts2SmartIDShardConfig = json.RawMessage(`{\n\t\t\"backends\": {\n\t\t\t\"3344472479136481280\": { \"backend_name\": \"jkt-a\", \"backend\": \"http://jkt.a.local\"},\n\t\t\t\"3346530764903677952\": { \"backend_name\": \"jkt-b\", \"backend\": \"http://jkt.b.local\"},\n\t\t\t\"default\": {\"backend_name\": \"jkt-c\", \"backend\": \"http://jkt.c.local\"}\n\t\t},\n\t\t\"shard_key_separator\": \"-\",\n\t\t\"shard_key_position\":2\n\t}`)\n\n\ts2SmartIDShardConfigBad = json.RawMessage(`{\n\t\t\"backends\": {\n\t\t\t\"3344472479136481280\": { \"backend_name\": \"jkt-a\", \"backend\": \"http://jkt.a.local\"},\n\t\t\t\"3346530764903677952\": { \"backend_name\": \"jkt-b\", \"backend\": \"http://jkt.b.local\"},\n\t\t\t\"default\": {\"backend_name\": \"jkt-c\", \"backend\": \"http://jkt.c.local\"}\n\t\t},\n\t\t\"shard_key_position\":2\n\t}`)\n)\n\nfunc TestNewS2StrategySuccess(t *testing.T) {\n\tstrategy, err := NewS2Strategy(s2ShardConfig)\n\tassert.NotNil(t, strategy)\n\tassert.Nil(t, err)\n}\n\nfunc TestNewS2StrategyFailure(t *testing.T) {\n\tsharder, err := NewS2Strategy(s2ShardConfigBad)\n\tassert.Nil(t, sharder)\n\tassert.NotNil(t, err)\n\tassert.Contains(t, err.Error(), \"json: cannot unmarshal string\")\n}\n\nfunc TestNewS2StrategyFailureWithMissingKeySeparator(t *testing.T) {\n\tsharder, err := NewS2Strategy(s2SmartIDShardConfigBad)\n\tassert.Nil(t, sharder)\n\tassert.NotNil(t, err)\n\tassert.Equal(t, \"[error] missing required config: shard_key_separator\", err.Error())\n}\n\nfunc TestNewS2StrategyFailureWithOverlappingShards(t *testing.T) {\n\tsharder, err := NewS2Strategy(s2ShardConfigOverlapping)\n\tassert.Nil(t, sharder)\n\tassert.NotNil(t, err)\n\tassert.Contains(t, err.Error(), \"[error] Overlapping S2 IDs found in backends:\")\n}\n\nfunc TestNewS2StrategyFailureWithInvalidS2ID(t *testing.T) {\n\tsharder, err := NewS2Strategy(s2ShardConfigInvalidS2ID)\n\tassert.Nil(t, sharder)\n\tassert.NotNil(t, err)\n\tassert.Contains(t, err.Error(), \"[error] Bad S2 ID found in backends:\")\n}\n\nfunc TestS2StrategyS2IDSuccess(t *testing.T) {\n\tstrategy := S2Strategy{shardKeySeparator: \",\", shardKeyPosition: -1}\n\tactualS2ID, err := strategy.s2ID(\"-6.1751,106.865\")\n\texpectedS2ID := uint64(3344474311656055761)\n\tassert.Nil(t, err)\n\tassert.Equal(t, expectedS2ID, actualS2ID)\n\n\tactualS2ID, err = strategy.s2ID(\"-6.1751,110.865\")\n\texpectedS2ID = uint64(3346531974082111711)\n\tassert.Nil(t, err)\n\tassert.Equal(t, expectedS2ID, actualS2ID)\n}\n\nfunc TestS2StrategyS2IDFailureForInvalidLat(t *testing.T) {\n\tstrategy := S2Strategy{shardKeySeparator: \",\", shardKeyPosition: -1}\n\t_, err := strategy.s2ID(\"-qwerty6.1751,32.865\")\n\tassert.NotNil(t, err)\n}\n\nfunc TestS2StrategyS2IDFailureForInvalidLng(t *testing.T) {\n\tstrategy := S2Strategy{shardKeySeparator: \",\", shardKeyPosition: -1}\n\t_, err := strategy.s2ID(\"-6.1751,qwerty1232.865\")\n\tassert.NotNil(t, err)\n}\n\nfunc TestS2StrategyS2IDFailureForInvalidLatLngObject(t *testing.T) {\n\tstrategy := S2Strategy{shardKeySeparator: \",\", shardKeyPosition: -1}\n\t_, err := strategy.s2ID(\"1116.1751,1232.865\")\n\tassert.NotNil(t, err)\n}\n\nfunc TestS2StrategyS2IDFailureForInvalidAlphanumeric(t *testing.T) {\n\tstrategy := S2Strategy{shardKeySeparator: \",\", shardKeyPosition: -1}\n\t_, err := strategy.s2ID(\"-6.17511232.865\")\n\tassert.NotNil(t, err)\n\n\t_, err = strategy.s2ID(\"abc,1232.865\")\n\tassert.NotNil(t, err)\n}\n\nfunc TestS2StrategyS2IDWithSmartIDSuccess(t *testing.T) {\n\tstrategy := S2Strategy{shardKeySeparator: \"-\", shardKeyPosition: 2}\n\tactualS2ID, err := strategy.s2ID(\"v1-foo-3344474403281829888\")\n\texpectedS2ID := uint64(3344474403281829888)\n\tassert.Nil(t, err)\n\tassert.Equal(t, expectedS2ID, actualS2ID)\n\n\tactualS2ID, err = strategy.s2ID(\"v1-foo-3346532139293212672\")\n\texpectedS2ID = uint64(3346532139293212672)\n\tassert.Nil(t, err)\n\tassert.Equal(t, expectedS2ID, actualS2ID)\n}\n\nfunc TestS2StrategyS2IDFailureForInvalidS2ID(t *testing.T) {\n\tstrategy := S2Strategy{shardKeySeparator: \"-\", shardKeyPosition: 2}\n\t_, err := strategy.s2ID(\"v1-foo-bar\")\n\tassert.NotNil(t, err)\n\tassert.Equal(t, \"[error] failed to parse s2id\", err.Error())\n}\n\nfunc TestS2StrategyS2IDFailureForInvalidSmartID(t *testing.T) {\n\tstrategy := S2Strategy{shardKeySeparator: \"-\", shardKeyPosition: 2}\n\t_, err := strategy.s2ID(\"booyeah\")\n\tassert.NotNil(t, err)\n\tassert.Equal(t, \"[error] failed to get location from smart-id\", err.Error())\n}\n\nfunc TestS2StrategyS2IDFailureForInvalidSeparatorConfig(t *testing.T) {\n\tstrategy := S2Strategy{shardKeySeparator: \"&\", shardKeyPosition: 2}\n\t_, err := strategy.s2ID(\"v1-foo-3344474403281829888\")\n\tassert.NotNil(t, err)\n\tassert.Equal(t, \"[error] failed to get location from smart-id\", err.Error())\n}\n\nfunc TestS2StrategyS2IDFailureForInvalidPositionConfig(t *testing.T) {\n\tstrategy := S2Strategy{shardKeySeparator: \"-\", shardKeyPosition: 3}\n\t_, err := strategy.s2ID(\"v1-foo-3344474403281829888\")\n\tassert.NotNil(t, err)\n\tassert.Equal(t, \"[error] failed to get location from smart-id\", err.Error())\n\n\tstrategy = S2Strategy{shardKeySeparator: \"-\", shardKeyPosition: 4}\n\t_, err = strategy.s2ID(\"v1-foo-3344474403281829888\")\n\tassert.NotNil(t, err)\n\tassert.Equal(t, \"[error] failed to get location from smart-id\", err.Error())\n}\n\nfunc TestS2StrategyShardSuccess(t *testing.T) {\n\tstrategy, _ := NewS2Strategy(s2ShardConfig)\n\tbackend, err := strategy.Shard(\"-6.1751,106.865\")\n\texpectedBackend := \"jkt-a\"\n\tassert.Nil(t, err)\n\tassert.Equal(t, expectedBackend, backend.Name)\n\n\tbackend, err = strategy.Shard(\"-6.1751,110.865\")\n\texpectedBackend = \"jkt-b\"\n\tassert.Nil(t, err)\n\tassert.Equal(t, expectedBackend, backend.Name)\n}\n\nfunc TestS2StrategySmartIDShardSuccess(t *testing.T) {\n\tstrategy, _ := NewS2Strategy(s2SmartIDShardConfig)\n\tbackend, err := strategy.Shard(\"v1-foo-3344474403281829888\")\n\texpectedBackend := \"jkt-a\"\n\tassert.Nil(t, err)\n\tassert.Equal(t, expectedBackend, backend.Name)\n\n\tbackend, err = strategy.Shard(\"v1-foo-3346532139293212672\")\n\texpectedBackend = \"jkt-b\"\n\tassert.Nil(t, err)\n\tassert.Equal(t, expectedBackend, backend.Name)\n\n\tbackend, err = strategy.Shard(\"v1-foo-2534\")\n\texpectedBackend = \"jkt-c\"\n\tassert.Nil(t, err)\n\tassert.Equal(t, expectedBackend, backend.Name)\n}\n\nfunc TestS2StrategyShardSuccessForDefaultBackend(t *testing.T) {\n\tstrategy, _ := NewS2Strategy(s2ShardConfig)\n\tbackend, err := strategy.Shard(\"-34.1751,106.865\")\n\texpectedBackend := \"jkt-c\"\n\tassert.Nil(t, err)\n\tassert.Equal(t, expectedBackend, backend.Name)\n}\n\nfunc TestS2StrategyShardFailure(t *testing.T) {\n\tstrategy, _ := NewS2Strategy(s2ShardConfig)\n\tbackendForWrongLatLng, err := strategy.Shard(\"-126.1751,906.865\")\n\tassert.NotNil(t, err)\n\tassert.Nil(t, backendForWrongLatLng)\n\n\tbackendForWrongInputNum, err := strategy.Shard(\"-126.1751865\")\n\tassert.NotNil(t, err)\n\tassert.Nil(t, backendForWrongInputNum)\n\n\tbackendForWrongInputAlpha, err := strategy.Shard(\"abc,xyz\")\n\tassert.NotNil(t, err)\n\tassert.Nil(t, backendForWrongInputAlpha)\n\n\tstrategy, _ = NewS2Strategy(s2ShardConfigWithoutDefault)\n\tnoBackendForCorrectInput, err := strategy.Shard(\"-6.1751,126.865\")\n\tassert.NotNil(t, err)\n\tassert.Equal(t, \"[error] fail to find backend\", err.Error())\n\tassert.Nil(t, noBackendForCorrectInput)\n}\n\nfunc TestGeneratesCustomError(t *testing.T) {\n\tce := CustomError{ExitMessage: \"Error for custom error\"}\n\terr := ce.Error()\n\tassert.Equal(t, err, \"[error] Error for custom error\")\n}\n" }, { "path": "pkg/shard/shard.go", "content": "package shard\n\nimport (\n\t\"encoding/json\"\n\t\"fmt\"\n\n\t\"github.com/gojektech/weaver\"\n)\n\nfunc New(name string, cfg json.RawMessage) (weaver.Sharder, error) {\n\tnewSharder, found := shardFuncTable[name]\n\tif !found {\n\t\treturn nil, fmt.Errorf(\"failed to find sharder with name '%s'\", name)\n\t}\n\n\treturn newSharder(cfg)\n}\n\ntype sharderGenerator func(json.RawMessage) (weaver.Sharder, error)\n\nvar shardFuncTable = map[string]sharderGenerator{\n\t\"lookup\": NewLookupStrategy,\n\t\"prefix-lookup\": NewPrefixLookupStrategy,\n\t\"none\": NewNoStrategy,\n\t\"modulo\": NewModuloStrategy,\n\t\"hashring\": NewHashRingStrategy,\n\t\"s2\": NewS2Strategy,\n}\n" }, { "path": "pkg/util/s2.go", "content": "package util\n\nimport (\n\t\"sort\"\n\n\t\"github.com/golang/geo/s2\"\n)\n\ntype s2List []s2.CellID\n\nfunc (s2l s2List) Len() int {\n\treturn len(s2l)\n}\n\nfunc (s2l s2List) Less(i int, j int) bool {\n\treturn s2l[i].Level() < s2l[j].Level()\n}\n\nfunc (s2l s2List) Swap(i int, j int) {\n\ts2l[i], s2l[j] = s2l[j], s2l[i]\n}\n\nfunc toS2List(ints []uint64) s2List {\n\tlst := s2List{}\n\tfor _, id := range ints {\n\t\tlst = append(lst, s2.CellID(id))\n\t}\n\treturn lst\n}\n\nfunc ContainsOverlappingS2IDs(ids []uint64) bool {\n\tlst := toS2List(ids)\n\n\tsort.Sort(lst)\n\n\tlength := lst.Len()\n\n\tfor i := 0; i < length; i++ {\n\t\thigher := lst[i]\n\t\tfor j := i + 1; j < length; j++ {\n\t\t\tlower := lst[j]\n\t\t\tif higher.Level() < lower.Level() && higher.Contains(lower) {\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\t}\n\treturn false\n}\n" }, { "path": "pkg/util/s2_test.go", "content": "package util\n\nimport (\n\t\"testing\"\n\n\t\"github.com/golang/geo/s2\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestS2Len(t *testing.T) {\n\tlist := s2List{\n\t\ts2.CellID(23458045904904),\n\t\ts2.CellID(23458222224904),\n\t\ts2.CellID(23888885904904),\n\t\ts2.CellID(23458999999904),\n\t}\n\tassert.Equal(t, list.Len(), 4)\n}\n\nfunc TestS2Less(t *testing.T) {\n\tlist := s2List{\n\t\ts2.CellID(4542091330435678208),\n\t\ts2.CellID(4542051748017078272),\n\t}\n\tassert.True(t, list.Less(0, 1))\n}\n\nfunc TestS2Swap(t *testing.T) {\n\tfirst := s2.CellID(4542091330435678208)\n\tsecond := s2.CellID(4542051748017078272)\n\tlist := s2List{\n\t\tfirst,\n\t\tsecond,\n\t}\n\tlist.Swap(0, 1)\n\tassert.Equal(t, list[0], second)\n\tassert.Equal(t, list[1], first)\n}\n\nfunc TestContainsOverlappingS2IDsTrueCase(t *testing.T) {\n\tlist := []uint64{\n\t\t4542051748017078272,\n\t\t4542091330435678208,\n\t}\n\tassert.True(t, ContainsOverlappingS2IDs(list))\n}\n\nfunc TestContainsOverlappingS2IDsFalseCase(t *testing.T) {\n\tlist := []uint64{\n\t\t4542051748017078272,\n\t\t1504976331727699968,\n\t}\n\tassert.False(t, ContainsOverlappingS2IDs(list))\n}\n\nfunc TestToS2List(t *testing.T) {\n\tidList := []uint64{\n\t\t4542051748017078272,\n\t\t4542091330435678208,\n\t}\n\texpectedS2List := s2List{\n\t\ts2.CellID(idList[0]),\n\t\ts2.CellID(idList[1]),\n\t}\n\tassert.Equal(t, expectedS2List, toS2List(idList))\n}\n" }, { "path": "pkg/util/util.go", "content": "package util\n\nimport (\n\t\"strings\"\n\t\"unicode\"\n)\n\nfunc ToSnake(in string) string {\n\trunes := []rune(in)\n\tlength := len(runes)\n\n\tvar out []rune\n\tfor i := 0; i < length; i++ {\n\t\tif i > 0 && unicode.IsUpper(runes[i]) && ((i+1 < length && unicode.IsLower(runes[i+1])) || unicode.IsLower(runes[i-1])) {\n\t\t\tout = append(out, '_')\n\t\t}\n\n\t\tout = append(out, unicode.ToLower(runes[i]))\n\t}\n\n\treturn strings.Replace(string(out), \"-\", \"\", -1)\n}\n\nfunc BoolToOnOff(on bool) string {\n\tif on {\n\t\treturn \"on\"\n\t}\n\n\treturn \"off\"\n}\n" }, { "path": "pkg/util/util_test.go", "content": "package util\n\nimport (\n\t\"testing\"\n)\n\ntype SnakeTest struct {\n\tinput string\n\toutput string\n}\n\nvar tests = []SnakeTest{\n\t{\"a\", \"a\"},\n\t{\"snake\", \"snake\"},\n\t{\"A\", \"a\"},\n\t{\"ID\", \"id\"},\n\t{\"MOTD\", \"motd\"},\n\t{\"Snake\", \"snake\"},\n\t{\"SnakeTest\", \"snake_test\"},\n\t{\"Snake-Test\", \"snake_test\"},\n\t{\"SnakeID\", \"snake_id\"},\n\t{\"Snake_ID\", \"snake_id\"},\n\t{\"SnakeIDGoogle\", \"snake_id_google\"},\n\t{\"LinuxMOTD\", \"linux_motd\"},\n\t{\"OMGWTFBBQ\", \"omgwtfbbq\"},\n\t{\"omg_wtf_bbq\", \"omg_wtf_bbq\"},\n}\n\nfunc TestToSnake(t *testing.T) {\n\tfor _, test := range tests {\n\t\tif ToSnake(test.input) != test.output {\n\t\t\tt.Errorf(`ToSnake(\"%s\"), wanted \"%s\", got \\%s\"`, test.input, test.output, ToSnake(test.input))\n\t\t}\n\t}\n}\n" }, { "path": "server/error.go", "content": "package server\n\nimport (\n\t\"encoding/json\"\n\t\"net/http\"\n\n\t\"github.com/gojektech/weaver/pkg/instrumentation\"\n)\n\ntype weaverResponse struct {\n\tErrors []errorDetails `json:\"errors\"`\n}\n\ntype errorDetails struct {\n\tCode string `json:\"code\"`\n\tMessage string `json:\"message\"`\n\tMessageTitle string `json:\"message_title\"`\n\tMessageSeverity string `json:\"message_severity\"`\n}\n\nfunc notFoundError(w http.ResponseWriter, r *http.Request) {\n\tinstrumentation.IncrementNotFound()\n\tw.Header().Set(\"Content-Type\", \"application/json\")\n\n\tw.WriteHeader(http.StatusNotFound)\n\n\terrorResponse := weaverResponse{\n\t\tErrors: []errorDetails{\n\t\t\t{\n\t\t\t\tCode: \"weaver:route:not_found\",\n\t\t\t\tMessage: \"Something went wrong\",\n\t\t\t\tMessageTitle: \"Failure\",\n\t\t\t\tMessageSeverity: \"failure\",\n\t\t\t},\n\t\t},\n\t}\n\n\tresponse, _ := json.Marshal(errorResponse)\n\tw.Write(response)\n}\n\nfunc internalServerError(w http.ResponseWriter, r *http.Request) {\n\tw.Header().Set(\"Content-Type\", \"application/json\")\n\n\tw.WriteHeader(http.StatusInternalServerError)\n\n\terrorResponse := weaverResponse{\n\t\tErrors: []errorDetails{\n\t\t\t{\n\t\t\t\tCode: \"weaver:service:unavailable\",\n\t\t\t\tMessage: \"Something went wrong\",\n\t\t\t\tMessageTitle: \"Internal error\",\n\t\t\t\tMessageSeverity: \"failure\",\n\t\t\t},\n\t\t},\n\t}\n\n\tresponse, _ := json.Marshal(errorResponse)\n\tw.Write(response)\n}\n\n// TODO: decouple instrumentation from this errors function\ntype err503Handler struct {\n\tACLName string\n}\n\nfunc (eh err503Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {\n\tfailureHTTPStatus := http.StatusServiceUnavailable\n\tinstrumentation.IncrementInternalAPIStatusCount(eh.ACLName, failureHTTPStatus)\n\n\terrorResponse := weaverResponse{\n\t\tErrors: []errorDetails{\n\t\t\t{\n\t\t\t\tCode: \"weaver:service:unavailable\",\n\t\t\t\tMessage: \"Something went wrong\",\n\t\t\t\tMessageTitle: \"Failure\",\n\t\t\t\tMessageSeverity: \"failure\",\n\t\t\t},\n\t\t},\n\t}\n\n\tresponse, _ := json.Marshal(errorResponse)\n\tw.WriteHeader(failureHTTPStatus)\n\tw.Write(response)\n\treturn\n}\n" }, { "path": "server/error_test.go", "content": "package server\n\nimport (\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"testing\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc Test404Handler(t *testing.T) {\n\tw := httptest.NewRecorder()\n\tr := httptest.NewRequest(\"GET\", \"/hello\", nil)\n\n\tnotFoundError(w, r)\n\n\tassert.Equal(t, http.StatusNotFound, w.Code)\n\tassert.Equal(t, \"{\\\"errors\\\":[{\\\"code\\\":\\\"weaver:route:not_found\\\",\\\"message\\\":\\\"Something went wrong\\\",\\\"message_title\\\":\\\"Failure\\\",\\\"message_severity\\\":\\\"failure\\\"}]}\", w.Body.String())\n}\n\nfunc Test500Handler(t *testing.T) {\n\tw := httptest.NewRecorder()\n\tr := httptest.NewRequest(\"GET\", \"/hello\", nil)\n\n\tinternalServerError(w, r)\n\n\tassert.Equal(t, http.StatusInternalServerError, w.Code)\n\tassert.Equal(t, \"{\\\"errors\\\":[{\\\"code\\\":\\\"weaver:service:unavailable\\\",\\\"message\\\":\\\"Something went wrong\\\",\\\"message_title\\\":\\\"Internal error\\\",\\\"message_severity\\\":\\\"failure\\\"}]}\", w.Body.String())\n}\n\nfunc Test503Handler(t *testing.T) {\n\tw := httptest.NewRecorder()\n\tr := httptest.NewRequest(\"GET\", \"/hello\", nil)\n\n\terr503Handler{}.ServeHTTP(w, r)\n\n\tassert.Equal(t, http.StatusServiceUnavailable, w.Code)\n\tassert.Equal(t, \"{\\\"errors\\\":[{\\\"code\\\":\\\"weaver:service:unavailable\\\",\\\"message\\\":\\\"Something went wrong\\\",\\\"message_title\\\":\\\"Failure\\\",\\\"message_severity\\\":\\\"failure\\\"}]}\", w.Body.String())\n}\n" }, { "path": "server/handler.go", "content": "package server\n\nimport (\n\t\"net/http\"\n\n\t\"github.com/gojektech/weaver/config\"\n\t\"github.com/gojektech/weaver/pkg/instrumentation\"\n\t\"github.com/gojektech/weaver/pkg/logger\"\n\tnewrelic \"github.com/newrelic/go-agent\"\n)\n\ntype proxy struct {\n\trouter *Router\n}\n\nfunc (proxy *proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {\n\trw := &wrapperResponseWriter{ResponseWriter: w}\n\n\tif r.URL.Path == \"/ping\" || r.URL.Path == \"/\" {\n\t\tproxy.pingHandler(rw, r)\n\t\treturn\n\t}\n\n\ttiming := instrumentation.NewTiming()\n\n\tdefer instrumentation.TimeTotalLatency(timing)\n\tinstrumentation.IncrementTotalRequestCount()\n\n\tacl, err := proxy.router.Route(r)\n\tif err != nil || acl == nil {\n\t\tlogger.Errorrf(r, \"failed to find route: %+v for request: %s\", err, r.URL.String())\n\n\t\tnotFoundError(rw, r)\n\t\treturn\n\t}\n\n\tbackend, err := acl.Endpoint.Shard(r)\n\tif backend == nil || err != nil {\n\t\tlogger.Errorrf(r, \"failed to find backend for acl %s for: %s, error: %s\", acl.ID, r.URL.String(), err)\n\n\t\terr503Handler{ACLName: acl.ID}.ServeHTTP(rw, r)\n\t\treturn\n\t}\n\n\tinstrumentation.IncrementAPIBackendRequestCount(acl.ID, backend.Name)\n\n\tinstrumentation.IncrementAPIRequestCount(acl.ID)\n\tapiTiming := instrumentation.NewTiming()\n\tdefer instrumentation.TimeAPILatency(acl.ID, apiTiming)\n\n\tapiBackendTiming := instrumentation.NewTiming()\n\tdefer instrumentation.TimeAPIBackendLatency(acl.ID, backend.Name, apiBackendTiming)\n\n\tvar s newrelic.ExternalSegment\n\tif txn, ok := w.(newrelic.Transaction); ok {\n\t\ts = newrelic.StartExternalSegment(txn, r)\n\t}\n\tbackend.Handler.ServeHTTP(rw, r)\n\n\ts.End()\n\n\tlogger.ProxyInfo(acl.ID, backend.Server.String(), r, rw.statusCode, rw)\n\tinstrumentation.IncrementAPIStatusCount(acl.ID, rw.statusCode)\n\tinstrumentation.IncrementAPIBackendStatusCount(acl.ID, backend.Name, rw.statusCode)\n}\n\nfunc (proxy *proxy) pingHandler(w http.ResponseWriter, r *http.Request) {\n\tw.Header().Set(\"Content-Type\", \"application/json\")\n\tw.WriteHeader(http.StatusOK)\n\tw.Write([]byte(\"{}\"))\n}\n\nfunc wrapNewRelicHandler(proxy *proxy) http.Handler {\n\tif !config.NewRelicConfig().Enabled {\n\t\treturn proxy\n\t}\n\n\treturn http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tpath := r.URL.Path\n\n\t\tif path == \"/ping\" {\n\t\t\tproxy.ServeHTTP(w, r)\n\t\t\treturn\n\t\t}\n\n\t\t_, next := newrelic.WrapHandleFunc(instrumentation.NewRelicApp(), path,\n\t\t\tfunc(w http.ResponseWriter, r *http.Request) {\n\t\t\t\tproxy.ServeHTTP(w, r)\n\t\t\t})\n\n\t\tnext(w, r)\n\t})\n}\n" }, { "path": "server/handler_test.go", "content": "package server\n\nimport (\n\t\"bytes\"\n\t\"encoding/json\"\n\t\"fmt\"\n\t\"github.com/gojektech/weaver/pkg/shard\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"testing\"\n\n\t\"github.com/gojektech/weaver\"\n\t\"github.com/gojektech/weaver/pkg/logger\"\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/require\"\n\t\"github.com/stretchr/testify/suite\"\n)\n\ntype ProxySuite struct {\n\tsuite.Suite\n\n\trtr *Router\n}\n\nfunc (ps *ProxySuite) SetupTest() {\n\tlogger.SetupLogger()\n\n\trouteLoader := &mockRouteLoader{}\n\n\tps.rtr = NewRouter(routeLoader)\n\trequire.NotNil(ps.T(), ps.rtr)\n}\n\nfunc TestProxySuite(t *testing.T) {\n\tsuite.Run(t, new(ProxySuite))\n}\n\nfunc (ps *ProxySuite) TestProxyHandlerOnSuccessfulRouting() {\n\n\tserver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tw.WriteHeader(http.StatusForbidden)\n\t\t_, _ = w.Write([]byte(\"foobar\"))\n\t}))\n\n\tacl := &weaver.ACL{\n\t\tID: \"svc-01\",\n\t\tCriterion: \"Method(`GET`) && PathRegexp(`/(GF-|R-).*`)\",\n\t\tEndpointConfig: &weaver.EndpointConfig{\n\t\t\tMatcher: \"path\",\n\t\t\tShardExpr: \"/(GF-|R-|).*\",\n\t\t\tShardFunc: \"lookup\",\n\t\t\tShardConfig: json.RawMessage(fmt.Sprintf(`{\n\t\t\t\t\"GF-\": {\n\t\t\t\t\t\"backend_name\": \"foo\",\n\t\t\t\t\t\"backend\": \"%s\"\n\t\t\t\t},\n\t\t\t\t\"R-\": {\n\t\t\t\t\t\"backend_name\": \"bar\",\n\t\t\t\t\t\"timeout\": 100.0,\n\t\t\t\t\t\"backend\": \"http://iamgone\"\n\t\t\t\t}\n\t\t\t}`, server.URL)),\n\t\t},\n\t}\n\n\tsharder, err := shard.New(acl.EndpointConfig.ShardFunc, acl.EndpointConfig.ShardConfig)\n\trequire.NoError(ps.T(), err, \"should not have failed to init a sharder\")\n\n\tacl.Endpoint, err = weaver.NewEndpoint(acl.EndpointConfig, sharder)\n\trequire.NoError(ps.T(), err, \"should not have failed to set endpoint\")\n\n\t_ = ps.rtr.UpsertRoute(acl.Criterion, acl)\n\n\tw := httptest.NewRecorder()\n\tr := httptest.NewRequest(\"GET\", \"/GF-1234\", nil)\n\n\tproxy := proxy{router: ps.rtr}\n\tproxy.ServeHTTP(w, r)\n\n\tassert.Equal(ps.T(), http.StatusForbidden, w.Code)\n\tassert.Equal(ps.T(), \"foobar\", w.Body.String())\n}\n\nfunc (ps *ProxySuite) TestProxyHandlerOnBodyBasedMatcherWithModuloSharding() {\n\n\tserver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tw.WriteHeader(http.StatusOK)\n\t\t_, _ = w.Write([]byte(\"foobar\"))\n\t}))\n\n\tacl := &weaver.ACL{\n\t\tID: \"svc-01\",\n\t\tCriterion: \"Method(`GET`) && PathRegexp(`/drivers`)\",\n\t\tEndpointConfig: &weaver.EndpointConfig{\n\t\t\tMatcher: \"body\",\n\t\t\tShardExpr: \".drivers.id\",\n\t\t\tShardFunc: \"modulo\",\n\t\t\tShardConfig: json.RawMessage(fmt.Sprintf(`{\n\t\t\t\t\"0\": {\n\t\t\t\t\t\"backend_name\": \"foo\",\n\t\t\t\t\t\"backend\": \"%s\"\n\t\t\t\t},\n\t\t\t\t\"1\": {\n\t\t\t\t\t\"backend_name\": \"bar\",\n\t\t\t\t\t\"timeout\": 100.0,\n\t\t\t\t\t\"backend\": \"http://shard01\"\n\t\t\t\t}\n\t\t\t}`, server.URL)),\n\t\t},\n\t}\n\n\tsharder, err := shard.New(acl.EndpointConfig.ShardFunc, acl.EndpointConfig.ShardConfig)\n\trequire.NoError(ps.T(), err, \"should not have failed to init a sharder\")\n\n\tacl.Endpoint, err = weaver.NewEndpoint(acl.EndpointConfig, sharder)\n\trequire.NoError(ps.T(), err, \"should not have failed to set endpoint\")\n\n\t_ = ps.rtr.UpsertRoute(acl.Criterion, acl)\n\n\tw := httptest.NewRecorder()\n\tbody := bytes.NewReader([]byte(`{ \"drivers\": { \"id\": \"122\" } }`))\n\tr := httptest.NewRequest(\"GET\", \"/drivers\", body)\n\n\tproxy := proxy{router: ps.rtr}\n\tproxy.ServeHTTP(w, r)\n\n\tassert.Equal(ps.T(), http.StatusOK, w.Code)\n\tassert.Equal(ps.T(), \"foobar\", w.Body.String())\n}\n\nfunc (ps *ProxySuite) TestProxyHandlerOnPathBasedMatcherWithModuloSharding() {\n\n\tserver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tw.WriteHeader(http.StatusOK)\n\t\t_, _ = w.Write([]byte(\"foobar\"))\n\t}))\n\n\tacl := &weaver.ACL{\n\t\tID: \"svc-01\",\n\t\tCriterion: \"Method(`GET`) && PathRegexp(`/drivers`)\",\n\t\tEndpointConfig: &weaver.EndpointConfig{\n\t\t\tMatcher: \"path\",\n\t\t\tShardExpr: `/drivers/(\\d+)`,\n\t\t\tShardFunc: \"modulo\",\n\t\t\tShardConfig: json.RawMessage(fmt.Sprintf(`{\n\t\t\t\t\"0\": {\n\t\t\t\t\t\"backend_name\": \"foo\",\n\t\t\t\t\t\"backend\": \"http://shard01\"\n\t\t\t\t},\n\t\t\t\t\"1\": {\n\t\t\t\t\t\"backend_name\": \"bar\",\n\t\t\t\t\t\"timeout\":100.0,\n\t\t\t\t\t\"backend\":\"%s\"\n\t\t\t\t}\n\t\t\t}`, server.URL)),\n\t\t},\n\t}\n\n\tsharder, err := shard.New(acl.EndpointConfig.ShardFunc, acl.EndpointConfig.ShardConfig)\n\trequire.NoError(ps.T(), err, \"should not have failed to init a sharder\")\n\n\tacl.Endpoint, err = weaver.NewEndpoint(acl.EndpointConfig, sharder)\n\trequire.NoError(ps.T(), err, \"should not have failed to set endpoint\")\n\n\t_ = ps.rtr.UpsertRoute(acl.Criterion, acl)\n\n\tw := httptest.NewRecorder()\n\tr := httptest.NewRequest(\"GET\", \"/drivers/123\", nil)\n\n\tproxy := proxy{router: ps.rtr}\n\tproxy.ServeHTTP(w, r)\n\n\tassert.Equal(ps.T(), http.StatusOK, w.Code)\n\tassert.Equal(ps.T(), \"foobar\", w.Body.String())\n}\n\nfunc (ps *ProxySuite) TestProxyHandlerOnFailureRouting() {\n\tw := httptest.NewRecorder()\n\tr := httptest.NewRequest(\"GET\", \"/GF-1234\", nil)\n\n\tproxy := proxy{router: ps.rtr}\n\tproxy.ServeHTTP(w, r)\n\n\tassert.Equal(ps.T(), http.StatusNotFound, w.Code)\n\tassert.Equal(ps.T(), \"{\\\"errors\\\":[{\\\"code\\\":\\\"weaver:route:not_found\\\",\\\"message\\\":\\\"Something went wrong\\\",\\\"message_title\\\":\\\"Failure\\\",\\\"message_severity\\\":\\\"failure\\\"}]}\", w.Body.String())\n}\n\nfunc (ps *ProxySuite) TestProxyHandlerOnMissingBackend() {\n\n\tacl := &weaver.ACL{\n\t\tID: \"svc-01\",\n\t\tCriterion: \"Method(`GET`) && PathRegexp(`/(GF-|R-).*`)\",\n\t\tEndpointConfig: &weaver.EndpointConfig{\n\t\t\tMatcher: \"path\",\n\t\t\tShardExpr: \"/(GF-|R-|).*\",\n\t\t\tShardFunc: \"lookup\",\n\t\t\tShardConfig: json.RawMessage(`{\n\t\t\t\t\"R-\": {\n\t\t\t\t\t\"backend_name\": \"foo\",\n\t\t\t\t\t\"timeout\": 100.0,\n\t\t\t\t\t\"backend\": \"http://iamgone\"\n\t\t\t\t}\n\t\t\t}`),\n\t\t},\n\t}\n\n\tsharder, err := shard.New(acl.EndpointConfig.ShardFunc, acl.EndpointConfig.ShardConfig)\n\trequire.NoError(ps.T(), err, \"should not have failed to init a sharder\")\n\n\tacl.Endpoint, err = weaver.NewEndpoint(acl.EndpointConfig, sharder)\n\trequire.NoError(ps.T(), err, \"should not have failed to set endpoint\")\n\n\t_ = ps.rtr.UpsertRoute(acl.Criterion, acl)\n\n\tw := httptest.NewRecorder()\n\tr := httptest.NewRequest(\"GET\", \"/GF-1234\", nil)\n\n\tproxy := proxy{router: ps.rtr}\n\tproxy.ServeHTTP(w, r)\n\n\tassert.Equal(ps.T(), http.StatusServiceUnavailable, w.Code)\n}\n\nfunc (ps *ProxySuite) TestHealthCheckWithPingRoute() {\n\tw := httptest.NewRecorder()\n\tr := httptest.NewRequest(\"GET\", \"/ping\", nil)\n\n\tproxy := proxy{router: ps.rtr}\n\tproxy.ServeHTTP(w, r)\n\n\tassert.Equal(ps.T(), http.StatusOK, w.Code)\n}\n\nfunc (ps *ProxySuite) TestHealthCheckWithDefaultRoute() {\n\tw := httptest.NewRecorder()\n\tr := httptest.NewRequest(\"GET\", \"/\", nil)\n\n\tproxy := proxy{router: ps.rtr}\n\tproxy.ServeHTTP(w, r)\n\n\tassert.Equal(ps.T(), http.StatusOK, w.Code)\n}\n" }, { "path": "server/loader.go", "content": "package server\n\nimport (\n\t\"context\"\n\n\t\"github.com/gojektech/weaver\"\n)\n\ntype UpsertRouteFunc func(*weaver.ACL) error\ntype DeleteRouteFunc func(*weaver.ACL) error\n\ntype RouteLoader interface {\n\tBootstrapRoutes(context.Context, UpsertRouteFunc) error\n\tWatchRoutes(context.Context, UpsertRouteFunc, DeleteRouteFunc)\n}\n" }, { "path": "server/mock.go", "content": "package server\n\nimport (\n\t\"context\"\n\n\t\"github.com/stretchr/testify/mock\"\n)\n\ntype mockRouteLoader struct {\n\tmock.Mock\n}\n\nfunc (mrl *mockRouteLoader) BootstrapRoutes(ctx context.Context, upsertRouteFunc UpsertRouteFunc) error {\n\targs := mrl.Called(ctx, upsertRouteFunc)\n\treturn args.Error(0)\n}\n\nfunc (mrl *mockRouteLoader) WatchRoutes(ctx context.Context, upsertRouteFunc UpsertRouteFunc, deleteRouteFunc DeleteRouteFunc) {\n\tmrl.Called(ctx, upsertRouteFunc, deleteRouteFunc)\n\treturn\n}\n" }, { "path": "server/recovery.go", "content": "package server\n\nimport (\n\t\"fmt\"\n\t\"net/http\"\n\n\traven \"github.com/getsentry/raven-go\"\n\t\"github.com/gojektech/weaver/pkg/instrumentation\"\n\t\"github.com/gojektech/weaver/pkg/logger\"\n)\n\nfunc Recover(next http.Handler) http.Handler {\n\treturn http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tw.Header().Set(\"Content-Type\", \"application/json\")\n\n\t\tdefer func() {\n\t\t\tif err := recover(); err != nil {\n\t\t\t\tinstrumentation.IncrementCrashCount()\n\n\t\t\t\tvar recoveredErr error\n\t\t\t\tswitch val := err.(type) {\n\t\t\t\tcase error:\n\t\t\t\t\trecoveredErr = val\n\t\t\t\tcase string:\n\t\t\t\t\trecoveredErr = fmt.Errorf(val)\n\t\t\t\t}\n\n\t\t\t\traven.CaptureError(recoveredErr, map[string]string{\"error\": recoveredErr.Error(), \"request_url\": r.URL.String()})\n\n\t\t\t\tlogger.Errorrf(r, \"failed to route request: %+v\", err)\n\t\t\t\tinternalServerError(w, r)\n\t\t\t\treturn\n\t\t\t}\n\t\t}()\n\n\t\tnext.ServeHTTP(w, r)\n\t})\n}\n" }, { "path": "server/recovery_test.go", "content": "package server\n\nimport (\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"testing\"\n\n\t\"github.com/gojektech/weaver/config\"\n\t\"github.com/gojektech/weaver/pkg/logger\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\ntype testHandler struct{}\n\nfunc (th testHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {\n\tpanic(\"failed\")\n}\n\nfunc TestRecoverMiddleware(t *testing.T) {\n\tconfig.Load()\n\tlogger.SetupLogger()\n\n\tr := httptest.NewRequest(\"GET\", \"/hello\", nil)\n\tw := httptest.NewRecorder()\n\n\tRecover(testHandler{}).ServeHTTP(w, r)\n\n\tassert.Equal(t, http.StatusInternalServerError, w.Code)\n\tassert.Equal(t, \"{\\\"errors\\\":[{\\\"code\\\":\\\"weaver:service:unavailable\\\",\\\"message\\\":\\\"Something went wrong\\\",\\\"message_title\\\":\\\"Internal error\\\",\\\"message_severity\\\":\\\"failure\\\"}]}\", w.Body.String())\n}\n" }, { "path": "server/router.go", "content": "package server\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"net/http\"\n\n\t\"github.com/gojektech/weaver\"\n\t\"github.com/pkg/errors\"\n\t\"github.com/vulcand/route\"\n)\n\ntype Router struct {\n\troute.Router\n\tloader RouteLoader\n}\n\ntype apiName string\n\nfunc (router *Router) Route(req *http.Request) (*weaver.ACL, error) {\n\trt, err := router.Router.Route(req)\n\tif err != nil {\n\t\treturn nil, errors.Wrapf(err, \"failed to find route with url: %s\", req.URL)\n\t}\n\n\tif rt == nil {\n\t\treturn nil, errors.WithStack(fmt.Errorf(\"route not found: %s\", req.URL))\n\t}\n\n\tacl, ok := rt.(*weaver.ACL)\n\tif !ok {\n\t\treturn nil, errors.WithStack(fmt.Errorf(\"error in casting %v to acl\", rt))\n\t}\n\n\treturn acl, nil\n}\n\nfunc NewRouter(loader RouteLoader) *Router {\n\treturn &Router{\n\t\tRouter: route.New(),\n\t\tloader: loader,\n\t}\n}\n\nfunc (router *Router) WatchRouteUpdates(routeSyncCtx context.Context) {\n\trouter.loader.WatchRoutes(routeSyncCtx, router.upsertACL, router.deleteACL)\n}\n\nfunc (router *Router) BootstrapRoutes(ctx context.Context) error {\n\treturn router.loader.BootstrapRoutes(ctx, router.upsertACL)\n}\n\nfunc (router *Router) upsertACL(acl *weaver.ACL) error {\n\treturn router.UpsertRoute(acl.Criterion, acl)\n}\n\nfunc (router *Router) deleteACL(acl *weaver.ACL) error {\n\treturn router.RemoveRoute(acl.Criterion)\n}\n" }, { "path": "server/router_test.go", "content": "package server\n\nimport (\n\t\"context\"\n\t\"encoding/json\"\n\t\"errors\"\n\t\"github.com/gojektech/weaver\"\n\t\"github.com/gojektech/weaver/pkg/shard\"\n\t\"net/http/httptest\"\n\t\"testing\"\n\n\t\"github.com/gojektech/weaver/config\"\n\t\"github.com/gojektech/weaver/pkg/logger\"\n\t\"github.com/stretchr/testify/assert\"\n\t\"github.com/stretchr/testify/mock\"\n\t\"github.com/stretchr/testify/require\"\n\t\"github.com/stretchr/testify/suite\"\n)\n\ntype RouterSuite struct {\n\tsuite.Suite\n\n\trtr *Router\n}\n\nfunc (rs *RouterSuite) SetupTest() {\n\tlogger.SetupLogger()\n\trouteLoader := &mockRouteLoader{}\n\n\trs.rtr = NewRouter(routeLoader)\n\trequire.NotNil(rs.T(), rs.rtr)\n}\n\nfunc TestRouterSuite(t *testing.T) {\n\tsuite.Run(t, new(RouterSuite))\n}\n\nfunc (rs *RouterSuite) TestRouteNotFound() {\n\treq := httptest.NewRequest(\"GET\",\n\t\t(\"http://\" + config.ProxyServerAddress() + \"/\"),\n\t\tnil)\n\t_, err := rs.rtr.Route(req)\n\tassert.Error(rs.T(), err)\n}\n\nfunc (rs *RouterSuite) TestRouteInvalidACL() {\n\treq := httptest.NewRequest(\"GET\",\n\t\t(\"http://\" + config.ProxyServerAddress() + \"/foobar\"),\n\t\tnil)\n\trs.rtr.UpsertRoute(\n\t\t\"Method(`GET`) && Path(`/foobar`)\",\n\t\t\"foobar\")\n\n\t_, err := rs.rtr.Route(req)\n\tassert.Error(rs.T(), err)\n}\n\nfunc (rs *RouterSuite) TestRouteReturnsACL() {\n\treq := httptest.NewRequest(\"GET\",\n\t\t(\"http://\" + config.ProxyServerAddress() + \"/R-1234\"),\n\t\tnil)\n\n\t// timeout is float64 because there are no integers in json\n\tacl := &weaver.ACL{\n\t\tID: \"svc-01\",\n\t\tCriterion: \"Method(`GET`) && PathRegexp(`/(GF-|R-).*`)\",\n\t\tEndpointConfig: &weaver.EndpointConfig{\n\t\t\tShardConfig: json.RawMessage(`{\n\t\t\t\t\"GF-\": {\n\t\t\t\t\t\"backend_name\": \"foobar\",\n\t\t\t\t\t\"backend\": \"http://customer-locations-primary\"\n\t\t\t\t},\n\t\t\t\t\"R-\": {\n\t\t\t\t\t\"timeout\": 100.0,\n\t\t\t\t\t\"backend_name\": \"foobar\",\n\t\t\t\t\t\"backend\": \"http://iamgone\"\n\t\t\t\t}\n\t\t\t}`),\n\t\t\tMatcher: \"path\",\n\t\t\tShardExpr: \"/(GF-|R-|).*\",\n\t\t\tShardFunc: \"lookup\",\n\t\t},\n\t}\n\n\tsharder, err := shard.New(acl.EndpointConfig.ShardFunc, acl.EndpointConfig.ShardConfig)\n\trequire.NoError(rs.T(), err, \"should not have failed to init a sharder\")\n\n\tacl.Endpoint, err = weaver.NewEndpoint(acl.EndpointConfig, sharder)\n\trequire.NoError(rs.T(), err, \"should not have failed to set endpoint\")\n\n\trs.rtr.UpsertRoute(acl.Criterion, acl)\n\n\tacl, err = rs.rtr.Route(req)\n\trequire.NoError(rs.T(), err, \"should not have failed to find a route handler\")\n\n\tassert.Equal(rs.T(), \"svc-01\", acl.ID)\n}\n\nfunc (rs *RouterSuite) TestBootstrapRoutesUseBootstrapRoutesOfRouteLoader() {\n\tctx := context.Background()\n\trouteLoader := &mockRouteLoader{}\n\n\trtr := NewRouter(routeLoader)\n\n\trouteLoader.On(\"BootstrapRoutes\", ctx, mock.AnythingOfType(\"UpsertRouteFunc\")).Return(nil)\n\n\terr := rtr.BootstrapRoutes(ctx)\n\trequire.NoError(rs.T(), err, \"should not have failed to bootstrap routes\")\n\n\trouteLoader.AssertExpectations(rs.T())\n}\n\nfunc (rs *RouterSuite) TestBootstrapRoutesUseBootstrapRoutesOfRouteLoaderFail() {\n\tctx := context.Background()\n\trouteLoader := &mockRouteLoader{}\n\n\trtr := NewRouter(routeLoader)\n\n\trouteLoader.On(\"BootstrapRoutes\", ctx, mock.AnythingOfType(\"UpsertRouteFunc\")).Return(errors.New(\"fail\"))\n\n\terr := rtr.BootstrapRoutes(ctx)\n\trequire.Error(rs.T(), err, \"should have failed to bootstrap routes\")\n\n\trouteLoader.AssertExpectations(rs.T())\n}\n\nfunc (rs *RouterSuite) TestWatchRouteUpdatesCallsWatchRoutesOfLoader() {\n\tctx := context.Background()\n\trouteLoader := &mockRouteLoader{}\n\n\trtr := NewRouter(routeLoader)\n\n\trouteLoader.On(\"WatchRoutes\", ctx, mock.AnythingOfType(\"UpsertRouteFunc\"), mock.AnythingOfType(\"DeleteRouteFunc\"))\n\n\trtr.WatchRouteUpdates(ctx)\n\n\trouteLoader.AssertExpectations(rs.T())\n}\n" }, { "path": "server/server.go", "content": "package server\n\nimport (\n\t\"context\"\n\t\"log\"\n\t\"net/http\"\n\n\t\"github.com/gojektech/weaver/config\"\n\t\"github.com/gojektech/weaver/pkg/util\"\n)\n\nvar server *Weaver\n\ntype Weaver struct {\n\thttpServer *http.Server\n}\n\nfunc ShutdownServer(ctx context.Context) {\n\tserver.httpServer.Shutdown(ctx)\n}\n\nfunc StartServer(ctx context.Context, routeLoader RouteLoader) {\n\tproxyRouter := NewRouter(routeLoader)\n\terr := proxyRouter.BootstrapRoutes(context.Background())\n\tif err != nil {\n\t\tlog.Printf(\"StartServer: failed to initialise proxy router: %s\", err)\n\t}\n\n\tlog.Printf(\"StartServer: bootstraped routes from etcd\")\n\n\tgo proxyRouter.WatchRouteUpdates(ctx)\n\n\tproxy := Recover(wrapNewRelicHandler(&proxy{\n\t\trouter: proxyRouter,\n\t}))\n\n\thttpServer := &http.Server{\n\t\tAddr: config.ProxyServerAddress(),\n\t\tHandler: proxy,\n\t\tReadTimeout: config.ServerReadTimeoutInMillis(),\n\t\tWriteTimeout: config.ServerWriteTimeoutInMillis(),\n\t}\n\n\tkeepAliveEnabled := config.Proxy().KeepAliveEnabled()\n\thttpServer.SetKeepAlivesEnabled(keepAliveEnabled)\n\n\tserver = &Weaver{\n\t\thttpServer: httpServer,\n\t}\n\n\tlog.Printf(\"StartServer: starting weaver on %s\", server.httpServer.Addr)\n\tlog.Printf(\"Keep-Alive: %s\", util.BoolToOnOff(keepAliveEnabled))\n\n\tif err := server.httpServer.ListenAndServe(); err != nil {\n\t\tlog.Fatalf(\"StartServer: starting weaver failed with %s\", err)\n\t}\n}\n" }, { "path": "server/wrapped_response_writer.go", "content": "package server\n\nimport \"net/http\"\n\ntype wrapperResponseWriter struct {\n\tstatusCode int\n\thttp.ResponseWriter\n}\n\nfunc (w *wrapperResponseWriter) Header() http.Header {\n\treturn w.ResponseWriter.Header()\n}\n\nfunc (w *wrapperResponseWriter) Write(data []byte) (int, error) {\n\treturn w.ResponseWriter.Write(data)\n}\n\nfunc (w *wrapperResponseWriter) WriteHeader(statusCode int) {\n\tw.statusCode = statusCode\n\tw.ResponseWriter.WriteHeader(statusCode)\n}\n" }, { "path": "sharder.go", "content": "package weaver\n\ntype Sharder interface {\n\tShard(key string) (*Backend, error)\n}\n" }, { "path": "weaver.conf.yaml.sample", "content": "SERVER_HOST: \"127.0.0.1\"\nSERVER_PORT: \"8080\"\nPROXY_HOST: \"127.0.0.1\"\nPROXY_PORT: \"8081\"\nPROXY_MAX_IDLE_CONNS: \"50\"\nPROXY_DIALER_TIMEOUT_IN_MS: \"1000\"\nPROXY_DIALER_KEEP_ALIVE_IN_MS: \"100\"\nPROXY_IDLE_CONN_TIMEOUT_IN_MS: \"100\"\nETCD_KEY_PREFIX: \"weaver\"\nLOGGER_LEVEL: \"debug\"\nETCD_ENDPOINTS: \"http://0.0.0.0:12379\"\nETCD_DIAL_TIMEOUT: \"5\"\nSTATSD_HOST: \"127.0.0.1\"\nSTATSD_PORT: \"18125\"\nSTATSD_PREFIX: \"weaver\"\nSTATSD_ENABLED: false\nSTATSD_FLUSH_PERIOD_IN_SECONDS: \"10\"\nNEW_RELIC_APP_NAME: \"weaver\"\nNEW_RELIC_LICENSE_KEY: \"__new_relic_fake_license_only_for_devs__\"\nNEW_RELIC_ENABLED: \"false\"\nSENTRY_DSN: \"hello\"\nSERVER_READ_TIMEOUT: 100\nSERVER_WRITE_TIMEOUT: 100\n" } ]