[ { "path": ".clabot", "content": "{\n \"contributors\": [\"buger\"]\n}\n" }, { "path": ".deepsource.toml", "content": "version = 1\n\nexclude_patterns = [\n \"vendor/**\"\n]\n\n[[analyzers]]\nname = \"go\"\nenabled = true\n\n [analyzers.meta]\n import_paths = [\"github.com/ankitdobhal/goreplay\"]\n\n[[analyzers]]\nname = \"docker\"\nenabled = true\n\n[[analyzers]]\nname = \"ruby\"\nenabled = true\n\n[[analyzers]]\nname = \"javascript\"\nenabled = true\n\n [analyzers.meta]\n environment = [\"nodejs\"]\n\n" }, { "path": ".dockerignore", "content": "*.tar.gz\ngor\ngor.test\n" }, { "path": ".github/dependabot.yaml", "content": "version: 2\nupdates:\n # Maintain dependencies for GitHub Actions\n - package-ecosystem: \"github-actions\"\n directory: \"/\"\n schedule:\n interval: \"weekly\"\n pull-request-branch-name:\n # Separate sections of the branch name with a hyphen\n separator: \"-\"\n\n - package-ecosystem: \"gomod\"\n directory: \"/\"\n schedule:\n interval: \"weekly\"\n \n - package-ecosystem: \"docker\"\n directory: \"/\"\n schedule:\n interval: \"weekly\"\n \n" }, { "path": ".github/workflows/ci-docker.yaml", "content": "name: ci\non:\n release:\n types: [published]\njobs:\n docker-build-and-push:\n runs-on: ubuntu-latest\n steps:\n -\n name: \"Checkout repository\"\n uses: actions/checkout@v3\n -\n name: \"Set up Docker Buildx\"\n uses: docker/setup-buildx-action@v2\n -\n name: \"Cache Docker layers\"\n uses: actions/cache@v3.0.5\n with:\n path: /tmp/.buildx-cache\n key: ${{ runner.os }}-buildx-${{ github.sha }}\n restore-keys: |\n ${{ runner.os }}-buildx-\n -\n name: \"Login to Container Registry\"\n uses: docker/login-action@v1\n with:\n username: ${{ secrets.DOCKERHUB_USERNAME }}\n password: ${{ secrets.DOCKERHUB_TOKEN }}\n -\n name: \"Build and push docker image\"\n uses: docker/build-push-action@v3\n with:\n context: .\n push: true\n tags: ${{ github.repository }}:${{ github.event.release.tag_name }}\n build-args: RELEASE_VERSION=${{ github.event.release.tag_name }}\n cache-from: type=local,src=/tmp/.buildx-cache\n cache-to: type=local,mode=max,dest=/tmp/.buildx-cache\n" }, { "path": ".github/workflows/ci-test.yaml", "content": "name: test\non: [push, pull_request]\njobs:\n test:\n strategy:\n matrix:\n go-version: [1.18.x, 1.19.x] # two latest minor versions\n runs-on: ubuntu-latest\n steps:\n - name: update package index\n run: sudo apt-get update\n - name: install libpcap\n run: sudo apt-get install libpcap-dev -y\n - name: install Go\n uses: actions/setup-go@v2\n with:\n go-version: ${{ matrix.go-version }}\n - name: checkout code\n uses: actions/checkout@v3\n - uses: actions/cache@v3.0.5\n with:\n path: |\n ~/go/pkg/mod # Module download cache\n ~/.cache/go-build # Build cache (Linux)\n key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}\n restore-keys: |\n ${{ runner.os }}-go-\n - name: test\n run: sudo go test ./... -v -timeout 120s\n" }, { "path": ".github/workflows/codesee-arch-diagram.yml", "content": "# This workflow was added by CodeSee. Learn more at https://codesee.io/\n# This is v2.0 of this workflow file\non:\n push:\n branches:\n - master\n pull_request_target:\n types: [opened, synchronize, reopened]\n\nname: CodeSee\n\npermissions: read-all\n\njobs:\n codesee:\n runs-on: ubuntu-latest\n continue-on-error: true\n name: Analyze the repo with CodeSee\n steps:\n - uses: Codesee-io/codesee-action@v2\n with:\n codesee-token: ${{ secrets.CODESEE_ARCH_DIAG_API_TOKEN }}\n codesee-url: https://app.codesee.io\n" }, { "path": ".github/workflows/probe.yaml", "content": "name: AI Comment Handler\n\n\non:\n pull_request:\n types: [opened] #[opened , labeled]\n issue_comment:\n types: [created] \n issues:\n types: [opened] #[opened, labeled] \n \n# Define permissions needed for the workflow\npermissions:\n issues: write\n pull-requests: write\n contents: read\n\njobs:\n trigger_probe_chat:\n # Uncomment if you want to run on on specific lables, in this example `probe`\n # if: |\n # (github.event_name == 'pull_request' && github.event.action == 'opened') || \n # (github.event_name == 'issues' && github.event.action == 'opened') || \n # (github.event_name == 'issue_comment' && github.event.action == 'created') || \n # ((github.event_name == 'pull_request' || github.event_name == 'issues') && \n # github.event.action == 'labeled' && github.event.label.name == 'probe')\n # Use the reusable workflow from your repository (replace and )\n uses: buger/probe/.github/workflows/probe.yml@main\n # Pass required inputs\n with:\n command_prefix: \"/probe\" # Or '/ai', '/ask', etc.\n # Optionally override the default npx command if the secret isn't set\n # default_probe_chat_command: 'node path/to/custom/script.js'\n # Pass ALL secrets from this repository to the reusable workflow\n # This includes GITHUB_TOKEN, PROBE_CHAT_COMMAND (if set), ANTHROPIC_API_KEY, etc.\n secrets:\n ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}\n ANTHROPIC_API_URL: ${{ secrets.ANTHROPIC_API_URL }}\n" }, { "path": ".github/workflows/semgrep.yml", "content": "on:\n pull_request: {}\n push:\n branches:\n - main\n - master\n paths:\n - .github/workflows/semgrep.yml\n schedule:\n # random HH:MM to avoid a load spike on GitHub Actions at 00:00\n - cron: 53 23 * * *\nname: Semgrep\njobs:\n semgrep:\n name: Scan\n runs-on: ubuntu-20.04\n env:\n SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }}\n container:\n image: returntocorp/semgrep\n steps:\n - uses: actions/checkout@v3\n - run: semgrep ci\n" }, { "path": ".gitignore", "content": "vendor\n*.swp\n*.gor\n*.rpm\n*.dep\n*.deb\n*.pkg\n*.exe\n*.pprof\n*.out\nhey\n\n*.bin\nlib/\noutput/\n*.gz\n*.zip\n.aider*\n\n*.class\n\n*.test\n.idea\n*.iml\ngor\n\n*.mprof\n\n*.pcap\n\n.DS_Store\n\ngoreplay\ncorpus\ncrashers\nsuppressions\ndist\n" }, { "path": ".gitmodules", "content": "" }, { "path": ".request", "content": "POST /post HTTP/1.1\nContent-Length: 7\nHost: www.w3.org\n\na=1&b=2\n" }, { "path": "COMM-LICENSE", "content": "END-USER LICENSE AGREEMENT\n\n------------------------------------------------------------------------------\n\nIMPORTANT: THIS SOFTWARE END-USER LICENSE AGREEMENT (\"EULA\") IS A LEGAL AGREEMENT (“Agreement”) BETWEEN YOU (THE CUSTOMER, EITHER AS AN INDIVIDUAL OR, IF PURCHASED OR OTHERWISE ACQUIRED BY OR FOR AN ENTITY, AS AN ENTITY), hereafter \"Customer\", AND Replay Software LLC (\"Licensor\"). READ IT CAREFULLY BEFORE COMPLETING THE INSTALLATION PROCESS AND USING GOREPLAY PRO AND RELATED SOFTWARE COMPONENTS (“SOFTWARE”). IT PROVIDES A LICENSE TO USE THE SOFTWARE AND CONTAINS WARRANTY INFORMATION AND LIABILITY DISCLAIMERS. BY INSTALLING AND USING THE SOFTWARE, YOU ARE CONFIRMING YOUR ACCEPTANCE OF THE SOFTWARE AND AGREEING TO BECOME BOUND BY THE TERMS OF THIS AGREEMENT.\n\n------------------------------------------------------------------------------\n\nIn order to use the Software under this Agreement, you must receive a “Source URL” at the time of purchase, in accordance with the scope of use and other terms specified for each type of Software and as set forth in this Section 1 of this Agreement. \n\n1. License Grant\n\n1.1 General Use. This Agreement grants you a non-exclusive, non-transferable, limited license to the use rights for the Software, without the right to grant sublicenses, subject to the terms and conditions in this Agreement. The Software is licensed, not sold.\n\n1.2 Unlimited Organization License. If you purchased an Organization License (included with the GoReplay Pro Software), you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. You may concurrently run the software on an unlimited number of Hosts.\n\n1.3 Appliance License. If you purchased an Appliance License, you may distribute the Software in any applications, frameworks, or elements (collectively referred to as an “Application” or “Applications”) that you develop using the Software in accordance with this EULA, provided that such distribution does not violate the restrictions set forth in section 3 of this EULA. You must not remove, obscure or interfere with any copyright, acknowledgment, attribution, trademark, warning or disclaimer statement affixed to, incorporated in or otherwise applied in connection with the Software. You are required to ensure that the Software is not reused by or with any applications other than those with which you distribute it as permitted herein. For example, if You install the Software on a customer’s server, that customer is not permitted to use the Software independently of your Application. You must inform Licensor of your knowledge of any infringing use of the Software by any of your customers. You are liable for compliance by those third parties with the terms and conditions of this EULA. You will not owe Licensor any royalties for your distribution of the Software in accordance with this EULA.\n\n1.4 Archive Copies. You are entitled to make a reasonable amount of copies of the Software for archival purposes. Each copy must reproduce all copyright and other proprietary rights notices on or in the Software Product.\n\n1.5 Electronic Delivery. All Software and license documentation shall be delivered by electronic means unless otherwise specified on the applicable invoice or at the time of purchase. Software shall be deemed delivered when it is made available for download by you (“Delivery”). \n\n2. Modifications. Licensor shall provide you with source code so that you can create Modifications of the original software. “Modification” means: (a) any addition to or deletion from the contents of a file included in the original Software or previous Modifications created by You, or (b) any new file that contains any part of the original Software or previous Modifications. While you retain all rights to any original work authored by you as part of the Modifications, We continue to own all copyright and other intellectual property rights in the Software.\n\n3. Restricted Uses. \n\n3.1 You shall not (and shall not allow any third party to): (a) decompile, disassemble, or otherwise reverse engineer the Software or attempt to reconstruct or discover any source code, underlying ideas, algorithms, file formats or programming interfaces of the Software by any means whatsoever (except and only to the extent that applicable law prohibits or restricts reverse engineering restrictions); (b) distribute, sell, sublicense, rent, lease or use the Software for time sharing, hosting, service provider or like purposes, except as expressly permitted under this Agreement; (c) redistribute the Software or Modifications other than by including the Software or a portion thereof within your own product, which must have substantially different functionality than the Software or Modifications and must not allow any third party to use the Software or Modifications, or any portions thereof, for software development or application development purposes; (d) redistribute the Software as part of a product, \"appliance\" or \"virtual server\"; (e) redistribute the Software on any server which is not directly under your control; (f) remove any product identification, proprietary, copyright or other notices contained in the Software; (g) modify any part of the Software, create a derivative work of any part of the Software (except as permitted in Section 4), or incorporate the Software, except to the extent expressly authorized in writing by Licensor; (h) publicly disseminate performance information or analysis (including, without limitation, benchmarks) from any source relating to the Software; (i) utilize any equipment, device, software, or other means designed to circumvent or remove any form of Source URL or copy protection used by Licensor in connection with the Software, or use the Software together with any authorization code, Source URL, serial number, or other copy protection device not supplied by Licensor; (j) use the Software to develop a product which is competitive with any Licensor product offerings; or (k) use unauthorized Source URLS or keycode(s) or distribute or publish Source URLs or keycode(s), except as may be expressly permitted by Licensor in writing. If your unique Source URL is ever published, Licensor reserves the right to terminate your access without notice.\n\n3.2 UNDER NO CIRCUMSTANCES MAY YOU USE THE SOFTWARE AS PART OF A PRODUCT OR SERVICE THAT PROVIDES SIMILAR FUNCTIONALITY TO THE SOFTWARE ITSELF.\n\nThe Open Source version of the Software (“LGPL Version”) is licensed\nunder the terms of the GNU Lesser General Public License versions 3.0\n(“LGPL”) and not under this EULA. \n\n4. Ownership. Notwithstanding anything to the contrary contained herein, except for the limited license rights expressly provided herein, Licensor and its suppliers have and will retain all rights, title and interest (including, without limitation, all patent, copyright, trademark, trade secret and other intellectual property rights) in and to the Software and all copies, modifications and derivative works thereof (including any changes which incorporate any of your ideas, feedback or suggestions). You acknowledge that you are obtaining only a limited license right to the Software, and that irrespective of any use of the words “purchase”, “sale” or like terms hereunder no ownership rights are being conveyed to you under this Agreement or otherwise. \n\n5. Fees and Payment. The Software license fees will be due and payable in full as set forth in the applicable invoice or at the time of purchase. If the Software does not function properly within two weeks of purchase, please contact us within those two weeks for a refund. You shall be responsible for all taxes, withholdings, duties and levies arising from the order (excluding taxes based on the net income of Licensor). \n\n6. Support, Maintenance and Services. Subject to the terms and conditions of this Agreement, as set forth in your invoice, and as set forth on the GoReplay Pro support page (https://github.com/buger/goreplay/wiki/Pro-Support), support and maintenance services may be included with the purchase of your license subscription.\n\n7. Term of Agreement. \n\n7.1 Term. This Agreement is effective as of the Delivery of the Software and expires at such time as all license and service subscriptions hereunder have expired in accordance with their own terms (the “Term”). For clarification, the term of your license under this Agreement may be perpetual, limited for Evaluation Version, or designated as a fixed-term license in the Invoice, and shall be specified at your time of purchase. Either party may terminate this Agreement (including all related Invoices) if the other party: (a) fails to cure any material breach of this Agreement within thirty (30) days after written notice of such breach, provided that Licensor may terminate this Agreement immediately upon any breach of Section 3 or if you exceed any other restrictions contained in Section 1, unless otherwise specified in this agreement; (b) ceases operation without a successor; or (c) seeks protection under any bankruptcy, receivership, trust deed, creditors arrangement, composition or comparable proceeding, or if any such proceeding is instituted against such party (and not dismissed within sixty (60) days)). Termination is not an exclusive remedy and the exercise by either party of any remedy under this Agreement will be without prejudice to any other remedies it may have under this Agreement, by law, or otherwise. \n\n7.2 Termination. Upon any termination of this Agreement, you shall cease any and all use of any Software and destroy all copies thereof. \n\n7.3 Expiration of License. Upon the expiration of any term under this Agreement, (a) all Software updates and services pursuant to the license shall cease, (b) you may only continue to run existing installations of the Software, (c) you may not install the Software on any additional Hosts, and (d) any new installation of the Software shall require the purchase of a new license subscription from Licensor.\n\n8. Disclaimer of Warranties. The Software is provided \"as is,\" with all faults, defects and errors, and without warranty of any kind. Licensor does not warrant that the Software will be free of bugs, errors, viruses or other defects, and Licensor shall have no liability of any kind for the use of or inability to use the Software, the Software content or any associated service, and you acknowledge that it is not technically practicable for Licensor to do so. \nTo the maximum extent permitted by applicable law, Licensor disclaims all warranties, express, implied, arising by law or otherwise, regarding the Software, the Software content and their respective performance or suitability for your intended use, including without limitation any implied warranty of merchantability, fitness for a particular purpose.\nNotwithstanding the foregoing, Licensor represents and warrants that it either owns the entire right to, title to, interest in, or has the right to license, the Software, and your proper use of the same will not violate any intellectual property or proprietary rights of another.\n\n9. Limitation of Liability. \n\nIn no event will either party be liable for any direct, indirect, consequential, incidental, special, exemplary, or punitive damages or liabilities whatsoever arising from or relating to the Software, the Software content or this Agreement, whether based on contract, tort (including negligence), strict liability or other theory, even if such party has been advised of the possibility of such damages. Except for indemnification obligations in Section 13.5, in no event will Licensor liability exceed the Software license price as indicated in the invoice. The existence of more than one claim will not enlarge or extend this limit.\n\n10. Remedies. Your exclusive remedy and Licensor’ entire liability for breach of this Agreement shall be limited, at Licensor’ sole and exclusive discretion, to (a) replacement of any defective software or documentation; or (b) refund of the license fee paid to Licensor, payable in accordance with Licensor' refund policy.\n\n11. Acknowledgements.\n\n11.1 Consent to the Use of Data. You agree that Licensor and its affiliates may collect and use technical information gathered as part of the product support services to be used in an anonymized manner. Licensor may use this information solely to improve products and services and will not disclose this information in a form that personally identifies you.\n\n11.2 Verification. We or a certified auditor acting on our behalf, may, upon its reasonable request and at its expense, audit you with respect to the use of the Software, subject to your reasonable information security and confidentiality protections. Such audit may be conducted by mail, electronic means or through an in-person visit to your place of business. Any such in-person audit shall be conducted during regular business hours at your facilities and shall not unreasonably interfere with your business activities. We shall not remove, copy, or redistribute any electronic material during the course of an audit. If an audit reveals that you are using the Software in a way that is in material violation of the terms of the EULA, then you shall pay our reasonable costs of conducting the audit. In the case of a material violation, you agree to pay Us any amounts owing that are attributable to the unauthorized use. In the alternative, We reserve the right, at our sole option, to terminate the licenses for the Software.\n\n11.3 Government End Users. If the Software and related documentation are supplied to or purchased by or on behalf of the United States Government, then the Software is deemed to be \"commercial software\" as that term is used in the Federal Acquisition Regulation system. Rights of the United States shall not exceed the minimum rights set forth in FAR 52.227-19 for \"restricted computer software\". All other terms and conditions of this Agreement apply.\n\n12. Third Party Software. Examples included in Software may provide links to third party libraries or code (collectively “Third Party Software”) to implement various functions. Third Party Software does not comprise part of the Software. In some cases, access to Third Party Software may be included along with the Software delivery as a convenience for demonstration purposes. Such source code and libraries may be included in the “…/examples” source tree delivered with the Software and do not comprise the Software. Licensee acknowledges (1) that some part of Third Party Software may require additional licensing of copyright and patents from the owners of such, and (2) that distribution of any of the Software referencing or including any portion of a Third Party Software may require appropriate licensing from such third parties.\n\n\n13. Miscellaneous\n\n13.1 Entire Agreement. This Agreement sets forth our entire agreement with respect to the Software and the subject matter hereof and supersedes all prior and contemporaneous understandings and agreements whether written or oral.\n\n13.2 Amendment. Licensor reserves the right, in its sole discretion, to amend this Agreement from time. Amendments to this Agreement can be located at: https://github.com/buger/goreplay/blob/master/COMM-LICENSE.\n\n13.3 Assignment. You may not assign this Agreement or any of its rights under this Agreement without the prior written consent of Licensor and any attempted assignment without such consent shall be void.\n\n13.4 Export Compliance. Each party agrees to comply with all applicable laws and regulations, including laws, regulations, orders or other restrictions on export, re-export or redistribution of software.\n\n13.5 Indemnification. Licensor agrees to defend, indemnify, and hold harmless You from and against any lawsuits, claims, losses, damages, fines and expenses (including attorneys' fees and costs) arising out of a claim brought by a third-party alleging that the Software directly infringes such third-party’s intellectual property rights; provided that Licensor shall have no indemnity obligation for claims arising out of your modification to the Software. You agree to defend, indemnify, and hold harmless Licensor from and against any lawsuits, claims, losses, damages, fines and expenses (including attorneys' fees and costs) brought by a third-party arising out of your modifications to the Software or breach of this Agreement.\n\n13.6 Governing Law. This Agreement is governed by the laws of the State of Oregon and the United States without regard to conflicts of laws provisions thereof, and without regard to the United Nations Convention on the International Sale of Goods or the Uniform Computer Information Transactions Act, as currently enacted by any jurisdiction or as may be codified or amended from time to time by any jurisdiction. The jurisdiction and venue for actions related to the subject matter hereof shall be the state of Oregon and United States federal courts located in Portland, Oregon, and both parties hereby submit to the personal jurisdiction of such courts. \n\n13.7 Attorneys’ Fees and Costs. The prevailing party in any action to enforce this Agreement will be entitled to recover its attorneys’ fees and costs in connection with such action. \n\n13.8 Severability. If any provision of this Agreement is held by a court of competent jurisdiction to be invalid, illegal, or unenforceable, the remainder of this Agreement will remain in full force and effect.\n\n13.9 Waiver. Failure or neglect by either party to enforce at any time any of the provisions of this licence Agreement shall not be construed or deemed to be a waiver of that party's rights under this Agreement.\n\n13.10 Headings. The headings of sections and paragraphs of this Agreement are for convenience of reference only and are not intended to restrict, affect or be of any weight in the interpretation or construction of the provisions of such sections or paragraphs.\n\n14. Contact Information. If you have any questions about this EULA, or if you want to contact Licensor for any reason, please direct correspondence to hello@goreplay.org\n" }, { "path": "Dockerfile", "content": "FROM alpine:3.16 as builder\n\nARG RELEASE_VERSION\n\nRUN apk add --no-cache ca-certificates openssl\nRUN wget https://github.com/buger/goreplay/releases/download/${RELEASE_VERSION}/gor_${RELEASE_VERSION}_x64.tar.gz -O gor.tar.gz\nRUN tar xzf gor.tar.gz\n\nFROM scratch\nCOPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/\nCOPY --from=builder /gor .\nENTRYPOINT [\"./gor\"]\n" }, { "path": "Dockerfile.dev", "content": "ARG BASE_IMAGE\nFROM ${BASE_IMAGE}\n\nRUN apk add --no-cache \\\n gcc \\\n g++ \\\n make \\\n linux-headers \\\n bison \\\n flex \\\n git \\\n wget\n\nRUN wget http://www.tcpdump.org/release/libpcap-1.10.0.tar.gz && tar xzf libpcap-1.10.0.tar.gz && cd libpcap-1.10.0 && ./configure && make install\n\nWORKDIR /go/src/github.com/buger/goreplay/\nADD . /go/src/github.com/buger/goreplay/\n\nRUN go get golang.org/x/lint/golint\nRUN go get\n" }, { "path": "ELASTICSEARCH.md", "content": "gor & elasticsearch\n===================\n\nPrerequisites\n-------------\n\n- elasticsearch\n- kibana (Get it here: http://www.elasticsearch.org/overview/kibana/)\n- gor\n\n\nelasticsearch\n-------------\n\nThe default elasticsearch configuration is just fine for most workloads. You won't need clustering, sharding or something like that.\n\nIn this example we're installing it on our gor replay server which gives us the elasticsearch listener on _http://localhost:9200_\n\n\nkibana\n------\n\nKibana (elasticsearch analytics web-ui) is just as simple. \nDownload it, extract it and serve it via a simple webserver.\n(Could be nginx or apache)\n\nYou could also use a shell, ```cd``` into the kibana directory and start a little quick and dirty python webserver with:\n\n```\npython -m SimpleHTTPServer 8000\n```\n\nIn this example we're also choosing the gor replay server as our kibana host. If you choose a different server you'll have to point kibana to your elasticsearch host.\n\n\ngor\n---\n\nStart your gor replay server with elasticsearch option:\n\n```\n./gor --input-raw :8000 --output-http http://staging.com --output-http-elasticsearch localhost:9200/gor\n```\n\n\n(You don't have to create the index upfront. That will be done for you automatically)\n\n\nNow visit your kibana url, load the predefined dashboard from the gist https://gist.github.com/gottwald/b2c875037f24719a9616 and watch the data rush in.\n\n\nTroubleshooting\n---------------\n\nThe replay process may complain about __too many open files__.\nThat's because your typical linux shell has a small open files soft limit at 1024.\nYou can easily raise that when you do this before starting your _gor replay_ process:\n\n```\nulimit -n 64000\n```\n\nPlease be aware, this is not a permanent setting. It's just valid for the following jobs you start from that shell.\n\nWe reached the 1024 limit in our tests with a ubuntu box replaying about 9000 requests per minute. (We had very slow responses there, should be way more with fast responses)\n" }, { "path": "LICENSE.txt", "content": "Copyright (c) 2011-present Leonid Bugaev\n\nPortions of this software are licensed as follows:\n\n* All content residing under the \"doc/\" directory of this repository is licensed under \"Creative Commons: CC BY-SA 4.0 license\".\n* The file \"pro.go\" and all files ending with the \"_pro.go\" suffix are released under the commercial license specified in the \"COMM-LICENSE\" file.\n* Content outside of the above mentioned directories or restrictions above is available under the \"LGPLv3\" license as defined below.\n\n\nGoReplay is an Open Source project licensed under the terms of\nthe LGPLv3 license. Please see \nfor license text.\n\nAs a special exception to the GNU Lesser General Public License version 3\n(\"LGPL3\"), the copyright holders of this Library give you permission to\nconvey to a third party a Combined Work that links statically or dynamically\nto this Library without providing any Minimal Corresponding Source or\nMinimal Application Code as set out in 4d or providing the installation\ninformation set out in section 4e, provided that you comply with the other\nprovisions of LGPL3 and provided that you meet, for the Application the\nterms and conditions of the license(s) which apply to the Application.\n\nTLDR: You are free to use Gor subpackages like `byteutils` or `proto` in your commercial projects.\n\n\nGoReplay Pro has a commercial-friendly license allowing private forks\nand modifications of GoReplay. Please see https://goreplay.org/pro.html for\nmore detail. You can find the commercial license terms in COMM-LICENSE.\n" }, { "path": "Makefile", "content": "SOURCE = $(shell ls -1 *.go | grep -v _test.go)\nPROJECT_NAME := goreplay\nSOURCE_PATH = /go/src/github.com/buger/goreplay/\nPORT = 8000\nFADDR = :8000\nDIST_PATH = dist\nCONTAINER_AMD=gor-amd64\nCONTAINER_ARM=gor-arm64\nRUN = docker run --rm -v `pwd`:$(SOURCE_PATH) -e AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) -e AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) -p 0.0.0.0:$(PORT):$(PORT) -t -i $(CONTAINER_AMD)\nBENCHMARK = BenchmarkRAWInput\nTEST = TestRawListenerBench\nBIN_NAME = gor\nVERSION := DEV-$(shell date +%s)\nCUSTOM_TAGS := --tags \"ngo$(if $(CUSTOM_BUILD_TAGS), $(CUSTOM_BUILD_TAGS),)\"\nLDFLAGS = -ldflags \"-X main.VERSION=$(VERSION) -extldflags \\\"-static\\\" -X main.DEMO=$(DEMO)\"\nMAC_LDFLAGS = -ldflags \"-X main.VERSION=$(VERSION) -X main.DEMO=$(DEMO)\"\nDOCKER_FPM_CMD := docker run --rm -t -v `pwd`:/src -w /src fleetdm/fpm\n\nFPM_COMMON= \\\n\t\t\t\t--name $(PROJECT_NAME) \\\n\t\t\t\t--description \"GoReplay is an open-source network monitoring tool which can record your live traffic, and use it for shadowing, load testing, monitoring and detailed analysis.\" \\\n\t\t\t\t-v $(VERSION) \\\n\t\t\t\t--vendor \"Leonid Bugaev\" \\\n\t\t\t\t-m \"\" \\\n\t\t\t\t--url \"https://goreplay.org\" \\\n\t\t\t\t-s dir\n\nrelease: clean release-linux-amd64 release-linux-arm64 release-mac-amd64 release-mac-arm64 release-windows\n\n.PHONY: vendor\nvendor:\n\tgo mod vendor\n\nrelease-bin-linux-amd64: vendor\n\tdocker run --platform linux/amd64 --rm -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 -i $(CONTAINER_AMD) go build -mod=vendor -o $(BIN_NAME) $(CUSTOM_TAGS) $(LDFLAGS) ./cmd/gor/\n\nrelease-bin-linux-arm64: vendor\n\tdocker run --platform linux/arm64 --rm -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=arm64 -i $(CONTAINER_ARM) go build -mod=vendor -o $(BIN_NAME) $(CUSTOM_TAGS) $(LDFLAGS) ./cmd/gor/\n\nrelease-bin-mac-amd64: vendor\n\tGOOS=darwin go build -mod=vendor -o $(BIN_NAME) $(CUSTOM_TAGS) $(MAC_LDFLAGS) ./cmd/gor/\n\nrelease-bin-mac-arm64: vendor\n\tGOOS=darwin GOARCH=arm64 CGO_ENABLED=1 go build -mod=vendor -o $(BIN_NAME) $(CUSTOM_TAGS) $(MAC_LDFLAGS)\n\nrelease-bin-windows: vendor\n\tdocker run -it --rm -v `pwd`:$(SOURCE_PATH) -w $(SOURCE_PATH) -e CGO_ENABLED=1 docker.elastic.co/beats-dev/golang-crossbuild:1.19.2-main --build-cmd \"make VERSION=$(VERSION) CUSTOM_BUILD_TAGS=$(CUSTOM_BUILD_TAGS) build\" -p \"windows/amd64\" ./cmd/gor/\n\tmv $(BIN_NAME) \"$(BIN_NAME).exe\"\n\nrelease-linux-amd64: dist release-bin-linux-amd64\n\ttar -czf $(DIST_PATH)/gor_$(VERSION)_linux_amd64.tar.gz $(BIN_NAME)\n\t$(DOCKER_FPM_CMD) $(FPM_COMMON) -f -t deb -a arm64 -p ./$(DIST_PATH) ./gor=/usr/local/bin\n\t$(DOCKER_FPM_CMD) $(FPM_COMMON) -f -t rpm -a arm64 -p ./$(DIST_PATH) ./gor=/usr/local/bin\n\trm -rf $(BIN_NAME)\n\nrelease-linux-arm64: dist release-bin-linux-arm64\n\ttar -czf $(DIST_PATH)/gor_$(VERSION)_linux_arm64.tar.gz $(BIN_NAME)\n\t$(DOCKER_FPM_CMD) $(FPM_COMMON) -f -t deb -a arm64 -p ./$(DIST_PATH) ./gor=/usr/local/bin\n\t$(DOCKER_FPM_CMD) $(FPM_COMMON) -f -t rpm -a arm64 -p ./$(DIST_PATH) ./gor=/usr/local/bin\n\trm -rf $(BIN_NAME)\n\nrelease-mac-amd64: dist release-bin-mac-amd64\n\ttar -czf $(DIST_PATH)/gor_$(VERSION)_darwin_amd64.tar.gz $(BIN_NAME)\n\tfpm $(FPM_COMMON) -f -t osxpkg -a amd64 -p ./$(DIST_PATH) ./gor=/usr/local/bin\n\tmv ./$(DIST_PATH)/$(PROJECT_NAME)-$(VERSION).pkg ./$(DIST_PATH)/$(PROJECT_NAME)-$(VERSION)-amd64.pkg\n\trm -rf $(BIN_NAME)\n\nrelease-mac-arm64: dist release-bin-mac-arm64\n\ttar -czf $(DIST_PATH)/gor_$(VERSION)_darwin_arm64.tar.gz $(BIN_NAME)\n\tfpm $(FPM_COMMON) -f -t osxpkg -a arm64 -p ./$(DIST_PATH) ./gor=/usr/local/bin\n\tmv ./$(DIST_PATH)/$(PROJECT_NAME)-$(VERSION).pkg ./$(DIST_PATH)/$(PROJECT_NAME)-$(VERSION)-arm64.pkg\n\trm -rf $(BIN_NAME)\n\nrelease-windows: dist release-bin-windows\n\tzip $(DIST_PATH)/gor-$(VERSION)_windows.zip \"$(BIN_NAME).exe\"\n\trm -rf \"$(BIN_NAME).exe\"\n\nclean:\n\trm -rf $(DIST_PATH)\n\nbuild:\n\tgo build -mod=vendor -o $(BIN_NAME) $(CUSTOM_TAGS) $(LDFLAGS)\n\ninstall:\n\tgo install $(CUSTOM_TAGS) $(MAC_LDFLAGS)\n\nbuild-env: build-amd64-env build-arm64-env\n\nbuild-amd64-env:\n\tdocker buildx build --build-arg BASE_IMAGE=golang:1.22-alpine --platform linux/amd64 -t $(CONTAINER_AMD) -f Dockerfile.dev --load .\n\nbuild-arm64-env:\n\tdocker buildx build --build-arg BASE_IMAGE=arm64v8/golang:1.22-alpine --platform linux/arm64 -t $(CONTAINER_ARM) -f Dockerfile.dev --load .\n\nbuild-docker:\n\tdocker build -t gor-dev -f Dockerfile .\n\nprofile:\n\tgo build && ./$(BIN_NAME) --output-http=\"http://localhost:9000\" --input-dummy 0 --input-raw :9000 --input-http :9000 --memprofile=./mem.out --cpuprofile=./cpu.out --stats --output-http-stats --output-http-timeout 100ms\n\nlint:\n\t$(RUN) golint $(PKG)\n\nrace:\n\t$(RUN) go test ./... $(ARGS) -v -race -timeout 15s\n\ntest:\n\t$(RUN) go test ./. -timeout 120s $(CUSTOM_TAGS) $(LDFLAGS) $(ARGS) -v\n\ntest_all:\n\t$(RUN) go test ./... -timeout 120s $(CUSTOM_TAGS) $(LDFLAGS) $(ARGS) -v\n\ntestone:\n\t$(RUN) go test ./. -timeout 60s $(CUSTOM_TAGS) $(LDFLAGS) -run $(TEST) $(ARGS) -v\n\ncover:\n\t$(RUN) go test $(ARGS) -race -v -timeout 15s -coverprofile=coverage.out\n\tgo tool cover -html=coverage.out\n\nfmt:\n\t$(RUN) gofmt -w -s ./..\n\nvet:\n\t$(RUN) go vet\n\nbench:\n\t$(RUN) go test $(CUSTOM_TAGS) $(LDFLAGS) -v -run NOT_EXISTING -bench $(BENCHMARK) -benchtime 5s\n\nprofile_test:\n\t$(RUN) go test $(CUSTOM_TAGS) $(LDFLAGS) -run $(TEST) ./capture/. $(ARGS) -memprofile mem.mprof -cpuprofile cpu.out\n\t$(RUN) go test $(CUSTOM_TAGS) $(LDFLAGS) -run $(TEST) ./capture/. $(ARGS) -c\n\n# Used mainly for debugging, because docker container do not have access to parent machine ports\nrun:\n\t$(RUN) go run $(CUSTOM_TAGS) $(LDFLAGS) $(SOURCE) --input-dummy=0 --output-http=\"http://localhost:9000\" --input-raw-track-response --input-raw 127.0.0.1:9000 --verbose 0 --middleware \"./examples/middleware/echo.sh\" --output-file requests.gor\n\nrun-2:\n\t$(RUN) go run $(CUSTOM_TAGS) $(LDFLAGS) $(SOURCE) --input-raw :8000 --input-raw-bpf-filter \"dst port 8000\" --output-stdout --output-http \"http://localhost:8000\" --input-dummy=0\n\nrun-3:\n\tsudo -E go run $(CUSTOM_TAGS) $(SOURCE) --input-tcp :27001 --output-stdout\n\nrun-arg:\n\tsudo -E go run $(CUSTOM_TAGS) $(SOURCE) $(ARGS)\n\nfile-server:\n\tgo run $(CUSTOM_TAGS) $(SOURCE) file-server $(FADDR)\n\nreadpcap:\n\tgo run $(CUSTOM_TAGS) $(SOURCE) --input-raw $(FILE) --input-raw-track-response --input-raw-engine pcap_file --output-stdout\n\nrecord:\n\t$(RUN) go run $(CUSTOM_TAGS) $(SOURCE) --input-dummy=0 --output-file=requests.gor --verbose --debug\n\nreplay:\n\t$(RUN) go run $(CUSTOM_TAGS) $(SOURCE) --input-file=requests.bin --output-tcp=:9000 --verbose -h\n\nbash:\n\t$(RUN) /bin/bash\n\ndist:\n\tmkdir -p $(DIST_PATH)\n" }, { "path": "Procfile", "content": "web: python -m SimpleHTTPServer 8000\nreplayed_web: python -m SimpleHTTPServer 8001\nlistener: sudo -E go run ./bin/gor.go --input-raw :8000 --output-tcp :8002 --verbose\nreplay: go run ./bin/gor.go --input-tcp :8002 --output-http localhost:8001 --verbose\n" }, { "path": "README.md", "content": " [![GitHub release](https://img.shields.io/github/release/buger/gor.svg?maxAge=3600)](https://github.com/buger/goreplay/releases) [![codebeat](https://codebeat.co/badges/6427d589-a78e-416c-a546-d299b4089893)](https://codebeat.co/projects/github-com-buger-gor) [![Go Report Card](https://goreportcard.com/badge/github.com/buger/gor)](https://goreportcard.com/report/github.com/buger/gor) [![Join the chat at https://gitter.im/buger/gor](https://badges.gitter.im/buger/gor.svg)](https://gitter.im/buger/gor?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Reviewed by Hound](https://img.shields.io/badge/Reviewed_by-Hound-8E64B0.svg)](https://houndci.com)\n\n![Go Replay](http://i.imgur.com/ZG2ki5n.png)\n\n## https://goreplay.org/\n\nGoReplay is an open-source network monitoring tool which can record your live traffic and use it for shadowing, load testing, monitoring and detailed analysis.\n\n## About\n\nAs your application grows, the effort required to test it also grows exponentially. GoReplay offers you the simple idea of reusing your existing traffic for testing, which makes it incredibly powerful. Our state of art technique allows you to analyze and record your application traffic without affecting it. This eliminates the risks that come with putting a third party component in the critical path. \n\nGoReplay increases your confidence in code deployments, configuration and infrastructure changes.\n\n\nGoReplay offers a unique approach for shadowing. Instead of being a proxy, GoReplay listens in the background for traffic on your network interfaces, requiring no changes in your production infrastructure, other than running GoReplay daemon on the same machine as your service.\n\n![Diagram](https://i.imgur.com/IN2xfDm.png)\n\nCheck [latest documentation](http://github.com/buger/goreplay/wiki).\n\n## Installation\nDownload the latest binary from https://github.com/buger/goreplay/releases or [compile by yourself](https://github.com/buger/goreplay/wiki/Compilation).\n\n## Getting started\n\nThe most basic setup will be `sudo ./gor --input-raw :8000 --output-stdout` which acts like tcpdump.\nIf you already have a test environment, you can start replaying by running: `sudo ./gor --input-raw :8000 --output-http http://staging.env`.\n\nSee our [documentation](https://github.com/buger/goreplay/wiki/) and the [Getting Started](https://github.com/buger/goreplay/wiki/Getting-Started) page for more info. \n\n## Newsletter\nSubscribe to our [newsletter](https://www.getdrip.com/forms/89690474/submissions/new) to stay informed about the latest features and changes to the Gor project.\n\n## Want to Upgrade?\n\nWe have created a [GoReplay PRO](https://goreplay.org/pro.html) extension which provides additional features such as support for binary protocols like Thrift or ProtocolBuffers, saving and replaying from cloud storage, TCP session replication, etc. The PRO version also includes a commercial-friendly license, dedicated support, and it also allows you to support high-quality open source development. \n\n\n## Problems?\nIf you have a problem, please review the [FAQ](https://github.com/buger/goreplay/wiki/FAQ) and [Troubleshooting](https://github.com/buger/goreplay/wiki/Troubleshooting) wiki pages. Searching the [issues](https://github.com/buger/goreplay/issues) for your problem is also a good idea.\n\nAll bug-reports and suggestions should go through Github Issues or our [Google Group](https://groups.google.com/forum/#!forum/gor-users) (you can just send email to gor-users@googlegroups.com).\nIf you have a private question feel free to send email to support@gortool.com.\n\n\n## Contributing\n\n1. Fork it\n2. Create your feature branch (git checkout -b my-new-feature)\n3. Commit your changes (git commit -am 'Added some feature')\n4. Push to the branch (git push origin my-new-feature)\n5. Create new Pull Request\n\n## Companies using Gor\n\n* [GOV.UK](https://www.gov.uk) - UK Government Digital Service\n* [theguardian.com](http://theguardian.com) - Most popular online newspaper in the UK\n* [TomTom](http://www.tomtom.com/) - Global leader in navigation, traffic and map products, GPS Sport Watches and fleet management solutions.\n* [3SCALE](http://www.3scale.net/) - API infrastructure to manage your APIs for internal or external users\n* [Optionlab](http://www.opinionlab.com) - Optimize customer experience and drive engagement across multiple channels\n* [TubeMogul](http://tubemogul.com) - Software for Brand Advertising\n* [Videology](http://www.videologygroup.com/) - Video advertising platform\n* [ForeksMobile](http://foreksmobile.com/) - One of the leading financial application development company in Turkey\n* [Granify](http://granify.com) - AI backed SaaS solution that enables online retailers to maximise their sales\n* And many more!\n\nIf you are using Gor, we are happy to add you to the list and share your story, just write to: hello@goreplay.org\n\n## Author\n\nLeonid Bugaev, [@buger](https://twitter.com/buger), https://leonsbox.com\n" }, { "path": "ce.go", "content": "//go:build !pro\n\npackage goreplay\n\nimport (\n\t\"fmt\"\n)\n\n// PRO this value indicates if goreplay is running in PRO mode.\nvar PRO = false\n\nfunc SettingsHook(settings *AppSettings) {\n\tif settings.RecognizeTCPSessions {\n\t\tsettings.RecognizeTCPSessions = false\n\t\tfmt.Println(\"[ERROR] TCP session recognition is not supported in the open-source version of GoReplay\")\n\t}\n}\n" }, { "path": "circle.yml", "content": "dependencies:\n pre:\n - sudo apt-get install libpcap-dev -y\n\ntest:\n override:\n - sudo bash -l -c \"export GOPATH='/home/ubuntu/.go_workspace:/usr/local/go_workspace:/home/ubuntu/.go_project' && GORACE='halt_on_error=1' /usr/local/go/bin/go test ./... -v -timeout 120s -race\"" }, { "path": "docs/CNAME", "content": "docs.goreplay.org" }, { "path": "docs/Capturing-and-replaying-traffic.md", "content": "Think about Gor more like a network analyzer or tcpdump on steroids, it is not a proxy and does not affect your app anyhow. You specify application port, and it will capture and replay incoming data.\n\nSimplest setup will be:\n```bash\n# Run on servers where you want to catch traffic. You can run it on every `web` machine.\nsudo gor --input-raw :80 --output-http http://staging.com\n```\nIt will record and replay traffic from the same machine. However, it is possible to use [[Aggregator-forwarder setup]], when Gor on your web machines forward traffic to Gor aggregator instance running on the separate server.\n\n> You may notice that it require `sudo`: to analyze network Gor need permissions which available only to root users. However, it is possible to configure Gor [beign run for non-root users](Running as a non-root user).\n\n\n### Forwarding to multiple addresses\n\nYou can forward traffic to multiple endpoints.\n```\ngor --input-tcp :28020 --output-http \"http://staging.com\" --output-http \"http://dev.com\"\n```\n\n### Splitting traffic\nBy default, it will send same traffic to all outputs, but you have options to equally split it (round-robin) using `--split-output` option.\n\n```\ngor --input-raw :80 --output-http \"http://staging.com\" --output-http \"http://dev.com\" --split-output true\n```\n\n### Tracking responses\nBy default `input-raw` does not intercept responses, only requests. You can turn response tracking using `--input-raw-track-response` option. When enable you will be able to access response information in middleware and `output-file`.\n\n\n### Traffic interception engine\nBy default, Gor will use `libpcap` for intercepting traffic, it should work in most cases. If you have any troubles with it, you may try alternative engine: `raw_socket`.\n\n```\nsudo gor --input-raw :80 --input-raw-engine \"raw_socket\" --output-http \"http://staging.com\"\n```\n\nYou can read more about [[Replaying HTTP traffic]].\n\nYou can use VXLAN or traffic mirroring from AWS to capture the traffic. The 4789 UDP port will be opened and that works as you are launched GoReplay on the source machine.\n\n```\ngor --input-raw :80 --input-raw-engine vxlan -output-stdout\n```\n\n### Tracking original IP addresses\nYou can use `--input-raw-realip-header` option to specify header name: If not blank, injects header with given name and real IP value to the request payload. Usually, this header should be named: `X-Real-IP`, but you can specify any name.\n\n`gor --input-raw :80 --input-raw-realip-header \"X-Real-IP\" ...`\n\n\n***\n\nAlso you may want to know about [[Rate limiting]], [[Request rewriting]] and [[Request filtering]]" }, { "path": "docs/Compilation.md", "content": "We provide pre-compiled binaries for Mac and Linux, but you are free to compile Gor by yourself.\n\nGor is written using Go, so first you need to download it from here https://golang.org/, use the latest stable version. \n\nThe only Gor dependency is [libpcap](https://github.com/the-tcpdump-group/libpcap), which is the interface to various kernel packet capture mechanisms, and https://github.com/google/gopacket, which is a Go wrapper around libpcap. Latest libpcap version can be obtained at http://www.tcpdump.org/release/. Libpcap itself depend on `flex` and `bison` packages, many operating systems already have them installed.\n\n```bash\n# Fetch libpcap dependencies. Depending on your OS, instead of `apt` you will use `yum` or `rpm`, or `brew` on Mac.\nsudo apt-get install flex bison -y\n\n# Download latest stable release, compile and install it\nwget http://www.tcpdump.org/release/libpcap-1.7.4.tar.gz && tar xzf libpcap-1.7.4.tar.gz\ncd libpcap-1.7.4\n./configure && make install\n\n\n# Lets fetch Gor source code\nmkdir $HOME/gocode\n# See more information about GOPATH https://github.com/golang/go/wiki/GOPATH\nexport GOPATH=$HOME/gocode\n# Fetch code from the Github\ngo get github.com/buger/gor\n\n# Compile from source\ncd $HOME/gocode/src/github.com/buger/gor\ngo build LDFLAGS = -ldflags \"-extldflags \\\"-static\\\"\"\n```\n\nAfter you finished, you should see `gor` binary in current directory. \n\n" }, { "path": "docs/Development-Setup.md", "content": "## STEP 1: Install Docker\nFor local development we recommend to use Docker.\n\nIf you don’t have it you can read how to install it here:\nhttps://docs.docker.com/engine/getstarted/step_one/#step-3-verify-your-installation\n\n## STEP 2: Download repository\n\n`git clone git@github.com:buger/goreplay.git`\n\n\n## STEP 3: Setup container\n\n```\ncd ./goreplay\nmake build\n\n```\n\n## Testing\nTo run tests execute next command:\n\n```\nmake test\n```\n\nYou can copy the command that is produced and modify it. For example, if you need to run one test copy the command and add `-run TestName`, e.g.:\n\n```\ndocker run -v `pwd`:/go/src/github.com/buger/gor/ -p 0.0.0.0:8000:8000 -t -i gor:go go test ./. -run TestEmitterFiltered -timeout 60s -ldflags \"-X main.VERSION=DEV-1482398347 -extldflags \\\"-static\\\"\" -v\n```\n\n\n## Building\nTo get a binary file run \n\n```\nmake release-bin\n```\n" }, { "path": "docs/Distributed-configuration.md", "content": "Sometimes it makes sense to use separate Gor instance for replaying traffic and performing things like load testing, so your production machines do not spend precious resources. It is possible to configure Gor on your web machines forward traffic to Gor aggregator instance running on the separate server.\n\n```bash\n# Run on servers where you want to catch traffic. You can run it on each `web` machine.\nsudo gor --input-raw :80 --output-tcp replay.local:28020\n\n# Replay server (replay.local).\ngor --input-tcp replay.local:28020 --output-http http://staging.com\n```\n\nIf you have multiple replay machines you can split traffic among them using `--split-output` option: it will equally split all incoming traffic to all outputs using round robin algorithm.\n```\ngor --input-raw :80 --split-output --output-tcp replay1.local:28020 --output-tcp replay2.local:28020\n```\n\n[GoReplay PRO](https://goreplay.org/pro.html) support accurate recording and replaying of tcp sessions, and when `--recognize-tcp-sessions` option is passed, instead of round-robin it will use a smarter algorithm which ensures that same sessions will be sent to the same replay instance.\n\n\nIn case if you are planning a large load testing, you may consider use separate master instance which will control Gor slaves which actually replay traffic. For example:\n```\n# This command will read multiple log files, replay them on 10x speed and loop them if needed for 30 seconds, and will distributed traffic (tcp session aware) among multiple workers\ngor --input-file logs_from_multiple_machines.*|1000% --input-file-loop --exit-after 30s --recognize-tcp-sessions --split-output --output-tcp worker1.local --output-tcp worker2.local:27017 --output-tcp worker3.local:27017 ... --output-tcp workerN.local:27017\n\n# worker \ngor --input-tcp :27017 --ouput-http load_test.target\n```\n" }, { "path": "docs/Exporting-to-ElasticSearch.md", "content": "Gor can export requests and replayed response data to ElasticSearch:\n\n```\n./gor --input-raw :8000 --output-http http://staging.com --output-http-elasticsearch localhost:9200/gor\n```\n\nYou don't have to create the index upfront. That will be done for you automatically.\n\n### Format\n\nFollowing structure represents ES format:\n\n```\ntype ESRequestResponse struct {\n\tReqURL string `json:\"Req_URL\"`\n\tReqMethod string `json:\"Req_Method\"`\n\tReqUserAgent string `json:\"Req_User-Agent\"`\n\tReqAcceptLanguage string `json:\"Req_Accept-Language,omitempty\"`\n\tReqAccept string `json:\"Req_Accept,omitempty\"`\n\tReqAcceptEncoding string `json:\"Req_Accept-Encoding,omitempty\"`\n\tReqIfModifiedSince string `json:\"Req_If-Modified-Since,omitempty\"`\n\tReqConnection string `json:\"Req_Connection,omitempty\"`\n\tReqCookies string `json:\"Req_Cookies,omitempty\"`\n\tRespStatus string `json:\"Resp_Status\"`\n\tRespStatusCode string `json:\"Resp_Status-Code\"`\n\tRespProto string `json:\"Resp_Proto,omitempty\"`\n\tRespContentLength string `json:\"Resp_Content-Length,omitempty\"`\n\tRespContentType string `json:\"Resp_Content-Type,omitempty\"`\n\tRespTransferEncoding string `json:\"Resp_Transfer-Encoding,omitempty\"`\n\tRespContentEncoding string `json:\"Resp_Content-Encoding,omitempty\"`\n\tRespExpires string `json:\"Resp_Expires,omitempty\"`\n\tRespCacheControl string `json:\"Resp_Cache-Control,omitempty\"`\n\tRespVary string `json:\"Resp_Vary,omitempty\"`\n\tRespSetCookie string `json:\"Resp_Set-Cookie,omitempty\"`\n\tRtt int64 `json:\"RTT\"`\n\tTimestamp time.Time\n}\n```" }, { "path": "docs/FAQ.md", "content": "### What OS are supported?\nGor will run everywhere where [libpcap](http://www.tcpdump.org/) works, and it works on most of the platforms. However, currently, we test it on Linux and Mac. See more about [[Compilation]].\n\n### Why does the `--input-raw` requires sudo or root access?\nListener works by sniffing traffic from a given port. It's accessible\nonly by using sudo or root access. But it is possible to [[Running as non root user]].\n\n### How do you deal with user session to replay the traffic correctly?\nYou can rewrite session related headers/params to match your staging environment. If you require custom logic (e.g random token based auth) follow this discussion: https://github.com/buger/gor/issues/154\n\n### Can I use Gor to intercept SSL traffic?\nBasic idea is that SSL was made to protect itself from traffic interception. There 2 options: \n1. Move SSL handling to proxy like Nginx or Amazon ELB. And allow Gor to listen on upstreams. \n2. Use `--input-http` so you can duplicate request payload directly from your app to Gor, but it will require your app modifications.\n\nMore can be find here: https://github.com/buger/gor/issues/85\n\n### Is there a limit for size of HTTP request when using output-http?\nDue to the fact that Gor can't guarantee interception of all packets, for large payloads > 200kb there is chance of missing some packets and corrupting body. Treat it as a feature and chance to test broken bodies handling :)\nThe only way to guarantee delivery is using `--input-http`, but you will miss some features.\n\n### I'm getting 'too many open files' error\nTypical Linux shell has a small open files soft limit at 1024. You can easily raise that when you do this before starting your gor replay process:\n \n ulimit -n 64000\n\nMore about ulimit: http://www.thecodingmachine.com/solving-the-too-many-open-files-exception-in-red5-or-any-other-application/\n\n### The CPU average across my load-balanced targets is higher than the source\nIf you are replaying traffic from multiple listeners to a load-balanced target and you use sticky sessions, you may observe that the target servers have a higher CPU load than the listener servers. This may be because the sticky session cookie of the original load balancer is not honored by the target load balancer thus resulting in requests that would normally hit the same target server hitting different servers on the backend thus reducing some caching benefits gained via the load balancing. Try running just one listener against one replay target and see if the CPU utilization comparison is more accurate.\n\nAlso see [[Troubleshooting]]." }, { "path": "docs/Middleware.md", "content": "#### Overview\nMiddleware is a program that accepts request and response payload at STDIN and emits modified requests at STDOUT. You can implement any custom logic like stripping private data, advanced rewriting, support for oAuth and etc. Check examples [included into our repo](https://github.com/buger/gor/tree/master/examples/middleware).\n\n\n```\n Original request +--------------+\n+-------------+----------STDIN---------->+ |\n| Gor input | | Middleware |\n+-------------+----------STDIN---------->+ |\n Original response (1) +------+---+---+\n | ^\n+-------------+ Modified request v |\n| Gor output +<---------STDOUT-----------------+ |\n+-----+-------+ |\n | |\n | Replayed response |\n +------------------STDIN----------------->----+\n```\n\n(1): Original responses will only be sent to the middleware if the `--input-raw-track-response` option is specified.\n\nMiddleware can be written in any language, see `examples/middleware` folder for examples.\nMiddleware program should accept the fact that all communication with Gor is asynchronous, there is no guarantee that original request and response messages will come one after each other. Your app should take care of the state if logic depends on original or replayed response, see `examples/middleware/token_modifier.go` as example.\n\nSimple bash echo middleware (returns same request) will look like this:\n```bash\nwhile read line; do\n echo $line\ndone\n```\n\nMiddleware can be enabled using `--middleware` option, by specifying path to executable file:\n```\ngor --input-raw :80 --middleware \"/opt/middleware_executable\" --output-http \"http://staging.server\"\n```\n\n#### Communication protocol\nAll messages should be hex encoded, new line character specifieds the end of the message, eg. new message per line.\n\nDecoded payload consist of 2 parts: header and HTTP payload, separated by new line character. \n\nExample request payload:\n\n```\n1 932079936fa4306fc308d67588178d17d823647c 1439818823587396305\nGET /a HTTP/1.1\nHost: 127.0.0.1\n\n```\n\nExample response payload (note: you will only receive this if you specify `--input-raw-track-response`)\n\n```\n2 8e091765ae902fef8a2b7d9dd960e9d52222bd8c 1439818823587996305 2782013\nHTTP/1.1 200 OK\nDate: Mon, 17 Aug 2015 13:40:23 GMT\nContent-Length: 0\nContent-Type: text/plain; charset=utf-8\n\n```\n\nHeader contains request meta information separated by spaces. First value is payload type, possible values: `1` - request, `2` - original response, `3` - replayed response.\nNext goes request id: unique among all requests (sha1 of time and Ack), but remain same for original and replayed response, so you can create associations between request and responses. The third argument is the time when request/response was initiated/received. Forth argument is populated only for responses and means latency.\n\nHTTP payload is unmodified HTTP requests/responses intercepted from network. You can read more about request format [here](http://www.jmarshall.com/easy/http/), [here](https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol) and [here](http://www.w3.org/Protocols/rfc2616/rfc2616.html). You can operate with payload as you want, add headers, change path, and etc. Basically you just editing a string, just ensure that it is RCF compliant.\n\nAt the end modified (or untouched) request should be emitted back to STDOUT, keeping original header, and hex-encoded. If you want to filter request, just not send it. Emitting responses back is required, even if you did not touch them.\n\n#### Advanced example\nImagine that you have auth system that randomly generate access tokens, which used later for accessing secure content. Since there is no pre-defined token value, naive approach without middleware (or if middleware use only request payloads) will fail, because replayed server have own tokens, not synced with origin. To fix this, our middleware should take in account responses of replayed and origin server, store `originalToken -> replayedToken` aliases and rewrite all requests using this token to use replayed alias. See [examples/middleware/token_modifier.go](https://github.com/buger/gor/tree/master/examples/middleware/token_modifier.go) and [middleware_test.go#TestTokenMiddleware](https://github.com/buger/gor/tree/master/middleware_test.go) as example of described scheme.\n\n***\n\nYou may also read about [[Request filtering]], [[Rate limiting]] and [[Request rewriting]].\n" }, { "path": "docs/Rate-limiting.md", "content": "Rate limiting can be useful if you only want to forward parts of incoming traffic, for example, to not overload your test environment. There are two strategies: dropping random requests or dropping fractions of requests based on Header or URL param value. \n\n### Dropping random requests\nEvery input and output support random rate limiting.\nThere are two limiting algorithms: absolute or percentage based. \n\n**Absolute**: If for current second it reached specified requests limit - disregard the rest, on next second counter reset.\n\n**Percentage**: For input-file it will slowdown or speedup request execution, for the rest it will use the random generator to decide if request pass or not based on the chance you specified. \n\nYou can specify your desired limit using the \"|\" operator after the server address, see examples below.\n\n#### Limiting replay using absolute number\n```\n# staging.server will not get more than ten requests per second\ngor --input-tcp :28020 --output-http \"http://staging.com|10\"\n```\n\n#### Limiting listener using percentage based limiter\n```\n# replay server will not get more than 10% of requests \n# useful for high-load environments\ngor --input-raw :80 --output-tcp \"replay.local:28020|10%\"\n```\n\n### Consistent limiting based on Header or URL param value\nIf you have unique user id (like API key) stored in header or URL you can consistently forward specified percent of traffic only for the fraction of this users. \nBasic formula looks like this: `FNV32-1A_hashing(value) % 100 >= chance`. Examples:\n```\n# Limit based on header value\ngor --input-raw :80 --output-tcp \"replay.local:28020|10%\" --http-header-limiter \"X-API-KEY: 10%\"\n\n# Limit based on header value\ngor --input-raw :80 --output-tcp \"replay.local:28020|10%\" --http-param-limiter \"api_key: 10%\"\n```\n\nWhen limiting based on header or param only percentage based limiting supported." }, { "path": "docs/Replaying-HTTP-traffic.md", "content": "Gor can replay HTTP traffic using `--output-http` option:\n\n```bash\nsudo ./gor --input-raw :8000 --output-http=\"http://staging.env\"\n```\n\nYou can [filter](Request filtering), [rate limit](Rate limiting) and [rewrite](Request rewriting) requests on the fly. \n\n### HTTP output workers\nBy default Gor creates a dynamic pool of workers: it starts with 10 and creates more HTTP output workers when the HTTP output queue length is greater than 10. The number of workers created (N) is equal to the queue length at the time which it is checked and found to have a length greater than 10. The queue length is checked every time a message is written to the HTTP output queue. No more workers will be spawned until that request to spawn N workers is satisfied. If a dynamic worker cannot process a message at that time, it will sleep for 100 milliseconds. If a dynamic worker cannot process a message for 2 seconds it dies.\nYou may specify fixed number of workers using `--output-http-workers=20` option.\n\n### Following redirects\nBy default Gor will ignore all redirects since they are handled by clients using your app, but in scenarios where your replayed environment introduces new redirects, you can enable them like this: \n```\ngor --input-tcp replay.local:28020 --output-http http://staging.com --output-http-redirects 2\n```\nThe given example will follow up to 2 redirects per request.\n\n### HTTP timeouts\nBy default http timeout for both request and response is 5 seconds. You can override it like this:\n```\ngor --input-tcp replay.local:28020 --output-http http://staging.com --output-http-timeout 30s\n```\n\n### Response buffer\nBy default, to reduce memory consumption, internal HTTP client will fetch max 200kb of the response body (used if you use middleware), by you can increase limit using `--output-http-response-buffer` option (accepts number of bytes).\n\n### Basic Auth\n\nIf your development or staging environment is protected by Basic Authentication then those credentials can be injected in during the replay:\n\n```\ngor --input-raw :80 --output-http \"http://user:pass@staging.com\"\n```\n\nNote: This will overwrite any Authorization headers in the original request.\n\n\n### Multiple domains support\n\nIf you app accepts traffic from multiple domains, and you want to keep original headers, there is specific `--http-original-host` with tells Gor do not touch Host header at all.\n\n\n***\nYou may also read about [[Saving and Replaying from file]]" }, { "path": "docs/Request-filtering.md", "content": "Filtering is useful when you need to capture only specific part of traffic, like API requests. It is possible to filter by URL, HTTP header or HTTP method.\n\n#### Allow url regexp\n```\n# only forward requests being sent to the /api endpoint\ngor --input-raw :8080 --output-http staging.com --http-allow-url /api\n```\n\n#### Disallow url regexp\n```\n# only forward requests NOT being sent to the /api... endpoint\ngor --input-raw :8080 --output-http staging.com --http-disallow-url /api\n```\n#### Filter based on regexp of header\n\n```\n# only forward requests with an api version of 1.0x\ngor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^1\\.0\\d\n\n# only forward requests NOT containing User-Agent header value \"Replayed by Gor\"\ngor --input-raw :8080 --output-http staging.com --http-disallow-header \"User-Agent: Replayed by Gor\"\n```\n\n#### Filter based on HTTP method\nRequests not matching a specified whitelist can be filtered out. For example to strip non-nullipotent requests:\n\n```\ngor --input-raw :80 --output-http \"http://staging.server\" \\\n --http-allow-method GET \\\n --http-allow-method OPTIONS\n```\n\n\n-----\nYou may also read about [[Request rewriting]], [[Rate limiting]] and [[Middleware]]" }, { "path": "docs/Request-rewriting.md", "content": "Gor supports rewriting of URLs, URL params and headers, see below.\n\nRewriting may be useful if you test environment does not have the same data as your production, and you want to perform all actions in the context of `test` user: for example rewrite all API tokens to some test value. Other possible use cases are toggling features on/off using custom headers or rewriting URL's if they changed in the new environment.\n\nFor more complex logic you can use [Middleware](middleware.md).\n\n#### Rewrite URL based on a mapping\n`--http-rewrite-url` expects value in \":\" format: \":\" is a dilimiter. In `` section you may use captured regexp group values. This works similar to `replace` method in Javascript or `gsub` in Ruby. \n\n```\n# Rewrites all `/v1/user//ping` requests to `/v2/user//ping`\ngor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\\\/]+)/ping:/v2/user/$1/ping\n```\n\n#### Set URL param\nSet request url param, if param already exists it will be overwritten.\n```\ngor --input-raw :8080 --output-http staging.com --http-set-param api_key=1\n```\n\n#### Set Header\nSet request header, if header already exists it will be overwritten. May be useful if you need to identify requests generated by Gor or enable feature flagged functionality in an application:\n\n```\ngor --input-raw :80 --output-http \"http://staging.server\" \\\n --http-set-header \"User-Agent: Replayed by Gor\" \\\n --http-set-header \"Enable-Feature-X: true\"\n```\n\n#### Host header\nHost header gets special treatment. By default Host get set to the value specified in --output-http. If you manually set --http-set-header \"Host: anonther.com\", Gor will not override Host value.\n\nIf you app accepts traffic from multiple domains, and you want to keep original headers, there is specific `--http-original-host` with tells Gor do not touch Host header at all.\n\n\n***\n\nYou may also read about [[Request filtering]], [[Rate limiting]] and [[Middleware]]\n" }, { "path": "docs/Running-as-non-root-user.md", "content": "You can enable Gor for non-root users in a secure method by using the following commands\n\n``` \n# Following commands assume that you put `gor` binary to /usr/local/bin\nadd gor\naddgroup gor\nchgrp gor /usr/local/bin/gor\nchmod 0750 /usr/local/bin/gor\nsetcap \"cap_net_raw,cap_net_admin+eip\" /usr/local/bin/gor\n```\n \nAs a brief explanation of the above.\n* We create a group called gor. \n* We then add the user you want to the new group so they will be able to use gor without sudo\n* We then change the user/group of gor binary the new group.\n* We then make sure the permissions are set on gor binary so that members of the group can execute it but other normal users cannot.\n* We then use `setcap` to give the CAP_NET_RAW and CAP_NET_ADMIN privilege to the executable when it runs. This is so that Gor can open its raw socket which is not normally permitted unless you are root." }, { "path": "docs/Saving-and-Replaying-from-file.md", "content": "You can save requests to file, and replay them later. While replaying it will preserve the original time differences between requests. If you apply [percentage based limiting](Rate Limiting) timing between requests will be reduced or increased appropriately: this approach opens possibilities like load testing, see below.\n\n```bash\n# write to file\ngor --input-raw :80 --output-file requests.log\n\n# read from file\ngor --input-file requests.gor --output-http \"http://staging.com\"\n```\n\nBy default Gor writes files in chunks. This configurable using `--output-file-append` option: the flushed chunk is appended to existence file or not. The default is **false**. By default, `--output-file` flushes each chunk to a different path.\n\n```bash\ngor ... --output-file %Y%m%d.log\n# append false\n20140608_0.log\n20140608_1.log\n20140609_0.log\n20140609_1.log\n```\n\nThis makes parallel file processing easy. But if you want to disable this behavior, you can disable it by adding `--output-file-append` option:\n\n```bash\ngor ... --output-file %Y%m%d.log --output-file-append\n# append true\n20140608.log\n20140609.log\n```\n\nIf you run gor multiple times, and it finds existing files, it will continue from last known index.\n\n### Chunk size\n\nYou can set chunk limits using `--output-file-size-limit` and `--output-file-queue-limit` options.\nThe length of the chunk queue and the size of each chunk, respectively. The default values are 256 and 32mb, respectively. The suffixes “k” (KB), “m” (MB), and “g” (GB) can be used for `output-file-size-limit`.\nIf you want to have only size constraint, you can set `--output-file-queue-limit` to 0, and vice versa.\n\n```bash\ngor --input-raw :80 --output-file %Y-%m-%d.gz --output-file-size-limit 256m --output-file-queue-limit 0\n```\n\n### Using date variables in file names\nFor example, you can tell to create new file each hour: `--output-file /mnt/logs/requests-%Y-%m-%d-%H.log`\nIt will create new file for each hour: requests-2016-06-01-12.log, requests-2016-06-01-13.log, ...\n\nThe time format used as part of the file name. The following characters are replaced with actual values when the file is created:\n\n* `%Y`: year including the century (at least 4 digits)\n* `%m`: month of the year (01..12)\n* `%d`: Day of the month (01..31)\n* `%H`: Hour of the day, 24-hour clock (00..23)\n* `%M`: Minute of the hour (00..59)\n* `%S`: Second of the minute (00..60)\n\nThe default format is `%Y%m%d%H`, which creates one file per hour.\n\n\n### GZIP compression\nTo read or write GZIP compressed files ensure that file extension ends with \".gz\": `--output-file log.gz`\n\n### Replaying from multiple files\n\n`--input-file` accepts file pattern, for example: `--input-file logs-2016-05-*`: it will replay all the files, sorting them in lexicographical order.\n\n### Buffered file output\nGor has memory buffer when it writes to file, and continuously flush changes to the file. Flushing to file happens if the buffer is filled, forced flush every 1 second, or if Gor is closed. You can change it using `--output-file-flush-interval` option. It most cases it should not be touched.\n\n### File format\nHTTP requests stored as it is, plain text: headers and bodies. Requests separated by `\\n🐵🙈🙉\\n` line (using such sequence for uniqueness and fun). Before each request goes single line with meta information containing payload type (1 - request, 2 - response, 3 - replayed response), unique request ID (request and response have the same) and timestamp when request was made. An example of 2 requests:\n\n```\n1 d7123dasd913jfd21312dasdhas31 127345969\\n\nGET / HTTP/1.1\\r\\n\n\\r\\n\n\\n\n🐵🙈🙉\n\\n\nPOST /upload HTTP/1.1\\r\\n\nContent-Length: 7\\r\\n\nHost: www.w3.org\\r\\n\n\\r\\n\na=1&b=2\n```\nNote that technically \\r and \\n symbols are invisible, and indicate new lines. I made them visible in example just to show how it looks on byte level.\n\nMaking it text friendly allows writing simple parsers and use console tools like `grep` to do an analysis. You can even edit them manually, but be sure that your file editor does not change line endings.\n\n## Performance testing\n\nCurrently, this functionality supported only by `input-file` and only when using percentage based limiter. Unlike default limiter for `input-file` instead of dropping requests it will slowdown or speedup request emitting. Note that **limiter is applied to input**:\n\n```\n# Replay from file on 2x speed \ngor --input-file \"requests.gor|200%\" --output-http \"staging.com\"\n```\n\nUse `--stats --output-http-stats` to see latency stats.\n\n### Looping files for replaying indefinitely\nYou can loop the same set of files, so when the last one replays all the requests, it will not stop, and will start from first one again. Having the only small amount of requests you can do extensive performance testing.\nPass `--input-file-loop` to make it work. \n\n***\nYou may also read about [[Capturing and replaying traffic]] and [[Rate limiting]]" }, { "path": "docs/Troubleshooting.md", "content": "Gor can report stats on the `output-tcp` and `output-http` request queues. Stats are reported to the console every 5 seconds in the form `latest,mean,max,count,count/second` by using the `--output-http-stats` and `--output-tcp-stats` options.\n\nExamples:\n\n```\n2014/04/23 21:17:50 output_tcp:latest,mean,max,count,count/second\n2014/04/23 21:17:50 output_tcp:0,0,0,0,0\n2014/04/23 21:17:55 output_tcp:1,1,2,68,13\n2014/04/23 21:18:00 output_tcp:1,1,2,92,18\n2014/04/23 21:18:05 output_tcp:1,1,2,119,23\n```\n\n```\n2014/04/23 21:19:46 output_http:latest,mean,max,count,count/second\n2014/04/23 21:19:46 output_http:0,0,0,0,0\n2014/04/23 21:19:51 output_http:0,0,0,0,0\n2014/04/23 21:19:56 output_http:0,0,0,0,0\n2014/04/23 21:20:01 output_http:1,0,1,50,10\n2014/04/23 21:20:06 output_http:1,1,4,72,14\n2014/04/23 21:20:11 output_http:1,0,1,179,35\n2014/04/23 21:20:16 output_http:1,0,1,148,29\n2014/04/23 21:20:21 output_http:1,1,2,91,18\n2014/04/23 21:20:26 output_http:1,1,2,150,30\n2014/04/23 21:18:15 output_http:100,99,100,70,14\n2014/04/23 21:18:21 output_http:100,99,100,55,11\n```\n\n### How can I tell if I have bottlenecks?\nKey areas that sometimes experience bottlenecks are the output-tcp and output-http functions which have internal queues for requests. Each queue has an upper limit of 100. Enable stats reporting to see if any queues are experiencing bottleneck behavior.\n \n#### Output HTTP bottlenecks\nWhen running a Gor replay the output-http feature may bottleneck if:\n\n * the replay has inadequate bandwidth. If the replay is receiving or sending more messages than its network adapter can handle the output-http-stats may report that the output-http queue is filling up. See if there is a way to upgrade the replay's bandwidth.\n * with `--output-http-workers` set to anything other than `-1` the `-output-http` target is unable to respond to messages in a timely manner. The http output workers which take messages off the output-http queue, process the request, and ensure that the request did not result in an error may not be able to keep up with the number of incoming requests. If the replay is not using dynamic worker scaling (`--output-http-workers=-1`) The optimal number of output-http-workers can be determined with the formula `output-workers = (Average number of requests per second)/(Average target response time per second)`.\n\n#### Output TCP bottlenecks\nWhen using the Gor listener the output-tcp feature may bottleneck if:\n\n * the replay is unable to accept and process more requests than the listener is able generate. Prior to troubleshooting the output-tcp bottleneck, ensure that the replay target is not experiencing any bottlenecks. \n * the replay target has inadequate bandwidth to handle all its incoming requests. If a replay target's incoming bandwidth is maxed out the output-tcp-stats may report that the output-tcp queue is filling up. See if there is a way to upgrade the replay's bandwidth.\n\n\n#### Tuning\n\nTo achieve the top most performance you should tune the source server system limits:\n\n net.ipv4.tcp_max_tw_buckets = 65536\n net.ipv4.tcp_tw_recycle = 1\n net.ipv4.tcp_tw_reuse = 0\n net.ipv4.tcp_max_syn_backlog = 131072\n net.ipv4.tcp_syn_retries = 3\n net.ipv4.tcp_synack_retries = 3\n net.ipv4.tcp_retries1 = 3\n net.ipv4.tcp_retries2 = 8\n net.ipv4.tcp_rmem = 16384 174760 349520\n net.ipv4.tcp_wmem = 16384 131072 262144\n net.ipv4.tcp_mem = 262144 524288 1048576\n net.ipv4.tcp_max_orphans = 65536\n net.ipv4.tcp_fin_timeout = 10\n net.ipv4.tcp_low_latency = 1\n net.ipv4.tcp_syncookies = 0\n***\n\n### Gor is crashing with following stacktrace\n```\nfatal error: unexpected signal during runtime execution\n[signal 0xb code=0x1 addr=0x63 pc=0x7ffcdfdf8b2c]\n\nruntime stack:\nruntime.throw(0xad8380, 0x2a)\n\t/usr/local/go/src/runtime/panic.go:547 +0x90\nruntime.sigpanic()\n\t/usr/local/go/src/runtime/sigpanic_unix.go:12 +0x5a\n\ngoroutine 103 [syscall, locked to thread]:\nruntime.cgocall(0x7b35a0, 0xc82121f1e8, 0x0)\n\t/usr/local/go/src/runtime/cgocall.go:123 +0x11b fp=0xc82121f188 sp=0xc82121f158\nnet._C2func_getaddrinfo(0x7ffcec0008c0, 0x0, 0xc821b221e0, 0xc8217b2b18, 0x0, 0x0, 0x0)\n\t??:0 +0x55 fp=0xc82121f1e8 sp=0xc82121f188\nnet.cgoLookupIPCNAME(0x7fffb17208ab, 0x12, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb17200)\n```\n\nThere is a chance that you hit Go bug. The crash comes from the CGO version of DNS resolver.\nBy default Go based version used, but ins some cases [it switches to CGO based](https://golang.org/pkg/net/#hdr-Name_Resolution). It is possible to force Go based DNS resolver using GODEBUG environment variable:\n`sudo GODEBUG=\"netdns=go\" ./gor --input-raw :80 --output-http staging.env`\n\n\n\nAlso, see [[FAQ]]" }, { "path": "docs/_Footer.md", "content": "[Website](https://goreplay.org) | [PRO version](https://goreplay.org/pro.html) | [[Getting started]] | [[FAQ]] | [Join newsletter](https://www.getdrip.com/forms/89690474/submissions/new)" }, { "path": "docs/_config.yml", "content": "theme: jekyll-theme-cayman" }, { "path": "docs/commercial/collaboration.md", "content": "Collaboration is difficult with commercial closed source but I do want to keep as much of the OSS ethos as possible available to customers who want to fix it themselves.\n\n## Legal\n\nIn order to unambiguously own and sell Gor commercial products, I must have the copyright associated with the entire codebase. Any code you create which is merged must be owned by me. That's not me trying to be a jerk, that's just the way it works.\n\n## Application\n\nIf you wish access to the product repository so you can send a PR, just open a new Gor issue and include the following info:\n\n1. the email address that bought the license, a max of one collaborator per license\n1. the following statement \"I assign all rights, including copyright, to any future Gor work by myself to Leonid Bugaev\"\n\nYou should be granted access to the private repo soon after.\n\n## Notes\n\n1. You should **never** work on the master branch. Only I may merge changes.\n1. I may revoke access for any reason at any time. Access is not guaranteed with purchase." }, { "path": "docs/commercial/faq.md", "content": "### What are GoReplay PRO and GoReplay Enterprise?\n\n[GoReplay PRO](https://goreplay.org/pro.html) and GoReplay Enterprise are extensions to GoReplay which add more functionality and provide additional support options for customers.\n\n### Is there a trial version?\n\nThere's no free trial but we do offer a 14 day period with full refund if it does not work for you.\n\n### What is the license?\n\nSee [COMM-LICENSE](https://github.com/buger/gor/blob/master/COMM-LICENSE) in the root of the GoReplay repo.\n\n### How does PRO licensing work?\n\nEvery organization running GoReplay Pro on its own servers must have a license. There's no limit to the amount of servers or environments used by that organization.\n\n### How does Enterprise licensing work?\n\nEvery organization running Gor Enterprise on its own servers must have a license. There's **no limit** to the amount of servers or environments used by that organization.\n\n### What happens if my subscription lapses?\n\nYou must have an active subscription to run GoReplay Pro or Enterprise. After a one week grace period, you'll lose access to binaries and priority support. You won't get any more updates or bug fixes.\n\n### How do I buy GoReplay Enterprise?\n\nSend email to [support@gortool.com](mailto:support@gortool.com) with your info. A PDF quote will be emailed to you with the price. Reply to that email with your purchase order or just \"Sounds good\" and we will send an invoice which can be paid with a credit card, ACH bank transfer or a paper check.\n\n### Can I upgrade from GoReplay Pro?\n\nYes! Current subscribers can upgrade by [requesting a quote](mailto:support@gortool.com). **Please note that you are an existing Pro subscriber.** We will add a one-time discount on your first invoice to reflect any remaining Pro subscription credit. If you purchased GoReplay Pro 6 months ago for $950, you'll get a $475 discount.\n\n### Can I distribute GoReplay PRO or Enterprise to my customers?\n\nThis is a common requirement for \"on-site installs\" or \"appliances\" sold to large corporations.\n\nThe standard license is appropriate for SaaS usage as it does not allow distribution. GoReplay PRO and Enterprise have an Appliance license option which **does** allow you to distribute them. The Appliance license is $9,500/yr for Pro and $19,500/yr for Enterprise. It allows you to distribute the Pro or Enterprise binaries as part of your application and each of your customers to run GoReplay Pro or Enterprise. Email [support@gortool.com](mailto:support@gortool.com) to purchase.\n\n### Can you transfer a license?\n\nLicenses are **not** transferable to another company. We will transfer the license from a user-specific email to a group email address (e.g. john_smith@example.com -> tech@example.com) but only for **the same domain**. It is strongly recommended that you buy the license using a group email address so the license is not attached to any one employee's email address.\n\n### What does the license require me to do?\n\nYour purchase gets you a unique access URL for downloading the Pro and/or Enterprise binaries. The license agreement requires you to keep this access URL private. If we find your access URL is ever publicized:\n\n1. We'll send you a warning email with details. You need to remove the content and send a new email address so we can generate a new access URL for you. The old access URL will stop working immediately so you'll need to update your apps.\n2. If your access URL is publicized a second time, we reserve the right to permanently remove access.\n\n### Can I get a refund?\n\nYes, up to two weeks after purchase. Let us know the reason and maybe we can help but either way it's not a problem. Email [support@gortool.com](mailto:support@gortool.com).\n\n### How do I update my credit card info?\n\nIf you purchased GoReplay Enterprise, there's nothing to do. Each annual invoice is paid separately.\n\nIf you purchased GoReplay PRO, log into [Gumroad](https://gumroad.com) with your email address, click the Billing tab and enter your new card. I can't provide support for the Gumroad website and don't have the ability to edit customer info - if you can't log in or change your credit card, you can always let your current subscription expire and purchase a new subscription." }, { "path": "docs/commercial/support.md", "content": "Gor offers only community support. Gor Pro and Enterprise offer priority support via email.\n\n## Priority Support\n\nCovers 1 incident per quarter, with a max response time of 2 working days. Scope is limited to Gor and Gor Pro and Enterprise features and APIs, not the application or infrastructure. For support, email **support** AT **gortool.com**. Please email using the same domain as the original license email or explain your connection to the licensed company.\n\nMore aggressive support contracts (phone, quicker response time) are available separately, email with your needs.\n\n## Onboarding\n\nEnterprise customers may request a one hour video chat session with @buger to discuss their application(s), requirements and how best to leverage the various Gor features. Contact support to set up your session." }, { "path": "docs/css/breadcrumbs.css", "content": ".wy-breadcrumbs li {\n display: inline-block;\n}\n\n.wy-breadcrumbs li.wy-breadcrumbs-aside {\n float: right;\n}\n\n.wy-breadcrumbs li a {\n display: inline-block;\n padding: 5px;\n}\n\n.wy-breadcrumbs li a:first-child {\n padding-left: 0;\n}\n\n.wy-breadcrumbs-extra {\n margin-bottom: 0;\n color: #b3b3b3;\n font-size: 80%;\n display: inline-block;\n}\n\n@media screen and (max-width: 480px) {\n .wy-breadcrumbs-extra {\n display: none;\n }\n\n .wy-breadcrumbs li.wy-breadcrumbs-aside {\n display: none;\n }\n}\n\n@media print {\n .wy-breadcrumbs li.wy-breadcrumbs-aside {\n display: none;\n }\n}" }, { "path": "docs/css/code.css", "content": ".codeblock-example {\n border: 1px solid #e1e4e5;\n border-bottom: none;\n padding: 24px;\n padding-top: 48px;\n font-weight: 500;\n background: #fff;\n position: relative;\n}\n\n.codeblock-example:after {\n content: \"Example\";\n position: absolute;\n top: 0;\n left: 0;\n background: #9B59B6;\n color: #fff;\n padding: 6px 12px;\n}\n\n.codeblock-example.prettyprint-example-only {\n border: 1px solid #e1e4e5;\n margin-bottom: 24px;\n}\n\n.codeblock,\npre.literal-block,\n.rst-content .literal-block,\n.rst-content pre.literal-block,\ndiv[class^='highlight'] {\n border: 1px solid #e1e4e5;\n padding: 0;\n overflow-x: auto;\n background: #fff;\n margin: 1px 0 24px;\n}\n\n.codeblock div[class^='highlight'],\npre.literal-block div[class^='highlight'],\n.rst-content .literal-block div[class^='highlight'],\ndiv[class^='highlight'] div[class^='highlight'] {\n border: none;\n background: none;\n margin: 0;\n}\n\ndiv[class^='highlight'] td.code {\n width: 100%;\n}\n\n.linenodiv pre {\n border-right: solid 1px #e6e9ea;\n margin: 0;\n padding: 12px;\n font-family: \"Source Code Pro\",\"Andale Mono WT\",\"Andale Mono\",\"Lucida Console\",\"Lucida Sans Typewriter\",\"DejaVu Sans Mono\",\"Bitstream Vera Sans Mono\",\"Liberation Mono\",\"Nimbus Mono L\",Monaco,\"Courier New\",Courier,monospace;\n font-size: 12px;\n line-height: 1.5;\n color: #d9d9d9;\n}\n\ndiv[class^='highlight'] pre {\n white-space: pre;\n margin: 0;\n padding: 12px;\n font-family: \"Source Code Pro\",\"Andale Mono WT\",\"Andale Mono\",\"Lucida Console\",\"Lucida Sans Typewriter\",\"DejaVu Sans Mono\",\"Bitstream Vera Sans Mono\",\"Liberation Mono\",\"Nimbus Mono L\",Monaco,\"Courier New\",Courier,monospace;\n font-size: 12px;\n line-height: 1.5;\n display: block;\n overflow: auto;\n color: #404040;\n}\n\n@media print {\n .codeblock,\n pre.literal-block,\n .rst-content .literal-block,\n .rst-content pre.literal-block,\n div[class^='highlight'],\n div[class^='highlight'] pre {\n white-space: pre-wrap;\n }\n\n}\n\n.hll {\n background-color: #ffc;\n margin: 0 -12px;\n padding: 0 12px;\n display: block;\n}\n\n.c {\n color: #998;\n font-style: italic;\n}\n\n.err {\n color: #a61717;\n background-color: #e3d2d2;\n}\n\n.k {\n font-weight: 700;\n}\n\n.o {\n font-weight: 700;\n}\n\n.cm {\n color: #998;\n font-style: italic;\n}\n\n.cp {\n color: #999;\n font-weight: 700;\n}\n\n.c1 {\n color: #998;\n font-style: italic;\n}\n\n.cs {\n color: #999;\n font-weight: 700;\n font-style: italic;\n}\n\n.gd {\n color: #000;\n background-color: #fdd;\n}\n\n.gd .x {\n color: #000;\n background-color: #faa;\n}\n\n.ge {\n font-style: italic;\n}\n\n.gr {\n color: #a00\n}\n\n.gh {\n color: #999;\n}\n\n.gi {\n color: #000;\n background-color: #dfd;\n}\n\n.gi .x {\n color: #000;\n background-color: #afa;\n}\n\n.go {\n color: #888;\n}\n\n.gp {\n color: #555;\n}\n\n.gs {\n font-weight: 700;\n}\n\n.gu {\n color: purple;\n font-weight: 700;\n}\n\n.gt {\n color: #a00;\n}\n\n.kc {\n font-weight: 700;\n}\n\n.kd {\n font-weight: 700;\n}\n\n.kn {\n font-weight: 700;\n}\n\n.kp {\n font-weight: 700;\n}\n\n.kr {\n font-weight: 700;\n}\n\n.kt {\n color: #458;\n font-weight: 700;\n}\n\n.m {\n color: #099;\n}\n\n.s {\n color: #d14;\n}\n\n.n {\n color: #333;\n}\n\n.na {\n color: teal;\n}\n\n.nb {\n color: #0086b3;\n}\n\n.nc {\n color: #458;\n font-weight: 700;\n}\n\n.no {\n color: teal;\n}\n\n.ni {\n color: purple;\n}\n\n.ne {\n color: #900;\n font-weight: 700;\n}\n\n.nf {\n color: #900;\n font-weight: 700;\n}\n\n.nn {\n color: #555;\n}\n\n.nt {\n color: navy;\n}\n\n.nv {\n color: teal;\n}\n\n.ow {\n font-weight: 700;\n}\n\n.w {\n color: #bbb;\n}\n\n.mf {\n color: #099;\n}\n\n.mh {\n color: #099;\n}\n\n.mi {\n color: #099;\n}\n\n.mo {\n color: #099;\n}\n\n.sb {\n color: #d14;\n}\n\n.sc {\n color: #d14;\n}\n\n.sd {\n color: #d14;\n}\n\n.s2 {\n color: #d14\n}\n\n.se {\n color: #d14;\n}\n\n.sh {\n color: #d14;\n}\n\n.si {\n color: #d14;\n}\n\n.sx {\n color: #d14;\n}\n\n.sr {\n color: #009926;\n}\n\n.s1 {\n color: #d14;\n}\n\n.ss {\n color: #990073;\n}\n\n.bp {\n color: #999;\n}\n\n.vc {\n color: teal;\n}\n\n.vg {\n color: teal;\n}\n\n.vi {\n color: teal;\n}\n\n.il {\n color: #099;\n}\n\n.gc {\n color: #999;\n background-color: #EAF2F5;\n}" }, { "path": "docs/css/fabric.css", "content": "/* Taken from https://docs.fabric.io/apple/fabric/overview.html */\n\n* {\n -webkit-box-sizing: border-box;\n -moz-box-sizing: border-box;\n box-sizing: border-box\n}\n\narticle,\naside,\ndetails,\nfigcaption,\nfigure,\nfooter,\nheader,\nhgroup,\nnav,\nsection {\n display: block\n}\n\naudio,\ncanvas,\nvideo {\n display: inline-block;\n *display: inline;\n *zoom: 1\n}\n\naudio:not([controls]) {\n display: none\n}\n\n[hidden] {\n display: none\n}\n\n* {\n -webkit-box-sizing: border-box;\n -moz-box-sizing: border-box;\n box-sizing: border-box\n}\n\nhtml {\n font-size: 100%;\n -webkit-text-size-adjust: 100%;\n -ms-text-size-adjust: 100%\n}\n\nbody {\n margin: 0\n}\n\na:hover,\na:active {\n outline: 0\n}\n\nabbr[title] {\n border-bottom: 1px dotted\n}\n\nb,\nstrong {\n font-weight: 700\n}\n\nblockquote {\n margin: 0\n}\n\ndfn {\n font-style: italic\n}\n\nins {\n background: #ff9;\n color: #000;\n text-decoration: none\n}\n\nmark {\n background: #ff0;\n color: #000;\n font-style: italic;\n font-weight: 700\n}\n\npre,\ncode,\n.rst-content tt,\n.rst-content code,\nkbd,\nsamp {\n font-family: monospace,serif;\n _font-family: \"courier new\",monospace;\n font-size: 1em\n}\n\npre {\n white-space: pre\n}\n\nq {\n quotes: none\n}\n\nq:before,\nq:after {\n content: \"\";\n content: none\n}\n\nsmall {\n font-size: 85%\n}\n\nsub,\nsup {\n font-size: 75%;\n line-height: 0;\n position: relative;\n vertical-align: baseline\n}\n\nsup {\n top: -.5em\n}\n\nsub {\n bottom: -.25em\n}\n\nul,\nol,\ndl {\n margin: 0;\n padding: 0;\n list-style: none;\n list-style-image: none\n}\n\nli {\n list-style: none\n}\n\ndd {\n margin: 0\n}\n\nimg {\n border: 0;\n -ms-interpolation-mode: bicubic;\n vertical-align: middle;\n max-width: 100%\n}\n\nsvg:not(:root) {\n overflow: hidden\n}\n\nfigure {\n margin: 0\n}\n\nform {\n margin: 0\n}\n\nfieldset {\n border: 0;\n margin: 0;\n padding: 0\n}\n\nlabel {\n cursor: pointer\n}\n\nlegend {\n border: 0;\n *margin-left: -7px;\n padding: 0;\n white-space: normal\n}\n\nbutton,\ninput,\nselect,\ntextarea {\n font-size: 100%;\n margin: 0;\n vertical-align: baseline;\n *vertical-align: middle\n}\n\nbutton,\ninput {\n line-height: normal\n}\n\nbutton,\ninput[type=\"button\"],\ninput[type=\"reset\"],\ninput[type=\"submit\"] {\n cursor: pointer;\n -webkit-appearance: button;\n *overflow: visible\n}\n\nbutton[disabled],\ninput[disabled] {\n cursor: default\n}\n\ninput[type=\"checkbox\"],\ninput[type=\"radio\"] {\n box-sizing: border-box;\n padding: 0;\n *width: 13px;\n *height: 13px\n}\n\ninput[type=\"search\"] {\n -webkit-appearance: textfield;\n -moz-box-sizing: content-box;\n -webkit-box-sizing: content-box;\n box-sizing: content-box\n}\n\ninput[type=\"search\"]::-webkit-search-decoration,\ninput[type=\"search\"]::-webkit-search-cancel-button {\n -webkit-appearance: none\n}\n\nbutton::-moz-focus-inner,\ninput::-moz-focus-inner {\n border: 0;\n padding: 0\n}\n\ntextarea {\n overflow: auto;\n vertical-align: top;\n resize: vertical\n}\n\ntable {\n border-collapse: collapse;\n border-spacing: 0\n}\n\ntd {\n vertical-align: top\n}\n\n.chromeframe {\n margin: .2em 0;\n background: #ccc;\n color: #000;\n padding: .2em 0\n}\n\n.ir {\n display: block;\n border: 0;\n text-indent: -999em;\n overflow: hidden;\n background-color: transparent;\n background-repeat: no-repeat;\n text-align: left;\n direction: ltr;\n *line-height: 0\n}\n\n.ir br {\n display: none\n}\n\n.hidden {\n display: none!important;\n visibility: hidden\n}\n\n.visuallyhidden {\n border: 0;\n clip: rect(0 0 0 0);\n height: 1px;\n margin: -1px;\n overflow: hidden;\n padding: 0;\n position: absolute;\n width: 1px\n}\n\n.visuallyhidden.focusable:active,\n.visuallyhidden.focusable:focus {\n clip: auto;\n height: auto;\n margin: 0;\n overflow: visible;\n position: static;\n width: auto\n}\n\n.invisible {\n visibility: hidden\n}\n\n.relative {\n position: relative\n}\n\nbig,\nsmall {\n font-size: 100%\n}\n\n@media print {\n html,\n body,\n section {\n background: none!important\n }\n\n * {\n box-shadow: none!important;\n text-shadow: none!important;\n filter: none!important;\n -ms-filter: none!important\n }\n\n a,\n .ir a:after,\n a[href^=\"javascript:\"]:after,\n a[href^=\"#\"]:after {\n content: \"\"\n }\n\n pre,\n blockquote {\n page-break-inside: avoid\n }\n\n thead {\n display: table-header-group\n }\n\n tr,\n img {\n page-break-inside: avoid\n }\n\n img {\n max-width: 100%!important\n }\n\n @page {\n margin: .5cm\n }\n\n p,\n h2,\n h3 {\n orphans: 3;\n widows: 3\n }\n\n h2,\n h3 {\n page-break-after: avoid\n }\n\n}\n\n.fa:before,\n.wy-menu-vertical li span.toctree-expand:before,\n.wy-menu-vertical li.on a span.toctree-expand:before,\n.wy-menu-vertical li.current>a span.toctree-expand:before,\n.rst-content .admonition-title:before,\n.rst-content h1 .headerlink:before,\n.rst-content h2 .headerlink:before,\n.rst-content h3 .headerlink:before,\n.rst-content h4 .headerlink:before,\n.rst-content h5 .headerlink:before,\n.rst-content h6 .headerlink:before,\n.rst-content dl dt .headerlink:before,\n.rst-content p.caption .headerlink:before,\n.rst-content tt.download span:first-child:before,\n.rst-content code.download span:first-child:before,\n.icon:before,\n.wy-dropdown .caret:before,\n.wy-inline-validate.wy-inline-validate-success .wy-input-context:before,\n.wy-inline-validate.wy-inline-validate-danger .wy-input-context:before,\n.wy-inline-validate.wy-inline-validate-warning .wy-input-context:before,\n.wy-inline-validate.wy-inline-validate-info .wy-input-context:before,\n.wy-alert,\n.rst-content .note,\n.rst-content .attention,\n.rst-content .caution,\n.rst-content .danger,\n.rst-content .error,\n.rst-content .hint,\n.rst-content .important,\n.rst-content .tip,\n.rst-content .warning,\n.rst-content .seealso,\n.rst-content .admonition-todo,\n.btn,\ninput[type=\"text\"],\ninput[type=\"password\"],\ninput[type=\"email\"],\ninput[type=\"url\"],\ninput[type=\"date\"],\ninput[type=\"month\"],\ninput[type=\"time\"],\ninput[type=\"datetime\"],\ninput[type=\"datetime-local\"],\ninput[type=\"week\"],\ninput[type=\"number\"],\ninput[type=\"search\"],\ninput[type=\"tel\"],\ninput[type=\"color\"],\nselect,\ntextarea,\n.wy-menu-vertical li.on a,\n.wy-menu-vertical li.current>a,\n.wy-side-nav-search>a,\n.wy-side-nav-search .wy-dropdown>a,\n.wy-nav-top a {\n -webkit-font-smoothing: antialiased\n}\n\n.clearfix {\n *zoom: 1\n}\n\n.clearfix:before,\n.clearfix:after {\n display: table;\n content: \"\"\n}\n\n.clearfix:after {\n clear: both\n}\n\n@font-face {\n font-family: 'FontAwesome';\n src: url(../fonts/fontawesome-webfont.eot?v=4.2.0);\n src: url(../fonts/fontawesome-webfont.eot?#iefix&v=4.2.0) format(\"embedded-opentype\"),url(../fonts/fontawesome-webfont.woff?v=4.2.0) format(\"woff\"),url(../fonts/fontawesome-webfont.ttf?v=4.2.0) format(\"truetype\"),url(../fonts/fontawesome-webfont.svg?v=4.2.0#fontawesomeregular) format(\"svg\");\n font-weight: 400;\n font-style: normal\n}\n\n.fa,\n.wy-menu-vertical li span.toctree-expand,\n.wy-menu-vertical li.on a span.toctree-expand,\n.wy-menu-vertical li.current>a span.toctree-expand,\n.rst-content .admonition-title,\n.rst-content h1 .headerlink,\n.rst-content h2 .headerlink,\n.rst-content h3 .headerlink,\n.rst-content h4 .headerlink,\n.rst-content h5 .headerlink,\n.rst-content h6 .headerlink,\n.rst-content dl dt .headerlink,\n.rst-content p.caption .headerlink,\n.rst-content tt.download span:first-child,\n.rst-content code.download span:first-child,\n.icon {\n display: inline-block;\n font: normal normal normal 14px/1 FontAwesome;\n font-size: inherit;\n text-rendering: auto;\n -webkit-font-smoothing: antialiased;\n -moz-osx-font-smoothing: grayscale\n}\n\n.fa-lg {\n font-size: 1.33333em;\n line-height: .75em;\n vertical-align: -15%\n}\n\n.fa-2x {\n font-size: 2em\n}\n\n.fa-3x {\n font-size: 3em\n}\n\n.fa-4x {\n font-size: 4em\n}\n\n.fa-5x {\n font-size: 5em\n}\n\n.fa-fw {\n width: 1.28571em;\n text-align: center\n}\n\n.fa-ul {\n padding-left: 0;\n margin-left: 2.14286em;\n list-style-type: none\n}\n\n.fa-ul>li {\n position: relative\n}\n\n.fa-li {\n position: absolute;\n left: -2.14286em;\n width: 2.14286em;\n top: .14286em;\n text-align: center\n}\n\n.fa-li.fa-lg {\n left: -1.85714em\n}\n\n.fa-border {\n padding: .2em .25em .15em;\n border: solid .08em #eee;\n border-radius: .1em\n}\n\n.pull-right {\n float: right\n}\n\n.pull-left {\n float: left\n}\n\n.fa.pull-left,\n.wy-menu-vertical li span.pull-left.toctree-expand,\n.wy-menu-vertical li.on a span.pull-left.toctree-expand,\n.wy-menu-vertical li.current>a span.pull-left.toctree-expand,\n.rst-content .pull-left.admonition-title,\n.rst-content h1 .pull-left.headerlink,\n.rst-content h2 .pull-left.headerlink,\n.rst-content h3 .pull-left.headerlink,\n.rst-content h4 .pull-left.headerlink,\n.rst-content h5 .pull-left.headerlink,\n.rst-content h6 .pull-left.headerlink,\n.rst-content dl dt .pull-left.headerlink,\n.rst-content p.caption .pull-left.headerlink,\n.rst-content tt.download span.pull-left:first-child,\n.rst-content code.download span.pull-left:first-child,\n.pull-left.icon {\n margin-right: .3em\n}\n\n.fa.pull-right,\n.wy-menu-vertical li span.pull-right.toctree-expand,\n.wy-menu-vertical li.on a span.pull-right.toctree-expand,\n.wy-menu-vertical li.current>a span.pull-right.toctree-expand,\n.rst-content .pull-right.admonition-title,\n.rst-content h1 .pull-right.headerlink,\n.rst-content h2 .pull-right.headerlink,\n.rst-content h3 .pull-right.headerlink,\n.rst-content h4 .pull-right.headerlink,\n.rst-content h5 .pull-right.headerlink,\n.rst-content h6 .pull-right.headerlink,\n.rst-content dl dt .pull-right.headerlink,\n.rst-content p.caption .pull-right.headerlink,\n.rst-content tt.download span.pull-right:first-child,\n.rst-content code.download span.pull-right:first-child,\n.pull-right.icon {\n margin-left: .3em\n}\n\n.fa,\n.wy-menu-vertical li span.toctree-expand,\n.wy-menu-vertical li.on a span.toctree-expand,\n.wy-menu-vertical li.current>a span.toctree-expand,\n.rst-content .admonition-title,\n.rst-content h1 .headerlink,\n.rst-content h2 .headerlink,\n.rst-content h3 .headerlink,\n.rst-content h4 .headerlink,\n.rst-content h5 .headerlink,\n.rst-content h6 .headerlink,\n.rst-content dl dt .headerlink,\n.rst-content p.caption .headerlink,\n.rst-content tt.download span:first-child,\n.rst-content code.download span:first-child,\n.icon,\n.wy-dropdown .caret,\n.wy-inline-validate.wy-inline-validate-success .wy-input-context,\n.wy-inline-validate.wy-inline-validate-danger .wy-input-context,\n.wy-inline-validate.wy-inline-validate-warning .wy-input-context,\n.wy-inline-validate.wy-inline-validate-info .wy-input-context {\n font-family: inherit\n}\n\n.fa:before,\n.wy-menu-vertical li span.toctree-expand:before,\n.wy-menu-vertical li.on a span.toctree-expand:before,\n.wy-menu-vertical li.current>a span.toctree-expand:before,\n.rst-content .admonition-title:before,\n.rst-content h1 .headerlink:before,\n.rst-content h2 .headerlink:before,\n.rst-content h3 .headerlink:before,\n.rst-content h4 .headerlink:before,\n.rst-content h5 .headerlink:before,\n.rst-content h6 .headerlink:before,\n.rst-content dl dt .headerlink:before,\n.rst-content p.caption .headerlink:before,\n.rst-content tt.download span:first-child:before,\n.rst-content code.download span:first-child:before,\n.icon:before,\n.wy-dropdown .caret:before,\n.wy-inline-validate.wy-inline-validate-success .wy-input-context:before,\n.wy-inline-validate.wy-inline-validate-danger .wy-input-context:before,\n.wy-inline-validate.wy-inline-validate-warning .wy-input-context:before,\n.wy-inline-validate.wy-inline-validate-info .wy-input-context:before {\n font-family: \"FontAwesome\";\n display: inline-block;\n font-style: normal;\n font-weight: 400;\n line-height: 1;\n text-decoration: inherit\n}\n\na .fa,\na .wy-menu-vertical li span.toctree-expand,\n.wy-menu-vertical li a span.toctree-expand,\n.wy-menu-vertical li.on a span.toctree-expand,\n.wy-menu-vertical li.current>a span.toctree-expand,\na .rst-content .admonition-title,\n.rst-content a .admonition-title,\na .rst-content h1 .headerlink,\n.rst-content h1 a .headerlink,\na .rst-content h2 .headerlink,\n.rst-content h2 a .headerlink,\na .rst-content h3 .headerlink,\n.rst-content h3 a .headerlink,\na .rst-content h4 .headerlink,\n.rst-content h4 a .headerlink,\na .rst-content h5 .headerlink,\n.rst-content h5 a .headerlink,\na .rst-content h6 .headerlink,\n.rst-content h6 a .headerlink,\na .rst-content dl dt .headerlink,\n.rst-content dl dt a .headerlink,\na .rst-content p.caption .headerlink,\n.rst-content p.caption a .headerlink,\na .rst-content tt.download span:first-child,\n.rst-content tt.download a span:first-child,\na .rst-content code.download span:first-child,\n.rst-content code.download a span:first-child,\na .icon {\n display: inline-block;\n text-decoration: inherit\n}\n\n.btn .fa,\n.btn .wy-menu-vertical li span.toctree-expand,\n.wy-menu-vertical li .btn span.toctree-expand,\n.btn .wy-menu-vertical li.on a span.toctree-expand,\n.wy-menu-vertical li.on a .btn span.toctree-expand,\n.btn .wy-menu-vertical li.current>a span.toctree-expand,\n.wy-menu-vertical li.current>a .btn span.toctree-expand,\n.btn .rst-content .admonition-title,\n.rst-content .btn .admonition-title,\n.btn .rst-content h1 .headerlink,\n.rst-content h1 .btn .headerlink,\n.btn .rst-content h2 .headerlink,\n.rst-content h2 .btn .headerlink,\n.btn .rst-content h3 .headerlink,\n.rst-content h3 .btn .headerlink,\n.btn .rst-content h4 .headerlink,\n.rst-content h4 .btn .headerlink,\n.btn .rst-content h5 .headerlink,\n.rst-content h5 .btn .headerlink,\n.btn .rst-content h6 .headerlink,\n.rst-content h6 .btn .headerlink,\n.btn .rst-content dl dt .headerlink,\n.rst-content dl dt .btn .headerlink,\n.btn .rst-content p.caption .headerlink,\n.rst-content p.caption .btn .headerlink,\n.btn .rst-content tt.download span:first-child,\n.rst-content tt.download .btn span:first-child,\n.btn .rst-content code.download span:first-child,\n.rst-content code.download .btn span:first-child,\n.btn .icon,\n.nav .fa,\n.nav .wy-menu-vertical li span.toctree-expand,\n.wy-menu-vertical li .nav span.toctree-expand,\n.nav .wy-menu-vertical li.on a span.toctree-expand,\n.wy-menu-vertical li.on a .nav span.toctree-expand,\n.nav .wy-menu-vertical li.current>a span.toctree-expand,\n.wy-menu-vertical li.current>a .nav span.toctree-expand,\n.nav .rst-content .admonition-title,\n.rst-content .nav .admonition-title,\n.nav .rst-content h1 .headerlink,\n.rst-content h1 .nav .headerlink,\n.nav .rst-content h2 .headerlink,\n.rst-content h2 .nav .headerlink,\n.nav .rst-content h3 .headerlink,\n.rst-content h3 .nav .headerlink,\n.nav .rst-content h4 .headerlink,\n.rst-content h4 .nav .headerlink,\n.nav .rst-content h5 .headerlink,\n.rst-content h5 .nav .headerlink,\n.nav .rst-content h6 .headerlink,\n.rst-content h6 .nav .headerlink,\n.nav .rst-content dl dt .headerlink,\n.rst-content dl dt .nav .headerlink,\n.nav .rst-content p.caption .headerlink,\n.rst-content p.caption .nav .headerlink,\n.nav .rst-content tt.download span:first-child,\n.rst-content tt.download .nav span:first-child,\n.nav .rst-content code.download span:first-child,\n.rst-content code.download .nav span:first-child,\n.nav .icon {\n display: inline\n}\n\n.btn .fa.fa-large,\n.btn .wy-menu-vertical li span.fa-large.toctree-expand,\n.wy-menu-vertical li .btn span.fa-large.toctree-expand,\n.btn .rst-content .fa-large.admonition-title,\n.rst-content .btn .fa-large.admonition-title,\n.btn .rst-content h1 .fa-large.headerlink,\n.rst-content h1 .btn .fa-large.headerlink,\n.btn .rst-content h2 .fa-large.headerlink,\n.rst-content h2 .btn .fa-large.headerlink,\n.btn .rst-content h3 .fa-large.headerlink,\n.rst-content h3 .btn .fa-large.headerlink,\n.btn .rst-content h4 .fa-large.headerlink,\n.rst-content h4 .btn .fa-large.headerlink,\n.btn .rst-content h5 .fa-large.headerlink,\n.rst-content h5 .btn .fa-large.headerlink,\n.btn .rst-content h6 .fa-large.headerlink,\n.rst-content h6 .btn .fa-large.headerlink,\n.btn .rst-content dl dt .fa-large.headerlink,\n.rst-content dl dt .btn .fa-large.headerlink,\n.btn .rst-content p.caption .fa-large.headerlink,\n.rst-content p.caption .btn .fa-large.headerlink,\n.btn .rst-content tt.download span.fa-large:first-child,\n.rst-content tt.download .btn span.fa-large:first-child,\n.btn .rst-content code.download span.fa-large:first-child,\n.rst-content code.download .btn span.fa-large:first-child,\n.btn .fa-large.icon,\n.nav .fa.fa-large,\n.nav .wy-menu-vertical li span.fa-large.toctree-expand,\n.wy-menu-vertical li .nav span.fa-large.toctree-expand,\n.nav .rst-content .fa-large.admonition-title,\n.rst-content .nav .fa-large.admonition-title,\n.nav .rst-content h1 .fa-large.headerlink,\n.rst-content h1 .nav .fa-large.headerlink,\n.nav .rst-content h2 .fa-large.headerlink,\n.rst-content h2 .nav .fa-large.headerlink,\n.nav .rst-content h3 .fa-large.headerlink,\n.rst-content h3 .nav .fa-large.headerlink,\n.nav .rst-content h4 .fa-large.headerlink,\n.rst-content h4 .nav .fa-large.headerlink,\n.nav .rst-content h5 .fa-large.headerlink,\n.rst-content h5 .nav .fa-large.headerlink,\n.nav .rst-content h6 .fa-large.headerlink,\n.rst-content h6 .nav .fa-large.headerlink,\n.nav .rst-content dl dt .fa-large.headerlink,\n.rst-content dl dt .nav .fa-large.headerlink,\n.nav .rst-content p.caption .fa-large.headerlink,\n.rst-content p.caption .nav .fa-large.headerlink,\n.nav .rst-content tt.download span.fa-large:first-child,\n.rst-content tt.download .nav span.fa-large:first-child,\n.nav .rst-content code.download span.fa-large:first-child,\n.rst-content code.download .nav span.fa-large:first-child,\n.nav .fa-large.icon {\n line-height: .9em\n}\n\n.btn .fa.fa-spin,\n.btn .wy-menu-vertical li span.fa-spin.toctree-expand,\n.wy-menu-vertical li .btn span.fa-spin.toctree-expand,\n.btn .rst-content .fa-spin.admonition-title,\n.rst-content .btn .fa-spin.admonition-title,\n.btn .rst-content h1 .fa-spin.headerlink,\n.rst-content h1 .btn .fa-spin.headerlink,\n.btn .rst-content h2 .fa-spin.headerlink,\n.rst-content h2 .btn .fa-spin.headerlink,\n.btn .rst-content h3 .fa-spin.headerlink,\n.rst-content h3 .btn .fa-spin.headerlink,\n.btn .rst-content h4 .fa-spin.headerlink,\n.rst-content h4 .btn .fa-spin.headerlink,\n.btn .rst-content h5 .fa-spin.headerlink,\n.rst-content h5 .btn .fa-spin.headerlink,\n.btn .rst-content h6 .fa-spin.headerlink,\n.rst-content h6 .btn .fa-spin.headerlink,\n.btn .rst-content dl dt .fa-spin.headerlink,\n.rst-content dl dt .btn .fa-spin.headerlink,\n.btn .rst-content p.caption .fa-spin.headerlink,\n.rst-content p.caption .btn .fa-spin.headerlink,\n.btn .rst-content tt.download span.fa-spin:first-child,\n.rst-content tt.download .btn span.fa-spin:first-child,\n.btn .rst-content code.download span.fa-spin:first-child,\n.rst-content code.download .btn span.fa-spin:first-child,\n.btn .fa-spin.icon,\n.nav .fa.fa-spin,\n.nav .wy-menu-vertical li span.fa-spin.toctree-expand,\n.wy-menu-vertical li .nav span.fa-spin.toctree-expand,\n.nav .rst-content .fa-spin.admonition-title,\n.rst-content .nav .fa-spin.admonition-title,\n.nav .rst-content h1 .fa-spin.headerlink,\n.rst-content h1 .nav .fa-spin.headerlink,\n.nav .rst-content h2 .fa-spin.headerlink,\n.rst-content h2 .nav .fa-spin.headerlink,\n.nav .rst-content h3 .fa-spin.headerlink,\n.rst-content h3 .nav .fa-spin.headerlink,\n.nav .rst-content h4 .fa-spin.headerlink,\n.rst-content h4 .nav .fa-spin.headerlink,\n.nav .rst-content h5 .fa-spin.headerlink,\n.rst-content h5 .nav .fa-spin.headerlink,\n.nav .rst-content h6 .fa-spin.headerlink,\n.rst-content h6 .nav .fa-spin.headerlink,\n.nav .rst-content dl dt .fa-spin.headerlink,\n.rst-content dl dt .nav .fa-spin.headerlink,\n.nav .rst-content p.caption .fa-spin.headerlink,\n.rst-content p.caption .nav .fa-spin.headerlink,\n.nav .rst-content tt.download span.fa-spin:first-child,\n.rst-content tt.download .nav span.fa-spin:first-child,\n.nav .rst-content code.download span.fa-spin:first-child,\n.rst-content code.download .nav span.fa-spin:first-child,\n.nav .fa-spin.icon {\n display: inline-block\n}\n\n.btn.fa:before,\n.wy-menu-vertical li span.btn.toctree-expand:before,\n.rst-content .btn.admonition-title:before,\n.rst-content h1 .btn.headerlink:before,\n.rst-content h2 .btn.headerlink:before,\n.rst-content h3 .btn.headerlink:before,\n.rst-content h4 .btn.headerlink:before,\n.rst-content h5 .btn.headerlink:before,\n.rst-content h6 .btn.headerlink:before,\n.rst-content dl dt .btn.headerlink:before,\n.rst-content p.caption .btn.headerlink:before,\n.rst-content tt.download span.btn:first-child:before,\n.rst-content code.download span.btn:first-child:before,\n.btn.icon:before {\n opacity: .5;\n -webkit-transition: opacity .05s ease-in;\n -moz-transition: opacity .05s ease-in;\n transition: opacity .05s ease-in\n}\n\n.btn.fa:hover:before,\n.wy-menu-vertical li span.btn.toctree-expand:hover:before,\n.rst-content .btn.admonition-title:hover:before,\n.rst-content h1 .btn.headerlink:hover:before,\n.rst-content h2 .btn.headerlink:hover:before,\n.rst-content h3 .btn.headerlink:hover:before,\n.rst-content h4 .btn.headerlink:hover:before,\n.rst-content h5 .btn.headerlink:hover:before,\n.rst-content h6 .btn.headerlink:hover:before,\n.rst-content dl dt .btn.headerlink:hover:before,\n.rst-content p.caption .btn.headerlink:hover:before,\n.rst-content tt.download span.btn:first-child:hover:before,\n.rst-content code.download span.btn:first-child:hover:before,\n.btn.icon:hover:before {\n opacity: 1\n}\n\n.btn-mini .fa:before,\n.btn-mini .wy-menu-vertical li span.toctree-expand:before,\n.wy-menu-vertical li .btn-mini span.toctree-expand:before,\n.btn-mini .rst-content .admonition-title:before,\n.rst-content .btn-mini .admonition-title:before,\n.btn-mini .rst-content h1 .headerlink:before,\n.rst-content h1 .btn-mini .headerlink:before,\n.btn-mini .rst-content h2 .headerlink:before,\n.rst-content h2 .btn-mini .headerlink:before,\n.btn-mini .rst-content h3 .headerlink:before,\n.rst-content h3 .btn-mini .headerlink:before,\n.btn-mini .rst-content h4 .headerlink:before,\n.rst-content h4 .btn-mini .headerlink:before,\n.btn-mini .rst-content h5 .headerlink:before,\n.rst-content h5 .btn-mini .headerlink:before,\n.btn-mini .rst-content h6 .headerlink:before,\n.rst-content h6 .btn-mini .headerlink:before,\n.btn-mini .rst-content dl dt .headerlink:before,\n.rst-content dl dt .btn-mini .headerlink:before,\n.btn-mini .rst-content p.caption .headerlink:before,\n.rst-content p.caption .btn-mini .headerlink:before,\n.btn-mini .rst-content tt.download span:first-child:before,\n.rst-content tt.download .btn-mini span:first-child:before,\n.btn-mini .rst-content code.download span:first-child:before,\n.rst-content code.download .btn-mini span:first-child:before,\n.btn-mini .icon:before {\n font-size: 14px;\n vertical-align: -15%\n}\n\n.wy-alert,\n.rst-content .note,\n.rst-content .attention,\n.rst-content .caution,\n.rst-content .danger,\n.rst-content .error,\n.rst-content .hint,\n.rst-content .important,\n.rst-content .tip,\n.rst-content .warning,\n.rst-content .seealso,\n.rst-content .admonition-todo {\n padding: 12px;\n line-height: 24px;\n margin-bottom: 24px;\n background: #e7f2fa\n}\n\n.wy-alert-title,\n.rst-content .admonition-title {\n color: #fff;\n font-weight: 700;\n display: block;\n color: #fff;\n background: #6ab0de;\n margin: -12px;\n padding: 6px 12px;\n margin-bottom: 12px\n}\n\n.wy-alert.wy-alert-danger,\n.rst-content .wy-alert-danger.note,\n.rst-content .wy-alert-danger.attention,\n.rst-content .wy-alert-danger.caution,\n.rst-content .danger,\n.rst-content .error,\n.rst-content .wy-alert-danger.hint,\n.rst-content .wy-alert-danger.important,\n.rst-content .wy-alert-danger.tip,\n.rst-content .wy-alert-danger.warning,\n.rst-content .wy-alert-danger.seealso,\n.rst-content .wy-alert-danger.admonition-todo {\n background: #fdf3f2\n}\n\n.wy-alert.wy-alert-danger .wy-alert-title,\n.rst-content .wy-alert-danger.note .wy-alert-title,\n.rst-content .wy-alert-danger.attention .wy-alert-title,\n.rst-content .wy-alert-danger.caution .wy-alert-title,\n.rst-content .danger .wy-alert-title,\n.rst-content .error .wy-alert-title,\n.rst-content .wy-alert-danger.hint .wy-alert-title,\n.rst-content .wy-alert-danger.important .wy-alert-title,\n.rst-content .wy-alert-danger.tip .wy-alert-title,\n.rst-content .wy-alert-danger.warning .wy-alert-title,\n.rst-content .wy-alert-danger.seealso .wy-alert-title,\n.rst-content .wy-alert-danger.admonition-todo .wy-alert-title,\n.wy-alert.wy-alert-danger .rst-content .admonition-title,\n.rst-content .wy-alert.wy-alert-danger .admonition-title,\n.rst-content .wy-alert-danger.note .admonition-title,\n.rst-content .wy-alert-danger.attention .admonition-title,\n.rst-content .wy-alert-danger.caution .admonition-title,\n.rst-content .danger .admonition-title,\n.rst-content .error .admonition-title,\n.rst-content .wy-alert-danger.hint .admonition-title,\n.rst-content .wy-alert-danger.important .admonition-title,\n.rst-content .wy-alert-danger.tip .admonition-title,\n.rst-content .wy-alert-danger.warning .admonition-title,\n.rst-content .wy-alert-danger.seealso .admonition-title,\n.rst-content .wy-alert-danger.admonition-todo .admonition-title {\n background: #f29f97\n}\n\n.wy-alert.wy-alert-warning,\n.rst-content .wy-alert-warning.note,\n.rst-content .attention,\n.rst-content .caution,\n.rst-content .wy-alert-warning.danger,\n.rst-content .wy-alert-warning.error,\n.rst-content .wy-alert-warning.hint,\n.rst-content .wy-alert-warning.important,\n.rst-content .wy-alert-warning.tip,\n.rst-content .warning,\n.rst-content .wy-alert-warning.seealso,\n.rst-content .admonition-todo {\n background: #ffedcc\n}\n\n.wy-alert.wy-alert-warning .wy-alert-title,\n.rst-content .wy-alert-warning.note .wy-alert-title,\n.rst-content .attention .wy-alert-title,\n.rst-content .caution .wy-alert-title,\n.rst-content .wy-alert-warning.danger .wy-alert-title,\n.rst-content .wy-alert-warning.error .wy-alert-title,\n.rst-content .wy-alert-warning.hint .wy-alert-title,\n.rst-content .wy-alert-warning.important .wy-alert-title,\n.rst-content .wy-alert-warning.tip .wy-alert-title,\n.rst-content .warning .wy-alert-title,\n.rst-content .wy-alert-warning.seealso .wy-alert-title,\n.rst-content .admonition-todo .wy-alert-title,\n.wy-alert.wy-alert-warning .rst-content .admonition-title,\n.rst-content .wy-alert.wy-alert-warning .admonition-title,\n.rst-content .wy-alert-warning.note .admonition-title,\n.rst-content .attention .admonition-title,\n.rst-content .caution .admonition-title,\n.rst-content .wy-alert-warning.danger .admonition-title,\n.rst-content .wy-alert-warning.error .admonition-title,\n.rst-content .wy-alert-warning.hint .admonition-title,\n.rst-content .wy-alert-warning.important .admonition-title,\n.rst-content .wy-alert-warning.tip .admonition-title,\n.rst-content .warning .admonition-title,\n.rst-content .wy-alert-warning.seealso .admonition-title,\n.rst-content .admonition-todo .admonition-title {\n background: #f0b37e\n}\n\n.wy-alert.wy-alert-info,\n.rst-content .note,\n.rst-content .wy-alert-info.attention,\n.rst-content .wy-alert-info.caution,\n.rst-content .wy-alert-info.danger,\n.rst-content .wy-alert-info.error,\n.rst-content .wy-alert-info.hint,\n.rst-content .wy-alert-info.important,\n.rst-content .wy-alert-info.tip,\n.rst-content .wy-alert-info.warning,\n.rst-content .seealso,\n.rst-content .wy-alert-info.admonition-todo {\n background: #e7f2fa\n}\n\n.wy-alert.wy-alert-info .wy-alert-title,\n.rst-content .note .wy-alert-title,\n.rst-content .wy-alert-info.attention .wy-alert-title,\n.rst-content .wy-alert-info.caution .wy-alert-title,\n.rst-content .wy-alert-info.danger .wy-alert-title,\n.rst-content .wy-alert-info.error .wy-alert-title,\n.rst-content .wy-alert-info.hint .wy-alert-title,\n.rst-content .wy-alert-info.important .wy-alert-title,\n.rst-content .wy-alert-info.tip .wy-alert-title,\n.rst-content .wy-alert-info.warning .wy-alert-title,\n.rst-content .seealso .wy-alert-title,\n.rst-content .wy-alert-info.admonition-todo .wy-alert-title,\n.wy-alert.wy-alert-info .rst-content .admonition-title,\n.rst-content .wy-alert.wy-alert-info .admonition-title,\n.rst-content .note .admonition-title,\n.rst-content .wy-alert-info.attention .admonition-title,\n.rst-content .wy-alert-info.caution .admonition-title,\n.rst-content .wy-alert-info.danger .admonition-title,\n.rst-content .wy-alert-info.error .admonition-title,\n.rst-content .wy-alert-info.hint .admonition-title,\n.rst-content .wy-alert-info.important .admonition-title,\n.rst-content .wy-alert-info.tip .admonition-title,\n.rst-content .wy-alert-info.warning .admonition-title,\n.rst-content .seealso .admonition-title,\n.rst-content .wy-alert-info.admonition-todo .admonition-title {\n background: #6ab0de\n}\n\n.wy-alert.wy-alert-success,\n.rst-content .wy-alert-success.note,\n.rst-content .wy-alert-success.attention,\n.rst-content .wy-alert-success.caution,\n.rst-content .wy-alert-success.danger,\n.rst-content .wy-alert-success.error,\n.rst-content .hint,\n.rst-content .important,\n.rst-content .tip,\n.rst-content .wy-alert-success.warning,\n.rst-content .wy-alert-success.seealso,\n.rst-content .wy-alert-success.admonition-todo {\n background: #dbfaf4\n}\n\n.wy-alert.wy-alert-success .wy-alert-title,\n.rst-content .wy-alert-success.note .wy-alert-title,\n.rst-content .wy-alert-success.attention .wy-alert-title,\n.rst-content .wy-alert-success.caution .wy-alert-title,\n.rst-content .wy-alert-success.danger .wy-alert-title,\n.rst-content .wy-alert-success.error .wy-alert-title,\n.rst-content .hint .wy-alert-title,\n.rst-content .important .wy-alert-title,\n.rst-content .tip .wy-alert-title,\n.rst-content .wy-alert-success.warning .wy-alert-title,\n.rst-content .wy-alert-success.seealso .wy-alert-title,\n.rst-content .wy-alert-success.admonition-todo .wy-alert-title,\n.wy-alert.wy-alert-success .rst-content .admonition-title,\n.rst-content .wy-alert.wy-alert-success .admonition-title,\n.rst-content .wy-alert-success.note .admonition-title,\n.rst-content .wy-alert-success.attention .admonition-title,\n.rst-content .wy-alert-success.caution .admonition-title,\n.rst-content .wy-alert-success.danger .admonition-title,\n.rst-content .wy-alert-success.error .admonition-title,\n.rst-content .hint .admonition-title,\n.rst-content .important .admonition-title,\n.rst-content .tip .admonition-title,\n.rst-content .wy-alert-success.warning .admonition-title,\n.rst-content .wy-alert-success.seealso .admonition-title,\n.rst-content .wy-alert-success.admonition-todo .admonition-title {\n background: #1abc9c\n}\n\n.wy-alert.wy-alert-neutral,\n.rst-content .wy-alert-neutral.note,\n.rst-content .wy-alert-neutral.attention,\n.rst-content .wy-alert-neutral.caution,\n.rst-content .wy-alert-neutral.danger,\n.rst-content .wy-alert-neutral.error,\n.rst-content .wy-alert-neutral.hint,\n.rst-content .wy-alert-neutral.important,\n.rst-content .wy-alert-neutral.tip,\n.rst-content .wy-alert-neutral.warning,\n.rst-content .wy-alert-neutral.seealso,\n.rst-content .wy-alert-neutral.admonition-todo {\n background: #f3f6f6\n}\n\n.wy-alert.wy-alert-neutral .wy-alert-title,\n.rst-content .wy-alert-neutral.note .wy-alert-title,\n.rst-content .wy-alert-neutral.attention .wy-alert-title,\n.rst-content .wy-alert-neutral.caution .wy-alert-title,\n.rst-content .wy-alert-neutral.danger .wy-alert-title,\n.rst-content .wy-alert-neutral.error .wy-alert-title,\n.rst-content .wy-alert-neutral.hint .wy-alert-title,\n.rst-content .wy-alert-neutral.important .wy-alert-title,\n.rst-content .wy-alert-neutral.tip .wy-alert-title,\n.rst-content .wy-alert-neutral.warning .wy-alert-title,\n.rst-content .wy-alert-neutral.seealso .wy-alert-title,\n.rst-content .wy-alert-neutral.admonition-todo .wy-alert-title,\n.wy-alert.wy-alert-neutral .rst-content .admonition-title,\n.rst-content .wy-alert.wy-alert-neutral .admonition-title,\n.rst-content .wy-alert-neutral.note .admonition-title,\n.rst-content .wy-alert-neutral.attention .admonition-title,\n.rst-content .wy-alert-neutral.caution .admonition-title,\n.rst-content .wy-alert-neutral.danger .admonition-title,\n.rst-content .wy-alert-neutral.error .admonition-title,\n.rst-content .wy-alert-neutral.hint .admonition-title,\n.rst-content .wy-alert-neutral.important .admonition-title,\n.rst-content .wy-alert-neutral.tip .admonition-title,\n.rst-content .wy-alert-neutral.warning .admonition-title,\n.rst-content .wy-alert-neutral.seealso .admonition-title,\n.rst-content .wy-alert-neutral.admonition-todo .admonition-title {\n color: #404040;\n background: #e1e4e5\n}\n\n.wy-alert.wy-alert-neutral a,\n.rst-content .wy-alert-neutral.note a,\n.rst-content .wy-alert-neutral.attention a,\n.rst-content .wy-alert-neutral.caution a,\n.rst-content .wy-alert-neutral.danger a,\n.rst-content .wy-alert-neutral.error a,\n.rst-content .wy-alert-neutral.hint a,\n.rst-content .wy-alert-neutral.important a,\n.rst-content .wy-alert-neutral.tip a,\n.rst-content .wy-alert-neutral.warning a,\n.rst-content .wy-alert-neutral.seealso a,\n.rst-content .wy-alert-neutral.admonition-todo a {\n color: #008BF3\n}\n\n.wy-alert p:last-child,\n.rst-content .note p:last-child,\n.rst-content .attention p:last-child,\n.rst-content .caution p:last-child,\n.rst-content .danger p:last-child,\n.rst-content .error p:last-child,\n.rst-content .hint p:last-child,\n.rst-content .important p:last-child,\n.rst-content .tip p:last-child,\n.rst-content .warning p:last-child,\n.rst-content .seealso p:last-child,\n.rst-content .admonition-todo p:last-child {\n margin-bottom: 0\n}\n\n.wy-tray-container {\n position: fixed;\n bottom: 0;\n left: 0;\n z-index: 600\n}\n\n.wy-tray-container li {\n display: block;\n width: 300px;\n background: transparent;\n color: #fff;\n text-align: center;\n box-shadow: 0 5px 5px 0 rgba(0,0,0,0.1);\n padding: 0 24px;\n min-width: 20%;\n opacity: 0;\n height: 0;\n line-height: 56px;\n overflow: hidden;\n -webkit-transition: all .3s ease-in;\n -moz-transition: all .3s ease-in;\n transition: all .3s ease-in\n}\n\n.wy-tray-container li.wy-tray-item-success {\n background: #27AE60\n}\n\n.wy-tray-container li.wy-tray-item-info {\n background: #2980B9\n}\n\n.wy-tray-container li.wy-tray-item-warning {\n background: #E67E22\n}\n\n.wy-tray-container li.wy-tray-item-danger {\n background: #E74C3C\n}\n\n.wy-tray-container li.on {\n opacity: 1;\n height: 56px\n}\n\n@media screen and (max-width: 768px) {\n .wy-tray-container {\n bottom: auto;\n top: 0;\n width: 100%\n }\n\n .wy-tray-container li {\n width: 100%\n }\n\n}\n\nbutton {\n font-size: 100%;\n margin: 0;\n vertical-align: baseline;\n *vertical-align: middle;\n cursor: pointer;\n line-height: normal;\n -webkit-appearance: button;\n *overflow: visible\n}\n\nbutton::-moz-focus-inner,\ninput::-moz-focus-inner {\n border: 0;\n padding: 0\n}\n\nbutton[disabled] {\n cursor: default\n}\n\n.btn {\n font-family: \"Source Sans Pro\",\"proxima-nova\",\"Helvetica Neue\",Arial,sans-serif;\n display: inline-block;\n padding: 10px 30px;\n margin-bottom: 0;\n font-weight: 400;\n text-align: center;\n vertical-align: middle;\n -ms-touch-action: manipulation;\n touch-action: manipulation;\n cursor: pointer;\n background-image: none;\n background-color: #4a90e2;\n border: 1px solid transparent;\n white-space: nowrap;\n color: rgba(255,255,255,0.95);\n border-radius: 3px;\n -webkit-user-select: none;\n -moz-user-select: none;\n -ms-user-select: none;\n user-select: none;\n outline: none\n}\n\n.btn-hover {\n background: #2e8ece;\n color: #fff\n}\n\n.btn:disabled {\n background-image: none;\n filter: progid: DXImageTransform.Microsoft.gradient(enabled=false);\n filter: alpha(opacity=40);\n opacity: .4;\n cursor: not-allowed;\n box-shadow: none\n}\n\n.btn-disabled {\n background-image: none;\n filter: progid: DXImageTransform.Microsoft.gradient(enabled=false);\n filter: alpha(opacity=40);\n opacity: .4;\n cursor: not-allowed;\n box-shadow: none\n}\n\n.btn-disabled:hover,\n.btn-disabled:focus,\n.btn-disabled:active {\n background-image: none;\n filter: progid: DXImageTransform.Microsoft.gradient(enabled=false);\n filter: alpha(opacity=40);\n opacity: .4;\n cursor: not-allowed;\n box-shadow: none\n}\n\n.btn::-moz-focus-inner {\n padding: 0;\n border: 0\n}\n\n.btn-small {\n font-size: 80%\n}\n\n.btn-info {\n background-color: #2980B9!important\n}\n\n.btn-info:hover {\n background-color: #2e8ece!important\n}\n\n.btn-neutral {\n font-size: 14px;\n color: #fff;\n font-weight: 300;\n /*background-color: #008bf3!important*/\n}\n\n.btn-success {\n background-color: #27AE60!important\n}\n\n.btn-success:hover {\n background-color: #295!important\n}\n\n.btn-danger {\n background-color: #E74C3C!important\n}\n\n.btn-danger:hover {\n background-color: #ea6153!important\n}\n\n.btn-warning {\n background-color: #E67E22!important\n}\n\n.btn-warning:hover {\n background-color: #e98b39!important\n}\n\n.btn-invert {\n background-color: #222\n}\n\n.btn-invert:hover {\n background-color: #2f2f2f!important\n}\n\n.btn-link {\n background-color: transparent!important;\n color: #2980B9;\n box-shadow: none;\n border-color: transparent!important\n}\n\n.btn-link:hover {\n background-color: transparent!important;\n color: #409ad5!important;\n box-shadow: none\n}\n\n.btn-link:active {\n background-color: transparent!important;\n color: #409ad5!important;\n box-shadow: none\n}\n\n.btn-link:visited {\n color: #9B59B6\n}\n\n.wy-btn-group .btn,\n.wy-control .btn {\n vertical-align: middle\n}\n\n.wy-btn-group {\n margin-bottom: 24px;\n *zoom: 1\n}\n\n.wy-btn-group:before,\n.wy-btn-group:after {\n display: table;\n content: \"\"\n}\n\n.wy-btn-group:after {\n clear: both\n}\n\n.wy-dropdown {\n position: relative;\n display: inline-block\n}\n\n.wy-dropdown-active .wy-dropdown-menu {\n display: block\n}\n\n.wy-dropdown-menu {\n position: absolute;\n left: 0;\n display: none;\n float: left;\n top: 100%;\n min-width: 100%;\n background: #fcfcfc;\n z-index: 100;\n border: solid 1px #cfd7dd;\n box-shadow: 0 2px 2px 0 rgba(0,0,0,0.1);\n padding: 12px\n}\n\n.wy-dropdown-menu>dd>a {\n display: block;\n clear: both;\n color: #404040;\n white-space: nowrap;\n font-size: 90%;\n padding: 0 12px;\n cursor: pointer\n}\n\n.wy-dropdown-menu>dd>a:hover {\n background: #2980B9;\n color: #fff\n}\n\n.wy-dropdown-menu>dd.divider {\n border-top: solid 1px #cfd7dd;\n margin: 6px 0\n}\n\n.wy-dropdown-menu>dd.search {\n padding-bottom: 12px\n}\n\n.wy-dropdown-menu>dd.search input[type=\"search\"] {\n width: 100%\n}\n\n.wy-dropdown-menu>dd.call-to-action {\n background: #e3e3e3;\n text-transform: uppercase;\n font-weight: 500;\n font-size: 80%\n}\n\n.wy-dropdown-menu>dd.call-to-action:hover {\n background: #e3e3e3\n}\n\n.wy-dropdown-menu>dd.call-to-action .btn {\n color: #fff\n}\n\n.wy-dropdown.wy-dropdown-up .wy-dropdown-menu {\n bottom: 100%;\n top: auto;\n left: auto;\n right: 0\n}\n\n.wy-dropdown.wy-dropdown-bubble .wy-dropdown-menu {\n background: #fcfcfc;\n margin-top: 2px\n}\n\n.wy-dropdown.wy-dropdown-bubble .wy-dropdown-menu a {\n padding: 6px 12px\n}\n\n.wy-dropdown.wy-dropdown-bubble .wy-dropdown-menu a:hover {\n background: #2980B9;\n color: #fff\n}\n\n.wy-dropdown.wy-dropdown-left .wy-dropdown-menu {\n right: 0;\n left: auto;\n text-align: right\n}\n\n.wy-dropdown-arrow:before {\n content: \" \";\n border-bottom: 5px solid #f5f5f5;\n border-left: 5px solid transparent;\n border-right: 5px solid transparent;\n position: absolute;\n display: block;\n top: -4px;\n left: 50%;\n margin-left: -3px\n}\n\n.wy-dropdown-arrow.wy-dropdown-arrow-left:before {\n left: 11px\n}\n\n.wy-form-stacked select {\n display: block\n}\n\n.wy-form-aligned input,\n.wy-form-aligned textarea,\n.wy-form-aligned select,\n.wy-form-aligned .wy-help-inline,\n.wy-form-aligned label {\n display: inline-block;\n *display: inline;\n *zoom: 1;\n vertical-align: middle\n}\n\n.wy-form-aligned .wy-control-group>label {\n display: inline-block;\n vertical-align: middle;\n width: 10em;\n margin: 6px 12px 0 0;\n float: left\n}\n\n.wy-form-aligned .wy-control {\n float: left\n}\n\n.wy-form-aligned .wy-control label {\n display: block\n}\n\n.wy-form-aligned .wy-control select {\n margin-top: 6px\n}\n\nfieldset {\n border: 0;\n margin: 0;\n padding: 0\n}\n\nlegend {\n display: block;\n width: 100%;\n border: 0;\n padding: 0;\n white-space: normal;\n margin-bottom: 24px;\n font-size: 150%;\n *margin-left: -7px\n}\n\nlabel {\n display: block;\n margin: 0 0 .3125em;\n color: #333;\n font-size: 90%\n}\n\ninput,\nselect,\ntextarea {\n font-size: 100%;\n margin: 0;\n vertical-align: baseline;\n *vertical-align: middle\n}\n\n.wy-control-group {\n margin-bottom: 24px;\n *zoom: 1;\n max-width: 68em;\n margin-left: auto;\n margin-right: auto;\n *zoom: 1\n}\n\n.wy-control-group:before,\n.wy-control-group:after {\n display: table;\n content: \"\"\n}\n\n.wy-control-group:after {\n clear: both\n}\n\n.wy-control-group:before,\n.wy-control-group:after {\n display: table;\n content: \"\"\n}\n\n.wy-control-group:after {\n clear: both\n}\n\n.wy-control-group.wy-control-group-required>label:after {\n content: \" *\";\n color: #E74C3C\n}\n\n.wy-control-group .wy-form-full,\n.wy-control-group .wy-form-halves,\n.wy-control-group .wy-form-thirds {\n padding-bottom: 12px\n}\n\n.wy-control-group .wy-form-full select,\n.wy-control-group .wy-form-halves select,\n.wy-control-group .wy-form-thirds select {\n width: 100%\n}\n\n.wy-control-group .wy-form-full input[type=\"text\"],\n.wy-control-group .wy-form-full input[type=\"password\"],\n.wy-control-group .wy-form-full input[type=\"email\"],\n.wy-control-group .wy-form-full input[type=\"url\"],\n.wy-control-group .wy-form-full input[type=\"date\"],\n.wy-control-group .wy-form-full input[type=\"month\"],\n.wy-control-group .wy-form-full input[type=\"time\"],\n.wy-control-group .wy-form-full input[type=\"datetime\"],\n.wy-control-group .wy-form-full input[type=\"datetime-local\"],\n.wy-control-group .wy-form-full input[type=\"week\"],\n.wy-control-group .wy-form-full input[type=\"number\"],\n.wy-control-group .wy-form-full input[type=\"search\"],\n.wy-control-group .wy-form-full input[type=\"tel\"],\n.wy-control-group .wy-form-full input[type=\"color\"],\n.wy-control-group .wy-form-halves input[type=\"text\"],\n.wy-control-group .wy-form-halves input[type=\"password\"],\n.wy-control-group .wy-form-halves input[type=\"email\"],\n.wy-control-group .wy-form-halves input[type=\"url\"],\n.wy-control-group .wy-form-halves input[type=\"date\"],\n.wy-control-group .wy-form-halves input[type=\"month\"],\n.wy-control-group .wy-form-halves input[type=\"time\"],\n.wy-control-group .wy-form-halves input[type=\"datetime\"],\n.wy-control-group .wy-form-halves input[type=\"datetime-local\"],\n.wy-control-group .wy-form-halves input[type=\"week\"],\n.wy-control-group .wy-form-halves input[type=\"number\"],\n.wy-control-group .wy-form-halves input[type=\"search\"],\n.wy-control-group .wy-form-halves input[type=\"tel\"],\n.wy-control-group .wy-form-halves input[type=\"color\"],\n.wy-control-group .wy-form-thirds input[type=\"text\"],\n.wy-control-group .wy-form-thirds input[type=\"password\"],\n.wy-control-group .wy-form-thirds input[type=\"email\"],\n.wy-control-group .wy-form-thirds input[type=\"url\"],\n.wy-control-group .wy-form-thirds input[type=\"date\"],\n.wy-control-group .wy-form-thirds input[type=\"month\"],\n.wy-control-group .wy-form-thirds input[type=\"time\"],\n.wy-control-group .wy-form-thirds input[type=\"datetime\"],\n.wy-control-group .wy-form-thirds input[type=\"datetime-local\"],\n.wy-control-group .wy-form-thirds input[type=\"week\"],\n.wy-control-group .wy-form-thirds input[type=\"number\"],\n.wy-control-group .wy-form-thirds input[type=\"search\"],\n.wy-control-group .wy-form-thirds input[type=\"tel\"],\n.wy-control-group .wy-form-thirds input[type=\"color\"] {\n width: 100%\n}\n\n.wy-control-group .wy-form-full {\n float: left;\n display: block;\n margin-right: 2.35765%;\n width: 100%;\n margin-right: 0\n}\n\n.wy-control-group .wy-form-full:last-child {\n margin-right: 0\n}\n\n.wy-control-group .wy-form-halves {\n float: left;\n display: block;\n margin-right: 2.35765%;\n width: 48.82117%\n}\n\n.wy-control-group .wy-form-halves:last-child {\n margin-right: 0\n}\n\n.wy-control-group .wy-form-halves:nth-of-type(2n) {\n margin-right: 0\n}\n\n.wy-control-group .wy-form-halves:nth-of-type(2n+1) {\n clear: left\n}\n\n.wy-control-group .wy-form-thirds {\n float: left;\n display: block;\n margin-right: 2.35765%;\n width: 31.76157%\n}\n\n.wy-control-group .wy-form-thirds:last-child {\n margin-right: 0\n}\n\n.wy-control-group .wy-form-thirds:nth-of-type(3n) {\n margin-right: 0\n}\n\n.wy-control-group .wy-form-thirds:nth-of-type(3n+1) {\n clear: left\n}\n\n.wy-control-group.wy-control-group-no-input .wy-control {\n margin: 6px 0 0;\n font-size: 90%\n}\n\n.wy-control-no-input {\n display: inline-block;\n margin: 6px 0 0;\n font-size: 90%\n}\n\n.wy-control-group.fluid-input input[type=\"text\"],\n.wy-control-group.fluid-input input[type=\"password\"],\n.wy-control-group.fluid-input input[type=\"email\"],\n.wy-control-group.fluid-input input[type=\"url\"],\n.wy-control-group.fluid-input input[type=\"date\"],\n.wy-control-group.fluid-input input[type=\"month\"],\n.wy-control-group.fluid-input input[type=\"time\"],\n.wy-control-group.fluid-input input[type=\"datetime\"],\n.wy-control-group.fluid-input input[type=\"datetime-local\"],\n.wy-control-group.fluid-input input[type=\"week\"],\n.wy-control-group.fluid-input input[type=\"number\"],\n.wy-control-group.fluid-input input[type=\"search\"],\n.wy-control-group.fluid-input input[type=\"tel\"],\n.wy-control-group.fluid-input input[type=\"color\"] {\n width: 100%\n}\n\n.wy-form-message-inline {\n display: inline-block;\n padding-left: .3em;\n color: #666;\n vertical-align: middle;\n font-size: 90%\n}\n\n.wy-form-message {\n display: block;\n color: #999;\n font-size: 70%;\n margin-top: .3125em;\n font-style: italic\n}\n\n.wy-form-message p {\n font-size: inherit;\n font-style: italic;\n margin-bottom: 6px\n}\n\n.wy-form-message p:last-child {\n margin-bottom: 0\n}\n\ninput {\n line-height: normal\n}\n\ninput[type=\"button\"],\ninput[type=\"reset\"],\ninput[type=\"submit\"] {\n -webkit-appearance: button;\n cursor: pointer;\n font-family: \"Source Sans Pro\",\"proxima-nova\",\"Helvetica Neue\",Arial,sans-serif;\n *overflow: visible\n}\n\ninput[type=\"text\"],\ninput[type=\"password\"],\ninput[type=\"email\"],\ninput[type=\"url\"],\ninput[type=\"date\"],\ninput[type=\"month\"],\ninput[type=\"time\"],\ninput[type=\"datetime\"],\ninput[type=\"datetime-local\"],\ninput[type=\"week\"],\ninput[type=\"number\"],\ninput[type=\"search\"],\ninput[type=\"tel\"],\ninput[type=\"color\"] {\n -webkit-appearance: none;\n padding: 6px;\n display: inline-block;\n border: 1px solid #ccc;\n font-size: 80%;\n font-family: \"Source Sans Pro\",\"proxima-nova\",\"Helvetica Neue\",Arial,sans-serif;\n box-shadow: inset 0 1px 3px #ddd;\n border-radius: 0;\n -webkit-transition: border .3s linear;\n -moz-transition: border .3s linear;\n transition: border .3s linear\n}\n\ninput[type=\"datetime-local\"] {\n padding: .34375em .625em\n}\n\ninput[disabled] {\n cursor: default\n}\n\ninput[type=\"checkbox\"],\ninput[type=\"radio\"] {\n -webkit-box-sizing: border-box;\n -moz-box-sizing: border-box;\n box-sizing: border-box;\n padding: 0;\n margin-right: .3125em;\n *height: 13px;\n *width: 13px\n}\n\ninput[type=\"search\"] {\n -webkit-box-sizing: border-box;\n -moz-box-sizing: border-box;\n box-sizing: border-box\n}\n\ninput[type=\"search\"]::-webkit-search-cancel-button,\ninput[type=\"search\"]::-webkit-search-decoration {\n -webkit-appearance: none\n}\n\ninput[type=\"text\"]:focus,\ninput[type=\"password\"]:focus,\ninput[type=\"email\"]:focus,\ninput[type=\"url\"]:focus,\ninput[type=\"date\"]:focus,\ninput[type=\"month\"]:focus,\ninput[type=\"time\"]:focus,\ninput[type=\"datetime\"]:focus,\ninput[type=\"datetime-local\"]:focus,\ninput[type=\"week\"]:focus,\ninput[type=\"number\"]:focus,\ninput[type=\"search\"]:focus,\ninput[type=\"tel\"]:focus,\ninput[type=\"color\"]:focus {\n outline: 0;\n outline: thin dotted \\9;\n border-color: #333\n}\n\ninput.no-focus:focus {\n border-color: #ccc!important\n}\n\ninput[type=\"file\"]:focus,\ninput[type=\"radio\"]:focus,\ninput[type=\"checkbox\"]:focus {\n outline: thin dotted #333;\n outline: 1px auto #129FEA\n}\n\ninput[type=\"text\"][disabled],\ninput[type=\"password\"][disabled],\ninput[type=\"email\"][disabled],\ninput[type=\"url\"][disabled],\ninput[type=\"date\"][disabled],\ninput[type=\"month\"][disabled],\ninput[type=\"time\"][disabled],\ninput[type=\"datetime\"][disabled],\ninput[type=\"datetime-local\"][disabled],\ninput[type=\"week\"][disabled],\ninput[type=\"number\"][disabled],\ninput[type=\"search\"][disabled],\ninput[type=\"tel\"][disabled],\ninput[type=\"color\"][disabled] {\n cursor: not-allowed;\n background-color: #fafafa\n}\n\ninput:focus:invalid,\ntextarea:focus:invalid,\nselect:focus:invalid {\n color: #E74C3C;\n border: 1px solid #E74C3C\n}\n\ninput:focus:invalid:focus,\ntextarea:focus:invalid:focus,\nselect:focus:invalid:focus {\n border-color: #E74C3C\n}\n\ninput[type=\"file\"]:focus:invalid:focus,\ninput[type=\"radio\"]:focus:invalid:focus,\ninput[type=\"checkbox\"]:focus:invalid:focus {\n outline-color: #E74C3C\n}\n\ninput.wy-input-large {\n padding: 12px;\n font-size: 100%\n}\n\ntextarea {\n overflow: auto;\n vertical-align: top;\n width: 100%;\n font-family: \"Source Sans Pro\",\"proxima-nova\",\"Helvetica Neue\",Arial,sans-serif\n}\n\nselect,\ntextarea {\n padding: .5em .625em;\n display: inline-block;\n border: 1px solid #ccc;\n font-size: 80%;\n box-shadow: inset 0 1px 3px #ddd;\n -webkit-transition: border .3s linear;\n -moz-transition: border .3s linear;\n transition: border .3s linear\n}\n\nselect {\n border: 1px solid #ccc;\n background-color: #fff\n}\n\nselect[multiple] {\n height: auto\n}\n\nselect:focus,\ntextarea:focus {\n outline: 0\n}\n\nselect[disabled],\ntextarea[disabled],\ninput[readonly],\nselect[readonly],\ntextarea[readonly] {\n cursor: not-allowed;\n background-color: #fafafa\n}\n\ninput[type=\"radio\"][disabled],\ninput[type=\"checkbox\"][disabled] {\n cursor: not-allowed\n}\n\n.wy-checkbox,\n.wy-radio {\n margin: 6px 0;\n color: #404040;\n display: block\n}\n\n.wy-checkbox input,\n.wy-radio input {\n vertical-align: baseline\n}\n\n.wy-form-message-inline {\n display: inline-block;\n *display: inline;\n *zoom: 1;\n vertical-align: middle\n}\n\n.wy-input-prefix,\n.wy-input-suffix {\n white-space: nowrap;\n padding: 6px\n}\n\n.wy-input-prefix .wy-input-context,\n.wy-input-suffix .wy-input-context {\n line-height: 27px;\n padding: 0 8px;\n display: inline-block;\n font-size: 80%;\n background-color: #f3f6f6;\n border: solid 1px #ccc;\n color: #999\n}\n\n.wy-input-suffix .wy-input-context {\n border-left: 0\n}\n\n.wy-input-prefix .wy-input-context {\n border-right: 0\n}\n\n.wy-switch {\n width: 36px;\n height: 12px;\n margin: 12px 0;\n position: relative;\n border-radius: 4px;\n background: #ccc;\n cursor: pointer;\n -webkit-transition: all .2s ease-in-out;\n -moz-transition: all .2s ease-in-out;\n transition: all .2s ease-in-out\n}\n\n.wy-switch:before {\n position: absolute;\n content: \"\";\n display: block;\n width: 18px;\n height: 18px;\n border-radius: 4px;\n background: #999;\n left: -3px;\n top: -3px;\n -webkit-transition: all .2s ease-in-out;\n -moz-transition: all .2s ease-in-out;\n transition: all .2s ease-in-out\n}\n\n.wy-switch:after {\n content: \"false\";\n position: absolute;\n left: 48px;\n display: block;\n font-size: 12px;\n color: #ccc\n}\n\n.wy-switch.active {\n background: #1e8449\n}\n\n.wy-switch.active:before {\n left: 24px;\n background: #27AE60\n}\n\n.wy-switch.active:after {\n content: \"true\"\n}\n\n.wy-switch.disabled,\n.wy-switch.active.disabled {\n cursor: not-allowed\n}\n\n.wy-control-group.wy-control-group-error .wy-form-message,\n.wy-control-group.wy-control-group-error>label {\n color: #E74C3C\n}\n\n.wy-control-group.wy-control-group-error input[type=\"text\"],\n.wy-control-group.wy-control-group-error input[type=\"password\"],\n.wy-control-group.wy-control-group-error input[type=\"email\"],\n.wy-control-group.wy-control-group-error input[type=\"url\"],\n.wy-control-group.wy-control-group-error input[type=\"date\"],\n.wy-control-group.wy-control-group-error input[type=\"month\"],\n.wy-control-group.wy-control-group-error input[type=\"time\"],\n.wy-control-group.wy-control-group-error input[type=\"datetime\"],\n.wy-control-group.wy-control-group-error input[type=\"datetime-local\"],\n.wy-control-group.wy-control-group-error input[type=\"week\"],\n.wy-control-group.wy-control-group-error input[type=\"number\"],\n.wy-control-group.wy-control-group-error input[type=\"search\"],\n.wy-control-group.wy-control-group-error input[type=\"tel\"],\n.wy-control-group.wy-control-group-error input[type=\"color\"] {\n border: solid 1px #E74C3C\n}\n\n.wy-control-group.wy-control-group-error textarea {\n border: solid 1px #E74C3C\n}\n\n.wy-inline-validate {\n white-space: nowrap\n}\n\n.wy-inline-validate .wy-input-context {\n padding: .5em .625em;\n display: inline-block;\n font-size: 80%\n}\n\n.wy-inline-validate.wy-inline-validate-success .wy-input-context {\n color: #27AE60\n}\n\n.wy-inline-validate.wy-inline-validate-danger .wy-input-context {\n color: #E74C3C\n}\n\n.wy-inline-validate.wy-inline-validate-warning .wy-input-context {\n color: #E67E22\n}\n\n.wy-inline-validate.wy-inline-validate-info .wy-input-context {\n color: #2980B9\n}\n\n.rotate-90 {\n -webkit-transform: rotate(90deg);\n -moz-transform: rotate(90deg);\n -ms-transform: rotate(90deg);\n -o-transform: rotate(90deg);\n transform: rotate(90deg)\n}\n\n.rotate-180 {\n -webkit-transform: rotate(180deg);\n -moz-transform: rotate(180deg);\n -ms-transform: rotate(180deg);\n -o-transform: rotate(180deg);\n transform: rotate(180deg)\n}\n\n.rotate-270 {\n -webkit-transform: rotate(270deg);\n -moz-transform: rotate(270deg);\n -ms-transform: rotate(270deg);\n -o-transform: rotate(270deg);\n transform: rotate(270deg)\n}\n\n.mirror {\n -webkit-transform: scaleX(-1);\n -moz-transform: scaleX(-1);\n -ms-transform: scaleX(-1);\n -o-transform: scaleX(-1);\n transform: scaleX(-1)\n}\n\n.mirror.rotate-90 {\n -webkit-transform: scaleX(-1) rotate(90deg);\n -moz-transform: scaleX(-1) rotate(90deg);\n -ms-transform: scaleX(-1) rotate(90deg);\n -o-transform: scaleX(-1) rotate(90deg);\n transform: scaleX(-1) rotate(90deg)\n}\n\n.mirror.rotate-180 {\n -webkit-transform: scaleX(-1) rotate(180deg);\n -moz-transform: scaleX(-1) rotate(180deg);\n -ms-transform: scaleX(-1) rotate(180deg);\n -o-transform: scaleX(-1) rotate(180deg);\n transform: scaleX(-1) rotate(180deg)\n}\n\n.mirror.rotate-270 {\n -webkit-transform: scaleX(-1) rotate(270deg);\n -moz-transform: scaleX(-1) rotate(270deg);\n -ms-transform: scaleX(-1) rotate(270deg);\n -o-transform: scaleX(-1) rotate(270deg);\n transform: scaleX(-1) rotate(270deg)\n}\n\n@media only screen and (max-width: 480px) {\n .wy-form button[type=\"submit\"] {\n margin: .7em 0 0\n }\n\n .wy-form input[type=\"text\"],\n .wy-form input[type=\"password\"],\n .wy-form input[type=\"email\"],\n .wy-form input[type=\"url\"],\n .wy-form input[type=\"date\"],\n .wy-form input[type=\"month\"],\n .wy-form input[type=\"time\"],\n .wy-form input[type=\"datetime\"],\n .wy-form input[type=\"datetime-local\"],\n .wy-form input[type=\"week\"],\n .wy-form input[type=\"number\"],\n .wy-form input[type=\"search\"],\n .wy-form input[type=\"tel\"],\n .wy-form input[type=\"color\"] {\n margin-bottom: .3em;\n display: block\n }\n\n .wy-form label {\n margin-bottom: .3em;\n display: block\n }\n\n .wy-form input[type=\"password\"],\n .wy-form input[type=\"email\"],\n .wy-form input[type=\"url\"],\n .wy-form input[type=\"date\"],\n .wy-form input[type=\"month\"],\n .wy-form input[type=\"time\"],\n .wy-form input[type=\"datetime\"],\n .wy-form input[type=\"datetime-local\"],\n .wy-form input[type=\"week\"],\n .wy-form input[type=\"number\"],\n .wy-form input[type=\"search\"],\n .wy-form input[type=\"tel\"],\n .wy-form input[type=\"color\"] {\n margin-bottom: 0\n }\n\n .wy-form-aligned .wy-control-group label {\n margin-bottom: .3em;\n text-align: left;\n display: block;\n width: 100%\n }\n\n .wy-form-aligned .wy-control {\n margin: 1.5em 0 0\n }\n\n .wy-form .wy-help-inline,\n .wy-form-message-inline,\n .wy-form-message {\n display: block;\n font-size: 80%;\n padding: 6px 0\n }\n\n}\n\n@media screen and (max-width: 768px) {\n .tablet-hide {\n display: none\n }\n\n}\n\n@media screen and (max-width: 480px) {\n .mobile-hide {\n display: none\n }\n\n}\n\n.float-left {\n float: left\n}\n\n.float-right {\n float: right\n}\n\n.full-width {\n width: 100%\n}\n\n.wy-table,\n.rst-content table.docutils,\n.rst-content table.field-list {\n border-collapse: collapse;\n border-spacing: 0;\n empty-cells: show;\n margin-bottom: 24px\n}\n\n.wy-table caption,\n.rst-content table.docutils caption,\n.rst-content table.field-list caption {\n color: #000;\n font: italic 85%/1 arial,sans-serif;\n padding: 1em 0;\n text-align: center\n}\n\n.wy-table td,\n.rst-content table.docutils td,\n.rst-content table.field-list td,\n.wy-table th,\n.rst-content table.docutils th,\n.rst-content table.field-list th {\n font-size: 16px;\n margin: 0;\n overflow: visible;\n padding: 6px 13px\n}\n\n.wy-table td:first-child,\n.rst-content table.docutils td:first-child,\n.rst-content table.field-list td:first-child,\n.wy-table th:first-child,\n.rst-content table.docutils th:first-child,\n.rst-content table.field-list th:first-child {\n border-left-width: 0\n}\n\n.wy-table thead,\n.rst-content table.docutils thead,\n.rst-content table.field-list thead {\n color: #000;\n text-align: center;\n vertical-align: bottom;\n white-space: nowrap\n}\n\n.wy-table thead th,\n.rst-content table.docutils thead th,\n.rst-content table.field-list thead th {\n font-weight: 700;\n border-bottom: solid 1px #ddd;\n}\n\n.wy-table td,\n.rst-content table.docutils td,\n.rst-content table.field-list td {\n background-color: transparent;\n vertical-align: middle\n}\n\n.wy-table td p,\n.rst-content table.docutils td p,\n.rst-content table.field-list td p {\n line-height: 18px\n}\n\n.wy-table td p:last-child,\n.rst-content table.docutils td p:last-child,\n.rst-content table.field-list td p:last-child {\n margin-bottom: 0\n}\n\n.wy-table .wy-table-cell-min,\n.rst-content table.docutils .wy-table-cell-min,\n.rst-content table.field-list .wy-table-cell-min {\n width: 1%;\n padding-right: 0\n}\n\n.wy-table .wy-table-cell-min input[type=checkbox],\n.rst-content table.docutils .wy-table-cell-min input[type=checkbox],\n.rst-content table.field-list .wy-table-cell-min input[type=checkbox],\n.wy-table .wy-table-cell-min input[type=checkbox],\n.rst-content table.docutils .wy-table-cell-min input[type=checkbox],\n.rst-content table.field-list .wy-table-cell-min input[type=checkbox] {\n margin: 0\n}\n\n.wy-table-secondary {\n color: gray;\n font-size: 90%\n}\n\n.wy-table-tertiary {\n color: gray;\n font-size: 80%\n}\n\n.wy-table-odd td,\n.wy-table-striped tr:nth-child(2n-1) td,\n.rst-content table.docutils:not(.field-list) tr:nth-child(2n-1) td {\n background-color: #fcfcfc;\n}\n\n.wy-table-odd td,\n.wy-table-striped tr:nth-child(2n) td,\n.rst-content table.docutils:not(.field-list) tr:nth-child(2n) td {\n background-color: #f8f8f8;\n}\n\n.wy-table-backed {\n background-color: #f8f8f8;\n}\n\n.wy-table-bordered-all,\n.rst-content table.docutils {\n border: 1px solid #ddd;\n line-height: 24px;\n}\n\n.wy-table-bordered-all td,\n.rst-content table.docutils td {\n border-bottom: 1px solid #ddd;\n border-left: 1px solid #ddd;\n}\n\n.wy-table-bordered-all tbody>tr:last-child td,\n.rst-content table.docutils tbody>tr:last-child td {\n border-bottom-width: 0\n}\n\n.wy-table-bordered {\n border: 1px solid #ddd;\n}\n\n.wy-table-bordered-rows td {\n border-bottom: 1px solid #ddd;\n}\n\n.wy-table-bordered-rows tbody>tr:last-child td {\n border-bottom-width: 0\n}\n\n.wy-table-horizontal tbody>tr:last-child td {\n border-bottom-width: 0\n}\n\n.wy-table-horizontal td,\n.wy-table-horizontal th {\n border-width: 0 0 1px;\n border-bottom: 1px solid #ddd;\n}\n\n.wy-table-horizontal tbody>tr:last-child td {\n border-bottom-width: 0\n}\n\n.wy-table-responsive {\n margin-bottom: 24px;\n max-width: 100%;\n overflow: auto\n}\n\n.wy-table-responsive table {\n margin-bottom: 0!important\n}\n\n.wy-table-responsive table td,\n.wy-table-responsive table th {\n white-space: nowrap\n}\n\na {\n text-decoration: none;\n cursor: pointer\n}\n\nhtml {\n height: 100%;\n overflow-x: hidden\n}\n\nbody {\n font-family: -apple-system, BlinkMacSystemFont, \"Segoe UI\", Roboto, Helvetica, Arial, sans-serif, \"Apple Color Emoji\", \"Segoe UI Emoji\", \"Segoe UI Symbol\";\n font-weight: 400;\n color: #333;\n min-height: 100%;\n overflow-x: hidden;\n background: #edf0f2\n}\n\n.wy-text-left {\n text-align: left\n}\n\n.wy-text-center {\n text-align: center\n}\n\n.wy-text-right {\n text-align: right\n}\n\n.wy-text-large {\n font-size: 120%\n}\n\n.wy-text-normal {\n font-size: 100%\n}\n\n.wy-text-small,\nsmall {\n font-size: 80%\n}\n\n.wy-text-strike {\n text-decoration: line-through\n}\n\n.wy-text-warning {\n color: #E67E22!important\n}\n\na.wy-text-warning:hover {\n color: #eb9950!important\n}\n\n.wy-text-info {\n color: #2980B9!important\n}\n\na.wy-text-info:hover {\n color: #409ad5!important\n}\n\n.wy-text-success {\n color: #27AE60!important\n}\n\na.wy-text-success:hover {\n color: #36d278!important\n}\n\n.wy-text-danger {\n color: #E74C3C!important\n}\n\na.wy-text-danger:hover {\n color: #ed7669!important\n}\n\n.wy-text-neutral {\n color: #404040!important\n}\n\na.wy-text-neutral:hover {\n color: #595959!important\n}\n\n.h1,\n.h2,\n.h3,\n.h4,\n.h5,\n.h6,\nh1,\nh2,\nh3,\nh4,\nh5,\nh6,\nlegend {\n margin-top: 0;\n font-weight: 700;\n font-family: \"Source Sans Pro\",\"ff-tisa-web-pro\",\"Georgia\",Arial,sans-serif;\n font-style: normal\n}\n\np {\n line-height: 24px;\n margin: 0;\n font-size: 16px;\n margin-bottom: 24px\n}\n\nh1,\n.h1 {\n font-size: 175%\n}\n\nh2,\n.h2 {\n font-size: 150%\n}\n\nh3,\n.h3 {\n font-size: 125%\n}\n\nh4,\n.h4 {\n font-size: 115%\n}\n\nh5,\n.h5 {\n font-size: 110%\n}\n\nh6,\n.h6 {\n font-size: 100%\n}\n\nhr {\n display: block;\n height: 1px;\n border: 0;\n border-top: 1px solid #e1e4e5;\n margin: 24px 0;\n padding: 0\n}\n\ncode,\n.rst-content tt,\n.rst-content code {\n max-width: 100%;\n background: #fff;\n border: solid 1px #e1e4e5;\n font-size: 75%;\n padding: 0 5px;\n font-family: \"Source Code Pro\",\"Andale Mono WT\",\"Andale Mono\",\"Lucida Console\",\"Lucida Sans Typewriter\",\"DejaVu Sans Mono\",\"Bitstream Vera Sans Mono\",\"Liberation Mono\",\"Nimbus Mono L\",Monaco,\"Courier New\",Courier,monospace;\n color: #E74C3C;\n overflow-x: auto\n}\n\ncode.code-large,\n.rst-content tt.code-large {\n font-size: 90%\n}\n\n.wy-plain-list-disc,\n.rst-content .section ul,\n.rst-content .toctree-wrapper ul,\narticle ul {\n list-style: disc;\n line-height: 24px;\n margin-bottom: 24px\n}\n\n.wy-plain-list-disc li,\n.rst-content .section ul li,\n.rst-content .toctree-wrapper ul li,\narticle ul li {\n list-style: disc;\n margin-left: 24px\n}\n\n.wy-plain-list-disc li p:last-child,\n.rst-content .section ul li p:last-child,\n.rst-content .toctree-wrapper ul li p:last-child,\narticle ul li p:last-child {\n margin-bottom: 0\n}\n\n.wy-plain-list-disc li ul,\n.rst-content .section ul li ul,\n.rst-content .toctree-wrapper ul li ul,\narticle ul li ul {\n margin-bottom: 0\n}\n\n.wy-plain-list-disc li li,\n.rst-content .section ul li li,\n.rst-content .toctree-wrapper ul li li,\narticle ul li li {\n list-style: circle\n}\n\n.wy-plain-list-disc li li li,\n.rst-content .section ul li li li,\n.rst-content .toctree-wrapper ul li li li,\narticle ul li li li {\n list-style: square\n}\n\n.wy-plain-list-disc li ol li,\n.rst-content .section ul li ol li,\n.rst-content .toctree-wrapper ul li ol li,\narticle ul li ol li {\n list-style: decimal\n}\n\n.wy-plain-list-decimal,\n.rst-content .section ol,\n.rst-content ol.arabic,\narticle ol {\n list-style: decimal;\n line-height: 24px;\n margin-bottom: 24px\n}\n\n.wy-plain-list-decimal li,\n.rst-content .section ol li,\n.rst-content ol.arabic li,\narticle ol li {\n list-style: decimal;\n margin-left: 24px\n}\n\n.wy-plain-list-decimal li p:last-child,\n.rst-content .section ol li p:last-child,\n.rst-content ol.arabic li p:last-child,\narticle ol li p:last-child {\n margin-bottom: 0\n}\n\n.wy-plain-list-decimal li ul,\n.rst-content .section ol li ul,\n.rst-content ol.arabic li ul,\narticle ol li ul {\n margin-bottom: 0\n}\n\n.wy-plain-list-decimal li ul li,\n.rst-content .section ol li ul li,\n.rst-content ol.arabic li ul li,\narticle ol li ul li {\n list-style: disc\n}\n\n\nfooter {\n color: #999\n}\n\nfooter p {\n margin-bottom: 12px\n}\n\n.rst-footer-buttons {\n *zoom: 1\n}\n\n.rst-footer-buttons:before,\n.rst-footer-buttons:after {\n display: table;\n content: \"\"\n}\n\n.rst-footer-buttons:after {\n clear: both\n}\n\n#search-results .search li {\n margin-bottom: 24px;\n border-bottom: solid 1px #e1e4e5;\n padding-bottom: 24px\n}\n\n#search-results .search li:first-child {\n border-top: solid 1px #e1e4e5;\n padding-top: 24px\n}\n\n#search-results .search li a {\n font-size: 120%;\n margin-bottom: 12px;\n display: inline-block\n}\n\n#search-results .context {\n color: gray;\n font-size: 90%\n}\n\n\n\n@media print {\n .rst-versions,\n footer,\n .wy-nav-side {\n display: none\n }\n\n .wy-nav-content-wrap {\n margin-left: 0\n }\n\n}\n\n.rst-versions {\n position: fixed;\n bottom: 0;\n left: 0;\n width: 300px;\n color: #fcfcfc;\n background: #1f1d1d;\n border-top: solid 10px #343131;\n font-family: \"Source Sans Pro\",\"proxima-nova\",\"Helvetica Neue\",Arial,sans-serif;\n z-index: 400\n}\n\n.rst-versions a {\n color: #2980B9;\n text-decoration: none\n}\n\n.rst-versions .rst-badge-small {\n display: none\n}\n\n.rst-versions .rst-current-version {\n padding: 12px;\n background-color: #272525;\n display: block;\n text-align: right;\n font-size: 90%;\n cursor: pointer;\n color: #27AE60;\n *zoom: 1\n}\n\n.rst-versions .rst-current-version:before,\n.rst-versions .rst-current-version:after {\n display: table;\n content: \"\"\n}\n\n.rst-versions .rst-current-version:after {\n clear: both\n}\n\n.rst-versions .rst-current-version .fa,\n.rst-versions .rst-current-version .wy-menu-vertical li span.toctree-expand,\n.wy-menu-vertical li .rst-versions .rst-current-version span.toctree-expand,\n.rst-versions .rst-current-version .rst-content .admonition-title,\n.rst-content .rst-versions .rst-current-version .admonition-title,\n.rst-versions .rst-current-version .rst-content h1 .headerlink,\n.rst-content h1 .rst-versions .rst-current-version .headerlink,\n.rst-versions .rst-current-version .rst-content h2 .headerlink,\n.rst-content h2 .rst-versions .rst-current-version .headerlink,\n.rst-versions .rst-current-version .rst-content h3 .headerlink,\n.rst-content h3 .rst-versions .rst-current-version .headerlink,\n.rst-versions .rst-current-version .rst-content h4 .headerlink,\n.rst-content h4 .rst-versions .rst-current-version .headerlink,\n.rst-versions .rst-current-version .rst-content h5 .headerlink,\n.rst-content h5 .rst-versions .rst-current-version .headerlink,\n.rst-versions .rst-current-version .rst-content h6 .headerlink,\n.rst-content h6 .rst-versions .rst-current-version .headerlink,\n.rst-versions .rst-current-version .rst-content dl dt .headerlink,\n.rst-content dl dt .rst-versions .rst-current-version .headerlink,\n.rst-versions .rst-current-version .rst-content p.caption .headerlink,\n.rst-content p.caption .rst-versions .rst-current-version .headerlink,\n.rst-versions .rst-current-version .rst-content tt.download span:first-child,\n.rst-content tt.download .rst-versions .rst-current-version span:first-child,\n.rst-versions .rst-current-version .rst-content code.download span:first-child,\n.rst-content code.download .rst-versions .rst-current-version span:first-child,\n.rst-versions .rst-current-version .icon {\n color: #fcfcfc\n}\n\n.rst-versions .rst-current-version .fa-book,\n.rst-versions .rst-current-version .icon-book {\n float: left\n}\n\n.rst-versions .rst-current-version .icon-book {\n float: left\n}\n\n.rst-versions .rst-current-version.rst-out-of-date {\n background-color: #E74C3C;\n color: #fff\n}\n\n.rst-versions .rst-current-version.rst-active-old-version {\n background-color: #F1C40F;\n color: #000\n}\n\n.rst-versions.shift-up .rst-other-versions {\n display: block\n}\n\n.rst-versions .rst-other-versions {\n font-size: 90%;\n padding: 12px;\n color: gray;\n display: none\n}\n\n.rst-versions .rst-other-versions hr {\n display: block;\n height: 1px;\n border: 0;\n margin: 20px 0;\n padding: 0;\n border-top: solid 1px #413d3d\n}\n\n.rst-versions .rst-other-versions dd {\n display: inline-block;\n margin: 0\n}\n\n.rst-versions .rst-other-versions dd a {\n display: inline-block;\n padding: 6px;\n color: #fcfcfc\n}\n\n.rst-versions.rst-badge {\n width: auto;\n bottom: 20px;\n right: 20px;\n left: auto;\n border: none;\n max-width: 300px\n}\n\n.rst-versions.rst-badge .icon-book {\n float: none\n}\n\n.rst-versions.rst-badge .fa-book,\n.rst-versions.rst-badge .icon-book {\n float: none\n}\n\n.rst-versions.rst-badge.shift-up .rst-current-version {\n text-align: right\n}\n\n.rst-versions.rst-badge.shift-up .rst-current-version .fa-book,\n.rst-versions.rst-badge.shift-up .rst-current-version .icon-book {\n float: left\n}\n\n.rst-versions.rst-badge.shift-up .rst-current-version .icon-book {\n float: left\n}\n\n.rst-versions.rst-badge .rst-current-version {\n width: auto;\n height: 30px;\n line-height: 30px;\n padding: 0 6px;\n display: block;\n text-align: center\n}\n\n@media screen and (max-width: 768px) {\n .rst-versions {\n width: 85%;\n display: none\n }\n\n .rst-versions.shift {\n display: block\n }\n\n img {\n width: 100%;\n height: auto\n }\n\n}\n\n.rst-content img {\n max-width: 100%;\n height: auto!important\n}\n\n.rst-content div.figure {\n margin-bottom: 24px\n}\n\n.rst-content div.figure.align-center {\n text-align: center\n}\n\n.rst-content .section>img,\n.rst-content .section>a>img {\n margin-bottom: 24px\n}\n\n.rst-content blockquote {\n margin-left: 24px;\n line-height: 24px;\n margin-bottom: 24px\n}\n\n.rst-content .note .last,\n.rst-content .attention .last,\n.rst-content .caution .last,\n.rst-content .danger .last,\n.rst-content .error .last,\n.rst-content .hint .last,\n.rst-content .important .last,\n.rst-content .tip .last,\n.rst-content .warning .last,\n.rst-content .seealso .last,\n.rst-content .admonition-todo .last {\n margin-bottom: 0\n}\n\n.rst-content .admonition-title:before {\n margin-right: 4px\n}\n\n.rst-content .admonition table {\n border-color: rgba(0,0,0,0.1)\n}\n\n.rst-content .admonition table td,\n.rst-content .admonition table th {\n background: transparent!important;\n border-color: rgba(0,0,0,0.1)!important\n}\n\n.rst-content .section ol.loweralpha,\n.rst-content .section ol.loweralpha li {\n list-style: lower-alpha\n}\n\n.rst-content .section ol.upperalpha,\n.rst-content .section ol.upperalpha li {\n list-style: upper-alpha\n}\n\n.rst-content .section ol p,\n.rst-content .section ul p {\n margin-bottom: 12px\n}\n\n.rst-content .line-block {\n margin-left: 24px\n}\n\n.rst-content .topic-title {\n font-weight: 700;\n margin-bottom: 12px\n}\n\n.rst-content .toc-backref {\n color: #404040\n}\n\n.rst-content .align-right {\n float: right;\n margin: 0 0 24px 24px\n}\n\n.rst-content .align-left {\n float: left;\n margin: 0 24px 24px 0\n}\n\n.rst-content .align-center {\n margin: auto;\n display: block\n}\n\n.rst-content h1 .headerlink,\n.rst-content h2 .headerlink,\n.rst-content h3 .headerlink,\n.rst-content h4 .headerlink,\n.rst-content h5 .headerlink,\n.rst-content h6 .headerlink,\n.rst-content dl dt .headerlink,\n.rst-content p.caption .headerlink {\n display: none;\n visibility: hidden;\n font-size: 14px\n}\n\n.rst-content h1 .headerlink:after,\n.rst-content h2 .headerlink:after,\n.rst-content h3 .headerlink:after,\n.rst-content h4 .headerlink:after,\n.rst-content h5 .headerlink:after,\n.rst-content h6 .headerlink:after,\n.rst-content dl dt .headerlink:after,\n.rst-content p.caption .headerlink:after {\n visibility: visible;\n content: \"\";\n font-family: FontAwesome;\n display: inline-block\n}\n\n.rst-content h1:hover .headerlink,\n.rst-content h2:hover .headerlink,\n.rst-content h3:hover .headerlink,\n.rst-content h4:hover .headerlink,\n.rst-content h5:hover .headerlink,\n.rst-content h6:hover .headerlink,\n.rst-content dl dt:hover .headerlink,\n.rst-content p.caption:hover .headerlink {\n display: inline-block\n}\n\n.rst-content .sidebar {\n float: right;\n width: 40%;\n display: block;\n margin: 0 0 24px 24px;\n padding: 24px;\n background: #f3f6f6;\n border: solid 1px #e1e4e5\n}\n\n.rst-content .sidebar p,\n.rst-content .sidebar ul,\n.rst-content .sidebar dl {\n font-size: 90%\n}\n\n.rst-content .sidebar .last {\n margin-bottom: 0\n}\n\n.rst-content .sidebar .sidebar-title {\n display: block;\n font-family: \"Source Sans Pro\",\"ff-tisa-web-pro\",\"Georgia\",Arial,sans-serif;\n font-weight: 700;\n background: #e1e4e5;\n padding: 6px 12px;\n margin: -24px;\n margin-bottom: 24px;\n font-size: 100%\n}\n\n.rst-content .highlighted {\n background: #F1C40F;\n display: inline-block;\n font-weight: 700;\n padding: 0 6px\n}\n\n.rst-content .footnote-reference,\n.rst-content .citation-reference {\n vertical-align: super;\n font-size: 90%\n}\n\n.rst-content table.docutils.citation,\n.rst-content table.docutils.footnote {\n background: none;\n border: none;\n color: #999\n}\n\n.rst-content table.docutils.citation td,\n.rst-content table.docutils.citation tr,\n.rst-content table.docutils.footnote td,\n.rst-content table.docutils.footnote tr {\n border: none;\n background-color: transparent!important;\n white-space: normal\n}\n\n.rst-content table.docutils.citation td.label,\n.rst-content table.docutils.footnote td.label {\n padding-left: 0;\n padding-right: 0;\n vertical-align: top\n}\n\n.rst-content table.field-list {\n border: none\n}\n\n.rst-content table.field-list td {\n border: none;\n padding-top: 5px\n}\n\n.rst-content table.field-list td>strong {\n display: inline-block;\n margin-top: 3px\n}\n\n.rst-content table.field-list .field-name {\n padding-right: 10px;\n text-align: left;\n white-space: nowrap\n}\n\n.rst-content table.field-list .field-body {\n text-align: left;\n padding-left: 0\n}\n\n.rst-content tt,\n.rst-content tt,\n.rst-content code {\n color: #000\n}\n\n.rst-content tt big,\n.rst-content tt em,\n.rst-content tt big,\n.rst-content code big,\n.rst-content tt em,\n.rst-content code em {\n font-size: 100%!important;\n line-height: normal\n}\n\n.rst-content tt .xref,\na .rst-content tt,\n.rst-content tt .xref,\n.rst-content code .xref,\na .rst-content tt,\na .rst-content code {\n font-weight: 700\n}\n\n.rst-content a tt,\n.rst-content a tt,\n.rst-content a code {\n color: #2980B9\n}\n\n.rst-content dl {\n margin-bottom: 24px\n}\n\n.rst-content dl dt {\n font-weight: 700\n}\n\n.rst-content dl p,\n.rst-content dl table,\n.rst-content dl ul,\n.rst-content dl ol {\n margin-bottom: 12px!important\n}\n\n.rst-content dl dd {\n margin: 0 0 12px 24px\n}\n\n.rst-content dl:not(.docutils) {\n margin-bottom: 24px\n}\n\n.rst-content dl:not(.docutils) dt {\n display: inline-block;\n margin: 6px 0;\n font-size: 90%;\n line-height: normal;\n background: #e7f2fa;\n color: #2980B9;\n border-top: solid 3px #6ab0de;\n padding: 6px;\n position: relative\n}\n\n.rst-content dl:not(.docutils) dt:before {\n color: #6ab0de\n}\n\n.rst-content dl:not(.docutils) dt .headerlink {\n color: #404040;\n font-size: 100%!important\n}\n\n.rst-content dl:not(.docutils) dl dt {\n margin-bottom: 6px;\n border: none;\n border-left: solid 3px #ccc;\n background: #f0f0f0;\n color: gray\n}\n\n.rst-content dl:not(.docutils) dl dt .headerlink {\n color: #404040;\n font-size: 100%!important\n}\n\n.rst-content dl:not(.docutils) dt:first-child {\n margin-top: 0\n}\n\n.rst-content dl:not(.docutils) tt,\n.rst-content dl:not(.docutils) tt,\n.rst-content dl:not(.docutils) code {\n font-weight: 700\n}\n\n.rst-content dl:not(.docutils) tt.descname,\n.rst-content dl:not(.docutils) tt.descclassname,\n.rst-content dl:not(.docutils) tt.descname,\n.rst-content dl:not(.docutils) code.descname,\n.rst-content dl:not(.docutils) tt.descclassname,\n.rst-content dl:not(.docutils) code.descclassname {\n background-color: transparent;\n border: none;\n padding: 0;\n font-size: 100%!important\n}\n\n.rst-content dl:not(.docutils) tt.descname,\n.rst-content dl:not(.docutils) tt.descname,\n.rst-content dl:not(.docutils) code.descname {\n font-weight: 700\n}\n\n.rst-content dl:not(.docutils) .optional {\n display: inline-block;\n padding: 0 4px;\n color: #000;\n font-weight: 700\n}\n\n.rst-content dl:not(.docutils) .property {\n display: inline-block;\n padding-right: 8px\n}\n\n.rst-content .viewcode-link,\n.rst-content .viewcode-back {\n display: inline-block;\n color: #27AE60;\n font-size: 80%;\n padding-left: 24px\n}\n\n.rst-content .viewcode-back {\n display: block;\n float: right\n}\n\n.rst-content p.rubric {\n margin-bottom: 12px;\n font-weight: 700\n}\n\n.rst-content tt.download,\n.rst-content code.download {\n background: inherit;\n padding: inherit;\n font-family: inherit;\n font-size: inherit;\n color: inherit;\n border: inherit;\n white-space: inherit\n}\n\n.rst-content tt.download span:first-child:before,\n.rst-content code.download span:first-child:before {\n margin-right: 4px\n}\n\n@media screen and (max-width: 480px) {\n .rst-content .sidebar {\n width: 100%\n }\n\n}\n\nspan[id*='MathJax-Span'] {\n color: #404040\n}\n\n.math {\n text-align: center\n}\n\n@font-face {\n font-family: 'Source Sans Pro';\n font-style: normal;\n font-weight: 100;\n src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_sans_pro/SourceSansPro-ExtraLight.ttf) format(\"truetype\")\n}\n\n@font-face {\n font-family: 'Source Sans Pro';\n font-style: italic;\n font-weight: 100;\n src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_sans_pro/SourceSansPro-ExtraLightItalic.ttf) format(\"truetype\")\n}\n\n@font-face {\n font-family: 'Source Sans Pro';\n font-style: normal;\n font-weight: 300;\n src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_sans_pro/SourceSansPro-Light.ttf) format(\"truetype\")\n}\n\n@font-face {\n font-family: 'Source Sans Pro';\n font-style: italic;\n font-weight: 300;\n src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_sans_pro/SourceSansPro-LightItalic.ttf) format(\"truetype\")\n}\n\n@font-face {\n font-family: 'Source Sans Pro';\n font-style: normal;\n font-weight: 400;\n src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_sans_pro/SourceSansPro-Regular.ttf) format(\"truetype\")\n}\n\n@font-face {\n font-family: 'Source Sans Pro';\n font-style: italic;\n font-weight: 400;\n src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_sans_pro/SourceSansPro-Italic.ttf) format(\"truetype\")\n}\n\n@font-face {\n font-family: 'Source Sans Pro';\n font-style: normal;\n font-weight: 500;\n src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_sans_pro/SourceSansPro-Semibold.ttf) format(\"truetype\")\n}\n\n@font-face {\n font-family: 'Source Sans Pro';\n font-style: italic;\n font-weight: 500;\n src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_sans_pro/SourceSansPro-SemiboldItalic.ttf) format(\"truetype\")\n}\n\n@font-face {\n font-family: 'Source Sans Pro';\n font-style: normal;\n font-weight: 700;\n src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_sans_pro/SourceSansPro-Bold.ttf) format(\"truetype\")\n}\n\n@font-face {\n font-family: 'Source Sans Pro';\n font-style: italic;\n font-weight: 700;\n src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_sans_pro/SourceSansPro-BoldItalic.ttf) format(\"truetype\")\n}\n\n@font-face {\n font-family: 'Source Code Pro';\n font-style: normal;\n font-weight: 300;\n src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_code_pro/SourceCodePro-Light.ttf) format(\"truetype\")\n}\n\n@font-face {\n font-family: 'Source Code Pro';\n font-style: normal;\n font-weight: 400;\n src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_code_pro/SourceCodePro-Regular.ttf) format(\"truetype\")\n}\n\n@font-face {\n font-family: 'Source Code Pro';\n font-style: normal;\n font-weight: 500;\n src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_code_pro/SourceCodePro-Medium.ttf) format(\"truetype\")\n}" }, { "path": "docs/css/goreplay.css", "content": "code {\n font-family: Consolas, \"Liberation Mono\", Menlo, Courier, monospace !important;\n border-bottom-left-radius: 3px;\n border-bottom-right-radius: 3px;\n border-top-left-radius: 3px;\n border-top-right-radius: 3px;\n background-color: #f7f7f7 !important;\n color: #333 !important;\n font-size: 14px !important;\n border: none !important;\n padding: 5px !important;\n}\n\npre > code {\n line-height: 20px;\n padding: 16px !important;\n}\n\nblockquote {\n color: #777 !important;\n border-left: 4px solid rgb(221, 221, 221);\n padding-left: 16px;\n padding-right: 16px;\n margin-left: 0px !important;\n}\n\n/* Visited links purple is not a good idea */\na:visited {\n color: #008BF3;\n}\n\na {\n color: #008BF3;\n}\n\nh3, h4, h5, h6 {\n color: #333 !important;\n}\n\n.rst-versions {\n display: none !important;\n}\n\n/* The next and previous button we don't need */\n.rst-footer-buttons {\n display: none;\n}\n\n/* Foldable parts */\ndetails > * {\n margin-left: 20px;\n}\ndetails > summary {\n margin-left: 0px; /* To overwrite the margin from above */\n\n font-size: 120%;\n cursor: pointer\n}\n\n/* Fix the design not having the correct spacing */\n.wy-menu-vertical .subnav li.current > a {\n padding: 0.4045em 2.427em;\n}\n\n.toctree-l3 {\n padding-left: 1.0em;\n}\n\nfooter .fastlane {\n margin: 20px 0;\n}\n\nfooter .fastlane iframe {\n vertical-align: middle;\n}\n\n/* Custom Syntax highlighting to look more like GitHub.com */\n.hljs-symbol {\n color: #0086b3;\n}\n\n.hljs-keyword {\n font-weight: normal;\n color: #a71d5d;\n}\n\n.hljs-string {\n color: #183691;\n}\n\n/* Header Anchors */\n/* From https://github.com/facelessuser/pymdown-extensions/blob/bf18d0635e9d91b0f98eacdcaa10f26e0ace0f20/doc_theme/css/theme_custom.css#L322-L395 */\n.rst-content .headeranchor-link {\n position: absolute;\n top: 0;\n bottom: 0;\n left: 0;\n display: block;\n padding-right: 6px;\n padding-left: 30px;\n margin-left: -30px;\n}\n\n.rst-content .headeranchor-link:focus {\n outline: none;\n}\n\n.rst-content h1,\n.rst-content h2,\n.rst-content h3,\n.rst-content h4,\n.rst-content h5,\n.rst-content h6 {\n position: relative;\n}\n\n.rst-content h1 .headeranchor,\n.rst-content h2 .headeranchor,\n.rst-content h3 .headeranchor,\n.rst-content h4 .headeranchor,\n.rst-content h5 .headeranchor,\n.rst-content h6 .headeranchor {\n display: none;\n color: #000;\n vertical-align: middle;\n}\n\n.rst-content h1:hover .headeranchor-link,\n.rst-content h2:hover .headeranchor-link,\n.rst-content h3:hover .headeranchor-link,\n.rst-content h4:hover .headeranchor-link,\n.rst-content h5:hover .headeranchor-link,\n.rst-content h6:hover .headeranchor-link {\n height: 1em;\n padding-left: 8px;\n margin-left: -30px;\n text-decoration: none;\n}\n\n.rst-content h1:hover .headeranchor-link .headeranchor,\n.rst-content h2:hover .headeranchor-link .headeranchor,\n.rst-content h3:hover .headeranchor-link .headeranchor,\n.rst-content h4:hover .headeranchor-link .headeranchor,\n.rst-content h5:hover .headeranchor-link .headeranchor,\n.rst-content h6:hover .headeranchor-link .headeranchor {\n display: inline-block;\n}\n\n.rst-content .headeranchor {\n font: normal normal 16px FontAwesome;\n line-height: 1;\n display: inline-block;\n text-decoration: none;\n -webkit-font-smoothing: antialiased;\n -moz-osx-font-smoothing: grayscale;\n -webkit-user-select: none;\n -moz-user-select: none;\n -ms-user-select: none;\n user-select: none;\n}\n\n.rst-content .headeranchor:before {\n content: '\\f0c1';\n}\n\n/* index.md badges */\n@media screen and (max-width: 768px) {\n .badge img {\n width: auto;\n }\n}" }, { "path": "docs/css/sidenav.css", "content": ".wy-affix {\n position: fixed;\n top: 1.618em;\n}\n\n.wy-menu a:hover {\n text-decoration: none;\n}\n\n.wy-menu-vertical header,\n.wy-menu-vertical p.caption {\n height: 32px;\n display: inline-block;\n line-height: 32px;\n padding: 0 1.618em;\n margin-bottom: 0;\n margin-top: 14px;\n display: block;\n font-weight: 700;\n text-transform: uppercase;\n font-size: 85%;\n color: #ccc;\n white-space: nowrap;\n}\n\n.wy-menu-vertical span {\n color: #666;\n}\n\n.wy-menu-vertical ul {\n margin-bottom: 0;\n}\n\n.wy-menu-vertical li.divide-top {\n border-top: solid 1px #404040;\n}\n\n.wy-menu-vertical li.divide-bottom {\n border-bottom: solid 1px #404040;\n}\n\n.wy-menu-vertical li.current {\n background-color: #e5e5e5;\n}\n\n.wy-menu-vertical li.current a {\n color: rgba(0, 93, 255, 0.7);\n border-right: none;\n}\n\n.wy-menu-vertical li.current a:hover {\n color: rgba(0, 93, 255, 0.9);\n}\n\n.wy-menu-vertical li code,\n.wy-menu-vertical li .rst-content tt,\n.rst-content .wy-menu-vertical li tt {\n border: none;\n background: inherit;\n color: inherit;\n padding-left: 0;\n padding-right: 0\n}\n\n.wy-menu-vertical li span.toctree-expand {\n display: block;\n float: left;\n margin-left: -1.2em;\n font-size: .8em;\n line-height: 1.6em;\n color: #999;\n}\n\n.wy-menu-vertical li.on a,\n.wy-menu-vertical li.current>a {\n color: rgba(0, 93, 255, 0.9);\n font-weight: 700;\n position: relative;\n background: #fafafa;\n border: none;\n}\n\n/*.wy-menu-vertical li.on a:hover span.toctree-expand,\n.wy-menu-vertical li.current>a:hover span.toctree-expand {\n color: gray;\n}*/\n\n.wy-menu-vertical li.on a span.toctree-expand,\n.wy-menu-vertical li.current>a span.toctree-expand {\n display: block;\n font-size: .8em;\n line-height: 1.6em;\n color: #333;\n}\n\n.wy-menu-vertical li.toctree-l1.current li.toctree-l2>ul,\n.wy-menu-vertical li.toctree-l2.current li.toctree-l3>ul {\n display: none;\n}\n\n.wy-menu-vertical li.toctree-l1.current li.toctree-l2.current>ul,\n.wy-menu-vertical li.toctree-l2.current li.toctree-l3.current>ul {\n display: block;\n}\n\n.wy-menu-vertical li.toctree-l2.current li.toctree-l3>a {\n display: block;\n padding: .4045em 4.045em;\n}\n\n.wy-menu-vertical li.toctree-l2 a:hover span.toctree-expand {\n color: #999;\n}\n\n.wy-menu-vertical li.toctree-l2 span.toctree-expand {\n color: #999;\n}\n\n.wy-menu-vertical li.toctree-l3 {\n background-color: #eee;\n font-size: .9em;\n}\n\n.wy-menu-vertical li.toctree-l3.current>a {\n padding: .4045em 4.045em;\n}\n\n.wy-menu-vertical li.toctree-l3.current li.toctree-l4>a {\n display: block;\n padding: .4045em 5.663em;\n border-top: none;\n border-bottom: none;\n}\n\n.wy-menu-vertical li.toctree-l3 a:hover span.toctree-expand {\n color: rgba(0, 93, 255, 0.9);\n}\n\n.wy-menu-vertical li.toctree-l3 span.toctree-expand {\n color: #999;\n}\n\n.wy-menu-vertical li.toctree-l4 {\n font-size: .9em;\n}\n\n.wy-menu-vertical li.current ul {\n display: block;\n}\n\n.wy-menu-vertical .local-toc li ul {\n display: block;\n}\n\n.wy-menu-vertical li ul li a {\n margin-bottom: 0;\n color: rgba(0, 93, 255, 0.7);\n font-weight: 400;\n}\n\n.wy-menu-vertical a {\n display: inline-block;\n line-height: 18px;\n padding: .4045em 1.618em;\n display: block;\n position: relative;\n font-size: 90%;\n color: rgba(0, 93, 255, 0.7);\n}\n\n.wy-menu-vertical li.on a:hover,\n.wy-menu-vertical li.current>a:hover {\n background-color: #fafafa;\n}\n\n.wy-menu-vertical a:hover {\n color: rgba(0, 93, 255, 0.9);\n cursor: pointer;\n background-color: #fafafa;\n}\n\n.wy-menu-vertical a:hover span.toctree-expand {\n color: rgba(0, 93, 255, 0.5);\n}\n\n.wy-menu-vertical a:active span.toctree-expand {\n color: rgba(0, 93, 255, 0.7);\n}\n\n/* Search */\n\n.wy-side-nav-search {\n z-index: 200;\n background-color: #fafafa;\n border-bottom: #333;\n text-align: center;\n padding: .809em;\n display: block;\n color: #333;\n margin-bottom: .809em\n}\n\n.wy-side-nav-search input[type=text] {\n width: 100%;\n color: #333;\n border-radius: 3px;\n outline: 0;\n padding: 10px;\n background-color: #fff;\n border: solid 1px #6d6d6d;\n box-shadow: none\n}\n\n.wy-side-nav-search img {\n display: block;\n margin: auto auto .809em;\n height: 45px;\n width: 45px;\n background-color: #2980B9;\n padding: 5px;\n border-radius: 100%\n}\n\n.wy-side-nav-search>a,\n.wy-side-nav-search .wy-dropdown>a {\n color: #333;\n font-size: 100%;\n font-weight: 700;\n display: inline-block;\n padding: 4px 6px;\n margin-bottom: .809em\n}\n\n.wy-side-nav-search>a:hover,\n.wy-side-nav-search .wy-dropdown>a:hover {\n background: rgba(255,255,255,0.1)\n}\n\n.wy-side-nav-search>a img.logo,\n.wy-side-nav-search .wy-dropdown>a img.logo {\n display: block;\n margin: 0 auto;\n height: auto;\n width: auto;\n border-radius: 0;\n max-width: 100%;\n background: transparent\n}\n\n.wy-side-nav-search>a.icon img.logo,\n.wy-side-nav-search .wy-dropdown>a.icon img.logo {\n margin-top: .85em\n}\n\n.wy-nav .wy-menu-vertical header {\n color: #2980B9\n}\n\n.wy-nav .wy-menu-vertical a {\n color: #b3b3b3\n}\n\n.wy-nav .wy-menu-vertical a:hover {\n background-color: #2980B9;\n color: #fff\n}\n\n[data-menu-wrap] {\n -webkit-transition: all .2s ease-in;\n -moz-transition: all .2s ease-in;\n transition: all .2s ease-in;\n position: absolute;\n opacity: 1;\n width: 100%;\n opacity: 0\n}\n\n[data-menu-wrap].move-center {\n left: 0;\n right: auto;\n opacity: 1\n}\n\n[data-menu-wrap].move-left {\n right: auto;\n left: -100%;\n opacity: 0\n}\n\n[data-menu-wrap].move-right {\n right: -100%;\n left: auto;\n opacity: 0\n}\n\n.wy-body-for-nav {\n background: left repeat-y #fcfcfc;\n background-image: url(data:image/png;\n base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyRpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMy1jMDExIDY2LjE0NTY2MSwgMjAxMi8wMi8wNi0xNDo1NjoyNyAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENTNiAoTWFjaW50b3NoKSIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDoxOERBMTRGRDBFMUUxMUUzODUwMkJCOThDMEVFNURFMCIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDoxOERBMTRGRTBFMUUxMUUzODUwMkJCOThDMEVFNURFMCI+IDx4bXBNTTpEZXJpdmVkRnJvbSBzdFJlZjppbnN0YW5jZUlEPSJ4bXAuaWlkOjE4REExNEZCMEUxRTExRTM4NTAyQkI5OEMwRUU1REUwIiBzdFJlZjpkb2N1bWVudElEPSJ4bXAuZGlkOjE4REExNEZDMEUxRTExRTM4NTAyQkI5OEMwRUU1REUwIi8+IDwvcmRmOkRlc2NyaXB0aW9uPiA8L3JkZjpSREY+IDwveDp4bXBtZXRhPiA8P3hwYWNrZXQgZW5kPSJyIj8+EwrlwAAAAA5JREFUeNpiMDU0BAgwAAE2AJgB9BnaAAAAAElFTkSuQmCC);\n background-size: 300px 1px\n}\n\n.wy-grid-for-nav {\n position: absolute;\n width: 100%;\n height: 100%\n}\n\n.wy-nav-side {\n position: fixed;\n top: 0;\n bottom: 0;\n left: 0;\n padding-bottom: 2em;\n width: 300px;\n overflow-x: hidden;\n overflow-y: scroll;\n min-height: 100%;\n background-color: #fafafa;\n z-index: 200;\n border-right: 2px solid #eee;\n}\n\n.wy-nav-top {\n display: none;\n background-color: #333;\n color: #fff;\n padding: .4045em .809em;\n position: relative;\n line-height: 50px;\n text-align: center;\n font-size: 100%;\n *zoom: 1\n}\n\n.wy-nav-top:before,\n.wy-nav-top:after {\n display: table;\n content: \"\"\n}\n\n.wy-nav-top:after {\n clear: both\n}\n\n.wy-nav-top a {\n color: #fff;\n font-weight: 700\n}\n\n.wy-nav-top img {\n margin-right: 12px;\n height: 45px;\n width: 45px;\n background-color: #2980B9;\n padding: 5px;\n border-radius: 100%\n}\n\n.wy-nav-top i {\n font-size: 30px;\n line-height: 50px;\n float: left;\n cursor: pointer\n}\n\n.wy-nav-content-wrap {\n margin-left: 300px;\n background: #fcfcfc;\n min-height: 100%\n}\n\n.wy-nav-content {\n padding: 1.618em 3.236em;\n height: 100%;\n max-width: 1100px;\n margin: auto\n}\n\n.wy-body-mask {\n position: fixed;\n width: 100%;\n height: 100%;\n background: rgba(0,0,0,0.2);\n display: none;\n z-index: 499\n}\n\n.wy-body-mask.on {\n display: block\n}\n\n@media screen and (max-width: 768px) {\n .wy-body-for-nav {\n background: #fcfcfc\n }\n\n .wy-nav-top {\n display: block\n }\n\n .wy-nav-side {\n left: -300px\n }\n\n .wy-nav-side.shift {\n width: 85%;\n left: 0\n }\n\n .wy-nav-content-wrap {\n margin-left: 0\n }\n\n .wy-nav-content-wrap .wy-nav-content {\n padding: 1.618em\n }\n\n .wy-nav-content-wrap.shift {\n position: fixed;\n min-width: 100%;\n left: 85%;\n top: 0;\n height: 100%;\n overflow: hidden\n }\n\n}\n\n@media screen and (min-width: 1400px) {\n .wy-nav-content {\n margin: 0;\n background: #fcfcfc\n }\n}" }, { "path": "docs/getting-started/basics.md", "content": "### Overview\nGor architecture tries to follow UNIX philosophy: everything made of pipes, various inputs multiplexing data to outputs.\n\nYou can [rate limit](/rate-limiting.md), [filter](request-filtering.md), [rewrite](request-rewriting.md) requests or even use your own [middleware](middleware.md) to implement custom logic. Also, it is possible to replay requests at the higher rate for [load testing](saving-and-replaying-from-file.md).\n\n### Available inputs\n\n* `--input-raw` - used to capture HTTP traffic, you should specify IP address or interface and application port. More about [[Capturing and replaying traffic]]. \n* `--input-file` - accepts file which previously was recorded using ` --output-file`. More about [[Saving and Replaying from file]]\n* `--input-tcp` - used by Gor aggregation instance if you decided forward traffic from multiple forwarder Gor instances to it. Read about using [[Aggregator-forwarder setup]].\n\n### Available outputs\n\n* `--output-http` - replay HTTP traffic to given endpoint, accepts base url. Read [more about it](Replaying HTTP traffic)\n* `--output-file` - records incoming traffic to the file. More about [[Saving and Replaying from file]]\n* `--output-tcp` - forward incoming data to another Gor instance, used in conjunction with `--input-tcp`. Read more about [[Aggregator-forwarder setup]].\n* `--output-stdout` - used for debugging, outputs all data to stdout." }, { "path": "docs/getting-started/tutorial.md", "content": "### Dependencies\nTo start working with Gor, you need to have a web server running on your machine, and a terminal to run commands. If you are just poking around, you can quickly start the server by calling `gor file-server :8000`, this will start a simple file server of the current directory on port `8000`. \n\n### Installing Gor\nDownload the latest Gor binary from https://github.com/buger/gor/releases (we provide precompiled binaries for Windows, Linux x64 and Mac OS), or you can compile by yourself [[Compilation]].\n\nOnce the archive is downloaded and uncompressed, you can run Gor from the current directory, or you may want to copy binary to your PATH (for Linux and Mac OS it can be `/usr/local/bin`).\n\n### Capturing web traffic\nNow run this command in terminal: `sudo ./gor --input-raw :8000 --output-stdout`\n\nThis command says to listen for all network activity happening on port 8000 and log it to stdout.\nIf you are familiar with `tcpdump`, we are going to implement similar functionality. \n\n> You may notice that it uses `sudo` and asks for the password: to analyze network, Gor needs permissions which are available only to super users.\n> However, it is possible to configure Gor [being run for non-root users](Running-as-non-root-user).\n\n\nMake a few requests by opening `http://localhost:8000` in your browser, or just by calling curl in terminal `curl http://localhost:8000`. You should see that `gor` outputs all the HTTP requests and responses right to the terminal window where it is running. \n\n\n**Gor is not a proxy:** you do not need to put 3-rd party tool to your critical path. Instead Gor just silently analyzes the traffic of your application and does not affect it anyhow.\n\n### Replaying\n\nNow it's time to replay your original traffic to another environment. Let's start the same file web server but on a different port: `gor file-server :8001`. \n\nInstead of `--output-stdout` we will use `--output-http` and provide URL of second server: `sudo ./gor --input-raw :8000 --output-http=\"http://localhost:8001\"`\n\nMake few requests to first server. You should see them replicated to the second one, voila! \n\n### Saving requests to file and replaying them later\nSometimes it's not possible to replay requests in real time; Gor allows you to save requests to the file and replay them later. \n\nFirst use `--output-file` to save them: `sudo ./gor --input-raw :8000 --output-file=requests.gor`. This will create new file and continuously write all captured requests to it. \n\nLet's re-run Gor, but now to replay requests from file: `./gor --input-file requests.gor --output-http=\"http://localhost:8001\"`. You should see all the recorded requests coming to the second server, and they will be replayed in the same order and with exactly same timing as they were recorded.\n\nNext: [[The Basics]]\n\n### Watch an overview:\n\n\n![YOUTUBE](https://www.youtube.com/watch?v=CxuKZcMKaW4)" }, { "path": "docs/index.md", "content": "Gor is an open-source tool for capturing and replaying live HTTP traffic into a test environment in order to continuously test your system with real data. It can be used to increase confidence in code deployments, configuration changes and infrastructure changes.\n\nRead for more info:\n\n* [[Getting Started]]\n* [[The Basics]]\n* [[Capturing and replaying traffic]]\n* [[Replaying HTTP traffic]]\n* [[[PRO] Replaying Binary protocols]]\n* [[[PRO] Recording and replaying keep alive TCP sessions]]\n* [[Saving and Replaying from file]]\n* [Performance testing](https://github.com/buger/gor/wiki/Saving-and-Replaying-from-file#performance-testing)\n* [[Rate limiting]]\n* [[Request filtering]]\n* [[Request rewriting]]\n* [[Middleware]]\n* [[Distributed configuration]]\n* [[Exporting to ElasticSearch]]\n* [[FAQ]]\n* [[Troubleshooting]]\n\n## Commercial Aspects\n\n* [[Commercial Support]]\n* [[Commercial FAQ]]\n* [[Commercial collaboration]]\n\n\nNext: [Getting Started](Getting-Started)\n" }, { "path": "docs/js/base.js", "content": "$(document).ready(function () {\n $('img[alt=\"YOUTUBE\"]').each(function () {\n var id = $(this).attr('src').split('/')[$(this).attr('src').split('/').length - 1].split(\"=\")[1];\n var video = '';\n $(this).replaceWith(video);\n });\n});" }, { "path": "docs/js/turbolinks.js", "content": "/*\nTurbolinks 5.0.0\nCopyright © 2016 Basecamp, LLC\n */\n(function(){(function(){(function(){this.Turbolinks={supported:function(){return null!=window.history.pushState&&null!=window.requestAnimationFrame}(),visit:function(e,r){return t.controller.visit(e,r)},clearCache:function(){return t.controller.clearCache()}}}).call(this)}).call(this);var t=this.Turbolinks;(function(){(function(){var e,r;t.copyObject=function(t){var e,r,n;r={};for(e in t)n=t[e],r[e]=n;return r},t.closest=function(t,r){return e.call(t,r)},e=function(){var t,e;return t=document.documentElement,null!=(e=t.closest)?e:function(t){var e;for(e=this;e;){if(e.nodeType===Node.ELEMENT_NODE&&r.call(e,t))return e;e=e.parentNode}}}(),t.defer=function(t){return setTimeout(t,1)},t.dispatch=function(t,e){var r,n,o,i,s;return i=null!=e?e:{},s=i.target,r=i.cancelable,n=i.data,o=document.createEvent(\"Events\"),o.initEvent(t,!0,r===!0),o.data=null!=n?n:{},(null!=s?s:document).dispatchEvent(o),o},t.match=function(t,e){return r.call(t,e)},r=function(){var t,e,r,n;return t=document.documentElement,null!=(e=null!=(r=null!=(n=t.matchesSelector)?n:t.webkitMatchesSelector)?r:t.msMatchesSelector)?e:t.mozMatchesSelector}(),t.uuid=function(){var t,e,r;for(r=\"\",t=e=1;36>=e;t=++e)r+=9===t||14===t||19===t||24===t?\"-\":15===t?\"4\":20===t?(Math.floor(4*Math.random())+8).toString(16):Math.floor(15*Math.random()).toString(16);return r}}).call(this),function(){t.Location=function(){function t(t){var e,r;null==t&&(t=\"\"),r=document.createElement(\"a\"),r.href=t.toString(),this.absoluteURL=r.href,e=r.hash.length,2>e?this.requestURL=this.absoluteURL:(this.requestURL=this.absoluteURL.slice(0,-e),this.anchor=r.hash.slice(1))}var e,r,n,o;return t.wrap=function(t){return t instanceof this?t:new this(t)},t.prototype.getOrigin=function(){return this.absoluteURL.split(\"/\",3).join(\"/\")},t.prototype.getPath=function(){var t,e;return null!=(t=null!=(e=this.absoluteURL.match(/\\/\\/[^\\/]*(\\/[^?;]*)/))?e[1]:void 0)?t:\"/\"},t.prototype.getPathComponents=function(){return this.getPath().split(\"/\").slice(1)},t.prototype.getLastPathComponent=function(){return this.getPathComponents().slice(-1)[0]},t.prototype.getExtension=function(){var t,e;return null!=(t=null!=(e=this.getLastPathComponent().match(/\\.[^.]*$/))?e[0]:void 0)?t:\"\"},t.prototype.isHTML=function(){return this.getExtension().match(/^(?:|\\.(?:htm|html|xhtml))$/)},t.prototype.isPrefixedBy=function(t){var e;return e=r(t),this.isEqualTo(t)||o(this.absoluteURL,e)},t.prototype.isEqualTo=function(t){return this.absoluteURL===(null!=t?t.absoluteURL:void 0)},t.prototype.toCacheKey=function(){return this.requestURL},t.prototype.toJSON=function(){return this.absoluteURL},t.prototype.toString=function(){return this.absoluteURL},t.prototype.valueOf=function(){return this.absoluteURL},r=function(t){return e(t.getOrigin()+t.getPath())},e=function(t){return n(t,\"/\")?t:t+\"/\"},o=function(t,e){return t.slice(0,e.length)===e},n=function(t,e){return t.slice(-e.length)===e},t}()}.call(this),function(){var e=function(t,e){return function(){return t.apply(e,arguments)}};t.HttpRequest=function(){function r(r,n,o){this.delegate=r,this.requestCanceled=e(this.requestCanceled,this),this.requestTimedOut=e(this.requestTimedOut,this),this.requestFailed=e(this.requestFailed,this),this.requestLoaded=e(this.requestLoaded,this),this.requestProgressed=e(this.requestProgressed,this),this.url=t.Location.wrap(n).requestURL,this.referrer=t.Location.wrap(o).absoluteURL,this.createXHR()}return r.NETWORK_FAILURE=0,r.TIMEOUT_FAILURE=-1,r.timeout=60,r.prototype.send=function(){var t;return this.xhr&&!this.sent?(this.notifyApplicationBeforeRequestStart(),this.setProgress(0),this.xhr.send(),this.sent=!0,\"function\"==typeof(t=this.delegate).requestStarted?t.requestStarted():void 0):void 0},r.prototype.cancel=function(){return this.xhr&&this.sent?this.xhr.abort():void 0},r.prototype.requestProgressed=function(t){return t.lengthComputable?this.setProgress(t.loaded/t.total):void 0},r.prototype.requestLoaded=function(){return this.endRequest(function(t){return function(){var e;return 200<=(e=t.xhr.status)&&300>e?t.delegate.requestCompletedWithResponse(t.xhr.responseText,t.xhr.getResponseHeader(\"Turbolinks-Location\")):(t.failed=!0,t.delegate.requestFailedWithStatusCode(t.xhr.status,t.xhr.responseText))}}(this))},r.prototype.requestFailed=function(){return this.endRequest(function(t){return function(){return t.failed=!0,t.delegate.requestFailedWithStatusCode(t.constructor.NETWORK_FAILURE)}}(this))},r.prototype.requestTimedOut=function(){return this.endRequest(function(t){return function(){return t.failed=!0,t.delegate.requestFailedWithStatusCode(t.constructor.TIMEOUT_FAILURE)}}(this))},r.prototype.requestCanceled=function(){return this.endRequest()},r.prototype.notifyApplicationBeforeRequestStart=function(){return t.dispatch(\"turbolinks:request-start\",{data:{url:this.url,xhr:this.xhr}})},r.prototype.notifyApplicationAfterRequestEnd=function(){return t.dispatch(\"turbolinks:request-end\",{data:{url:this.url,xhr:this.xhr}})},r.prototype.createXHR=function(){return this.xhr=new XMLHttpRequest,this.xhr.open(\"GET\",this.url,!0),this.xhr.timeout=1e3*this.constructor.timeout,this.xhr.setRequestHeader(\"Accept\",\"text/html, application/xhtml+xml\"),this.xhr.setRequestHeader(\"Turbolinks-Referrer\",this.referrer),this.xhr.onprogress=this.requestProgressed,this.xhr.onload=this.requestLoaded,this.xhr.onerror=this.requestFailed,this.xhr.ontimeout=this.requestTimedOut,this.xhr.onabort=this.requestCanceled},r.prototype.endRequest=function(t){return this.xhr?(this.notifyApplicationAfterRequestEnd(),null!=t&&t.call(this),this.destroy()):void 0},r.prototype.setProgress=function(t){var e;return this.progress=t,\"function\"==typeof(e=this.delegate).requestProgressed?e.requestProgressed(this.progress):void 0},r.prototype.destroy=function(){var t;return this.setProgress(1),\"function\"==typeof(t=this.delegate).requestFinished&&t.requestFinished(),this.delegate=null,this.xhr=null},r}()}.call(this),function(){var e=function(t,e){return function(){return t.apply(e,arguments)}};t.ProgressBar=function(){function t(){this.trickle=e(this.trickle,this),this.stylesheetElement=this.createStylesheetElement(),this.progressElement=this.createProgressElement()}var r;return r=300,t.defaultCSS=\".turbolinks-progress-bar {\\n position: fixed;\\n display: block;\\n top: 0;\\n left: 0;\\n height: 3px;\\n background: #0076ff;\\n z-index: 9999;\\n transition: width \"+r+\"ms ease-out, opacity \"+r/2+\"ms \"+r/2+\"ms ease-in;\\n transform: translate3d(0, 0, 0);\\n}\",t.prototype.show=function(){return this.visible?void 0:(this.visible=!0,this.installStylesheetElement(),this.installProgressElement(),this.startTrickling())},t.prototype.hide=function(){return this.visible&&!this.hiding?(this.hiding=!0,this.fadeProgressElement(function(t){return function(){return t.uninstallProgressElement(),t.stopTrickling(),t.visible=!1,t.hiding=!1}}(this))):void 0},t.prototype.setValue=function(t){return this.value=t,this.refresh()},t.prototype.installStylesheetElement=function(){return document.head.insertBefore(this.stylesheetElement,document.head.firstChild)},t.prototype.installProgressElement=function(){return this.progressElement.style.width=0,this.progressElement.style.opacity=1,document.documentElement.insertBefore(this.progressElement,document.body),this.refresh()},t.prototype.fadeProgressElement=function(t){return this.progressElement.style.opacity=0,setTimeout(t,1.5*r)},t.prototype.uninstallProgressElement=function(){return this.progressElement.parentNode?document.documentElement.removeChild(this.progressElement):void 0},t.prototype.startTrickling=function(){return null!=this.trickleInterval?this.trickleInterval:this.trickleInterval=setInterval(this.trickle,r)},t.prototype.stopTrickling=function(){return clearInterval(this.trickleInterval),this.trickleInterval=null},t.prototype.trickle=function(){return this.setValue(this.value+Math.random()/100)},t.prototype.refresh=function(){return requestAnimationFrame(function(t){return function(){return t.progressElement.style.width=10+90*t.value+\"%\"}}(this))},t.prototype.createStylesheetElement=function(){var t;return t=document.createElement(\"style\"),t.type=\"text/css\",t.textContent=this.constructor.defaultCSS,t},t.prototype.createProgressElement=function(){var t;return t=document.createElement(\"div\"),t.className=\"turbolinks-progress-bar\",t},t}()}.call(this),function(){var e=function(t,e){return function(){return t.apply(e,arguments)}};t.BrowserAdapter=function(){function r(r){this.controller=r,this.showProgressBar=e(this.showProgressBar,this),this.progressBar=new t.ProgressBar}var n,o,i,s;return s=t.HttpRequest,n=s.NETWORK_FAILURE,i=s.TIMEOUT_FAILURE,o=500,r.prototype.visitProposedToLocationWithAction=function(t,e){return this.controller.startVisitToLocationWithAction(t,e)},r.prototype.visitStarted=function(t){return t.issueRequest(),t.changeHistory(),t.loadCachedSnapshot()},r.prototype.visitRequestStarted=function(t){return this.progressBar.setValue(0),t.hasCachedSnapshot()||\"restore\"!==t.action?this.showProgressBarAfterDelay():this.showProgressBar()},r.prototype.visitRequestProgressed=function(t){return this.progressBar.setValue(t.progress)},r.prototype.visitRequestCompleted=function(t){return t.loadResponse()},r.prototype.visitRequestFailedWithStatusCode=function(t,e){switch(e){case n:case i:return this.reload();default:return t.loadResponse()}},r.prototype.visitRequestFinished=function(t){return this.hideProgressBar()},r.prototype.visitCompleted=function(t){return t.followRedirect()},r.prototype.pageInvalidated=function(){return this.reload()},r.prototype.showProgressBarAfterDelay=function(){return this.progressBarTimeout=setTimeout(this.showProgressBar,o)},r.prototype.showProgressBar=function(){return this.progressBar.show()},r.prototype.hideProgressBar=function(){return this.progressBar.hide(),clearTimeout(this.progressBarTimeout)},r.prototype.reload=function(){return window.location.reload()},r}()}.call(this),function(){var e,r=function(t,e){return function(){return t.apply(e,arguments)}};e=!1,addEventListener(\"load\",function(){return t.defer(function(){return e=!0})},!1),t.History=function(){function n(t){this.delegate=t,this.onPopState=r(this.onPopState,this)}return n.prototype.start=function(){return this.started?void 0:(addEventListener(\"popstate\",this.onPopState,!1),this.started=!0)},n.prototype.stop=function(){return this.started?(removeEventListener(\"popstate\",this.onPopState,!1),this.started=!1):void 0},n.prototype.push=function(e,r){return e=t.Location.wrap(e),this.update(\"push\",e,r)},n.prototype.replace=function(e,r){return e=t.Location.wrap(e),this.update(\"replace\",e,r)},n.prototype.onPopState=function(e){var r,n,o,i;return this.shouldHandlePopState()&&(i=null!=(n=e.state)?n.turbolinks:void 0)?(r=t.Location.wrap(window.location),o=i.restorationIdentifier,this.delegate.historyPoppedToLocationWithRestorationIdentifier(r,o)):void 0},n.prototype.shouldHandlePopState=function(){return e===!0},n.prototype.update=function(t,e,r){var n;return n={turbolinks:{restorationIdentifier:r}},history[t+\"State\"](n,null,e)},n}()}.call(this),function(){t.Snapshot=function(){function e(t){var e,r;r=t.head,e=t.body,this.head=null!=r?r:document.createElement(\"head\"),this.body=null!=e?e:document.createElement(\"body\")}return e.wrap=function(t){return t instanceof this?t:this.fromHTML(t)},e.fromHTML=function(t){var e;return e=document.createElement(\"html\"),e.innerHTML=t,this.fromElement(e)},e.fromElement=function(t){return new this({head:t.querySelector(\"head\"),body:t.querySelector(\"body\")})},e.prototype.clone=function(){return new e({head:this.head.cloneNode(!0),body:this.body.cloneNode(!0)})},e.prototype.getRootLocation=function(){var e,r;return r=null!=(e=this.getSetting(\"root\"))?e:\"/\",new t.Location(r)},e.prototype.getCacheControlValue=function(){return this.getSetting(\"cache-control\")},e.prototype.hasAnchor=function(t){try{return null!=this.body.querySelector(\"[id='\"+t+\"']\")}catch(e){}},e.prototype.isPreviewable=function(){return\"no-preview\"!==this.getCacheControlValue()},e.prototype.isCacheable=function(){return\"no-cache\"!==this.getCacheControlValue()},e.prototype.getSetting=function(t){var e,r;return r=this.head.querySelectorAll(\"meta[name='turbolinks-\"+t+\"']\"),e=r[r.length-1],null!=e?e.getAttribute(\"content\"):void 0},e}()}.call(this),function(){var e=[].slice;t.Renderer=function(){function t(){}var r;return t.render=function(){var t,r,n,o;return n=arguments[0],r=arguments[1],t=3<=arguments.length?e.call(arguments,2):[],o=function(t,e,r){r.prototype=t.prototype;var n=new r,o=t.apply(n,e);return Object(o)===o?o:n}(this,t,function(){}),o.delegate=n,o.render(r),o},t.prototype.renderView=function(t){return this.delegate.viewWillRender(this.newBody),t(),this.delegate.viewRendered(this.newBody)},t.prototype.invalidateView=function(){return this.delegate.viewInvalidated()},t.prototype.createScriptElement=function(t){var e;return\"false\"===t.getAttribute(\"data-turbolinks-eval\")?t:(e=document.createElement(\"script\"),e.textContent=t.textContent,r(e,t),e)},r=function(t,e){var r,n,o,i,s,a,u;for(i=e.attributes,a=[],r=0,n=i.length;n>r;r++)s=i[r],o=s.name,u=s.value,a.push(t.setAttribute(o,u));return a},t}()}.call(this),function(){t.HeadDetails=function(){function t(t){var e,r,i,s,a,u,c;for(this.element=t,this.elements={},c=this.element.childNodes,s=0,u=c.length;u>s;s++)i=c[s],i.nodeType===Node.ELEMENT_NODE&&(a=i.outerHTML,r=null!=(e=this.elements)[a]?e[a]:e[a]={type:o(i),tracked:n(i),elements:[]},r.elements.push(i))}var e,r,n,o;return t.prototype.hasElementWithKey=function(t){return t in this.elements},t.prototype.getTrackedElementSignature=function(){var t,e;return function(){var r,n;r=this.elements,n=[];for(t in r)e=r[t].tracked,e&&n.push(t);return n}.call(this).join(\"\")},t.prototype.getScriptElementsNotInDetails=function(t){return this.getElementsMatchingTypeNotInDetails(\"script\",t)},t.prototype.getStylesheetElementsNotInDetails=function(t){return this.getElementsMatchingTypeNotInDetails(\"stylesheet\",t)},t.prototype.getElementsMatchingTypeNotInDetails=function(t,e){var r,n,o,i,s,a;o=this.elements,s=[];for(n in o)i=o[n],a=i.type,r=i.elements,a!==t||e.hasElementWithKey(n)||s.push(r[0]);return s},t.prototype.getProvisionalElements=function(){var t,e,r,n,o,i,s;r=[],n=this.elements;for(e in n)o=n[e],s=o.type,i=o.tracked,t=o.elements,null!=s||i?t.length>1&&r.push.apply(r,t.slice(1)):r.push.apply(r,t);return r},o=function(t){return e(t)?\"script\":r(t)?\"stylesheet\":void 0},n=function(t){return\"reload\"===t.getAttribute(\"data-turbolinks-track\")},e=function(t){var e;return e=t.tagName.toLowerCase(),\"script\"===e},r=function(t){var e;return e=t.tagName.toLowerCase(),\"style\"===e||\"link\"===e&&\"stylesheet\"===t.getAttribute(\"rel\")},t}()}.call(this),function(){var e=function(t,e){function n(){this.constructor=t}for(var o in e)r.call(e,o)&&(t[o]=e[o]);return n.prototype=e.prototype,t.prototype=new n,t.__super__=e.prototype,t},r={}.hasOwnProperty;t.SnapshotRenderer=function(r){function n(e,r){this.currentSnapshot=e,this.newSnapshot=r,this.currentHeadDetails=new t.HeadDetails(this.currentSnapshot.head),this.newHeadDetails=new t.HeadDetails(this.newSnapshot.head),this.newBody=this.newSnapshot.body}return e(n,r),n.prototype.render=function(t){return this.trackedElementsAreIdentical()?(this.mergeHead(),this.renderView(function(e){return function(){return e.replaceBody(),e.focusFirstAutofocusableElement(),t()}}(this))):this.invalidateView()},n.prototype.mergeHead=function(){return this.copyNewHeadStylesheetElements(),this.copyNewHeadScriptElements(),this.removeCurrentHeadProvisionalElements(),this.copyNewHeadProvisionalElements()},n.prototype.replaceBody=function(){return this.activateBodyScriptElements(),this.importBodyPermanentElements(),this.assignNewBody()},n.prototype.trackedElementsAreIdentical=function(){return this.currentHeadDetails.getTrackedElementSignature()===this.newHeadDetails.getTrackedElementSignature()},n.prototype.copyNewHeadStylesheetElements=function(){var t,e,r,n,o;for(n=this.getNewHeadStylesheetElements(),o=[],e=0,r=n.length;r>e;e++)t=n[e],o.push(document.head.appendChild(t));return o},n.prototype.copyNewHeadScriptElements=function(){var t,e,r,n,o;for(n=this.getNewHeadScriptElements(),o=[],e=0,r=n.length;r>e;e++)t=n[e],o.push(document.head.appendChild(this.createScriptElement(t)));return o},n.prototype.removeCurrentHeadProvisionalElements=function(){var t,e,r,n,o;for(n=this.getCurrentHeadProvisionalElements(),o=[],e=0,r=n.length;r>e;e++)t=n[e],o.push(document.head.removeChild(t));return o},n.prototype.copyNewHeadProvisionalElements=function(){var t,e,r,n,o;for(n=this.getNewHeadProvisionalElements(),o=[],e=0,r=n.length;r>e;e++)t=n[e],o.push(document.head.appendChild(t));return o},n.prototype.importBodyPermanentElements=function(){var t,e,r,n,o,i;for(n=this.getNewBodyPermanentElements(),i=[],e=0,r=n.length;r>e;e++)o=n[e],(t=this.findCurrentBodyPermanentElement(o))?i.push(o.parentNode.replaceChild(t,o)):i.push(void 0);return i},n.prototype.activateBodyScriptElements=function(){var t,e,r,n,o,i;for(n=this.getNewBodyScriptElements(),i=[],e=0,r=n.length;r>e;e++)o=n[e],t=this.createScriptElement(o),i.push(o.parentNode.replaceChild(t,o));return i},n.prototype.assignNewBody=function(){return document.body=this.newBody},n.prototype.focusFirstAutofocusableElement=function(){var t;return null!=(t=this.findFirstAutofocusableElement())?t.focus():void 0},n.prototype.getNewHeadStylesheetElements=function(){return this.newHeadDetails.getStylesheetElementsNotInDetails(this.currentHeadDetails)},n.prototype.getNewHeadScriptElements=function(){return this.newHeadDetails.getScriptElementsNotInDetails(this.currentHeadDetails)},n.prototype.getCurrentHeadProvisionalElements=function(){return this.currentHeadDetails.getProvisionalElements()},n.prototype.getNewHeadProvisionalElements=function(){return this.newHeadDetails.getProvisionalElements()},n.prototype.getNewBodyPermanentElements=function(){return this.newBody.querySelectorAll(\"[id][data-turbolinks-permanent]\")},n.prototype.findCurrentBodyPermanentElement=function(t){return document.body.querySelector(\"#\"+t.id+\"[data-turbolinks-permanent]\")},n.prototype.getNewBodyScriptElements=function(){return this.newBody.querySelectorAll(\"script\")},n.prototype.findFirstAutofocusableElement=function(){return document.body.querySelector(\"[autofocus]\")},n}(t.Renderer)}.call(this),function(){var e=function(t,e){function n(){this.constructor=t}for(var o in e)r.call(e,o)&&(t[o]=e[o]);return n.prototype=e.prototype,t.prototype=new n,t.__super__=e.prototype,t},r={}.hasOwnProperty;t.ErrorRenderer=function(t){function r(t){this.html=t}return e(r,t),r.prototype.render=function(t){return this.renderView(function(e){return function(){return e.replaceDocumentHTML(),e.activateBodyScriptElements(),t()}}(this))},r.prototype.replaceDocumentHTML=function(){return document.documentElement.innerHTML=this.html},r.prototype.activateBodyScriptElements=function(){var t,e,r,n,o,i;for(n=this.getScriptElements(),i=[],e=0,r=n.length;r>e;e++)o=n[e],t=this.createScriptElement(o),i.push(o.parentNode.replaceChild(t,o));return i},r.prototype.getScriptElements=function(){return document.documentElement.querySelectorAll(\"script\")},r}(t.Renderer)}.call(this),function(){t.View=function(){function e(t){this.delegate=t,this.element=document.documentElement}return e.prototype.getRootLocation=function(){return this.getSnapshot().getRootLocation()},e.prototype.getSnapshot=function(){return t.Snapshot.fromElement(this.element)},e.prototype.render=function(t,e){var r,n,o;return o=t.snapshot,r=t.error,n=t.isPreview,this.markAsPreview(n),null!=o?this.renderSnapshot(o,e):this.renderError(r,e)},e.prototype.markAsPreview=function(t){return t?this.element.setAttribute(\"data-turbolinks-preview\",\"\"):this.element.removeAttribute(\"data-turbolinks-preview\")},e.prototype.renderSnapshot=function(e,r){return t.SnapshotRenderer.render(this.delegate,r,this.getSnapshot(),t.Snapshot.wrap(e))},e.prototype.renderError=function(e,r){return t.ErrorRenderer.render(this.delegate,r,e)},e}()}.call(this),function(){var e=function(t,e){return function(){return t.apply(e,arguments)}};t.ScrollManager=function(){function t(t){this.delegate=t,this.onScroll=e(this.onScroll,this)}return t.prototype.start=function(){return this.started?void 0:(addEventListener(\"scroll\",this.onScroll,!1),this.onScroll(),this.started=!0)},t.prototype.stop=function(){return this.started?(removeEventListener(\"scroll\",this.onScroll,!1),this.started=!1):void 0},t.prototype.scrollToElement=function(t){return t.scrollIntoView()},t.prototype.scrollToPosition=function(t){var e,r;return e=t.x,r=t.y,window.scrollTo(e,r)},t.prototype.onScroll=function(t){return this.updatePosition({x:window.pageXOffset,y:window.pageYOffset})},t.prototype.updatePosition=function(t){var e;return this.position=t,null!=(e=this.delegate)?e.scrollPositionChanged(this.position):void 0},t}()}.call(this),function(){t.SnapshotCache=function(){function e(t){this.size=t,this.keys=[],this.snapshots={}}var r;return e.prototype.has=function(t){var e;return e=r(t),e in this.snapshots},e.prototype.get=function(t){var e;if(this.has(t))return e=this.read(t),this.touch(t),e},e.prototype.put=function(t,e){return this.write(t,e),this.touch(t),e},e.prototype.read=function(t){var e;return e=r(t),this.snapshots[e]},e.prototype.write=function(t,e){var n;return n=r(t),this.snapshots[n]=e},e.prototype.touch=function(t){var e,n;return n=r(t),e=this.keys.indexOf(n),e>-1&&this.keys.splice(e,1),this.keys.unshift(n),this.trim()},e.prototype.trim=function(){var t,e,r,n,o;for(n=this.keys.splice(this.size),o=[],t=0,r=n.length;r>t;t++)e=n[t],o.push(delete this.snapshots[e]);return o},r=function(e){return t.Location.wrap(e).toCacheKey()},e}()}.call(this),function(){var e=function(t,e){return function(){return t.apply(e,arguments)}};t.Visit=function(){function r(r,n,o){this.controller=r,this.action=o,this.performScroll=e(this.performScroll,this),this.identifier=t.uuid(),this.location=t.Location.wrap(n),this.adapter=this.controller.adapter,this.state=\"initialized\",this.timingMetrics={}}var n;return r.prototype.start=function(){return\"initialized\"===this.state?(this.recordTimingMetric(\"visitStart\"),this.state=\"started\",this.adapter.visitStarted(this)):void 0},r.prototype.cancel=function(){var t;return\"started\"===this.state?(null!=(t=this.request)&&t.cancel(),this.cancelRender(),this.state=\"canceled\"):void 0},r.prototype.complete=function(){var t;return\"started\"===this.state?(this.recordTimingMetric(\"visitEnd\"),this.state=\"completed\",\"function\"==typeof(t=this.adapter).visitCompleted&&t.visitCompleted(this),this.controller.visitCompleted(this)):void 0},r.prototype.fail=function(){var t;return\"started\"===this.state?(this.state=\"failed\",\"function\"==typeof(t=this.adapter).visitFailed?t.visitFailed(this):void 0):void 0},r.prototype.changeHistory=function(){var t,e;return this.historyChanged?void 0:(t=this.location.isEqualTo(this.referrer)?\"replace\":this.action,e=n(t),this.controller[e](this.location,this.restorationIdentifier),this.historyChanged=!0)},r.prototype.issueRequest=function(){return this.shouldIssueRequest()&&null==this.request?(this.progress=0,this.request=new t.HttpRequest(this,this.location,this.referrer),this.request.send()):void 0},r.prototype.getCachedSnapshot=function(){var t;return!(t=this.controller.getCachedSnapshotForLocation(this.location))||null!=this.location.anchor&&!t.hasAnchor(this.location.anchor)||\"restore\"!==this.action&&!t.isPreviewable()?void 0:t},r.prototype.hasCachedSnapshot=function(){return null!=this.getCachedSnapshot()},r.prototype.loadCachedSnapshot=function(){var t,e;return(e=this.getCachedSnapshot())?(t=this.shouldIssueRequest(),this.render(function(){var r;return this.cacheSnapshot(),this.controller.render({snapshot:e,isPreview:t},this.performScroll),\"function\"==typeof(r=this.adapter).visitRendered&&r.visitRendered(this),t?void 0:this.complete()})):void 0},r.prototype.loadResponse=function(){return null!=this.response?this.render(function(){var t,e;return this.cacheSnapshot(),this.request.failed?(this.controller.render({error:this.response},this.performScroll),\"function\"==typeof(t=this.adapter).visitRendered&&t.visitRendered(this),this.fail()):(this.controller.render({snapshot:this.response},this.performScroll),\"function\"==typeof(e=this.adapter).visitRendered&&e.visitRendered(this),this.complete())}):void 0},r.prototype.followRedirect=function(){return this.redirectedToLocation&&!this.followedRedirect?(this.location=this.redirectedToLocation,this.controller.replaceHistoryWithLocationAndRestorationIdentifier(this.redirectedToLocation,this.restorationIdentifier),this.followedRedirect=!0):void 0},r.prototype.requestStarted=function(){var t;return this.recordTimingMetric(\"requestStart\"),\"function\"==typeof(t=this.adapter).visitRequestStarted?t.visitRequestStarted(this):void 0},r.prototype.requestProgressed=function(t){var e;return this.progress=t,\"function\"==typeof(e=this.adapter).visitRequestProgressed?e.visitRequestProgressed(this):void 0},r.prototype.requestCompletedWithResponse=function(e,r){return this.response=e,null!=r&&(this.redirectedToLocation=t.Location.wrap(r)),this.adapter.visitRequestCompleted(this)},r.prototype.requestFailedWithStatusCode=function(t,e){return this.response=e,this.adapter.visitRequestFailedWithStatusCode(this,t)},r.prototype.requestFinished=function(){var t;return this.recordTimingMetric(\"requestEnd\"),\"function\"==typeof(t=this.adapter).visitRequestFinished?t.visitRequestFinished(this):void 0},r.prototype.performScroll=function(){return this.scrolled?void 0:(\"restore\"===this.action?this.scrollToRestoredPosition()||this.scrollToTop():this.scrollToAnchor()||this.scrollToTop(),this.scrolled=!0)},r.prototype.scrollToRestoredPosition=function(){var t,e;return t=null!=(e=this.restorationData)?e.scrollPosition:void 0,null!=t?(this.controller.scrollToPosition(t),!0):void 0},r.prototype.scrollToAnchor=function(){return null!=this.location.anchor?(this.controller.scrollToAnchor(this.location.anchor),!0):void 0},r.prototype.scrollToTop=function(){return this.controller.scrollToPosition({x:0,y:0})},r.prototype.recordTimingMetric=function(t){var e;return null!=(e=this.timingMetrics)[t]?e[t]:e[t]=(new Date).getTime()},r.prototype.getTimingMetrics=function(){return t.copyObject(this.timingMetrics)},n=function(t){switch(t){case\"replace\":return\"replaceHistoryWithLocationAndRestorationIdentifier\";case\"advance\":case\"restore\":return\"pushHistoryWithLocationAndRestorationIdentifier\"}},r.prototype.shouldIssueRequest=function(){return\"restore\"===this.action?!this.hasCachedSnapshot():!0},r.prototype.cacheSnapshot=function(){return this.snapshotCached?void 0:(this.controller.cacheSnapshot(),this.snapshotCached=!0)},r.prototype.render=function(t){return this.cancelRender(),this.frame=requestAnimationFrame(function(e){return function(){return e.frame=null,t.call(e)}}(this))},r.prototype.cancelRender=function(){return this.frame?cancelAnimationFrame(this.frame):void 0},r}()}.call(this),function(){var e=function(t,e){return function(){return t.apply(e,arguments)}};t.Controller=function(){function r(){this.clickBubbled=e(this.clickBubbled,this),this.clickCaptured=e(this.clickCaptured,this),this.pageLoaded=e(this.pageLoaded,this),this.history=new t.History(this),this.view=new t.View(this),this.scrollManager=new t.ScrollManager(this),this.restorationData={},this.clearCache()}return r.prototype.start=function(){return t.supported&&!this.started?(addEventListener(\"click\",this.clickCaptured,!0),addEventListener(\"DOMContentLoaded\",this.pageLoaded,!1),this.scrollManager.start(),this.startHistory(),this.started=!0,this.enabled=!0):void 0},r.prototype.disable=function(){return this.enabled=!1},r.prototype.stop=function(){return this.started?(removeEventListener(\"click\",this.clickCaptured,!0),removeEventListener(\"DOMContentLoaded\",this.pageLoaded,!1),this.scrollManager.stop(),this.stopHistory(),this.started=!1):void 0},r.prototype.clearCache=function(){return this.cache=new t.SnapshotCache(10)},r.prototype.visit=function(e,r){var n,o;return null==r&&(r={}),e=t.Location.wrap(e),this.applicationAllowsVisitingLocation(e)?this.locationIsVisitable(e)?(n=null!=(o=r.action)?o:\"advance\",this.adapter.visitProposedToLocationWithAction(e,n)):window.location=e:void 0},r.prototype.startVisitToLocationWithAction=function(e,r,n){var o;return t.supported?(o=this.getRestorationDataForIdentifier(n),this.startVisit(e,r,{restorationData:o})):window.location=e},r.prototype.startHistory=function(){return this.location=t.Location.wrap(window.location),this.restorationIdentifier=t.uuid(),this.history.start(),this.history.replace(this.location,this.restorationIdentifier)},r.prototype.stopHistory=function(){return this.history.stop()},r.prototype.pushHistoryWithLocationAndRestorationIdentifier=function(e,r){return this.restorationIdentifier=r,this.location=t.Location.wrap(e),this.history.push(this.location,this.restorationIdentifier)},r.prototype.replaceHistoryWithLocationAndRestorationIdentifier=function(e,r){return this.restorationIdentifier=r,this.location=t.Location.wrap(e),this.history.replace(this.location,this.restorationIdentifier)},r.prototype.historyPoppedToLocationWithRestorationIdentifier=function(e,r){var n;return this.restorationIdentifier=r,this.enabled?(n=this.getRestorationDataForIdentifier(this.restorationIdentifier),this.startVisit(e,\"restore\",{restorationIdentifier:this.restorationIdentifier,restorationData:n,historyChanged:!0}),this.location=t.Location.wrap(e)):this.adapter.pageInvalidated()},r.prototype.getCachedSnapshotForLocation=function(t){var e;return e=this.cache.get(t),e?e.clone():void 0},r.prototype.shouldCacheSnapshot=function(){return this.view.getSnapshot().isCacheable()},r.prototype.cacheSnapshot=function(){var t;return this.shouldCacheSnapshot()?(this.notifyApplicationBeforeCachingSnapshot(),t=this.view.getSnapshot(),this.cache.put(this.lastRenderedLocation,t.clone())):void 0},r.prototype.scrollToAnchor=function(t){var e;return(e=document.getElementById(t))?this.scrollToElement(e):this.scrollToPosition({x:0,y:0})},r.prototype.scrollToElement=function(t){return this.scrollManager.scrollToElement(t)},r.prototype.scrollToPosition=function(t){return this.scrollManager.scrollToPosition(t)},r.prototype.scrollPositionChanged=function(t){var e;return e=this.getCurrentRestorationData(),e.scrollPosition=t},r.prototype.render=function(t,e){return this.view.render(t,e)},r.prototype.viewInvalidated=function(){return this.adapter.pageInvalidated()},r.prototype.viewWillRender=function(t){return this.notifyApplicationBeforeRender(t)},r.prototype.viewRendered=function(){return this.lastRenderedLocation=this.currentVisit.location,this.notifyApplicationAfterRender()},r.prototype.pageLoaded=function(){return this.lastRenderedLocation=this.location,this.notifyApplicationAfterPageLoad()},r.prototype.clickCaptured=function(){return removeEventListener(\"click\",this.clickBubbled,!1),addEventListener(\"click\",this.clickBubbled,!1)},r.prototype.clickBubbled=function(t){var e,r,n;return this.enabled&&this.clickEventIsSignificant(t)&&(r=this.getVisitableLinkForNode(t.target))&&(n=this.getVisitableLocationForLink(r))&&this.applicationAllowsFollowingLinkToLocation(r,n)?(t.preventDefault(),e=this.getActionForLink(r),this.visit(n,{action:e})):void 0},r.prototype.applicationAllowsFollowingLinkToLocation=function(t,e){var r;return r=this.notifyApplicationAfterClickingLinkToLocation(t,e),!r.defaultPrevented},r.prototype.applicationAllowsVisitingLocation=function(t){var e;return e=this.notifyApplicationBeforeVisitingLocation(t),!e.defaultPrevented},r.prototype.notifyApplicationAfterClickingLinkToLocation=function(e,r){return t.dispatch(\"turbolinks:click\",{target:e,data:{url:r.absoluteURL},cancelable:!0})},r.prototype.notifyApplicationBeforeVisitingLocation=function(e){return t.dispatch(\"turbolinks:before-visit\",{data:{url:e.absoluteURL},cancelable:!0})},r.prototype.notifyApplicationAfterVisitingLocation=function(e){return t.dispatch(\"turbolinks:visit\",{data:{url:e.absoluteURL}})},r.prototype.notifyApplicationBeforeCachingSnapshot=function(){return t.dispatch(\"turbolinks:before-cache\")},r.prototype.notifyApplicationBeforeRender=function(e){return t.dispatch(\"turbolinks:before-render\",{data:{newBody:e}})},r.prototype.notifyApplicationAfterRender=function(){return t.dispatch(\"turbolinks:render\")},r.prototype.notifyApplicationAfterPageLoad=function(e){return null==e&&(e={}),t.dispatch(\"turbolinks:load\",{data:{url:this.location.absoluteURL,timing:e}})},r.prototype.startVisit=function(t,e,r){var n;return null!=(n=this.currentVisit)&&n.cancel(),this.currentVisit=this.createVisit(t,e,r),this.currentVisit.start(),this.notifyApplicationAfterVisitingLocation(t)},r.prototype.createVisit=function(e,r,n){\nvar o,i,s,a,u;return i=null!=n?n:{},a=i.restorationIdentifier,s=i.restorationData,o=i.historyChanged,u=new t.Visit(this,e,r),u.restorationIdentifier=null!=a?a:t.uuid(),u.restorationData=t.copyObject(s),u.historyChanged=o,u.referrer=this.location,u},r.prototype.visitCompleted=function(t){return this.notifyApplicationAfterPageLoad(t.getTimingMetrics())},r.prototype.clickEventIsSignificant=function(t){return!(t.defaultPrevented||t.target.isContentEditable||t.which>1||t.altKey||t.ctrlKey||t.metaKey||t.shiftKey)},r.prototype.getVisitableLinkForNode=function(e){return this.nodeIsVisitable(e)?t.closest(e,\"a[href]:not([target])\"):void 0},r.prototype.getVisitableLocationForLink=function(e){var r;return r=new t.Location(e.getAttribute(\"href\")),this.locationIsVisitable(r)?r:void 0},r.prototype.getActionForLink=function(t){var e;return null!=(e=t.getAttribute(\"data-turbolinks-action\"))?e:\"advance\"},r.prototype.nodeIsVisitable=function(e){var r;return(r=t.closest(e,\"[data-turbolinks]\"))?\"false\"!==r.getAttribute(\"data-turbolinks\"):!0},r.prototype.locationIsVisitable=function(t){return t.isPrefixedBy(this.view.getRootLocation())&&t.isHTML()},r.prototype.getCurrentRestorationData=function(){return this.getRestorationDataForIdentifier(this.restorationIdentifier)},r.prototype.getRestorationDataForIdentifier=function(t){var e;return null!=(e=this.restorationData)[t]?e[t]:e[t]={}},r}()}.call(this),function(){var e,r,n;t.start=function(){return r()?(null==t.controller&&(t.controller=e()),t.controller.start()):void 0},r=function(){return null==window.Turbolinks&&(window.Turbolinks=t),n()},e=function(){var e;return e=new t.Controller,e.adapter=new t.BrowserAdapter(e),e},n=function(){return window.Turbolinks===t},n()&&t.start()}.call(this)}).call(this),\"object\"==typeof module&&module.exports?module.exports=t:false}).call(this);" }, { "path": "docs/pro/recording-and-replaying-keep-alive-tcp-sessions.md", "content": "> **This feature available only in PRO version. See https://goreplay.org/pro.html for details.**\n\nBy default, GoReplay does not guarantee that when you record keep-alive TCP session, it will be replayed in the same TCP connection as well. This is ok for most of the cases, but it does not give an accurate number of TCP sessions while replaying, also may cause issues if your application state depends on TCP session (do not mess with HTTP session).\n\n[GoReplay PRO](https://goreplay.org/pro.html) extension adds support for accurate recording and replaying of keep-alive TCP sessions. Separate connection to your server is created per original session and it makes benchmarks and tests incredibly accurate. To enable session recognition you just need to pass `--recognize-tcp-sessions` option. \n\n```\ngor --input-raw :80 --recognize-tcp-sessions --output-http http://test.target\n```\n\nNote that enabling this option also change algorithm of distributing traffic when using `--split-output`, see [Distributed configuration]." }, { "path": "docs/pro/replaying-binary-protocols.md", "content": "> **This feature available only in PRO version. See https://goreplay.org/pro.html for details.**\n\nGor includes basic support for working with binary formats like `thrift` or `protocol-buffers`. To start set `--input-raw-protocol` to 'binary' (by default 'http'). For replaying, you should use `--output-binary`, example:\n\n```\ngor --input-raw :80 --input-raw-protocol binary --output-binary staging:8081\n```\n\nWhile working with `--input-raw` you may notice a 2-second delay before messages are emitted to the outputs. This behaviour is expected and happening because for general binary protocol it is impossible to know when TCP message ends, so Gor has to set inactivity timeout. Each protocol has own rules (for example write message length as first bytes), and require individual handling to know message length. We consider improving detailed protocol support for `thrift`, `protocol-buffer` and etc.\n\nNote that you can use all load testing features for binary protocols. For example, the following command will loop and replay recorded payload on 10x speed for 30 seconds:\n```\ngor --input-file './binary*.gor|1000%' --output-binary staging:9091 --input-file-loop --exit-after 30s\n```" }, { "path": "elasticsearch.go", "content": "package goreplay\n\nimport (\n\t\"encoding/json\"\n\t\"github.com/buger/goreplay/proto\"\n\t\"log\"\n\t\"net/url\"\n\t\"strings\"\n\t\"time\"\n\n\telastigo \"github.com/mattbaird/elastigo/lib\"\n)\n\ntype ESUriErorr struct{}\n\nfunc (e *ESUriErorr) Error() string {\n\treturn \"Wrong ElasticSearch URL format. Expected to be: scheme://host/index_name\"\n}\n\ntype ESPlugin struct {\n\tActive bool\n\tApiPort string\n\teConn *elastigo.Conn\n\tHost string\n\tIndex string\n\tindexor *elastigo.BulkIndexer\n\tdone chan bool\n}\n\ntype ESRequestResponse struct {\n\tReqURL string `json:\"Req_URL\"`\n\tReqMethod string `json:\"Req_Method\"`\n\tReqUserAgent string `json:\"Req_User-Agent\"`\n\tReqAcceptLanguage string `json:\"Req_Accept-Language,omitempty\"`\n\tReqAccept string `json:\"Req_Accept,omitempty\"`\n\tReqAcceptEncoding string `json:\"Req_Accept-Encoding,omitempty\"`\n\tReqIfModifiedSince string `json:\"Req_If-Modified-Since,omitempty\"`\n\tReqConnection string `json:\"Req_Connection,omitempty\"`\n\tReqCookies string `json:\"Req_Cookies,omitempty\"`\n\tRespStatus string `json:\"Resp_Status\"`\n\tRespStatusCode string `json:\"Resp_Status-Code\"`\n\tRespProto string `json:\"Resp_Proto,omitempty\"`\n\tRespContentLength string `json:\"Resp_Content-Length,omitempty\"`\n\tRespContentType string `json:\"Resp_Content-Type,omitempty\"`\n\tRespTransferEncoding string `json:\"Resp_Transfer-Encoding,omitempty\"`\n\tRespContentEncoding string `json:\"Resp_Content-Encoding,omitempty\"`\n\tRespExpires string `json:\"Resp_Expires,omitempty\"`\n\tRespCacheControl string `json:\"Resp_Cache-Control,omitempty\"`\n\tRespVary string `json:\"Resp_Vary,omitempty\"`\n\tRespSetCookie string `json:\"Resp_Set-Cookie,omitempty\"`\n\tRtt int64 `json:\"RTT\"`\n\tTimestamp time.Time\n}\n\n// Parse ElasticSearch URI\n//\n// Proper format is: scheme://[userinfo@]host/index_name\n// userinfo is: user[:password]\n// net/url.Parse() does not fail if scheme is not provided but actually does not\n// handle URI properly.\n// So we must 'validate' URI format to match requirements to use net/url.Parse()\nfunc parseURI(URI string) (err error, index string) {\n\n\tparsedUrl, parseErr := url.Parse(URI)\n\n\tif parseErr != nil {\n\t\terr = new(ESUriErorr)\n\t\treturn\n\t}\n\n\t//\tcheck URL validity by extracting host and index values.\n\thost := parsedUrl.Host\n\turlPathParts := strings.Split(parsedUrl.Path, \"/\")\n\tindex = urlPathParts[len(urlPathParts)-1]\n\n\t// force index specification in uri : ie no implicit index\n\tif host == \"\" || index == \"\" {\n\t\terr = new(ESUriErorr)\n\t}\n\n\treturn\n}\n\nfunc (p *ESPlugin) Init(URI string) {\n\tvar err error\n\n\terr, p.Index = parseURI(URI)\n\n\tif err != nil {\n\t\tlog.Fatal(\"Can't initialize ElasticSearch plugin.\", err)\n\t}\n\n\tp.eConn = elastigo.NewConn()\n\n\tp.eConn.SetFromUrl(URI)\n\n\tp.indexor = p.eConn.NewBulkIndexerErrors(50, 60)\n\tp.done = make(chan bool)\n\tp.indexor.Start()\n\n\tgo p.ErrorHandler()\n\n\tDebug(1, \"Initialized Elasticsearch Plugin\")\n\treturn\n}\n\nfunc (p *ESPlugin) IndexerShutdown() {\n\tp.indexor.Stop()\n\treturn\n}\n\nfunc (p *ESPlugin) ErrorHandler() {\n\tfor {\n\t\terrBuf := <-p.indexor.ErrorChannel\n\t\tDebug(1, \"[ELASTICSEARCH]\", errBuf.Err)\n\t}\n}\n\nfunc (p *ESPlugin) RttDurationToMs(d time.Duration) int64 {\n\tsec := d / time.Second\n\tnsec := d % time.Second\n\tfl := float64(sec) + float64(nsec)*1e-6\n\treturn int64(fl)\n}\n\n// ResponseAnalyze send req and resp to ES\nfunc (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) {\n\tif len(resp) == 0 {\n\t\t// nil http response - skipped elasticsearch export for this request\n\t\treturn\n\t}\n\tt := time.Now()\n\trtt := p.RttDurationToMs(stop.Sub(start))\n\n\tesResp := ESRequestResponse{\n\t\tReqURL: string(proto.Path(req)),\n\t\tReqMethod: string(proto.Method(req)),\n\t\tReqUserAgent: string(proto.Header(req, []byte(\"User-Agent\"))),\n\t\tReqAcceptLanguage: string(proto.Header(req, []byte(\"Accept-Language\"))),\n\t\tReqAccept: string(proto.Header(req, []byte(\"Accept\"))),\n\t\tReqAcceptEncoding: string(proto.Header(req, []byte(\"Accept-Encoding\"))),\n\t\tReqIfModifiedSince: string(proto.Header(req, []byte(\"If-Modified-Since\"))),\n\t\tReqConnection: string(proto.Header(req, []byte(\"Connection\"))),\n\t\tReqCookies: string(proto.Header(req, []byte(\"Cookie\"))),\n\t\tRespStatus: string(proto.Status(resp)),\n\t\tRespStatusCode: string(proto.Status(resp)),\n\t\tRespProto: string(proto.Method(resp)),\n\t\tRespContentLength: string(proto.Header(resp, []byte(\"Content-Length\"))),\n\t\tRespContentType: string(proto.Header(resp, []byte(\"Content-Type\"))),\n\t\tRespTransferEncoding: string(proto.Header(resp, []byte(\"Transfer-Encoding\"))),\n\t\tRespContentEncoding: string(proto.Header(resp, []byte(\"Content-Encoding\"))),\n\t\tRespExpires: string(proto.Header(resp, []byte(\"Expires\"))),\n\t\tRespCacheControl: string(proto.Header(resp, []byte(\"Cache-Control\"))),\n\t\tRespVary: string(proto.Header(resp, []byte(\"Vary\"))),\n\t\tRespSetCookie: string(proto.Header(resp, []byte(\"Set-Cookie\"))),\n\t\tRtt: rtt,\n\t\tTimestamp: t,\n\t}\n\tj, err := json.Marshal(&esResp)\n\tif err != nil {\n\t\tDebug(0, \"[ELASTIC-RESPONSE]\", err)\n\t} else {\n\t\tp.indexor.Index(p.Index, \"RequestResponse\", \"\", \"\", \"\", &t, j)\n\t}\n\treturn\n}\n" }, { "path": "elasticsearch_test.go", "content": "package goreplay\n\nimport (\n\t\"testing\"\n)\n\nconst expectedIndex = \"gor\"\n\nfunc assertExpectedGorIndex(index string, t *testing.T) {\n\tif expectedIndex != index {\n\t\tt.Fatalf(\"Expected index %s but got %s\", expectedIndex, index)\n\t}\n}\n\nfunc assertExpectedIndex(expectedIndex string, index string, t *testing.T) {\n\tif expectedIndex != index {\n\t\tt.Fatalf(\"Expected index %s but got %s\", expectedIndex, index)\n\t}\n}\n\nfunc assertExpectedError(returnedError error, t *testing.T) {\n\texpectedError := new(ESUriErorr)\n\n\tif expectedError != returnedError {\n\t\tt.Errorf(\"Expected err %s but got %s\", expectedError, returnedError)\n\t}\n}\n\nfunc assertNoError(returnedError error, t *testing.T) {\n\tif nil != returnedError {\n\t\tt.Errorf(\"Expected no err but got %s\", returnedError)\n\t}\n}\n\n// Argument host:port/index_name\n// i.e : localhost:9200/gor\n// Fail because scheme is mandatory\nfunc TestElasticConnectionBuildFailWithoutScheme(t *testing.T) {\n\turi := \"localhost:9200/\" + expectedIndex\n\n\terr, _ := parseURI(uri)\n\tassertExpectedError(err, t)\n}\n\n// Argument scheme://Host:port\n// i.e : http://localhost:9200\n// Fail : explicit index is required\nfunc TestElasticConnectionBuildFailWithoutIndex(t *testing.T) {\n\turi := \"http://localhost:9200\"\n\n\terr, index := parseURI(uri)\n\n\tassertExpectedIndex(\"\", index, t)\n\n\tassertExpectedError(err, t)\n}\n\n// Argument scheme://Host/index_name\n// i.e : http://localhost/gor\nfunc TestElasticConnectionBuildFailWithoutPort(t *testing.T) {\n\turi := \"http://localhost/\" + expectedIndex\n\n\terr, index := parseURI(uri)\n\n\tassertNoError(err, t)\n\n\tassertExpectedGorIndex(index, t)\n}\n\n// Argument scheme://Host:port/index_name\n// i.e : http://localhost:9200/gor\nfunc TestElasticLocalConnectionBuild(t *testing.T) {\n\turi := \"http://localhost:9200/\" + expectedIndex\n\n\terr, index := parseURI(uri)\n\n\tassertNoError(err, t)\n\n\tassertExpectedGorIndex(index, t)\n}\n\n// Argument scheme://Host:port/index_name\n// i.e : http://localhost.local:9200/gor or https://localhost.local:9200/gor\nfunc TestElasticSimpleLocalWithSchemeConnectionBuild(t *testing.T) {\n\turi := \"http://localhost.local:9200/\" + expectedIndex\n\n\terr, index := parseURI(uri)\n\n\tassertNoError(err, t)\n\n\tassertExpectedGorIndex(index, t)\n}\n\n// Argument scheme://Host:port/index_name\n// i.e : http://localhost.local:9200/gor or https://localhost.local:9200/gor\nfunc TestElasticSimpleLocalWithHTTPSConnectionBuild(t *testing.T) {\n\turi := \"https://localhost.local:9200/\" + expectedIndex\n\n\terr, index := parseURI(uri)\n\n\tassertNoError(err, t)\n\n\tassertExpectedGorIndex(index, t)\n}\n\n// Argument scheme://Host:port/index_name\n// i.e : localhost.local:9200/pathtoElastic/gor\nfunc TestElasticLongPathConnectionBuild(t *testing.T) {\n\turi := \"http://localhost.local:9200/pathtoElastic/\" + expectedIndex\n\n\terr, index := parseURI(uri)\n\n\tassertNoError(err, t)\n\n\tassertExpectedGorIndex(index, t)\n}\n\n// Argument scheme://Host:userinfo@port/index_name\n// i.e : http://user:password@localhost.local:9200/gor\nfunc TestElasticBasicAuthConnectionBuild(t *testing.T) {\n\turi := \"http://user:password@localhost.local:9200/\" + expectedIndex\n\n\terr, index := parseURI(uri)\n\n\tassertNoError(err, t)\n\n\tassertExpectedGorIndex(index, t)\n}\n\n// Argument scheme://Host:port/path/index_name\n// i.e : http://localhost.local:9200/path/gor or https://localhost.local:9200/path/gor\nfunc TestElasticComplexPathConnectionBuild(t *testing.T) {\n\turi := \"http://localhost.local:9200/path/\" + expectedIndex\n\n\terr, index := parseURI(uri)\n\n\tassertNoError(err, t)\n\n\tassertExpectedGorIndex(index, t)\n}\n" }, { "path": "emitter.go", "content": "package goreplay\n\nimport (\n\t\"fmt\"\n\t\"github.com/buger/goreplay/internal/byteutils\"\n\t\"hash/fnv\"\n\t\"io\"\n\t\"log\"\n\t\"sync\"\n\n\t\"github.com/coocood/freecache\"\n)\n\n// Emitter represents an abject to manage plugins communication\ntype Emitter struct {\n\tsync.WaitGroup\n\tplugins *InOutPlugins\n}\n\n// NewEmitter creates and initializes new Emitter object.\nfunc NewEmitter() *Emitter {\n\treturn &Emitter{}\n}\n\n// Start initialize loop for sending data from inputs to outputs\nfunc (e *Emitter) Start(plugins *InOutPlugins, middlewareCmd string) {\n\tif Settings.CopyBufferSize < 1 {\n\t\tSettings.CopyBufferSize = 5 << 20\n\t}\n\te.plugins = plugins\n\n\tif middlewareCmd != \"\" {\n\t\tmiddleware := NewMiddleware(middlewareCmd)\n\n\t\tfor _, in := range plugins.Inputs {\n\t\t\tmiddleware.ReadFrom(in)\n\t\t}\n\n\t\te.plugins.Inputs = append(e.plugins.Inputs, middleware)\n\t\te.plugins.All = append(e.plugins.All, middleware)\n\t\te.Add(1)\n\t\tgo func() {\n\t\t\tdefer e.Done()\n\t\t\tif err := CopyMulty(middleware, plugins.Outputs...); err != nil {\n\t\t\t\tDebug(2, fmt.Sprintf(\"[EMITTER] error during copy: %q\", err))\n\t\t\t}\n\t\t}()\n\t} else {\n\t\tfor _, in := range plugins.Inputs {\n\t\t\te.Add(1)\n\t\t\tgo func(in PluginReader) {\n\t\t\t\tdefer e.Done()\n\t\t\t\tif err := CopyMulty(in, plugins.Outputs...); err != nil {\n\t\t\t\t\tDebug(2, fmt.Sprintf(\"[EMITTER] error during copy: %q\", err))\n\t\t\t\t}\n\t\t\t}(in)\n\t\t}\n\t}\n}\n\n// Close closes all the goroutine and waits for it to finish.\nfunc (e *Emitter) Close() {\n\tfor _, p := range e.plugins.All {\n\t\tif cp, ok := p.(io.Closer); ok {\n\t\t\tcp.Close()\n\t\t}\n\t}\n\tif len(e.plugins.All) > 0 {\n\t\t// wait for everything to stop\n\t\te.Wait()\n\t}\n\te.plugins.All = nil // avoid Close to make changes again\n}\n\n// CopyMulty copies from 1 reader to multiple writers\nfunc CopyMulty(src PluginReader, writers ...PluginWriter) error {\n\twIndex := 0\n\tmodifier := NewHTTPModifier(&Settings.ModifierConfig)\n\tfilteredRequests := freecache.NewCache(200 * 1024 * 1024) // 200M\n\n\tfor {\n\t\tmsg, err := src.PluginRead()\n\t\tif err != nil {\n\t\t\tif err == ErrorStopped || err == io.EOF {\n\t\t\t\treturn nil\n\t\t\t}\n\t\t\treturn err\n\t\t}\n\t\tif msg != nil && len(msg.Data) > 0 {\n\t\t\tif len(msg.Data) > int(Settings.CopyBufferSize) {\n\t\t\t\tmsg.Data = msg.Data[:Settings.CopyBufferSize]\n\t\t\t}\n\t\t\tmeta := payloadMeta(msg.Meta)\n\t\t\tif len(meta) < 3 {\n\t\t\t\tDebug(2, fmt.Sprintf(\"[EMITTER] Found malformed record %q from %q\", msg.Meta, src))\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\trequestID := meta[1]\n\t\t\t// start a subroutine only when necessary\n\t\t\tif Settings.Verbose >= 3 {\n\t\t\t\tDebug(3, \"[EMITTER] input: \", byteutils.SliceToString(msg.Meta[:len(msg.Meta)-1]), \" from: \", src)\n\t\t\t}\n\t\t\tif modifier != nil {\n\t\t\t\tDebug(3, \"[EMITTER] modifier:\", requestID, \"from:\", src)\n\t\t\t\tif isRequestPayload(msg.Meta) {\n\t\t\t\t\tmsg.Data = modifier.Rewrite(msg.Data)\n\t\t\t\t\t// If modifier tells to skip request\n\t\t\t\t\tif len(msg.Data) == 0 {\n\t\t\t\t\t\tfilteredRequests.Set(requestID, []byte{}, 60) //\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t\tDebug(3, \"[EMITTER] Rewritten input:\", requestID, \"from:\", src)\n\n\t\t\t\t} else {\n\t\t\t\t\t_, err := filteredRequests.Get(requestID)\n\t\t\t\t\tif err == nil {\n\t\t\t\t\t\tfilteredRequests.Del(requestID)\n\t\t\t\t\t\tcontinue\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif Settings.PrettifyHTTP {\n\t\t\t\tmsg.Data = prettifyHTTP(msg.Data)\n\t\t\t\tif len(msg.Data) == 0 {\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif Settings.SplitOutput {\n\t\t\t\tif Settings.RecognizeTCPSessions {\n\t\t\t\t\tif !PRO {\n\t\t\t\t\t\tlog.Fatal(\"Detailed TCP sessions work only with PRO license\")\n\t\t\t\t\t}\n\t\t\t\t\thasher := fnv.New32a()\n\t\t\t\t\thasher.Write(meta[1])\n\n\t\t\t\t\twIndex = int(hasher.Sum32()) % len(writers)\n\t\t\t\t\tif _, err := writers[wIndex].PluginWrite(msg); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\t// Simple round robin\n\t\t\t\t\tif _, err := writers[wIndex].PluginWrite(msg); err != nil {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\n\t\t\t\t\twIndex = (wIndex + 1) % len(writers)\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tfor _, dst := range writers {\n\t\t\t\t\tif _, err := dst.PluginWrite(msg); err != nil && err != io.ErrClosedPipe {\n\t\t\t\t\t\treturn err\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n" }, { "path": "emitter_test.go", "content": "package goreplay\n\nimport (\n\t\"fmt\"\n\t\"os\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestMain(m *testing.M) {\n\tPRO = true\n\tcode := m.Run()\n\tos.Exit(code)\n}\n\nfunc TestEmitter(t *testing.T) {\n\twg := new(sync.WaitGroup)\n\n\tinput := NewTestInput()\n\toutput := NewTestOutput(func(*Message) {\n\t\twg.Done()\n\t})\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output},\n\t}\n\tplugins.All = append(plugins.All, input, output)\n\n\temitter := NewEmitter()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\tfor i := 0; i < 1000; i++ {\n\t\twg.Add(1)\n\t\tinput.EmitGET()\n\t}\n\n\twg.Wait()\n\temitter.Close()\n}\n\nfunc TestEmitterFiltered(t *testing.T) {\n\twg := new(sync.WaitGroup)\n\n\tinput := NewTestInput()\n\tinput.skipHeader = true\n\n\toutput := NewTestOutput(func(*Message) {\n\t\twg.Done()\n\t})\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output},\n\t}\n\tplugins.All = append(plugins.All, input, output)\n\n\tmethods := HTTPMethods{[]byte(\"GET\")}\n\tSettings.ModifierConfig = HTTPModifierConfig{Methods: methods}\n\n\temitter := &Emitter{}\n\tgo emitter.Start(plugins, \"\")\n\n\twg.Add(2)\n\n\tid := uuid()\n\treqh := payloadHeader(RequestPayload, id, time.Now().UnixNano(), -1)\n\treqb := append(reqh, []byte(\"POST / HTTP/1.1\\r\\nHost: www.w3.org\\r\\nUser-Agent: Go 1.1 package http\\r\\nAccept-Encoding: gzip\\r\\n\\r\\n\")...)\n\n\tresh := payloadHeader(ResponsePayload, id, time.Now().UnixNano()+1, 1)\n\trespb := append(resh, []byte(\"HTTP/1.1 200 OK\\r\\nContent-Length: 0\\r\\n\\r\\n\")...)\n\n\tinput.EmitBytes(reqb)\n\tinput.EmitBytes(respb)\n\n\tid = uuid()\n\treqh = payloadHeader(RequestPayload, id, time.Now().UnixNano(), -1)\n\treqb = append(reqh, []byte(\"GET / HTTP/1.1\\r\\nHost: www.w3.org\\r\\nUser-Agent: Go 1.1 package http\\r\\nAccept-Encoding: gzip\\r\\n\\r\\n\")...)\n\n\tresh = payloadHeader(ResponsePayload, id, time.Now().UnixNano()+1, 1)\n\trespb = append(resh, []byte(\"HTTP/1.1 200 OK\\r\\nContent-Length: 0\\r\\n\\r\\n\")...)\n\n\tinput.EmitBytes(reqb)\n\tinput.EmitBytes(respb)\n\n\twg.Wait()\n\temitter.Close()\n\n\tSettings.ModifierConfig = HTTPModifierConfig{}\n}\n\nfunc TestEmitterSplitRoundRobin(t *testing.T) {\n\twg := new(sync.WaitGroup)\n\n\tinput := NewTestInput()\n\n\tvar counter1, counter2 int32\n\n\toutput1 := NewTestOutput(func(*Message) {\n\t\tatomic.AddInt32(&counter1, 1)\n\t\twg.Done()\n\t})\n\n\toutput2 := NewTestOutput(func(*Message) {\n\t\tatomic.AddInt32(&counter2, 1)\n\t\twg.Done()\n\t})\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output1, output2},\n\t}\n\n\tSettings.SplitOutput = true\n\n\temitter := NewEmitter()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\tfor i := 0; i < 1000; i++ {\n\t\twg.Add(1)\n\t\tinput.EmitGET()\n\t}\n\n\twg.Wait()\n\n\temitter.Close()\n\n\tif counter1 == 0 || counter2 == 0 || counter1 != counter2 {\n\t\tt.Errorf(\"Round robin should split traffic equally: %d vs %d\", counter1, counter2)\n\t}\n\n\tSettings.SplitOutput = false\n}\n\nfunc TestEmitterRoundRobin(t *testing.T) {\n\twg := new(sync.WaitGroup)\n\n\tinput := NewTestInput()\n\n\tvar counter1, counter2 int32\n\n\toutput1 := NewTestOutput(func(*Message) {\n\t\tcounter1++\n\t\twg.Done()\n\t})\n\n\toutput2 := NewTestOutput(func(*Message) {\n\t\tcounter2++\n\t\twg.Done()\n\t})\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output1, output2},\n\t}\n\tplugins.All = append(plugins.All, input, output1, output2)\n\n\tSettings.SplitOutput = true\n\n\temitter := NewEmitter()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\tfor i := 0; i < 1000; i++ {\n\t\twg.Add(1)\n\t\tinput.EmitGET()\n\t}\n\n\twg.Wait()\n\temitter.Close()\n\n\tif counter1 == 0 || counter2 == 0 {\n\t\tt.Errorf(\"Round robin should split traffic equally: %d vs %d\", counter1, counter2)\n\t}\n\n\tSettings.SplitOutput = false\n}\n\nfunc TestEmitterSplitSession(t *testing.T) {\n\twg := new(sync.WaitGroup)\n\twg.Add(200)\n\n\tinput := NewTestInput()\n\tinput.skipHeader = true\n\n\tvar counter1, counter2 int32\n\n\toutput1 := NewTestOutput(func(msg *Message) {\n\t\tif payloadID(msg.Meta)[0] == 'a' {\n\t\t\tcounter1++\n\t\t}\n\t\twg.Done()\n\t})\n\n\toutput2 := NewTestOutput(func(msg *Message) {\n\t\tif payloadID(msg.Meta)[0] == 'b' {\n\t\t\tcounter2++\n\t\t}\n\t\twg.Done()\n\t})\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output1, output2},\n\t}\n\n\tSettings.SplitOutput = true\n\tSettings.RecognizeTCPSessions = true\n\n\temitter := NewEmitter()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\tfor i := 0; i < 200; i++ {\n\t\t// Keep session but randomize\n\t\tid := make([]byte, 20)\n\t\tif i&1 == 0 { // for recognizeTCPSessions one should be odd and other will be even number\n\t\t\tid[0] = 'a'\n\t\t} else {\n\t\t\tid[0] = 'b'\n\t\t}\n\t\tinput.EmitBytes([]byte(fmt.Sprintf(\"1 %s 1 1\\nGET / HTTP/1.1\\r\\n\\r\\n\", id[:20])))\n\t}\n\n\twg.Wait()\n\n\tif counter1 != counter2 {\n\t\tt.Errorf(\"Round robin should split traffic equally: %d vs %d\", counter1, counter2)\n\t}\n\n\tSettings.SplitOutput = false\n\tSettings.RecognizeTCPSessions = false\n\temitter.Close()\n}\n\nfunc BenchmarkEmitter(b *testing.B) {\n\twg := new(sync.WaitGroup)\n\n\tinput := NewTestInput()\n\n\toutput := NewTestOutput(func(*Message) {\n\t\twg.Done()\n\t})\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output},\n\t}\n\tplugins.All = append(plugins.All, input, output)\n\n\temitter := NewEmitter()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\tb.ResetTimer()\n\n\tfor i := 0; i < b.N; i++ {\n\t\twg.Add(1)\n\t\tinput.EmitGET()\n\t}\n\n\twg.Wait()\n\temitter.Close()\n}\n" }, { "path": "examples/middleware/echo.clj", "content": "(ns echo.core\n (:gen-class)\n (:require [clojure.string :as cs]\n [clojure.java.io :as io])\n (:import org.apache.commons.codec.binary.Hex\n java.io.BufferedReader\n java.io.IOException\n java.io.InputStreamReader))\n\n\n(defn transform-http-msg\n \"Function that transforms/filters the incoming HTTP messages.\"\n [headers body]\n ;; do actual transformations here\n [headers body])\n\n\n(defn decode-hex-string\n \"Decode an Hex-encoded string.\"\n [s]\n (String. (Hex/decodeHex (.toCharArray s))))\n\n\n(defn encode-hex-string\n \"Encode a string to a hex-encoded string.\"\n [^String s]\n (String. (Hex/encodeHex (.getBytes s))))\n\n\n(defn -main\n [& args]\n (let [br (BufferedReader. (InputStreamReader. System/in))]\n (try\n (loop [hex-line (.readLine br)]\n (let [decoded-req (decode-hex-string hex-line)\n\n ;; empty line separates headers from body\n http-request (partition-by empty? (cs/split-lines decoded-req))\n headers (first http-request)\n\n ;; HTTP messages can contain no body:\n body (when (= 3 (count http-request)) (last http-request))\n [new-headers new-body] (transform-http-msg headers body)]\n\n (println (encode-hex-string (str (cs/join \"\\n\" headers)\n (when body\n (str \"\\n\\n\"\n (cs/join \"\\n\" body)))))))\n (when-let [line (.readLine br)]\n (recur line)))\n (catch IOException e nil))))\n\n\n" }, { "path": "examples/middleware/echo.java", "content": "import java.io.BufferedReader;\nimport java.io.IOException;\nimport java.io.InputStreamReader;\n\nimport org.apache.commons.codec.DecoderException;\nimport org.apache.commons.codec.binary.Hex;\n\n\nclass Echo {\n public static String decodeHexString(String s) throws DecoderException {\n return new String(Hex.decodeHex(s.toCharArray()));\n }\n\n public static String encodeHexString(String s) {\n return new String(Hex.encodeHex(s.getBytes()));\n }\n\n public static String transformHTTPMessage(String req) {\n // do actual transformations here\n return req;\n }\n\n public static void main(String[] args) throws DecoderException {\n if(args != null){\n for(String arg : args){\n System.out.println(arg);\n }\n\n }\n\n BufferedReader stdin = new BufferedReader(new InputStreamReader(\n System.in));\n String line = null;\n\n try {\n while ((line = stdin.readLine()) != null) {\n String decodedLine = decodeHexString(line);\n\n String transformedLine = transformHTTPMessage(decodedLine);\n\n String encodedLine = encodeHexString(transformedLine);\n System.out.println(encodedLine);\n\n }\n } catch (IOException e) {\n }\n }\n}\n" }, { "path": "examples/middleware/echo.js", "content": "#!/usr/bin/env node\nconst readline = require(\"readline\");\nconst StringDecoder = require(\"string_decoder\").StringDecoder\n\nconst rl = readline.createInterface({\n input: process.stdin\n});\n\nvar ignoreIds = new Set();\nvar ignoreAddresses = \"/api\";\nconst decoder = new StringDecoder(\"utf8\");\n\nfunction convertHexString(hex) {\n var bytes = [];\n for (var i = 0; i < hex.length - 1; i += 2) {\n bytes.push(parseInt(hex.substr(i, 2), 16));\n }\n return decoder.write(Buffer.from(bytes));\n}\n\nfunction log(output) {\n\tconsole.error(\"===================\");\n\tconsole.error(output);\n}\n\nfunction shouldOutputLine(request) {\n const components = request.split(\"\\n\");\n const header = components[0].split(\" \");\n const type = parseInt(header[0]);\n const tag = header[1];\n\n if (type === 3) {\n return true;\n }\n if (type === 1) {\n // Check if it's oauth\n const endpoint = components[1].split(\" \")[1];\n if (!endpoint.startsWith(ignoreAddresses)) {\n ignoreIds.add(tag);\n return false;\n }\n } else if (type === 2) {\n if (ignoreIds.has(tag)) {\n ignoreIds.delete(tag);\n return false;\n }\n }\n return true;\n}\n\nrl.on(\"line\", (input) => {\n const str = convertHexString(input);\n console.log(input);\n if (shouldOutputLine(str)) {\n log(str);\n\t}\n});\n" }, { "path": "examples/middleware/echo.py", "content": "#! /usr/bin/env python3\n# -*- coding: utf-8 -*-\n\nimport sys\nimport fileinput\nimport binascii\n\n# Used to find end of the Headers section\nEMPTY_LINE = b'\\r\\n\\r\\n'\n\n\ndef log(msg):\n \"\"\"\n Logging to STDERR as STDOUT and STDIN used for data transfer\n @type msg: str or byte string\n @param msg: Message to log to STDERR\n \"\"\"\n try:\n msg = str(msg) + '\\n'\n except:\n pass\n sys.stderr.write(msg)\n sys.stderr.flush()\n\n\ndef find_end_of_headers(byte_data):\n \"\"\"\n Finds where the header portion ends and the content portion begins.\n @type byte_data: str or byte string\n @param byte_data: Hex decoded req or resp string\n \"\"\"\n return byte_data.index(EMPTY_LINE) + 4\n\n\ndef process_stdin():\n \"\"\"\n Process STDIN and output to STDOUT\n \"\"\"\n for raw_line in fileinput.input():\n\n line = raw_line.rstrip()\n\n # Decode base64 encoded line\n decoded = bytes.fromhex(line)\n\n # Split into metadata and payload, the payload is headers + body\n (raw_metadata, payload) = decoded.split(b'\\n', 1)\n\n # Split into headers and payload\n headers_pos = find_end_of_headers(payload)\n raw_headers = payload[:headers_pos]\n raw_content = payload[headers_pos:]\n\n log('===================================')\n request_type_id = int(raw_metadata.split(b' ')[0])\n log('Request type: {}'.format({\n 1: 'Request',\n 2: 'Original Response',\n 3: 'Replayed Response'\n }[request_type_id]))\n log('===================================')\n\n log('Original data:')\n log(line)\n\n log('Decoded request:')\n log(decoded)\n\n encoded = binascii.hexlify(raw_metadata + b'\\n' + raw_headers + raw_content).decode('ascii')\n log('Encoded data:')\n log(encoded)\n\n sys.stdout.write(encoded + '\\n')\n\nif __name__ == '__main__':\n process_stdin()\n" }, { "path": "examples/middleware/echo.rb", "content": "#!/usr/bin/env ruby\n# encoding: utf-8\nwhile data = STDIN.gets # continuously read line from STDIN\n next unless data\n data = data.chomp # remove end of line symbol\n \n decoded = [data].pack(\"H*\") # decode base64 encoded request\n \n # decoded value is raw HTTP payload, example:\n # \n # POST /post HTTP/1.1\n # Content-Length: 7\n # Host: www.w3.org\n #\n # a=1&b=2\"\n \n encoded = decoded.unpack(\"H*\").first # encoding back to base64\n \n # Emit request back\n # You can skip this if want to filter out request\n STDOUT.puts encoded\nend\n" }, { "path": "examples/middleware/echo.sh", "content": "#!/usr/bin/env bash\n#\n# `xxd` utility included into vim-common package\n# It allow hex decoding/encoding\n# \n# This example may broke if you request contains `null` string, you may consider using pipes instead.\n# See: https://github.com/buger/gor/issues/309\n# \n\nfunction log {\n if [[ -n \"$GOR_TEST\" ]]; then # if we are not testing\n # Logging to stderr, because stdout/stdin used for data transfer\n >&2 echo \"[DEBUG][ECHO] $1\"\n fi\n}\n\nwhile read line; do\n decoded=$(echo -e \"$line\" | xxd -r -p)\n\n header=$(echo -e \"$decoded\" | head -n +1)\n payload=$(echo -e \"$decoded\" | tail -n +2)\n\n encoded=$(echo -e \"$header\\n$payload\" | xxd -p | tr -d \"\\\\n\")\n\n log \"\"\n log \"===================================\"\n\n case ${header:0:1} in\n \"1\")\n log \"Request type: Request\"\n ;;\n \"2\")\n log \"Request type: Original Response\"\n ;;\n \"3\")\n log \"Request type: Replayed Response\"\n ;;\n *)\n log \"Unknown request type $header\"\n esac\n echo \"$encoded\"\n\n log \"===================================\"\n\n log \"Original data: $line\"\n log \"Decoded request: $decoded\"\n log \"Encoded data: $encoded\"\ndone;\n" }, { "path": "examples/middleware/token_modifier.go", "content": "/*\nThis middleware made for auth system that randomly generate access tokens, which used later for accessing secure content. Since there is no pre-defined token value, naive approach without middleware (or if middleware use only request payloads) will fail, because replayed server have own tokens, not synced with origin. To fix this, our middleware should take in account responses of replayed and origin server, store `originalToken -> replayedToken` aliases and rewrite all requests using this token to use replayed alias. See `middleware_test.go#TestTokenMiddleware` test for examples of using this middleware.\n\nHow middleware works:\n\n Original request +--------------+\n+-------------+----------STDIN---------->+ |\n| Gor input | | Middleware |\n+-------------+----------STDIN---------->+ |\n Original response +------+---+---+\n | ^\n+-------------+ Modified request v |\n| Gor output +<---------STDOUT-----------------+ |\n+-----+-------+ |\n | |\n | Replayed response |\n +------------------STDIN----------------->----+\n*/\n\npackage main\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"github.com/buger/goreplay/proto\"\n\t\"os\"\n)\n\n// requestID -> originalToken\nvar originalTokens map[string][]byte\n\n// originalToken -> replayedToken\nvar tokenAliases map[string][]byte\n\nfunc main() {\n\toriginalTokens = make(map[string][]byte)\n\ttokenAliases = make(map[string][]byte)\n\n\tscanner := bufio.NewScanner(os.Stdin)\n\n\tfor scanner.Scan() {\n\t\tencoded := scanner.Bytes()\n\t\tbuf := make([]byte, len(encoded)/2)\n\t\thex.Decode(buf, encoded)\n\n\t\tprocess(buf)\n\t}\n}\n\nfunc process(buf []byte) {\n\t// First byte indicate payload type, possible values:\n\t// 1 - Request\n\t// 2 - Response\n\t// 3 - ReplayedResponse\n\tpayloadType := buf[0]\n\theaderSize := bytes.IndexByte(buf, '\\n') + 1\n\theader := buf[:headerSize-1]\n\n\t// Header contains space separated values of: request type, request id, and request start time (or round-trip time for responses)\n\tmeta := bytes.Split(header, []byte(\" \"))\n\t// For each request you should receive 3 payloads (request, response, replayed response) with same request id\n\treqID := string(meta[1])\n\tpayload := buf[headerSize:]\n\n\tDebug(\"Received payload:\", string(buf))\n\n\tswitch payloadType {\n\tcase '1': // Request\n\t\tif bytes.Equal(proto.Path(payload), []byte(\"/token\")) {\n\t\t\toriginalTokens[reqID] = []byte{}\n\t\t\tDebug(\"Found token request:\", reqID)\n\t\t} else {\n\t\t\ttoken, vs, _ := proto.PathParam(payload, []byte(\"token\"))\n\n\t\t\tif vs != -1 { // If there is GET token param\n\t\t\t\tif alias, ok := tokenAliases[string(token)]; ok {\n\t\t\t\t\t// Rewrite original token to alias\n\t\t\t\t\tpayload = proto.SetPathParam(payload, []byte(\"token\"), alias)\n\n\t\t\t\t\t// Copy modified payload to our buffer\n\t\t\t\t\tbuf = append(buf[:headerSize], payload...)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Emitting data back\n\t\tos.Stdout.Write(encode(buf))\n\tcase '2': // Original response\n\t\tif _, ok := originalTokens[reqID]; ok {\n\t\t\t// Token is inside response body\n\t\t\tsecureToken := proto.Body(payload)\n\t\t\toriginalTokens[reqID] = secureToken\n\t\t\tDebug(\"Remember origial token:\", string(secureToken))\n\t\t}\n\tcase '3': // Replayed response\n\t\tif originalToken, ok := originalTokens[reqID]; ok {\n\t\t\tdelete(originalTokens, reqID)\n\t\t\tsecureToken := proto.Body(payload)\n\t\t\ttokenAliases[string(originalToken)] = secureToken\n\n\t\t\tDebug(\"Create alias for new token token, was:\", string(originalToken), \"now:\", string(secureToken))\n\t\t}\n\t}\n}\n\nfunc encode(buf []byte) []byte {\n\tdst := make([]byte, len(buf)*2+1)\n\thex.Encode(dst, buf)\n\tdst[len(dst)-1] = '\\n'\n\n\treturn dst\n}\n\nfunc Debug(args ...interface{}) {\n\tif os.Getenv(\"GOR_TEST\") == \"\" { // if we are not testing\n\t\tfmt.Fprint(os.Stderr, \"[DEBUG][TOKEN-MOD] \")\n\t\tfmt.Fprintln(os.Stderr, args...)\n\t}\n}\n" }, { "path": "go.mod", "content": "module github.com/buger/goreplay\n\ngo 1.21\n\nrequire (\n\tgithub.com/Shopify/sarama v1.38.1\n\tgithub.com/aws/aws-sdk-go v1.44.262\n\tgithub.com/coocood/freecache v1.2.3\n\tgithub.com/google/gopacket v1.1.20-0.20210429153827-3eaba0894325\n\tgithub.com/gorilla/websocket v1.5.0\n\tgithub.com/klauspost/compress v1.16.5 // indirect\n\tgithub.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b\n\tgithub.com/stretchr/testify v1.8.2\n\tgithub.com/xdg-go/scram v1.1.2\n\tgolang.org/x/net v0.34.0\n\tgolang.org/x/sys v0.29.0\n\tk8s.io/apimachinery v0.27.1\n\tk8s.io/client-go v0.27.1\n)\n\nrequire (\n\tgithub.com/araddon/gou v0.0.0-20211019181548-e7d08105776c // indirect\n\tgithub.com/bitly/go-hostpool v0.1.0 // indirect\n\tgithub.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect\n\tgithub.com/cespare/xxhash/v2 v2.2.0 // indirect\n\tgithub.com/davecgh/go-spew v1.1.1 // indirect\n\tgithub.com/eapache/go-resiliency v1.3.0 // indirect\n\tgithub.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect\n\tgithub.com/eapache/queue v1.1.0 // indirect\n\tgithub.com/emicklei/go-restful/v3 v3.10.2 // indirect\n\tgithub.com/go-logr/logr v1.2.4 // indirect\n\tgithub.com/go-openapi/jsonpointer v0.19.6 // indirect\n\tgithub.com/go-openapi/jsonreference v0.20.2 // indirect\n\tgithub.com/go-openapi/swag v0.22.3 // indirect\n\tgithub.com/gogo/protobuf v1.3.2 // indirect\n\tgithub.com/golang/protobuf v1.5.3 // indirect\n\tgithub.com/golang/snappy v0.0.4 // indirect\n\tgithub.com/google/gnostic v0.6.9 // indirect\n\tgithub.com/google/go-cmp v0.5.9 // indirect\n\tgithub.com/google/gofuzz v1.2.0 // indirect\n\tgithub.com/google/uuid v1.3.0 // indirect\n\tgithub.com/hashicorp/errwrap v1.1.0 // indirect\n\tgithub.com/hashicorp/go-multierror v1.1.1 // indirect\n\tgithub.com/hashicorp/go-uuid v1.0.3 // indirect\n\tgithub.com/jcmturner/aescts/v2 v2.0.0 // indirect\n\tgithub.com/jcmturner/dnsutils/v2 v2.0.0 // indirect\n\tgithub.com/jcmturner/gofork v1.7.6 // indirect\n\tgithub.com/jcmturner/gokrb5/v8 v8.4.4 // indirect\n\tgithub.com/jcmturner/rpc/v2 v2.0.3 // indirect\n\tgithub.com/jmespath/go-jmespath v0.4.0 // indirect\n\tgithub.com/josharian/intern v1.0.0 // indirect\n\tgithub.com/json-iterator/go v1.1.12 // indirect\n\tgithub.com/mailru/easyjson v0.7.7 // indirect\n\tgithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect\n\tgithub.com/modern-go/reflect2 v1.0.2 // indirect\n\tgithub.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect\n\tgithub.com/pierrec/lz4/v4 v4.1.17 // indirect\n\tgithub.com/pmezard/go-difflib v1.0.0 // indirect\n\tgithub.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect\n\tgithub.com/smartystreets/goconvey v1.7.2 // indirect\n\tgithub.com/xdg-go/pbkdf2 v1.0.0 // indirect\n\tgithub.com/xdg-go/stringprep v1.0.4 // indirect\n\tgolang.org/x/crypto v0.32.0 // indirect\n\tgolang.org/x/oauth2 v0.8.0 // indirect\n\tgolang.org/x/term v0.28.0 // indirect\n\tgolang.org/x/text v0.21.0 // indirect\n\tgolang.org/x/time v0.3.0 // indirect\n\tgoogle.golang.org/appengine v1.6.7 // indirect\n\tgoogle.golang.org/protobuf v1.30.0 // indirect\n\tgopkg.in/inf.v0 v0.9.1 // indirect\n\tgopkg.in/yaml.v2 v2.4.0 // indirect\n\tgopkg.in/yaml.v3 v3.0.1 // indirect\n\tk8s.io/api v0.27.1 // indirect\n\tk8s.io/klog/v2 v2.100.1 // indirect\n\tk8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect\n\tk8s.io/utils v0.0.0-20230505201702-9f6742963106 // indirect\n\tsigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect\n\tsigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect\n\tsigs.k8s.io/yaml v1.3.0 // indirect\n)\n" }, { "path": "go.sum", "content": "cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=\ncloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=\ngithub.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=\ngithub.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=\ngithub.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A=\ngithub.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g=\ngithub.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc=\ngithub.com/Shopify/toxiproxy/v2 v2.5.0/go.mod h1:yhM2epWtAmel9CB8r2+L+PCmhH6yH2pITaPAo7jxJl0=\ngithub.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=\ngithub.com/araddon/gou v0.0.0-20211019181548-e7d08105776c h1:XUqw//RExYoxW4Eie8MuKp8sEDAZI1gMHX/daUFgZww=\ngithub.com/araddon/gou v0.0.0-20211019181548-e7d08105776c/go.mod h1:ikc1XA58M+Rx7SEbf0bLJCfBkwayZ8T5jBo5FXK8Uz8=\ngithub.com/aws/aws-sdk-go v1.44.262 h1:gyXpcJptWoNkK+DiAiaBltlreoWKQXjAIh6FRh60F+I=\ngithub.com/aws/aws-sdk-go v1.44.262/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=\ngithub.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2ioR0=\ngithub.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw=\ngithub.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=\ngithub.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=\ngithub.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=\ngithub.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=\ngithub.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=\ngithub.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=\ngithub.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=\ngithub.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=\ngithub.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=\ngithub.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=\ngithub.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=\ngithub.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=\ngithub.com/coocood/freecache v1.2.3 h1:lcBwpZrwBZRZyLk/8EMyQVXRiFl663cCuMOrjCALeto=\ngithub.com/coocood/freecache v1.2.3/go.mod h1:RBUWa/Cy+OHdfTGFEhEuE1pMCMX51Ncizj7rthiQ3vk=\ngithub.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=\ngithub.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=\ngithub.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=\ngithub.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=\ngithub.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0=\ngithub.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho=\ngithub.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 h1:8yY/I9ndfrgrXUbOGObLHKBR4Fl3nZXwM2c7OYTT8hM=\ngithub.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0=\ngithub.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc=\ngithub.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=\ngithub.com/emicklei/go-restful/v3 v3.10.2 h1:hIovbnmBTLjHXkqEBUz3HGpXZdM7ZrE9fJIZIqlJLqE=\ngithub.com/emicklei/go-restful/v3 v3.10.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=\ngithub.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=\ngithub.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=\ngithub.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=\ngithub.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=\ngithub.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=\ngithub.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=\ngithub.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0=\ngithub.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=\ngithub.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=\ngithub.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=\ngithub.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=\ngithub.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ=\ngithub.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=\ngithub.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE=\ngithub.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=\ngithub.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=\ngithub.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=\ngithub.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=\ngithub.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=\ngithub.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=\ngithub.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=\ngithub.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=\ngithub.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=\ngithub.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=\ngithub.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=\ngithub.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=\ngithub.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=\ngithub.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=\ngithub.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=\ngithub.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=\ngithub.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=\ngithub.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=\ngithub.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=\ngithub.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=\ngithub.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=\ngithub.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=\ngithub.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=\ngithub.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=\ngithub.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=\ngithub.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=\ngithub.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=\ngithub.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0=\ngithub.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E=\ngithub.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=\ngithub.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=\ngithub.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=\ngithub.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=\ngithub.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=\ngithub.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=\ngithub.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=\ngithub.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=\ngithub.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=\ngithub.com/google/gopacket v1.1.20-0.20210429153827-3eaba0894325 h1:YmIcZ5Var3BAQ64AW98Iiys5Ih4fiU0xK41+8isC5Ec=\ngithub.com/google/gopacket v1.1.20-0.20210429153827-3eaba0894325/go.mod h1:riddUzxTSBpJXk3qBHtYr4qOhFhT6k/1c0E3qkQjQpA=\ngithub.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec=\ngithub.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=\ngithub.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=\ngithub.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=\ngithub.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=\ngithub.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=\ngithub.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=\ngithub.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=\ngithub.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=\ngithub.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=\ngithub.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=\ngithub.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=\ngithub.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=\ngithub.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=\ngithub.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=\ngithub.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=\ngithub.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=\ngithub.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=\ngithub.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=\ngithub.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=\ngithub.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8=\ngithub.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=\ngithub.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo=\ngithub.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=\ngithub.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg=\ngithub.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo=\ngithub.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o=\ngithub.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=\ngithub.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8=\ngithub.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=\ngithub.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=\ngithub.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=\ngithub.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=\ngithub.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=\ngithub.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=\ngithub.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=\ngithub.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=\ngithub.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=\ngithub.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=\ngithub.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=\ngithub.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=\ngithub.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=\ngithub.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=\ngithub.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=\ngithub.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=\ngithub.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=\ngithub.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=\ngithub.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=\ngithub.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=\ngithub.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=\ngithub.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=\ngithub.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=\ngithub.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=\ngithub.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=\ngithub.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=\ngithub.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=\ngithub.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b h1:v29yPGHhOqw7VHEnTeQFAth3SsBrmwc8JfuhNY0G34k=\ngithub.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b/go.mod h1:5MWrJXKRQyhQdUCF+vu6U5c4nQpg70vW3eHaU0/AYbU=\ngithub.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=\ngithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=\ngithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=\ngithub.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=\ngithub.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=\ngithub.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=\ngithub.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=\ngithub.com/onsi/ginkgo/v2 v2.9.1 h1:zie5Ly042PD3bsCvsSOPvRnFwyo3rKe64TJlD6nu0mk=\ngithub.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo=\ngithub.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E=\ngithub.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ=\ngithub.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc=\ngithub.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=\ngithub.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=\ngithub.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=\ngithub.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=\ngithub.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=\ngithub.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=\ngithub.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=\ngithub.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=\ngithub.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=\ngithub.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=\ngithub.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=\ngithub.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=\ngithub.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg=\ngithub.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM=\ngithub.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=\ngithub.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=\ngithub.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=\ngithub.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=\ngithub.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=\ngithub.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=\ngithub.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=\ngithub.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=\ngithub.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=\ngithub.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=\ngithub.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=\ngithub.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=\ngithub.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=\ngithub.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=\ngithub.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=\ngithub.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=\ngithub.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=\ngithub.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=\ngithub.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=\ngithub.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=\ngithub.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=\ngithub.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=\ngithub.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=\ngithub.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=\ngithub.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=\ngithub.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=\ngithub.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=\ngithub.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngithub.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=\ngithub.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=\ngo.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=\ngolang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=\ngolang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=\ngolang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=\ngolang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=\ngolang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=\ngolang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=\ngolang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=\ngolang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=\ngolang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=\ngolang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=\ngolang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=\ngolang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=\ngolang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=\ngolang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=\ngolang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=\ngolang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=\ngolang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=\ngolang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=\ngolang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=\ngolang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=\ngolang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=\ngolang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=\ngolang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=\ngolang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=\ngolang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=\ngolang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=\ngolang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=\ngolang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=\ngolang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=\ngolang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0=\ngolang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=\ngolang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=\ngolang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=\ngolang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8=\ngolang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE=\ngolang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=\ngolang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=\ngolang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=\ngolang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=\ngolang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=\ngolang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=\ngolang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=\ngolang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=\ngolang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=\ngolang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=\ngolang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=\ngolang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=\ngolang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg=\ngolang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=\ngolang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=\ngolang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=\ngolang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=\ngolang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=\ngolang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=\ngolang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=\ngolang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=\ngolang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=\ngolang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=\ngolang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=\ngolang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=\ngolang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=\ngolang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=\ngolang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=\ngolang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=\ngolang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=\ngolang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=\ngolang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=\ngolang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=\ngolang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=\ngolang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=\ngolang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=\ngolang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngolang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=\ngoogle.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=\ngoogle.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=\ngoogle.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=\ngoogle.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=\ngoogle.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=\ngoogle.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=\ngoogle.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=\ngoogle.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=\ngoogle.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=\ngoogle.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=\ngoogle.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=\ngoogle.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=\ngoogle.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=\ngoogle.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=\ngoogle.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=\ngoogle.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=\ngoogle.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=\ngoogle.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=\ngoogle.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=\ngoogle.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=\ngoogle.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=\ngoogle.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=\ngoogle.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=\ngoogle.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=\ngoogle.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=\ngoogle.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=\ngoogle.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=\ngoogle.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=\ngopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=\ngopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=\ngopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=\ngopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=\ngopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=\ngopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=\ngopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=\ngopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=\ngopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\ngopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\ngopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=\ngopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=\nhonnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nhonnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=\nk8s.io/api v0.27.1 h1:Z6zUGQ1Vd10tJ+gHcNNNgkV5emCyW+v2XTmn+CLjSd0=\nk8s.io/api v0.27.1/go.mod h1:z5g/BpAiD+f6AArpqNjkY+cji8ueZDU/WV1jcj5Jk4E=\nk8s.io/apimachinery v0.27.1 h1:EGuZiLI95UQQcClhanryclaQE6xjg1Bts6/L3cD7zyc=\nk8s.io/apimachinery v0.27.1/go.mod h1:5ikh59fK3AJ287GUvpUsryoMFtH9zj/ARfWCo3AyXTM=\nk8s.io/client-go v0.27.1 h1:oXsfhW/qncM1wDmWBIuDzRHNS2tLhK3BZv512Nc59W8=\nk8s.io/client-go v0.27.1/go.mod h1:f8LHMUkVb3b9N8bWturc+EDtVVVwZ7ueTVquFAJb2vA=\nk8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg=\nk8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=\nk8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg=\nk8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg=\nk8s.io/utils v0.0.0-20230505201702-9f6742963106 h1:EObNQ3TW2D+WptiYXlApGNLVy0zm/JIBVY9i+M4wpAU=\nk8s.io/utils v0.0.0-20230505201702-9f6742963106/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=\nsigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=\nsigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=\nsigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=\nsigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=\nsigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=\nsigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=\n" }, { "path": "gor_stat.go", "content": "package goreplay\n\nimport (\n\t\"runtime\"\n\t\"strconv\"\n\t\"time\"\n)\n\ntype GorStat struct {\n\tstatName string\n\trateMs int\n\tlatest int\n\tmean int\n\tmax int\n\tcount int\n}\n\nfunc NewGorStat(statName string, rateMs int) (s *GorStat) {\n\ts = new(GorStat)\n\ts.statName = statName\n\ts.rateMs = rateMs\n\ts.latest = 0\n\ts.mean = 0\n\ts.max = 0\n\ts.count = 0\n\n\tif Settings.Stats {\n\t\tgo s.reportStats()\n\t}\n\treturn\n}\n\nfunc (s *GorStat) Write(latest int) {\n\tif Settings.Stats {\n\t\tif latest > s.max {\n\t\t\ts.max = latest\n\t\t}\n\t\tif latest != 0 {\n\t\t\ts.mean = ((s.mean * s.count) + latest) / (s.count + 1)\n\t\t}\n\t\ts.latest = latest\n\t\ts.count = s.count + 1\n\t}\n}\n\nfunc (s *GorStat) Reset() {\n\ts.latest = 0\n\ts.max = 0\n\ts.mean = 0\n\ts.count = 0\n}\n\nfunc (s *GorStat) String() string {\n\treturn s.statName + \":\" + strconv.Itoa(s.latest) + \",\" + strconv.Itoa(s.mean) + \",\" + strconv.Itoa(s.max) + \",\" + strconv.Itoa(s.count) + \",\" + strconv.Itoa(s.count/(s.rateMs/1000.0)) + \",\" + strconv.Itoa(runtime.NumGoroutine())\n}\n\nfunc (s *GorStat) reportStats() {\n\tDebug(0, \"\\n\", s.statName+\":latest,mean,max,count,count/second,gcount\")\n\tfor {\n\t\tDebug(0, \"\\n\", s)\n\t\ts.Reset()\n\t\ttime.Sleep(time.Duration(s.rateMs) * time.Millisecond)\n\t}\n}\n" }, { "path": "homebrew/gor.rb", "content": "require \"language/go\"\n\nclass Gor < Formula\n desc \"Real-time HTTP traffic replay tool written in Go\"\n homepage \"https://gortool.com\"\n url \"https://github.com/buger/gor/archive/v0.14.0.tar.gz\"\n sha256 \"62260a6f5cabde571b91d5762fba9c47691643df0a58565cbe808854cd064dc8\"\n head \"https://github.com/buger/gor.git\"\n\n bottle do\n cellar :any_skip_relocation\n sha256 \"c382403de70a41b7445920a02051f5e82030704aaaae70cfcd4e8f401cc87f6a\" => :el_capitan\n sha256 \"4b76b3785584897800e87967f1af9510208faefe46f57d7bd6f8b40a7133c19b\" => :yosemite\n sha256 \"d186cb1566d33ab8f78215e69934f49dd96becb1c236905b4502d94399ae1974\" => :mavericks\n end\n\n depends_on \"go\" => :build\n\n go_resource \"github.com/bitly/go-hostpool\" do\n url \"https://github.com/bitly/go-hostpool.git\",\n :revision => \"d0e59c22a56e8dadfed24f74f452cea5a52722d2\"\n end\n\n go_resource \"github.com/buger/elastigo\" do\n url \"https://github.com/buger/elastigo.git\",\n :revision => \"23fcfd9db0d8be2189a98fdab77a4c90fcc3a1e9\"\n end\n\n go_resource \"github.com/google/gopacket\" do\n url \"https://github.com/google/gopacket.git\",\n :revision => \"aa09ced736460d76535444c825932a0742975f7d\"\n end\n\n def install\n ENV[\"GOPATH\"] = buildpath\n mkdir_p buildpath/\"src/github.com/buger/\"\n ln_sf buildpath, buildpath/\"src/github.com/buger/gor\"\n Language::Go.stage_deps resources, buildpath/\"src\"\n\n system \"go\", \"build\", \"-o\", \"#{bin}/gor\", \"-ldflags\", \"-X main.VERSION \\\"#{version}\\\"\"\n end\n\n test do\n assert_match version.to_s, shell_output(\"#{bin}/gor\", 1)\n end\nend\n" }, { "path": "http_modifier.go", "content": "package goreplay\n\nimport (\n\t\"bytes\"\n\t\"encoding/base64\"\n\t\"github.com/buger/goreplay/proto\"\n\t\"hash/fnv\"\n\t\"strings\"\n)\n\ntype HTTPModifier struct {\n\tconfig *HTTPModifierConfig\n}\n\nfunc NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier {\n\t// Optimization to skip modifier completely if we do not need it\n\tif len(config.URLRegexp) == 0 &&\n\t\tlen(config.URLNegativeRegexp) == 0 &&\n\t\tlen(config.URLRewrite) == 0 &&\n\t\tlen(config.HeaderRewrite) == 0 &&\n\t\tlen(config.HeaderFilters) == 0 &&\n\t\tlen(config.HeaderNegativeFilters) == 0 &&\n\t\tlen(config.HeaderBasicAuthFilters) == 0 &&\n\t\tlen(config.HeaderHashFilters) == 0 &&\n\t\tlen(config.ParamHashFilters) == 0 &&\n\t\tlen(config.Params) == 0 &&\n\t\tlen(config.Headers) == 0 &&\n\t\tlen(config.Methods) == 0 {\n\t\treturn nil\n\t}\n\n\treturn &HTTPModifier{config: config}\n}\n\nfunc (m *HTTPModifier) Rewrite(payload []byte) (response []byte) {\n\tif !proto.HasRequestTitle(payload) {\n\t\treturn payload\n\t}\n\n\tif len(m.config.Methods) > 0 {\n\t\tmethod := proto.Method(payload)\n\n\t\tmatched := false\n\n\t\tfor _, m := range m.config.Methods {\n\t\t\tif bytes.Equal(method, m) {\n\t\t\t\tmatched = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\tif !matched {\n\t\t\treturn\n\t\t}\n\t}\n\n\tif len(m.config.Headers) > 0 {\n\t\tfor _, header := range m.config.Headers {\n\t\t\tpayload = proto.SetHeader(payload, []byte(header.Name), []byte(header.Value))\n\t\t}\n\t}\n\n\tif len(m.config.Params) > 0 {\n\t\tfor _, param := range m.config.Params {\n\t\t\tpayload = proto.SetPathParam(payload, param.Name, param.Value)\n\t\t}\n\t}\n\n\tif len(m.config.URLRegexp) > 0 {\n\t\tpath := proto.Path(payload)\n\n\t\tmatched := false\n\n\t\tfor _, f := range m.config.URLRegexp {\n\t\t\tif f.regexp.Match(path) {\n\t\t\t\tmatched = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\tif !matched {\n\t\t\treturn\n\t\t}\n\t}\n\n\tif len(m.config.URLNegativeRegexp) > 0 {\n\t\tpath := proto.Path(payload)\n\n\t\tfor _, f := range m.config.URLNegativeRegexp {\n\t\t\tif f.regexp.Match(path) {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\n\tif len(m.config.HeaderFilters) > 0 {\n\t\tfor _, f := range m.config.HeaderFilters {\n\t\t\tvalue := proto.Header(payload, f.name)\n\n\t\t\tif len(value) == 0 {\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tif !f.regexp.Match(value) {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\n\tif len(m.config.HeaderNegativeFilters) > 0 {\n\t\tfor _, f := range m.config.HeaderNegativeFilters {\n\t\t\tvalue := proto.Header(payload, f.name)\n\n\t\t\tif len(value) > 0 && f.regexp.Match(value) {\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}\n\n\tif len(m.config.HeaderBasicAuthFilters) > 0 {\n\t\tfor _, f := range m.config.HeaderBasicAuthFilters {\n\t\t\tvalue := proto.Header(payload, []byte(\"Authorization\"))\n\n\t\t\tif len(value) > 0 {\n\t\t\t\tvalueString := string(value)\n\t\t\t\ttrimmedBasicAuthEncoded := strings.TrimPrefix(valueString, \"Basic \")\n\t\t\t\tif strings.Compare(valueString, trimmedBasicAuthEncoded) != 0 {\n\t\t\t\t\tdecodedAuth, _ := base64.StdEncoding.DecodeString(trimmedBasicAuthEncoded)\n\t\t\t\t\tif !f.regexp.Match(decodedAuth) {\n\t\t\t\t\t\treturn\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tif len(m.config.HeaderHashFilters) > 0 {\n\t\tfor _, f := range m.config.HeaderHashFilters {\n\t\t\tvalue := proto.Header(payload, f.name)\n\n\t\t\tif len(value) > 0 {\n\t\t\t\thasher := fnv.New32a()\n\t\t\t\thasher.Write(value)\n\n\t\t\t\tif (hasher.Sum32() % 100) >= f.percent {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tif len(m.config.ParamHashFilters) > 0 {\n\t\tfor _, f := range m.config.ParamHashFilters {\n\t\t\tvalue, s, _ := proto.PathParam(payload, f.name)\n\n\t\t\tif s != -1 {\n\t\t\t\thasher := fnv.New32a()\n\t\t\t\thasher.Write(value)\n\n\t\t\t\tif (hasher.Sum32() % 100) >= f.percent {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\tif len(m.config.URLRewrite) > 0 {\n\t\tpath := proto.Path(payload)\n\n\t\tfor _, f := range m.config.URLRewrite {\n\t\t\tif f.src.Match(path) {\n\t\t\t\tpath = f.src.ReplaceAll(path, f.target)\n\t\t\t\tpayload = proto.SetPath(payload, path)\n\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\n\tif len(m.config.HeaderRewrite) > 0 {\n\t\tfor _, f := range m.config.HeaderRewrite {\n\t\t\tvalue := proto.Header(payload, f.header)\n\t\t\tif len(value) == 0 {\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t\tif f.src.Match(value) {\n\t\t\t\tnewValue := f.src.ReplaceAll(value, f.target)\n\t\t\t\tpayload = proto.SetHeader(payload, f.header, newValue)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn payload\n}\n" }, { "path": "http_modifier_settings.go", "content": "package goreplay\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"regexp\"\n\t\"strconv\"\n\t\"strings\"\n)\n\n// HTTPModifierConfig holds configuration options for built-in traffic modifier\ntype HTTPModifierConfig struct {\n\tURLNegativeRegexp HTTPURLRegexp `json:\"http-disallow-url\"`\n\tURLRegexp HTTPURLRegexp `json:\"http-allow-url\"`\n\tURLRewrite URLRewriteMap `json:\"http-rewrite-url\"`\n\tHeaderRewrite HeaderRewriteMap `json:\"http-rewrite-header\"`\n\tHeaderFilters HTTPHeaderFilters `json:\"http-allow-header\"`\n\tHeaderNegativeFilters HTTPHeaderFilters `json:\"http-disallow-header\"`\n\tHeaderBasicAuthFilters HTTPHeaderBasicAuthFilters `json:\"http-basic-auth-filter\"`\n\tHeaderHashFilters HTTPHashFilters `json:\"http-header-limiter\"`\n\tParamHashFilters HTTPHashFilters `json:\"http-param-limiter\"`\n\tParams HTTPParams `json:\"http-set-param\"`\n\tHeaders HTTPHeaders `json:\"http-set-header\"`\n\tMethods HTTPMethods `json:\"http-allow-method\"`\n}\n\n// Handling of --http-allow-header, --http-disallow-header options\ntype headerFilter struct {\n\tname []byte\n\tregexp *regexp.Regexp\n}\n\n// HTTPHeaderFilters holds list of headers and their regexps\ntype HTTPHeaderFilters []headerFilter\n\nfunc (h *HTTPHeaderFilters) String() string {\n\treturn fmt.Sprint(*h)\n}\n\n// Set method to implement flags.Value\nfunc (h *HTTPHeaderFilters) Set(value string) error {\n\tvalArr := strings.SplitN(value, \":\", 2)\n\tif len(valArr) < 2 {\n\t\treturn errors.New(\"need both header and value, colon-delimited (ex. user_id:^169$)\")\n\t}\n\tval := strings.TrimSpace(valArr[1])\n\tr, err := regexp.Compile(val)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t*h = append(*h, headerFilter{name: []byte(valArr[0]), regexp: r})\n\n\treturn nil\n}\n\n// Handling of --http-basic-auth-filter option\ntype basicAuthFilter struct {\n\tregexp *regexp.Regexp\n}\n\n// HTTPHeaderBasicAuthFilters holds list of regxp to match basic Auth header values\ntype HTTPHeaderBasicAuthFilters []basicAuthFilter\n\nfunc (h *HTTPHeaderBasicAuthFilters) String() string {\n\treturn fmt.Sprint(*h)\n}\n\n// Set method to implement flags.Value\nfunc (h *HTTPHeaderBasicAuthFilters) Set(value string) error {\n\tr, err := regexp.Compile(value)\n\tif err != nil {\n\t\treturn err\n\t}\n\n\t*h = append(*h, basicAuthFilter{regexp: r})\n\n\treturn nil\n}\n\n// Handling of --http-allow-header-hash and --http-allow-param-hash options\ntype hashFilter struct {\n\tname []byte\n\tpercent uint32\n}\n\n// HTTPHashFilters represents a slice of header hash filters\ntype HTTPHashFilters []hashFilter\n\nfunc (h *HTTPHashFilters) String() string {\n\treturn fmt.Sprint(*h)\n}\n\n// Set method to implement flags.Value\nfunc (h *HTTPHashFilters) Set(value string) error {\n\tvalArr := strings.SplitN(value, \":\", 2)\n\tif len(valArr) < 2 {\n\t\treturn errors.New(\"need both header and value, colon-delimited (ex. user_id:50%)\")\n\t}\n\n\tf := hashFilter{name: []byte(valArr[0])}\n\n\tval := strings.TrimSpace(valArr[1])\n\n\tif strings.Contains(val, \"%\") {\n\t\tp, _ := strconv.ParseInt(val[:len(val)-1], 0, 0)\n\t\tf.percent = uint32(p)\n\t} else if strings.Contains(val, \"/\") {\n\t\t// DEPRECATED format\n\t\tvar num, den uint64\n\n\t\tfracArr := strings.Split(val, \"/\")\n\t\tnum, _ = strconv.ParseUint(fracArr[0], 10, 64)\n\t\tden, _ = strconv.ParseUint(fracArr[1], 10, 64)\n\n\t\tf.percent = uint32((float64(num) / float64(den)) * 100)\n\t} else {\n\t\treturn errors.New(\"Value should be percent and contain '%'\")\n\t}\n\n\t*h = append(*h, f)\n\n\treturn nil\n}\n\n// Handling of --http-set-header option\ntype httpHeader struct {\n\tName string\n\tValue string\n}\n\n// HTTPHeaders is a slice of headers that must appended\ntype HTTPHeaders []httpHeader\n\nfunc (h *HTTPHeaders) String() string {\n\treturn fmt.Sprint(*h)\n}\n\n// Set method to implement flags.Value\nfunc (h *HTTPHeaders) Set(value string) error {\n\tv := strings.SplitN(value, \":\", 2)\n\tif len(v) != 2 {\n\t\treturn errors.New(\"Expected `Key: Value`\")\n\t}\n\n\theader := httpHeader{\n\t\tstrings.TrimSpace(v[0]),\n\t\tstrings.TrimSpace(v[1]),\n\t}\n\n\t*h = append(*h, header)\n\treturn nil\n}\n\n// Handling of --http-set-param option\ntype httpParam struct {\n\tName []byte\n\tValue []byte\n}\n\n// HTTPParams filters for --http-set-param\ntype HTTPParams []httpParam\n\nfunc (h *HTTPParams) String() string {\n\treturn fmt.Sprint(*h)\n}\n\n// Set method to implement flags.Value\nfunc (h *HTTPParams) Set(value string) error {\n\tv := strings.SplitN(value, \"=\", 2)\n\tif len(v) != 2 {\n\t\treturn errors.New(\"Expected `Key=Value`\")\n\t}\n\n\tparam := httpParam{\n\t\t[]byte(strings.TrimSpace(v[0])),\n\t\t[]byte(strings.TrimSpace(v[1])),\n\t}\n\n\t*h = append(*h, param)\n\treturn nil\n}\n\n//\n// Handling of --http-allow-method option\n//\n\n// HTTPMethods holds values for method allowed\ntype HTTPMethods [][]byte\n\nfunc (h *HTTPMethods) String() string {\n\treturn fmt.Sprint(*h)\n}\n\n// Set method to implement flags.Value\nfunc (h *HTTPMethods) Set(value string) error {\n\t*h = append(*h, []byte(value))\n\treturn nil\n}\n\n// Handling of --http-rewrite-url option\ntype urlRewrite struct {\n\tsrc *regexp.Regexp\n\ttarget []byte\n}\n\n// URLRewriteMap holds regexp and data to modify URL\ntype URLRewriteMap []urlRewrite\n\nfunc (r *URLRewriteMap) String() string {\n\treturn fmt.Sprint(*r)\n}\n\n// Set method to implement flags.Value\nfunc (r *URLRewriteMap) Set(value string) error {\n\tvalArr := strings.SplitN(value, \":\", 2)\n\tif len(valArr) < 2 {\n\t\treturn errors.New(\"need both src and target, colon-delimited (ex. /a:/b)\")\n\t}\n\tregexp, err := regexp.Compile(valArr[0])\n\tif err != nil {\n\t\treturn err\n\t}\n\t*r = append(*r, urlRewrite{src: regexp, target: []byte(valArr[1])})\n\treturn nil\n}\n\n// Handling of --http-rewrite-header option\ntype headerRewrite struct {\n\theader []byte\n\tsrc *regexp.Regexp\n\ttarget []byte\n}\n\n// HeaderRewriteMap holds regexp and data to rewrite headers\ntype HeaderRewriteMap []headerRewrite\n\nfunc (r *HeaderRewriteMap) String() string {\n\treturn fmt.Sprint(*r)\n}\n\n// Set method to implement flags.Value\nfunc (r *HeaderRewriteMap) Set(value string) error {\n\theaderArr := strings.SplitN(value, \":\", 2)\n\tif len(headerArr) < 2 {\n\t\treturn errors.New(\"need both header, regexp and rewrite target, colon-delimited (ex. Header: regexp,target)\")\n\t}\n\n\theader := headerArr[0]\n\tvalArr := strings.SplitN(strings.TrimSpace(headerArr[1]), \",\", 2)\n\n\tif len(valArr) < 2 {\n\t\treturn errors.New(\"need both header, regexp and rewrite target, colon-delimited (ex. Header: regexp,target)\")\n\t}\n\n\tregexp, err := regexp.Compile(valArr[0])\n\tif err != nil {\n\t\treturn err\n\t}\n\t*r = append(*r, headerRewrite{header: []byte(header), src: regexp, target: []byte(valArr[1])})\n\treturn nil\n}\n\n// Handling of --http-allow-url option\ntype urlRegexp struct {\n\tregexp *regexp.Regexp\n}\n\n// HTTPURLRegexp a slice of regexp to match URLs\ntype HTTPURLRegexp []urlRegexp\n\nfunc (r *HTTPURLRegexp) String() string {\n\treturn fmt.Sprint(*r)\n}\n\n// Set method to implement flags.Value\nfunc (r *HTTPURLRegexp) Set(value string) error {\n\tregexp, err := regexp.Compile(value)\n\n\t*r = append(*r, urlRegexp{regexp: regexp})\n\n\treturn err\n}\n" }, { "path": "http_modifier_settings_test.go", "content": "package goreplay\n\nimport (\n\t\"testing\"\n)\n\nfunc TestHTTPHeaderFilters(t *testing.T) {\n\tfilters := HTTPHeaderFilters{}\n\n\terr := filters.Set(\"Header1:^$\")\n\tif err != nil {\n\t\tt.Error(\"Should not error on Header1:^$\")\n\t}\n\n\terr = filters.Set(\"Header2:^:$\")\n\tif err != nil {\n\t\tt.Error(\"Should not error on Header2:^:$\")\n\t}\n\n\t// Missing colon\n\terr = filters.Set(\"Header3-^$\")\n\tif err == nil {\n\t\tt.Error(\"Should error on Header2:^:$\")\n\t}\n}\n\nfunc TestHTTPHashFilters(t *testing.T) {\n\tfilters := HTTPHashFilters{}\n\n\terr := filters.Set(\"Header1:1/2\")\n\tif err != nil {\n\t\tt.Error(\"Should support old syntax\")\n\t}\n\n\tif filters[0].percent != 50 {\n\t\tt.Error(\"Wrong percentage\", filters[0].percent)\n\t}\n\n\terr = filters.Set(\"Header2:1\")\n\tif err == nil {\n\t\tt.Error(\"Should error on Header2 because no % symbol\")\n\t}\n\n\terr = filters.Set(\"Header2:10%\")\n\tif err != nil {\n\t\tt.Error(\"Should pass\")\n\t}\n\n\tif filters[1].percent != 10 {\n\t\tt.Error(\"Wrong percentage\", filters[1].percent)\n\t}\n}\n\nfunc TestUrlRewriteMap(t *testing.T) {\n\tvar err error\n\trewrites := URLRewriteMap{}\n\n\tif err = rewrites.Set(\"/v1/user/([^\\\\/]+)/ping:/v2/user/$1/ping\"); err != nil {\n\t\tt.Error(\"Should set mapping\", err)\n\t}\n\n\tif err = rewrites.Set(\"/v1/user/([^\\\\/]+)/ping\"); err == nil {\n\t\tt.Error(\"Should not set mapping without :\")\n\t}\n}\n" }, { "path": "http_modifier_test.go", "content": "package goreplay\n\nimport (\n\t\"bytes\"\n\t\"github.com/buger/goreplay/proto\"\n\t\"testing\"\n)\n\nfunc TestHTTPModifierWithoutConfig(t *testing.T) {\n\tif NewHTTPModifier(&HTTPModifierConfig{}) != nil {\n\t\tt.Error(\"If no config specified should not be initialized\")\n\t}\n}\n\nfunc TestHTTPModifierHeaderFilters(t *testing.T) {\n\tfilters := HTTPHeaderFilters{}\n\tfilters.Set(\"Host:^www.w3.org$\")\n\n\tmodifier := NewHTTPModifier(&HTTPModifierConfig{\n\t\tHeaderFilters: filters,\n\t})\n\n\tpayload := []byte(\"POST /post HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\n\tif len(modifier.Rewrite(payload)) == 0 {\n\t\tt.Error(\"Request should pass filters\")\n\t}\n\n\tfilters = HTTPHeaderFilters{}\n\t// Setting filter that not match our header\n\tfilters.Set(\"Host:^www.w4.org$\")\n\n\tmodifier = NewHTTPModifier(&HTTPModifierConfig{\n\t\tHeaderFilters: filters,\n\t})\n\n\tif len(modifier.Rewrite(payload)) != 0 {\n\t\tt.Error(\"Request should not pass filters\")\n\t}\n}\n\nfunc TestHTTPModifierHeaderNegativeFilters(t *testing.T) {\n\tfilters := HTTPHeaderFilters{}\n\tfilters.Set(\"Host:^www.w3.org$\")\n\n\tmodifier := NewHTTPModifier(&HTTPModifierConfig{\n\t\tHeaderNegativeFilters: filters,\n\t})\n\n\tpayload := []byte(\"POST /post HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w4.org\\r\\n\\r\\na=1&b=2\")\n\n\tif len(modifier.Rewrite(payload)) == 0 {\n\t\tt.Error(\"Request should pass filters\")\n\t}\n\n\tfilters = HTTPHeaderFilters{}\n\t// Setting filter that not match our header\n\tfilters.Set(\"Host:^www.w4.org$\")\n\n\tmodifier = NewHTTPModifier(&HTTPModifierConfig{\n\t\tHeaderNegativeFilters: filters,\n\t})\n\n\tif len(modifier.Rewrite(payload)) != 0 {\n\t\tt.Error(\"Request should not pass filters\")\n\t}\n\n\tfilters = HTTPHeaderFilters{}\n\t// Setting filter that not match our header\n\tfilters.Set(\"Host: www*\")\n\n\tmodifier = NewHTTPModifier(&HTTPModifierConfig{\n\t\tHeaderNegativeFilters: filters,\n\t})\n\n\tif len(modifier.Rewrite(payload)) != 0 {\n\t\tt.Error(\"Request should not pass filters\")\n\t}\n}\n\nfunc TestHTTPHeaderBasicAuthFilters(t *testing.T) {\n\tfilters := HTTPHeaderBasicAuthFilters{}\n\tfilters.Set(\"^customer[0-9].*\")\n\n\tmodifier := NewHTTPModifier(&HTTPModifierConfig{\n\t\tHeaderBasicAuthFilters: filters,\n\t})\n\n\t//Encoded UserId:Password = customer3:welcome\n\tpayload := []byte(\"POST /post HTTP/1.1\\r\\nContent-Length: 7\\r\\nAuthorization: Basic Y3VzdG9tZXIzOndlbGNvbWU=\\r\\n\\r\\na=1&b=2\")\n\tif len(modifier.Rewrite(payload)) == 0 {\n\t\tt.Error(\"Request should pass filters\")\n\t}\n\n\t//customer6:rest@123^TEST\n\tpayload = []byte(\"POST /post HTTP/1.1\\r\\nContent-Length: 88\\r\\nAuthorization: Basic Y3VzdG9tZXI2OnJlc3RAMTIzXlRFU1Q==\\r\\n\\r\\na=1&b=2\")\n\tif len(modifier.Rewrite(payload)) == 0 {\n\t\tt.Error(\"Request should pass filters\")\n\t}\n\n\tfilters = HTTPHeaderBasicAuthFilters{}\n\t// Setting filter that not match our header\n\tfilters.Set(\"^(homer simpson|mickey mouse).*\")\n\n\tmodifier = NewHTTPModifier(&HTTPModifierConfig{\n\t\tHeaderBasicAuthFilters: filters,\n\t})\n\n\tif len(modifier.Rewrite(payload)) != 0 {\n\t\tt.Error(\"Request should not pass filters\")\n\t}\n\n\t//mickey mouse:happy123\n\tpayload = []byte(\"POST /post HTTP/1.1\\r\\nContent-Length: 88\\r\\nAuthorization: Basic bWlja2V5IG1vdXNlOmhhcHB5MTIz\\r\\n\\r\\na=1&b=2\")\n\tif len(modifier.Rewrite(payload)) == 0 {\n\t\tt.Error(\"Request should pass filters\")\n\t}\n}\n\nfunc TestHTTPModifierURLRewrite(t *testing.T) {\n\tvar url, newURL []byte\n\n\trewrites := URLRewriteMap{}\n\n\tpayload := func(url []byte) []byte {\n\t\treturn []byte(\"POST \" + string(url) + \" HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\t}\n\n\terr := rewrites.Set(\"/v1/user/([^\\\\/]+)/ping:/v2/user/$1/ping\")\n\tif err != nil {\n\t\tt.Error(\"Should not error on /v1/user/([^\\\\/]+)/ping:/v2/user/$1/ping\")\n\t}\n\n\tmodifier := NewHTTPModifier(&HTTPModifierConfig{\n\t\tURLRewrite: rewrites,\n\t})\n\n\turl = []byte(\"/v1/user/joe/ping\")\n\tif newURL = proto.Path(modifier.Rewrite(payload(url))); bytes.Equal(newURL, url) {\n\t\tt.Error(\"Request url should have been rewritten, wasn't\", string(newURL))\n\t}\n\n\turl = []byte(\"/v1/user/ping\")\n\tif newURL = proto.Path(modifier.Rewrite(payload(url))); !bytes.Equal(newURL, url) {\n\t\tt.Error(\"Request url should have been rewritten, wasn't\", string(newURL))\n\t}\n}\n\nfunc TestHTTPModifierHeaderRewrite(t *testing.T) {\n\tvar header, newHeader []byte\n\n\trewrites := HeaderRewriteMap{}\n\tpayload := []byte(\"GET / HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\n\terr := rewrites.Set(\"Host: (.*).w3.org,$1.beta.w3.org\")\n\tif err != nil {\n\t\tt.Error(\"Should not error\", err)\n\t}\n\n\tmodifier := NewHTTPModifier(&HTTPModifierConfig{\n\t\tHeaderRewrite: rewrites,\n\t})\n\n\theader = []byte(\"www.beta.w3.org\")\n\tif newHeader = proto.Header(modifier.Rewrite(payload), []byte(\"Host\")); !bytes.Equal(newHeader, header) {\n\t\tt.Error(\"Request header should have been rewritten, wasn't\", string(newHeader), string(header))\n\t}\n}\n\nfunc TestHTTPModifierHeaderHashFilters(t *testing.T) {\n\tfilters := HTTPHashFilters{}\n\tfilters.Set(\"Header2:1/2\")\n\n\tmodifier := NewHTTPModifier(&HTTPModifierConfig{\n\t\tHeaderHashFilters: filters,\n\t})\n\n\tpayload := func(header []byte) []byte {\n\t\treturn []byte(\"POST / HTTP/1.1\\r\\n\" + string(header) + \"Content-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\t}\n\n\tif p := modifier.Rewrite(payload([]byte(\"\"))); len(p) == 0 {\n\t\tt.Error(\"Request should pass filters if Header does not exist\")\n\t}\n\n\tif p := modifier.Rewrite(payload([]byte(\"Header2: 3\\r\\n\"))); len(p) > 0 {\n\t\tt.Error(\"Request should not pass filters, Header2 hash too high\")\n\t}\n\n\tif p := modifier.Rewrite(payload([]byte(\"Header2: 1\\r\\n\"))); len(p) == 0 {\n\t\tt.Error(\"Request should pass filters\")\n\t}\n}\n\nfunc TestHTTPModifierParamHashFilters(t *testing.T) {\n\tfilters := HTTPHashFilters{}\n\tfilters.Set(\"user_id:1/2\")\n\n\tmodifier := NewHTTPModifier(&HTTPModifierConfig{\n\t\tParamHashFilters: filters,\n\t})\n\n\tpayload := func(value []byte) []byte {\n\t\treturn []byte(\"POST /\" + string(value) + \" HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\t}\n\n\tif p := modifier.Rewrite(payload([]byte(\"\"))); len(p) == 0 {\n\t\tt.Error(\"Request should pass filters if param does not exist\")\n\t}\n\n\tif p := modifier.Rewrite(payload([]byte(\"?user_id=3\"))); len(p) > 0 {\n\t\tt.Error(\"Request should not pass filters\", string(p))\n\t}\n\n\tif p := modifier.Rewrite(payload([]byte(\"?user_id=1\"))); len(p) == 0 {\n\t\tt.Error(\"Request should pass filters\")\n\t}\n}\n\nfunc TestHTTPModifierHeaders(t *testing.T) {\n\theaders := HTTPHeaders{}\n\theaders.Set(\"Header1:1\")\n\theaders.Set(\"Host:localhost\")\n\n\tmodifier := NewHTTPModifier(&HTTPModifierConfig{\n\t\tHeaders: headers,\n\t})\n\n\tpayload := []byte(\"POST /post HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\tnewPayload := []byte(\"POST /post HTTP/1.1\\r\\nHeader1: 1\\r\\nContent-Length: 7\\r\\nHost: localhost\\r\\n\\r\\na=1&b=2\")\n\n\tif payload = modifier.Rewrite(payload); !bytes.Equal(payload, newPayload) {\n\t\tt.Error(\"Should update request headers\", string(payload))\n\t}\n}\n\nfunc TestHTTPModifierURLRegexp(t *testing.T) {\n\tfilters := HTTPURLRegexp{}\n\tfilters.Set(\"/v1/app\")\n\tfilters.Set(\"/v1/api\")\n\n\tmodifier := NewHTTPModifier(&HTTPModifierConfig{\n\t\tURLRegexp: filters,\n\t})\n\n\tpayload := func(url string) []byte {\n\t\treturn []byte(\"POST \" + url + \" HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\t}\n\n\tif len(modifier.Rewrite(payload(\"/v1/app/test\"))) == 0 {\n\t\tt.Error(\"Should pass url\")\n\t}\n\n\tif len(modifier.Rewrite(payload(\"/v1/api/test\"))) == 0 {\n\t\tt.Error(\"Should pass url\")\n\t}\n\n\tif len(modifier.Rewrite(payload(\"/other\"))) > 0 {\n\t\tt.Error(\"Should not pass url\")\n\t}\n}\n\nfunc TestHTTPModifierURLNegativeRegexp(t *testing.T) {\n\tfilters := HTTPURLRegexp{}\n\tfilters.Set(\"/restricted1\")\n\tfilters.Set(\"/some/restricted2\")\n\n\tmodifier := NewHTTPModifier(&HTTPModifierConfig{\n\t\tURLNegativeRegexp: filters,\n\t})\n\n\tpayload := func(url string) []byte {\n\t\treturn []byte(\"POST \" + url + \" HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\t}\n\n\tif len(modifier.Rewrite(payload(\"/v1/app/test\"))) == 0 {\n\t\tt.Error(\"Should pass url\")\n\t}\n\n\tif len(modifier.Rewrite(payload(\"/restricted1\"))) > 0 {\n\t\tt.Error(\"Should not pass url\")\n\t}\n\n\tif len(modifier.Rewrite(payload(\"/some/restricted2\"))) > 0 {\n\t\tt.Error(\"Should not pass url\")\n\t}\n}\n\nfunc TestHTTPModifierSetHeader(t *testing.T) {\n\tfilters := HTTPHeaders{}\n\tfilters.Set(\"User-Agent:Gor\")\n\n\tmodifier := NewHTTPModifier(&HTTPModifierConfig{\n\t\tHeaders: filters,\n\t})\n\n\tpayload := []byte(\"POST /post HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\tpayloadAfter := []byte(\"POST /post HTTP/1.1\\r\\nUser-Agent: Gor\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\n\tif payload = modifier.Rewrite(payload); !bytes.Equal(payloadAfter, payload) {\n\t\tt.Error(\"Should add new header\", string(payload))\n\t}\n}\n\nfunc TestHTTPModifierSetParam(t *testing.T) {\n\tfilters := HTTPParams{}\n\tfilters.Set(\"api_key=1\")\n\n\tmodifier := NewHTTPModifier(&HTTPModifierConfig{\n\t\tParams: filters,\n\t})\n\n\tpayload := []byte(\"POST /post?api_key=1234 HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\tpayloadAfter := []byte(\"POST /post?api_key=1 HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\n\tif payload = modifier.Rewrite(payload); !bytes.Equal(payloadAfter, payload) {\n\t\tt.Error(\"Should override param\", string(payload))\n\t}\n}\n" }, { "path": "http_prettifier.go", "content": "package goreplay\n\nimport (\n\t\"bytes\"\n\t\"compress/gzip\"\n\t\"fmt\"\n\t\"github.com/buger/goreplay/proto\"\n\t\"io/ioutil\"\n\t\"net/http/httputil\"\n\t\"strconv\"\n)\n\nfunc prettifyHTTP(p []byte) []byte {\n\n\ttEnc := bytes.Equal(proto.Header(p, []byte(\"Transfer-Encoding\")), []byte(\"chunked\"))\n\tcEnc := bytes.Equal(proto.Header(p, []byte(\"Content-Encoding\")), []byte(\"gzip\"))\n\n\tif !(tEnc || cEnc) {\n\t\treturn p\n\t}\n\n\theadersPos := proto.MIMEHeadersEndPos(p)\n\n\tif headersPos < 5 || headersPos > len(p) {\n\t\treturn p\n\t}\n\n\theaders := p[:headersPos]\n\tcontent := p[headersPos:]\n\n\tif tEnc {\n\t\tbuf := bytes.NewReader(content)\n\t\tr := httputil.NewChunkedReader(buf)\n\t\tcontent, _ = ioutil.ReadAll(r)\n\n\t\theaders = proto.DeleteHeader(headers, []byte(\"Transfer-Encoding\"))\n\n\t\tnewLen := strconv.Itoa(len(content))\n\t\theaders = proto.SetHeader(headers, []byte(\"Content-Length\"), []byte(newLen))\n\t}\n\n\tif cEnc {\n\t\tbuf := bytes.NewReader(content)\n\t\tg, err := gzip.NewReader(buf)\n\n\t\tif err != nil {\n\t\t\tDebug(1, \"[Prettifier] GZIP encoding error:\", err)\n\t\t\treturn []byte{}\n\t\t}\n\n\t\tcontent, err = ioutil.ReadAll(g)\n\t\tif err != nil {\n\t\t\tDebug(1, fmt.Sprintf(\"[HTTP-PRETTIFIER] %q\", err))\n\t\t\treturn p\n\t\t}\n\n\t\theaders = proto.DeleteHeader(headers, []byte(\"Content-Encoding\"))\n\n\t\tnewLen := strconv.Itoa(len(content))\n\t\theaders = proto.SetHeader(headers, []byte(\"Content-Length\"), []byte(newLen))\n\t}\n\n\tnewPayload := append(headers, content...)\n\n\treturn newPayload\n}\n" }, { "path": "http_prettifier_test.go", "content": "package goreplay\n\nimport (\n\t\"bytes\"\n\t\"compress/gzip\"\n\t\"github.com/buger/goreplay/proto\"\n\t\"strconv\"\n\t\"testing\"\n)\n\nfunc TestHTTPPrettifierGzip(t *testing.T) {\n\tb := bytes.NewBufferString(\"\")\n\tw := gzip.NewWriter(b)\n\tw.Write([]byte(\"test\"))\n\tw.Close()\n\n\tsize := strconv.Itoa(len(b.Bytes()))\n\n\tpayload := []byte(\"HTTP/1.1 200 OK\\r\\nContent-Length: \" + size + \"\\r\\nContent-Encoding: gzip\\r\\n\\r\\n\")\n\tpayload = append(payload, b.Bytes()...)\n\n\tnewPayload := prettifyHTTP(payload)\n\n\tif string(newPayload) != \"HTTP/1.1 200 OK\\r\\nContent-Length: 4\\r\\n\\r\\ntest\" {\n\t\tt.Errorf(\"Payload not match %q\", string(newPayload))\n\t}\n}\n\nfunc TestHTTPPrettifierChunked(t *testing.T) {\n\tpayload := []byte(\"POST / HTTP/1.1\\r\\nHost: www.w3.org\\r\\nTransfer-Encoding: chunked\\r\\n\\r\\n4\\r\\nWiki\\r\\n5\\r\\npedia\\r\\ne\\r\\n in\\r\\n\\r\\nchunks.\\r\\n0\\r\\n\\r\\n\")\n\n\tpayload = prettifyHTTP(payload)\n\tif string(proto.Header(payload, []byte(\"Content-Length\"))) != \"23\" {\n\t\tt.Errorf(\"payload should have content length of 23\")\n\t}\n}\n" }, { "path": "input_dummy.go", "content": "package goreplay\n\nimport (\n\t\"time\"\n)\n\n// DummyInput used for debugging. It generate 1 \"GET /\"\" request per second.\ntype DummyInput struct {\n\tdata chan []byte\n\tquit chan struct{}\n}\n\n// NewDummyInput constructor for DummyInput\nfunc NewDummyInput(options string) (di *DummyInput) {\n\tdi = new(DummyInput)\n\tdi.data = make(chan []byte)\n\tdi.quit = make(chan struct{})\n\n\tgo di.emit()\n\n\treturn\n}\n\n// PluginRead reads message from this plugin\nfunc (i *DummyInput) PluginRead() (*Message, error) {\n\tvar msg Message\n\tselect {\n\tcase <-i.quit:\n\t\treturn nil, ErrorStopped\n\tcase buf := <-i.data:\n\t\tmsg.Meta, msg.Data = payloadMetaWithBody(buf)\n\t\treturn &msg, nil\n\t}\n}\n\nfunc (i *DummyInput) emit() {\n\tticker := time.NewTicker(time.Second)\n\n\tfor range ticker.C {\n\t\tuuid := uuid()\n\t\treqh := payloadHeader(RequestPayload, uuid, time.Now().UnixNano(), -1)\n\t\ti.data <- append(reqh, []byte(\"GET / HTTP/1.1\\r\\nHost: www.w3.org\\r\\nUser-Agent: Go 1.1 package http\\r\\nAccept-Encoding: gzip\\r\\n\\r\\n\")...)\n\n\t\tresh := payloadHeader(ResponsePayload, uuid, time.Now().UnixNano()+1, 1)\n\t\ti.data <- append(resh, []byte(\"HTTP/1.1 200 OK\\r\\nContent-Length: 0\\r\\n\\r\\n\")...)\n\t}\n}\n\nfunc (i *DummyInput) String() string {\n\treturn \"Dummy Input\"\n}\n\n// Close closes this plugins\nfunc (i *DummyInput) Close() error {\n\tclose(i.quit)\n\treturn nil\n}\n" }, { "path": "input_file.go", "content": "package goreplay\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"compress/gzip\"\n\t\"container/heap\"\n\t\"errors\"\n\t\"expvar\"\n\t\"fmt\"\n\t\"io\"\n\t\"math\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/aws/session\"\n\t\"github.com/aws/aws-sdk-go/service/s3\"\n)\n\ntype filePayload struct {\n\tdata []byte\n\ttimestamp int64\n}\n\n// An IntHeap is a min-heap of ints.\ntype payloadQueue struct {\n\tsync.RWMutex\n\ts []*filePayload\n}\n\nfunc (h payloadQueue) Len() int { return len(h.s) }\nfunc (h payloadQueue) Less(i, j int) bool { return h.s[i].timestamp < h.s[j].timestamp }\nfunc (h payloadQueue) Swap(i, j int) { h.s[i], h.s[j] = h.s[j], h.s[i] }\n\nfunc (h *payloadQueue) Push(x interface{}) {\n\t// Push and Pop use pointer receivers because they modify the slice's length,\n\t// not just its contents.\n\th.s = append(h.s, x.(*filePayload))\n}\n\nfunc (h *payloadQueue) Pop() interface{} {\n\told := h.s\n\tn := len(old)\n\tx := old[n-1]\n\th.s = old[0 : n-1]\n\treturn x\n}\n\nfunc (h payloadQueue) Idx(i int) *filePayload {\n\treturn h.s[i]\n}\n\ntype fileInputReader struct {\n\treader *bufio.Reader\n\tfile io.ReadCloser\n\tclosed int32 // Value of 0 indicates that the file is still open.\n\ts3 bool\n\tqueue payloadQueue\n\treadDepth int\n\tdryRun bool\n\tpath string\n}\n\nfunc (f *fileInputReader) parse(init chan struct{}) error {\n\tpayloadSeparatorAsBytes := []byte(payloadSeparator)\n\tvar buffer bytes.Buffer\n\tvar initialized bool\n\n\tlineNum := 0\n\n\tfor {\n\t\tline, err := f.reader.ReadBytes('\\n')\n\t\tlineNum++\n\n\t\tif err != nil {\n\t\t\tif err != io.EOF {\n\t\t\t\tDebug(1, err)\n\t\t\t}\n\n\t\t\tf.Close()\n\n\t\t\tif !initialized {\n\t\t\t\tclose(init)\n\t\t\t\tinitialized = true\n\t\t\t}\n\n\t\t\treturn err\n\t\t}\n\n\t\tif bytes.Equal(payloadSeparatorAsBytes[1:], line) {\n\t\t\tasBytes := buffer.Bytes()\n\t\t\tmeta := payloadMeta(asBytes)\n\n\t\t\tif len(meta) < 3 {\n\t\t\t\tDebug(1, fmt.Sprintf(\"Found malformed record, file: %s, line %d\", f.path, lineNum))\n\t\t\t\tbuffer = bytes.Buffer{}\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\ttimestamp, _ := strconv.ParseInt(string(meta[2]), 10, 64)\n\t\t\tdata := asBytes[:len(asBytes)-1]\n\n\t\t\tf.queue.Lock()\n\t\t\theap.Push(&f.queue, &filePayload{\n\t\t\t\ttimestamp: timestamp,\n\t\t\t\tdata: data,\n\t\t\t})\n\t\t\tf.queue.Unlock()\n\n\t\t\tfor {\n\t\t\t\tif f.queue.Len() < f.readDepth {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\n\t\t\t\tif !initialized {\n\t\t\t\t\tclose(init)\n\t\t\t\t\tinitialized = true\n\t\t\t\t}\n\n\t\t\t\tif !f.dryRun {\n\t\t\t\t\ttime.Sleep(100 * time.Millisecond)\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tbuffer = bytes.Buffer{}\n\t\t\tcontinue\n\t\t}\n\n\t\tbuffer.Write(line)\n\t}\n}\n\nfunc (f *fileInputReader) wait() {\n\tfor {\n\t\tif atomic.LoadInt32(&f.closed) == 1 {\n\t\t\treturn\n\t\t}\n\n\t\tif f.queue.Len() > 0 {\n\t\t\treturn\n\t\t}\n\n\t\tif !f.dryRun {\n\t\t\ttime.Sleep(100 * time.Millisecond)\n\t\t}\n\t}\n\n\treturn\n}\n\n// Close closes this plugin\nfunc (f *fileInputReader) Close() error {\n\tif atomic.LoadInt32(&f.closed) == 0 {\n\t\tatomic.StoreInt32(&f.closed, 1)\n\t\tf.file.Close()\n\t}\n\n\treturn nil\n}\n\nfunc newFileInputReader(path string, readDepth int, dryRun bool) *fileInputReader {\n\tvar file io.ReadCloser\n\tvar err error\n\n\tif strings.HasPrefix(path, \"s3://\") {\n\t\tfile = NewS3ReadCloser(path)\n\t} else {\n\t\tfile, err = os.Open(path)\n\t}\n\n\tif err != nil {\n\t\tDebug(0, fmt.Sprintf(\"[INPUT-FILE] err: %q\", err))\n\t\treturn nil\n\t}\n\n\tr := &fileInputReader{path: path, file: file, closed: 0, readDepth: readDepth, dryRun: dryRun}\n\tif strings.HasSuffix(path, \".gz\") {\n\t\tgzReader, err := gzip.NewReader(file)\n\t\tif err != nil {\n\t\t\tDebug(0, fmt.Sprintf(\"[INPUT-FILE] err: %q\", err))\n\t\t\treturn nil\n\t\t}\n\t\tr.reader = bufio.NewReader(gzReader)\n\t} else {\n\t\tr.reader = bufio.NewReader(file)\n\t}\n\n\theap.Init(&r.queue)\n\n\tinit := make(chan struct{})\n\tgo r.parse(init)\n\t<-init\n\n\treturn r\n}\n\n// FileInput can read requests generated by FileOutput\ntype FileInput struct {\n\tmu sync.Mutex\n\tdata chan []byte\n\texit chan bool\n\tpath string\n\treaders []*fileInputReader\n\tspeedFactor float64\n\tloop bool\n\treadDepth int\n\tdryRun bool\n\tmaxWait time.Duration\n\n\tstats *expvar.Map\n}\n\n// NewFileInput constructor for FileInput. Accepts file path as argument.\nfunc NewFileInput(path string, loop bool, readDepth int, maxWait time.Duration, dryRun bool) (i *FileInput) {\n\ti = new(FileInput)\n\ti.data = make(chan []byte, 1000)\n\ti.exit = make(chan bool)\n\ti.path = path\n\ti.speedFactor = 1\n\ti.loop = loop\n\ti.readDepth = readDepth\n\ti.stats = expvar.NewMap(\"file-\" + path)\n\ti.dryRun = dryRun\n\ti.maxWait = maxWait\n\n\tif err := i.init(); err != nil {\n\t\treturn\n\t}\n\n\tgo i.emit()\n\n\treturn\n}\n\nfunc parseS3Url(path string) (bucket, key string) {\n\tpath = path[5:] // stripping `s3://`\n\tsep := strings.IndexByte(path, '/')\n\n\tbucket = path[:sep]\n\tkey = path[sep+1:]\n\n\treturn bucket, key\n}\n\nfunc (i *FileInput) init() (err error) {\n\tdefer i.mu.Unlock()\n\ti.mu.Lock()\n\n\tvar matches []string\n\n\tif strings.HasPrefix(i.path, \"s3://\") {\n\t\tsess := session.Must(session.NewSession(awsConfig()))\n\t\tsvc := s3.New(sess)\n\n\t\tbucket, key := parseS3Url(i.path)\n\n\t\tparams := &s3.ListObjectsInput{\n\t\t\tBucket: aws.String(bucket),\n\t\t\tPrefix: aws.String(key),\n\t\t}\n\n\t\tresp, err := svc.ListObjects(params)\n\t\tif err != nil {\n\t\t\tDebug(2, \"[INPUT-FILE] Error while retrieving list of files from S3\", i.path, err)\n\t\t\treturn err\n\t\t}\n\n\t\tfor _, c := range resp.Contents {\n\t\t\tmatches = append(matches, \"s3://\"+bucket+\"/\"+(*c.Key))\n\t\t}\n\t} else if matches, err = filepath.Glob(i.path); err != nil {\n\t\tDebug(2, \"[INPUT-FILE] Wrong file pattern\", i.path, err)\n\t\treturn\n\t}\n\n\tif len(matches) == 0 {\n\t\tDebug(2, \"[INPUT-FILE] No files match pattern: \", i.path)\n\t\treturn errors.New(\"no matching files\")\n\t}\n\n\ti.readers = make([]*fileInputReader, len(matches))\n\n\tfor idx, p := range matches {\n\t\ti.readers[idx] = newFileInputReader(p, i.readDepth, i.dryRun)\n\t}\n\n\ti.stats.Add(\"reader_count\", int64(len(matches)))\n\n\treturn nil\n}\n\n// PluginRead reads message from this plugin\nfunc (i *FileInput) PluginRead() (*Message, error) {\n\tvar msg Message\n\tselect {\n\tcase <-i.exit:\n\t\treturn nil, ErrorStopped\n\tcase buf := <-i.data:\n\t\ti.stats.Add(\"read_from\", 1)\n\t\tmsg.Meta, msg.Data = payloadMetaWithBody(buf)\n\t\treturn &msg, nil\n\t}\n}\n\nfunc (i *FileInput) String() string {\n\treturn \"File input: \" + i.path\n}\n\n// Find reader with smallest timestamp e.g next payload in row\nfunc (i *FileInput) nextReader() (next *fileInputReader) {\n\tfor _, r := range i.readers {\n\t\tif r == nil {\n\t\t\tcontinue\n\t\t}\n\n\t\tr.wait()\n\n\t\tif r.queue.Len() == 0 {\n\t\t\tcontinue\n\t\t}\n\n\t\tif next == nil || r.queue.Idx(0).timestamp < next.queue.Idx(0).timestamp {\n\t\t\tnext = r\n\t\t\tcontinue\n\t\t}\n\t}\n\n\treturn\n}\n\nfunc (i *FileInput) emit() {\n\tvar lastTime int64 = -1\n\n\tvar maxWait, firstWait, minWait int64\n\tminWait = math.MaxInt64\n\n\ti.stats.Add(\"negative_wait\", 0)\n\n\tfor {\n\t\tselect {\n\t\tcase <-i.exit:\n\t\t\treturn\n\t\tdefault:\n\t\t}\n\n\t\treader := i.nextReader()\n\n\t\tif reader == nil {\n\t\t\tif i.loop {\n\t\t\t\ti.init()\n\t\t\t\tlastTime = -1\n\t\t\t\tcontinue\n\t\t\t} else {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\treader.queue.RLock()\n\t\tpayload := heap.Pop(&reader.queue).(*filePayload)\n\t\ti.stats.Add(\"total_counter\", 1)\n\t\ti.stats.Add(\"total_bytes\", int64(len(payload.data)))\n\t\treader.queue.RUnlock()\n\n\t\tif lastTime != -1 {\n\t\t\tdiff := payload.timestamp - lastTime\n\n\t\t\tif firstWait == 0 {\n\t\t\t\tfirstWait = diff\n\t\t\t}\n\n\t\t\tif i.speedFactor != 1 {\n\t\t\t\tdiff = int64(float64(diff) / i.speedFactor)\n\t\t\t}\n\n\t\t\tif i.maxWait > 0 && diff > int64(i.maxWait) {\n\t\t\t\tdiff = int64(i.maxWait)\n\t\t\t}\n\n\t\t\tif diff >= 0 {\n\t\t\t\tlastTime = payload.timestamp\n\n\t\t\t\tif !i.dryRun {\n\t\t\t\t\ttime.Sleep(time.Duration(diff))\n\t\t\t\t}\n\n\t\t\t\ti.stats.Add(\"total_wait\", diff)\n\n\t\t\t\tif diff > maxWait {\n\t\t\t\t\tmaxWait = diff\n\t\t\t\t}\n\n\t\t\t\tif diff < minWait {\n\t\t\t\t\tminWait = diff\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\ti.stats.Add(\"negative_wait\", 1)\n\t\t\t}\n\t\t} else {\n\t\t\tlastTime = payload.timestamp\n\t\t}\n\n\t\t// Recheck if we have exited since last check.\n\t\tselect {\n\t\tcase <-i.exit:\n\t\t\treturn\n\t\tdefault:\n\t\t\tif !i.dryRun {\n\t\t\t\ti.data <- payload.data\n\t\t\t}\n\t\t}\n\t}\n\n\ti.stats.Set(\"first_wait\", time.Duration(firstWait))\n\ti.stats.Set(\"max_wait\", time.Duration(maxWait))\n\ti.stats.Set(\"min_wait\", time.Duration(minWait))\n\n\tDebug(2, fmt.Sprintf(\"[INPUT-FILE] FileInput: end of file '%s'\\n\", i.path))\n\n\tif i.dryRun {\n\t\tfmt.Printf(\"Records found: %v\\nFiles processed: %v\\nBytes processed: %v\\nMax wait: %v\\nMin wait: %v\\nFirst wait: %v\\nIt will take `%v` to replay at current speed.\\nFound %v records with out of order timestamp\\n\",\n\t\t\ti.stats.Get(\"total_counter\"),\n\t\t\ti.stats.Get(\"reader_count\"),\n\t\t\ti.stats.Get(\"total_bytes\"),\n\t\t\ti.stats.Get(\"max_wait\"),\n\t\t\ti.stats.Get(\"min_wait\"),\n\t\t\ti.stats.Get(\"first_wait\"),\n\t\t\ttime.Duration(i.stats.Get(\"total_wait\").(*expvar.Int).Value()),\n\t\t\ti.stats.Get(\"negative_wait\"),\n\t\t)\n\t}\n}\n\n// Close closes this plugin\nfunc (i *FileInput) Close() error {\n\tdefer i.mu.Unlock()\n\ti.mu.Lock()\n\n\tclose(i.exit)\n\tfor _, r := range i.readers {\n\t\tr.Close()\n\t}\n\n\treturn nil\n}\n" }, { "path": "input_file_test.go", "content": "package goreplay\n\nimport (\n\t\"bytes\"\n\t\"errors\"\n\t\"fmt\"\n\t\"io/ioutil\"\n\t\"math/rand\"\n\t\"os\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestInputFileWithGET(t *testing.T) {\n\tinput := NewTestInput()\n\trg := NewRequestGenerator([]PluginReader{input}, func() { input.EmitGET() }, 1)\n\treadPayloads := []*Message{}\n\n\t// Given a capture file with a GET request\n\texpectedCaptureFile := CreateCaptureFile(rg)\n\tdefer expectedCaptureFile.TearDown()\n\n\t// When the request is read from the capture file\n\terr := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(msg *Message) {\n\t\treadPayloads = append(readPayloads, msg)\n\t})\n\n\t// The read request should match the original request\n\tif err != nil {\n\t\tt.Error(err)\n\t} else if !expectedCaptureFile.PayloadsEqual(readPayloads) {\n\t\tt.Error(\"Request read back from file should match\")\n\n\t}\n}\n\nfunc TestInputFileWithPayloadLargerThan64Kb(t *testing.T) {\n\tinput := NewTestInput()\n\trg := NewRequestGenerator([]PluginReader{input}, func() { input.EmitSizedPOST(64 * 1024) }, 1)\n\treadPayloads := []*Message{}\n\n\t// Given a capture file with a request over 64Kb\n\texpectedCaptureFile := CreateCaptureFile(rg)\n\tdefer expectedCaptureFile.TearDown()\n\n\t// When the request is read from the capture file\n\terr := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(msg *Message) {\n\t\treadPayloads = append(readPayloads, msg)\n\t})\n\n\t// The read request should match the original request\n\tif err != nil {\n\t\tt.Error(err)\n\t} else if !expectedCaptureFile.PayloadsEqual(readPayloads) {\n\t\tt.Error(\"Request read back from file should match\")\n\n\t}\n\n}\n\nfunc TestInputFileWithGETAndPOST(t *testing.T) {\n\n\tinput := NewTestInput()\n\trg := NewRequestGenerator([]PluginReader{input}, func() {\n\t\tinput.EmitGET()\n\t\tinput.EmitPOST()\n\t}, 2)\n\treadPayloads := []*Message{}\n\n\t// Given a capture file with a GET request\n\texpectedCaptureFile := CreateCaptureFile(rg)\n\tdefer expectedCaptureFile.TearDown()\n\n\t// When the requests are read from the capture file\n\terr := ReadFromCaptureFile(expectedCaptureFile.file, 2, func(msg *Message) {\n\t\treadPayloads = append(readPayloads, msg)\n\t})\n\n\t// The read requests should match the original request\n\tif err != nil {\n\t\tt.Error(err)\n\t} else if !expectedCaptureFile.PayloadsEqual(readPayloads) {\n\t\tt.Error(\"Request read back from file should match\")\n\n\t}\n\n}\n\nfunc TestInputFileMultipleFilesWithRequestsOnly(t *testing.T) {\n\trnd := rand.Int63()\n\n\tfile1, _ := os.OpenFile(fmt.Sprintf(\"/tmp/%d_0\", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)\n\tfile1.Write([]byte(\"1 1 1\\ntest1\"))\n\tfile1.Write([]byte(payloadSeparator))\n\tfile1.Write([]byte(\"1 1 3\\ntest2\"))\n\tfile1.Write([]byte(payloadSeparator))\n\tfile1.Close()\n\n\tfile2, _ := os.OpenFile(fmt.Sprintf(\"/tmp/%d_1\", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)\n\tfile2.Write([]byte(\"1 1 2\\ntest3\"))\n\tfile2.Write([]byte(payloadSeparator))\n\tfile2.Write([]byte(\"1 1 4\\ntest4\"))\n\tfile2.Write([]byte(payloadSeparator))\n\tfile2.Close()\n\n\tinput := NewFileInput(fmt.Sprintf(\"/tmp/%d*\", rnd), false, 100, 0, false)\n\n\tfor i := '1'; i <= '4'; i++ {\n\t\tmsg, _ := input.PluginRead()\n\t\tif msg.Meta[4] != byte(i) {\n\t\t\tt.Error(\"Should emit requests in right order\", string(msg.Meta))\n\t\t}\n\t}\n\n\tos.Remove(file1.Name())\n\tos.Remove(file2.Name())\n}\n\nfunc TestInputFileRequestsWithLatency(t *testing.T) {\n\trnd := rand.Int63()\n\n\tfile, _ := os.OpenFile(fmt.Sprintf(\"/tmp/%d\", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)\n\tdefer file.Close()\n\n\tfile.Write([]byte(\"1 1 100000000\\nrequest1\"))\n\tfile.Write([]byte(payloadSeparator))\n\tfile.Write([]byte(\"1 2 150000000\\nrequest2\"))\n\tfile.Write([]byte(payloadSeparator))\n\tfile.Write([]byte(\"1 3 250000000\\nrequest3\"))\n\tfile.Write([]byte(payloadSeparator))\n\n\tinput := NewFileInput(fmt.Sprintf(\"/tmp/%d\", rnd), false, 100, 0, false)\n\n\tstart := time.Now().UnixNano()\n\tfor i := 0; i < 3; i++ {\n\t\tinput.PluginRead()\n\t}\n\tend := time.Now().UnixNano()\n\n\tvar expectedLatency int64 = 300000000 - 100000000\n\trealLatency := end - start\n\tif realLatency > expectedLatency {\n\t\tt.Errorf(\"Should emit requests respecting latency. Expected: %v, real: %v\", expectedLatency, realLatency)\n\t}\n}\n\nfunc TestInputFileMultipleFilesWithRequestsAndResponses(t *testing.T) {\n\trnd := rand.Int63()\n\n\tfile1, _ := os.OpenFile(fmt.Sprintf(\"/tmp/%d_0\", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)\n\tfile1.Write([]byte(\"1 1 1\\nrequest1\"))\n\tfile1.Write([]byte(payloadSeparator))\n\tfile1.Write([]byte(\"2 1 1\\nresponse1\"))\n\tfile1.Write([]byte(payloadSeparator))\n\tfile1.Write([]byte(\"1 2 3\\nrequest2\"))\n\tfile1.Write([]byte(payloadSeparator))\n\tfile1.Write([]byte(\"2 2 3\\nresponse2\"))\n\tfile1.Write([]byte(payloadSeparator))\n\tfile1.Close()\n\n\tfile2, _ := os.OpenFile(fmt.Sprintf(\"/tmp/%d_1\", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)\n\tfile2.Write([]byte(\"1 3 2\\nrequest3\"))\n\tfile2.Write([]byte(payloadSeparator))\n\tfile2.Write([]byte(\"2 3 2\\nresponse3\"))\n\tfile2.Write([]byte(payloadSeparator))\n\tfile2.Write([]byte(\"1 4 4\\nrequest4\"))\n\tfile2.Write([]byte(payloadSeparator))\n\tfile2.Write([]byte(\"2 4 4\\nresponse4\"))\n\tfile2.Write([]byte(payloadSeparator))\n\tfile2.Close()\n\n\tinput := NewFileInput(fmt.Sprintf(\"/tmp/%d*\", rnd), false, 100, 0, false)\n\n\tfor i := '1'; i <= '4'; i++ {\n\t\tmsg, _ := input.PluginRead()\n\t\tif msg.Meta[0] != '1' && msg.Meta[4] != byte(i) {\n\t\t\tt.Error(\"Shound emit requests in right order\", string(msg.Meta))\n\t\t}\n\n\t\tmsg, _ = input.PluginRead()\n\t\tif msg.Meta[0] != '2' && msg.Meta[4] != byte(i) {\n\t\t\tt.Error(\"Shound emit responses in right order\", string(msg.Meta))\n\t\t}\n\t}\n\n\tos.Remove(file1.Name())\n\tos.Remove(file2.Name())\n}\n\nfunc TestInputFileLoop(t *testing.T) {\n\trnd := rand.Int63()\n\n\tfile, _ := os.OpenFile(fmt.Sprintf(\"/tmp/%d\", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)\n\tfile.Write([]byte(\"1 1 1\\ntest1\"))\n\tfile.Write([]byte(payloadSeparator))\n\tfile.Write([]byte(\"1 1 2\\ntest2\"))\n\tfile.Write([]byte(payloadSeparator))\n\tfile.Close()\n\n\tinput := NewFileInput(fmt.Sprintf(\"/tmp/%d\", rnd), true, 100, 0, false)\n\n\t// Even if we have just 2 requests in file, it should indifinitly loop\n\tfor i := 0; i < 1000; i++ {\n\t\tinput.PluginRead()\n\t}\n\n\tinput.Close()\n\tos.Remove(file.Name())\n}\n\nfunc TestInputFileCompressed(t *testing.T) {\n\trnd := rand.Int63()\n\n\toutput := NewFileOutput(fmt.Sprintf(\"/tmp/%d_0.gz\", rnd), &FileOutputConfig{FlushInterval: time.Minute, Append: true})\n\tfor i := 0; i < 1000; i++ {\n\t\toutput.PluginWrite(&Message{Meta: []byte(\"1 1 1\\r\\n\"), Data: []byte(\"test\")})\n\t}\n\tname1 := output.file.Name()\n\toutput.Close()\n\n\toutput2 := NewFileOutput(fmt.Sprintf(\"/tmp/%d_1.gz\", rnd), &FileOutputConfig{FlushInterval: time.Minute, Append: true})\n\tfor i := 0; i < 1000; i++ {\n\t\toutput2.PluginWrite(&Message{Meta: []byte(\"1 1 1\\r\\n\"), Data: []byte(\"test\")})\n\t}\n\tname2 := output2.file.Name()\n\toutput2.Close()\n\n\tinput := NewFileInput(fmt.Sprintf(\"/tmp/%d*\", rnd), false, 100, 0, false)\n\tfor i := 0; i < 2000; i++ {\n\t\tinput.PluginRead()\n\t}\n\n\tos.Remove(name1)\n\tos.Remove(name2)\n}\n\ntype CaptureFile struct {\n\tmsgs []*Message\n\tfile *os.File\n}\n\nfunc NewExpectedCaptureFile(msgs []*Message, file *os.File) *CaptureFile {\n\tecf := new(CaptureFile)\n\tecf.file = file\n\tecf.msgs = msgs\n\treturn ecf\n}\n\nfunc (expectedCaptureFile *CaptureFile) TearDown() {\n\tif expectedCaptureFile.file != nil {\n\t\tos.Remove(expectedCaptureFile.file.Name())\n\t}\n}\n\ntype RequestGenerator struct {\n\tinputs []PluginReader\n\temit func()\n\twg *sync.WaitGroup\n}\n\nfunc NewRequestGenerator(inputs []PluginReader, emit func(), count int) (rg *RequestGenerator) {\n\trg = new(RequestGenerator)\n\trg.inputs = inputs\n\trg.emit = emit\n\trg.wg = new(sync.WaitGroup)\n\trg.wg.Add(count)\n\treturn\n}\n\nfunc (expectedCaptureFile *CaptureFile) PayloadsEqual(other []*Message) bool {\n\n\tif len(expectedCaptureFile.msgs) != len(other) {\n\t\treturn false\n\t}\n\n\tfor i, payload := range other {\n\t\tif !bytes.Equal(expectedCaptureFile.msgs[i].Meta, payload.Meta) {\n\t\t\treturn false\n\t\t}\n\t\tif !bytes.Equal(expectedCaptureFile.msgs[i].Data, payload.Data) {\n\t\t\treturn false\n\t\t}\n\t}\n\n\treturn true\n\n}\n\nfunc CreateCaptureFile(requestGenerator *RequestGenerator) *CaptureFile {\n\tf, err := ioutil.TempFile(\"\", \"testmainconf\")\n\tif err != nil {\n\t\tpanic(err)\n\t}\n\n\treadPayloads := []*Message{}\n\toutput := NewTestOutput(func(msg *Message) {\n\t\treadPayloads = append(readPayloads, msg)\n\t\trequestGenerator.wg.Done()\n\t})\n\n\toutputFile := NewFileOutput(f.Name(), &FileOutputConfig{FlushInterval: time.Second, Append: true})\n\n\tplugins := &InOutPlugins{\n\t\tInputs: requestGenerator.inputs,\n\t\tOutputs: []PluginWriter{output, outputFile},\n\t}\n\tfor _, input := range requestGenerator.inputs {\n\t\tplugins.All = append(plugins.All, input)\n\t}\n\tplugins.All = append(plugins.All, output, outputFile)\n\n\temitter := NewEmitter()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\trequestGenerator.emit()\n\trequestGenerator.wg.Wait()\n\n\ttime.Sleep(100 * time.Millisecond)\n\temitter.Close()\n\n\treturn NewExpectedCaptureFile(readPayloads, f)\n\n}\n\nfunc ReadFromCaptureFile(captureFile *os.File, count int, callback writeCallback) (err error) {\n\twg := new(sync.WaitGroup)\n\n\tinput := NewFileInput(captureFile.Name(), false, 100, 0, false)\n\toutput := NewTestOutput(func(msg *Message) {\n\t\tcallback(msg)\n\t\twg.Done()\n\t})\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output},\n\t}\n\tplugins.All = append(plugins.All, input, output)\n\n\twg.Add(count)\n\temitter := NewEmitter()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\tdone := make(chan int, 1)\n\tgo func() {\n\t\twg.Wait()\n\t\tdone <- 1\n\t}()\n\n\tselect {\n\tcase <-done:\n\t\tbreak\n\tcase <-time.After(2 * time.Second):\n\t\terr = errors.New(\"Timed out\")\n\t}\n\temitter.Close()\n\treturn\n\n}\n" }, { "path": "input_http.go", "content": "package goreplay\n\nimport (\n\t\"log\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/http/httputil\"\n\t\"time\"\n)\n\n// HTTPInput used for sending requests to Gor via http\ntype HTTPInput struct {\n\tdata chan []byte\n\taddress string\n\tlistener net.Listener\n\tstop chan bool // Channel used only to indicate goroutine should shutdown\n}\n\n// NewHTTPInput constructor for HTTPInput. Accepts address with port which it will listen on.\nfunc NewHTTPInput(address string) (i *HTTPInput) {\n\ti = new(HTTPInput)\n\ti.data = make(chan []byte, 1000)\n\ti.stop = make(chan bool)\n\n\ti.listen(address)\n\n\treturn\n}\n\n// PluginRead reads message from this plugin\nfunc (i *HTTPInput) PluginRead() (*Message, error) {\n\tvar msg Message\n\tselect {\n\tcase <-i.stop:\n\t\treturn nil, ErrorStopped\n\tcase buf := <-i.data:\n\t\tmsg.Data = buf\n\t\tmsg.Meta = payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)\n\t\treturn &msg, nil\n\t}\n}\n\n// Close closes this plugin\nfunc (i *HTTPInput) Close() error {\n\tclose(i.stop)\n\treturn nil\n}\n\nfunc (i *HTTPInput) handler(w http.ResponseWriter, r *http.Request) {\n\tr.URL.Scheme = \"http\"\n\tr.URL.Host = i.address\n\n\tbuf, _ := httputil.DumpRequestOut(r, true)\n\thttp.Error(w, http.StatusText(200), 200)\n\ti.data <- buf\n}\n\nfunc (i *HTTPInput) listen(address string) {\n\tvar err error\n\n\tmux := http.NewServeMux()\n\n\tmux.HandleFunc(\"/\", i.handler)\n\n\ti.listener, err = net.Listen(\"tcp\", address)\n\tif err != nil {\n\t\tlog.Fatal(\"HTTP input listener failure:\", err)\n\t}\n\ti.address = i.listener.Addr().String()\n\n\tgo func() {\n\t\terr = http.Serve(i.listener, mux)\n\t\tif err != nil && err != http.ErrServerClosed {\n\t\t\tlog.Fatal(\"HTTP input serve failure \", err)\n\t\t}\n\t}()\n}\n\nfunc (i *HTTPInput) String() string {\n\treturn \"HTTP input: \" + i.address\n}\n" }, { "path": "input_http_test.go", "content": "package goreplay\n\nimport (\n\t\"bytes\"\n\t\"net/http\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestHTTPInput(t *testing.T) {\n\twg := new(sync.WaitGroup)\n\n\tinput := NewHTTPInput(\"127.0.0.1:0\")\n\ttime.Sleep(time.Millisecond)\n\toutput := NewTestOutput(func(*Message) {\n\t\twg.Done()\n\t})\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output},\n\t}\n\tplugins.All = append(plugins.All, input, output)\n\n\temitter := NewEmitter()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\taddress := strings.Replace(input.address, \"[::]\", \"127.0.0.1\", -1)\n\n\tfor i := 0; i < 100; i++ {\n\t\twg.Add(1)\n\t\thttp.Get(\"http://\" + address + \"/\")\n\t}\n\n\twg.Wait()\n\temitter.Close()\n}\n\nfunc TestInputHTTPLargePayload(t *testing.T) {\n\twg := new(sync.WaitGroup)\n\tconst n = 10 << 20 // 10MB\n\tvar large [n]byte\n\tlarge[n-1] = '0'\n\n\tinput := NewHTTPInput(\"127.0.0.1:0\")\n\toutput := NewTestOutput(func(msg *Message) {\n\t\t_len := len(msg.Data)\n\t\tif _len >= n { // considering http body CRLF\n\t\t\tt.Errorf(\"expected body to be >= %d\", n)\n\t\t}\n\t\twg.Done()\n\t})\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output},\n\t}\n\tplugins.All = append(plugins.All, input, output)\n\n\temitter := NewEmitter()\n\tdefer emitter.Close()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\taddress := strings.Replace(input.address, \"[::]\", \"127.0.0.1\", -1)\n\tvar req *http.Request\n\tvar err error\n\treq, err = http.NewRequest(\"POST\", \"http://\"+address, bytes.NewBuffer(large[:]))\n\tif err != nil {\n\t\tt.Error(err)\n\t\treturn\n\t}\n\twg.Add(1)\n\t_, err = http.DefaultClient.Do(req)\n\tif err != nil {\n\t\tt.Error(err)\n\t\treturn\n\t}\n\twg.Wait()\n}\n" }, { "path": "input_kafka.go", "content": "package goreplay\n\nimport (\n\t\"encoding/json\"\n\t\"log\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/Shopify/sarama\"\n\t\"github.com/Shopify/sarama/mocks\"\n)\n\n// KafkaInput is used for receiving Kafka messages and\n// transforming them into HTTP payloads.\ntype KafkaInput struct {\n\tconfig *InputKafkaConfig\n\tconsumers []sarama.PartitionConsumer\n\tmessages chan *sarama.ConsumerMessage\n\tspeedFactor float64\n\tquit chan struct{}\n\tkafkaTimer *kafkaTimer\n}\n\nfunc getOffsetOfPartitions(offsetCfg string) int64 {\n\toffset, err := strconv.ParseInt(offsetCfg, 10, 64)\n\tif err != nil || offset < -2 {\n\t\tlog.Fatalln(\"Failed to parse offset: \"+offsetCfg, err)\n\t}\n\treturn offset\n}\n\n// NewKafkaInput creates instance of kafka consumer client with TLS config\nfunc NewKafkaInput(offsetCfg string, config *InputKafkaConfig, tlsConfig *KafkaTLSConfig) *KafkaInput {\n\tc := NewKafkaConfig(&config.SASLConfig, tlsConfig)\n\n\tvar con sarama.Consumer\n\n\tif mock, ok := config.consumer.(*mocks.Consumer); ok && mock != nil {\n\t\tcon = config.consumer\n\t} else {\n\t\tvar err error\n\t\tcon, err = sarama.NewConsumer(strings.Split(config.Host, \",\"), c)\n\n\t\tif err != nil {\n\t\t\tlog.Fatalln(\"Failed to start Sarama(Kafka) consumer:\", err)\n\t\t}\n\t}\n\n\tpartitions, err := con.Partitions(config.Topic)\n\tif err != nil {\n\t\tlog.Fatalln(\"Failed to collect Sarama(Kafka) partitions:\", err)\n\t}\n\n\ti := &KafkaInput{\n\t\tconfig: config,\n\t\tconsumers: make([]sarama.PartitionConsumer, len(partitions)),\n\t\tmessages: make(chan *sarama.ConsumerMessage, 256),\n\t\tspeedFactor: 1,\n\t\tquit: make(chan struct{}),\n\t\tkafkaTimer: new(kafkaTimer),\n\t}\n\ti.config.Offset = offsetCfg\n\n\tfor index, partition := range partitions {\n\t\tconsumer, err := con.ConsumePartition(config.Topic, partition, getOffsetOfPartitions(offsetCfg))\n\t\tif err != nil {\n\t\t\tlog.Fatalln(\"Failed to start Sarama(Kafka) partition consumer:\", err)\n\t\t}\n\n\t\tgo func(consumer sarama.PartitionConsumer) {\n\t\t\tdefer consumer.Close()\n\n\t\t\tfor message := range consumer.Messages() {\n\t\t\t\ti.messages <- message\n\t\t\t}\n\t\t}(consumer)\n\n\t\tgo i.ErrorHandler(consumer)\n\n\t\ti.consumers[index] = consumer\n\t}\n\n\treturn i\n}\n\n// ErrorHandler should receive errors\nfunc (i *KafkaInput) ErrorHandler(consumer sarama.PartitionConsumer) {\n\tfor err := range consumer.Errors() {\n\t\tDebug(1, \"Failed to read access log entry:\", err)\n\t}\n}\n\n// PluginRead a reads message from this plugin\nfunc (i *KafkaInput) PluginRead() (*Message, error) {\n\tvar message *sarama.ConsumerMessage\n\tvar msg Message\n\tselect {\n\tcase <-i.quit:\n\t\treturn nil, ErrorStopped\n\tcase message = <-i.messages:\n\t}\n\n\tinputTs := \"\"\n\n\tmsg.Data = message.Value\n\tif i.config.UseJSON {\n\n\t\tvar kafkaMessage KafkaMessage\n\t\tjson.Unmarshal(message.Value, &kafkaMessage)\n\n\t\tinputTs = kafkaMessage.ReqTs\n\t\tvar err error\n\t\tmsg.Data, err = kafkaMessage.Dump()\n\t\tif err != nil {\n\t\t\tDebug(1, \"[INPUT-KAFKA] failed to decode access log entry:\", err)\n\t\t\treturn nil, err\n\t\t}\n\t}\n\n\t// does it have meta\n\tif isOriginPayload(msg.Data) {\n\t\tmsg.Meta, msg.Data = payloadMetaWithBody(msg.Data)\n\t\tinputTs = string(payloadMeta(msg.Meta)[2])\n\t}\n\n\ti.timeWait(inputTs)\n\n\treturn &msg, nil\n\n}\n\nfunc (i *KafkaInput) String() string {\n\treturn \"Kafka Input: \" + i.config.Host + \"/\" + i.config.Topic\n}\n\n// Close closes this plugin\nfunc (i *KafkaInput) Close() error {\n\tclose(i.quit)\n\treturn nil\n}\n\nfunc (i *KafkaInput) timeWait(curInputTs string) {\n\tif i.config.Offset == \"-1\" || curInputTs == \"\" {\n\t\treturn\n\t}\n\n\t// implement for Kafka input showdown or speedup emitting\n\ttimer := i.kafkaTimer\n\tcurTs := time.Now().UnixNano()\n\n\tcurInput, err := strconv.ParseInt(curInputTs, 10, 64)\n\n\tif timer.latestInputTs == 0 || timer.latestOutputTs == 0 {\n\t\ttimer.latestInputTs = curInput\n\t\ttimer.latestOutputTs = curTs\n\t\treturn\n\t}\n\n\tif err != nil {\n\t\tlog.Fatalln(\"Fatal to parse timestamp err: \", err)\n\t}\n\n\tdiffTs := curInput - timer.latestInputTs\n\tpastTs := curTs - timer.latestOutputTs\n\n\tdiff := diffTs - pastTs\n\tif i.speedFactor != 1 {\n\t\tdiff = int64(float64(diff) / i.speedFactor)\n\t}\n\n\tif diff > 0 {\n\t\ttime.Sleep(time.Duration(diff))\n\t}\n\n\ttimer.latestInputTs = curInput\n\ttimer.latestOutputTs = curTs\n}\n\ntype kafkaTimer struct {\n\tlatestInputTs int64\n\tlatestOutputTs int64\n}\n" }, { "path": "input_kafka_test.go", "content": "package goreplay\n\nimport (\n\t\"testing\"\n\n\t\"github.com/Shopify/sarama\"\n\t\"github.com/Shopify/sarama/mocks\"\n)\n\nfunc TestInputKafkaRAW(t *testing.T) {\n\tconsumer := mocks.NewConsumer(t, nil)\n\tdefer consumer.Close()\n\n\tconsumer.ExpectConsumePartition(\"test\", 0, mocks.AnyOffset).YieldMessage(&sarama.ConsumerMessage{Value: []byte(\"1 2 3\\nGET / HTTP1.1\\r\\nHeader: 1\\r\\n\\r\\n\")})\n\tconsumer.SetTopicMetadata(\n\t\tmap[string][]int32{\"test\": {0}},\n\t)\n\n\tinput := NewKafkaInput(\"-1\", &InputKafkaConfig{\n\t\tconsumer: consumer,\n\t\tTopic: \"test\",\n\t\tUseJSON: false,\n\t}, nil)\n\n\tmsg, err := input.PluginRead()\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif string(append(msg.Meta, msg.Data...)) != \"1 2 3\\nGET / HTTP1.1\\r\\nHeader: 1\\r\\n\\r\\n\" {\n\t\tt.Error(\"Message not properly decoded\")\n\t}\n}\n\nfunc TestInputKafkaJSON(t *testing.T) {\n\tconsumer := mocks.NewConsumer(t, nil)\n\tdefer consumer.Close()\n\n\tconsumer.ExpectConsumePartition(\"test\", 0, mocks.AnyOffset).YieldMessage(&sarama.ConsumerMessage{Value: []byte(`{\"Req_URL\":\"/\",\"Req_Type\":\"1\",\"Req_ID\":\"2\",\"Req_Ts\":\"3\",\"Req_Method\":\"GET\",\"Req_Headers\":{\"Header\":\"1\"}}`)})\n\tconsumer.SetTopicMetadata(\n\t\tmap[string][]int32{\"test\": {0}},\n\t)\n\n\tinput := NewKafkaInput(\"-1\", &InputKafkaConfig{\n\t\tconsumer: consumer,\n\t\tTopic: \"test\",\n\t\tUseJSON: true,\n\t}, nil)\n\n\tmsg, err := input.PluginRead()\n\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\n\tif string(append(msg.Meta, msg.Data...)) != \"1 2 3\\nGET / HTTP/1.1\\r\\nHeader: 1\\r\\n\\r\\n\" {\n\t\tt.Error(\"Message not properly decoded\")\n\t}\n}\n" }, { "path": "input_raw.go", "content": "package goreplay\n\nimport (\n\t\"context\"\n\t\"fmt\"\n\t\"github.com/buger/goreplay/internal/capture\"\n\t\"github.com/buger/goreplay/internal/tcp\"\n\t\"github.com/buger/goreplay/proto\"\n\t\"log\"\n\t\"net\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n)\n\n// RAWInputConfig represents configuration that can be applied on raw input\ntype RAWInputConfig = capture.PcapOptions\n\n// RAWInput used for intercepting traffic for given address\ntype RAWInput struct {\n\tsync.Mutex\n\tconfig RAWInputConfig\n\tmessageStats []tcp.Stats\n\tlistener *capture.Listener\n\tmessageParser *tcp.MessageParser\n\tcancelListener context.CancelFunc\n\tclosed bool\n\n\tquit chan bool // Channel used only to indicate goroutine should shutdown\n\thost string\n\tports []uint16\n}\n\n// NewRAWInput constructor for RAWInput. Accepts raw input config as arguments.\nfunc NewRAWInput(address string, config RAWInputConfig) (i *RAWInput) {\n\ti = new(RAWInput)\n\ti.config = config\n\ti.quit = make(chan bool)\n\n\thost, _ports, err := net.SplitHostPort(address)\n\tif err != nil {\n\t\t// If we are reading pcap file, no port needed\n\t\tif strings.HasSuffix(address, \"pcap\") {\n\t\t\thost = address\n\t\t\t_ports = \"0\"\n\t\t\terr = nil\n\t\t} else if strings.HasPrefix(address, \"k8s://\") {\n\t\t\tportIndex := strings.LastIndex(address, \":\")\n\t\t\thost = address[:portIndex]\n\t\t\t_ports = address[portIndex+1:]\n\t\t} else {\n\t\t\tlog.Fatalf(\"input-raw: error while parsing address: %s\", err)\n\t\t}\n\t}\n\n\tif strings.HasSuffix(host, \"pcap\") {\n\t\ti.config.Engine = capture.EnginePcapFile\n\t}\n\n\tvar ports []uint16\n\tif _ports != \"\" {\n\t\tportsStr := strings.Split(_ports, \",\")\n\n\t\tfor _, portStr := range portsStr {\n\t\t\tport, err := strconv.Atoi(strings.TrimSpace(portStr))\n\t\t\tif err != nil {\n\t\t\t\tlog.Fatalf(\"parsing port error: %v\", err)\n\t\t\t}\n\t\t\tports = append(ports, uint16(port))\n\n\t\t}\n\t}\n\n\ti.host = host\n\ti.ports = ports\n\n\ti.listen(address)\n\n\treturn\n}\n\n// PluginRead reads meassage from this plugin\nfunc (i *RAWInput) PluginRead() (*Message, error) {\n\tvar msgTCP *tcp.Message\n\tvar msg Message\n\tselect {\n\tcase <-i.quit:\n\t\treturn nil, ErrorStopped\n\tcase msgTCP = <-i.listener.Messages():\n\t\tmsg.Data = msgTCP.Data()\n\t}\n\n\tvar msgType byte = ResponsePayload\n\tif msgTCP.Direction == tcp.DirIncoming {\n\t\tmsgType = RequestPayload\n\t\tif i.config.RealIPHeader != \"\" {\n\t\t\tmsg.Data = proto.SetHeader(msg.Data, []byte(i.config.RealIPHeader), []byte(msgTCP.SrcAddr))\n\t\t}\n\t}\n\tmsg.Meta = payloadHeader(msgType, msgTCP.UUID(), msgTCP.Start.UnixNano(), msgTCP.End.UnixNano()-msgTCP.Start.UnixNano())\n\n\t// to be removed....\n\tif msgTCP.Truncated {\n\t\tDebug(2, \"[INPUT-RAW] message truncated, increase copy-buffer-size\")\n\t}\n\t// to be removed...\n\tif msgTCP.TimedOut {\n\t\tDebug(2, \"[INPUT-RAW] message timeout reached, increase input-raw-expire\")\n\t}\n\tif i.config.Stats {\n\t\tstat := msgTCP.Stats\n\t\tgo i.addStats(stat)\n\t}\n\tmsgTCP = nil\n\treturn &msg, nil\n}\n\nfunc (i *RAWInput) listen(address string) {\n\tvar err error\n\ti.listener, err = capture.NewListener(i.host, i.ports, i.config)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\terr = i.listener.Activate()\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tvar ctx context.Context\n\tctx, i.cancelListener = context.WithCancel(context.Background())\n\terrCh := i.listener.ListenBackground(ctx)\n\t<-i.listener.Reading\n\tDebug(1, i)\n\tgo func() {\n\t\t<-errCh // the listener closed voluntarily\n\t\ti.Close()\n\t}()\n}\n\nfunc (i *RAWInput) String() string {\n\treturn fmt.Sprintf(\"Intercepting traffic from: %s:%s\", i.host, strings.Join(strings.Fields(fmt.Sprint(i.ports)), \",\"))\n}\n\n// GetStats returns the stats so far and reset the stats\nfunc (i *RAWInput) GetStats() []tcp.Stats {\n\ti.Lock()\n\tdefer func() {\n\t\ti.messageStats = []tcp.Stats{}\n\t\ti.Unlock()\n\t}()\n\treturn i.messageStats\n}\n\n// Close closes the input raw listener\nfunc (i *RAWInput) Close() error {\n\ti.Lock()\n\tdefer i.Unlock()\n\tif i.closed {\n\t\treturn nil\n\t}\n\ti.cancelListener()\n\tclose(i.quit)\n\ti.closed = true\n\treturn nil\n}\n\nfunc (i *RAWInput) addStats(mStats tcp.Stats) {\n\ti.Lock()\n\tif len(i.messageStats) >= 10000 {\n\t\ti.messageStats = []tcp.Stats{}\n\t}\n\ti.messageStats = append(i.messageStats, mStats)\n\ti.Unlock()\n}\n" }, { "path": "input_raw_test.go", "content": "package goreplay\n\nimport (\n\t\"bytes\"\n\t\"github.com/buger/goreplay/internal/capture\"\n\t\"github.com/buger/goreplay/internal/tcp\"\n\t\"github.com/buger/goreplay/proto\"\n\t\"io/ioutil\"\n\t\"net\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t\"net/http/httputil\"\n\t\"os/exec\"\n\t\"strings\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n)\n\nconst testRawExpire = time.Millisecond * 200\n\nfunc TestRAWInputIPv4(t *testing.T) {\n\twg := new(sync.WaitGroup)\n\n\tlistener, err := net.Listen(\"tcp4\", \"127.0.0.1:0\")\n\tif err != nil {\n\t\tt.Error(err)\n\t\treturn\n\t}\n\torigin := &http.Server{\n\t\tHandler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\t\tw.Write([]byte(\"ab\"))\n\t\t}),\n\t\tReadTimeout: 10 * time.Second,\n\t\tWriteTimeout: 10 * time.Second,\n\t}\n\tgo origin.Serve(listener)\n\tdefer listener.Close()\n\t_, port, _ := net.SplitHostPort(listener.Addr().String())\n\n\tvar respCounter, reqCounter int64\n\tconf := RAWInputConfig{\n\t\tEngine: capture.EnginePcap,\n\t\tExpire: 0,\n\t\tProtocol: tcp.ProtocolHTTP,\n\t\tTrackResponse: true,\n\t\tRealIPHeader: \"X-Real-IP\",\n\t}\n\tinput := NewRAWInput(listener.Addr().String(), conf)\n\n\toutput := NewTestOutput(func(msg *Message) {\n\t\tif msg.Meta[0] == '1' {\n\t\t\tif len(proto.Header(msg.Data, []byte(\"X-Real-IP\"))) == 0 {\n\t\t\t\tt.Error(\"Should have X-Real-IP header\")\n\t\t\t}\n\t\t\treqCounter++\n\t\t} else {\n\t\t\trespCounter++\n\t\t}\n\n\t\twg.Done()\n\t})\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output},\n\t}\n\tplugins.All = append(plugins.All, input, output)\n\n\taddr := \"http://127.0.0.1:\" + port\n\temitter := NewEmitter()\n\tdefer emitter.Close()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\t// time.Sleep(time.Second)\n\tfor i := 0; i < 1; i++ {\n\t\twg.Add(2)\n\t\t_, err = http.Get(addr)\n\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t\treturn\n\t\t}\n\t}\n\n\twg.Wait()\n\tconst want = 10\n\tif reqCounter != respCounter && reqCounter != want {\n\t\tt.Errorf(\"want %d requests and %d responses, got %d requests and %d responses\", want, want, reqCounter, respCounter)\n\t}\n}\n\nfunc TestRAWInputNoKeepAlive(t *testing.T) {\n\twg := new(sync.WaitGroup)\n\n\tlistener, err := net.Listen(\"tcp\", \"127.0.0.1:0\")\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\torigin := &http.Server{\n\t\tHandler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\t\tw.Write([]byte(\"ab\"))\n\t\t}),\n\t\tReadTimeout: 10 * time.Second,\n\t\tWriteTimeout: 10 * time.Second,\n\t}\n\torigin.SetKeepAlivesEnabled(false)\n\tgo origin.Serve(listener)\n\tdefer listener.Close()\n\t_, port, _ := net.SplitHostPort(listener.Addr().String())\n\n\tconf := RAWInputConfig{\n\t\tEngine: capture.EnginePcap,\n\t\tExpire: testRawExpire,\n\t\tProtocol: tcp.ProtocolHTTP,\n\t\tTrackResponse: true,\n\t}\n\tinput := NewRAWInput(\":\"+port, conf)\n\tvar respCounter, reqCounter int64\n\toutput := NewTestOutput(func(msg *Message) {\n\t\tif msg.Meta[0] == '1' {\n\t\t\tatomic.AddInt64(&reqCounter, 1)\n\t\t\twg.Done()\n\t\t} else {\n\t\t\tatomic.AddInt64(&respCounter, 1)\n\t\t\twg.Done()\n\t\t}\n\t})\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output},\n\t}\n\tplugins.All = append(plugins.All, input, output)\n\n\taddr := \"http://127.0.0.1:\" + port\n\n\temitter := NewEmitter()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\tfor i := 0; i < 10; i++ {\n\t\t// request + response\n\t\twg.Add(2)\n\t\t_, err = http.Get(addr)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t\treturn\n\t\t}\n\t}\n\n\twg.Wait()\n\tconst want = 10\n\tif reqCounter != respCounter && reqCounter != want {\n\t\tt.Errorf(\"want %d requests and %d responses, got %d requests and %d responses\", want, want, reqCounter, respCounter)\n\t}\n\temitter.Close()\n}\n\nfunc TestRAWInputIPv6(t *testing.T) {\n\twg := new(sync.WaitGroup)\n\n\tlistener, err := net.Listen(\"tcp\", \"[::1]:0\")\n\tif err != nil {\n\t\treturn\n\t}\n\torigin := &http.Server{\n\t\tHandler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\t\tw.Write([]byte(\"ab\"))\n\t\t}),\n\t\tReadTimeout: 10 * time.Second,\n\t\tWriteTimeout: 10 * time.Second,\n\t}\n\tgo origin.Serve(listener)\n\tdefer listener.Close()\n\t_, port, _ := net.SplitHostPort(listener.Addr().String())\n\toriginAddr := \"[::1]:\" + port\n\n\tvar respCounter, reqCounter int64\n\tconf := RAWInputConfig{\n\t\tEngine: capture.EnginePcap,\n\t\tProtocol: tcp.ProtocolHTTP,\n\t\tTrackResponse: true,\n\t}\n\tinput := NewRAWInput(originAddr, conf)\n\n\toutput := NewTestOutput(func(msg *Message) {\n\t\tif msg.Meta[0] == '1' {\n\t\t\tatomic.AddInt64(&reqCounter, 1)\n\t\t} else {\n\t\t\tatomic.AddInt64(&respCounter, 1)\n\t\t}\n\t\twg.Done()\n\t})\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output},\n\t}\n\n\temitter := NewEmitter()\n\taddr := \"http://\" + originAddr\n\tgo emitter.Start(plugins, Settings.Middleware)\n\tfor i := 0; i < 10; i++ {\n\t\t// request + response\n\t\twg.Add(2)\n\t\t_, err = http.Get(addr)\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t\treturn\n\t\t}\n\t}\n\n\twg.Wait()\n\tconst want = 10\n\tif reqCounter != respCounter && reqCounter != want {\n\t\tt.Errorf(\"want %d requests and %d responses, got %d requests and %d responses\", want, want, reqCounter, respCounter)\n\t}\n\temitter.Close()\n}\n\nfunc TestInputRAWChunkedEncoding(t *testing.T) {\n\twg := new(sync.WaitGroup)\n\n\tfileContent, _ := ioutil.ReadFile(\"README.md\")\n\n\t// Origing and Replay server initialization\n\torigin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tdefer r.Body.Close()\n\t\tioutil.ReadAll(r.Body)\n\n\t\twg.Done()\n\t}))\n\n\toriginAddr := strings.Replace(origin.Listener.Addr().String(), \"[::]\", \"127.0.0.1\", -1)\n\tconf := RAWInputConfig{\n\t\tEngine: capture.EnginePcap,\n\t\tExpire: time.Second,\n\t\tProtocol: tcp.ProtocolHTTP,\n\t\tTrackResponse: true,\n\t\tAllowIncomplete: true,\n\t}\n\tinput := NewRAWInput(originAddr, conf)\n\n\treplay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\tdefer r.Body.Close()\n\t\tbody, _ := ioutil.ReadAll(r.Body)\n\n\t\tif !bytes.Equal(body, fileContent) {\n\t\t\tbuf, _ := httputil.DumpRequest(r, true)\n\t\t\tt.Error(\"Wrong POST body:\", string(buf))\n\t\t}\n\n\t\twg.Done()\n\t}))\n\tdefer replay.Close()\n\n\thttpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{})\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{httpOutput},\n\t}\n\tplugins.All = append(plugins.All, input, httpOutput)\n\n\temitter := NewEmitter()\n\tdefer emitter.Close()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\twg.Add(2)\n\n\tcurl := exec.Command(\"curl\", \"http://\"+originAddr, \"--header\", \"Transfer-Encoding: chunked\", \"--header\", \"Expect:\", \"--data-binary\", \"@README.md\")\n\terr := curl.Run()\n\tif err != nil {\n\t\tt.Error(err)\n\t\treturn\n\t}\n\n\twg.Wait()\n}\n\nfunc BenchmarkRAWInputWithReplay(b *testing.B) {\n\tvar respCounter, reqCounter, replayCounter uint32\n\twg := &sync.WaitGroup{}\n\n\tlistener, err := net.Listen(\"tcp4\", \"127.0.0.1:0\")\n\tif err != nil {\n\t\tb.Error(err)\n\t\treturn\n\t}\n\tlistener0, err := net.Listen(\"tcp4\", \"127.0.0.1:0\")\n\tif err != nil {\n\t\tb.Error(err)\n\t\treturn\n\t}\n\n\torigin := http.Server{\n\t\tHandler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\t\tw.Write([]byte(\"ab\"))\n\t\t}),\n\t}\n\tgo origin.Serve(listener)\n\tdefer origin.Close()\n\toriginAddr := listener.Addr().String()\n\n\treplay := http.Server{\n\t\tHandler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\t\tatomic.AddUint32(&replayCounter, 1)\n\t\t\tw.Write(nil)\n\t\t\twg.Done()\n\t\t}),\n\t}\n\tgo replay.Serve(listener0)\n\tdefer replay.Close()\n\treplayAddr := listener0.Addr().String()\n\n\tconf := RAWInputConfig{\n\t\tEngine: capture.EnginePcap,\n\t\tExpire: testRawExpire,\n\t\tProtocol: tcp.ProtocolHTTP,\n\t\tTrackResponse: true,\n\t}\n\tinput := NewRAWInput(originAddr, conf)\n\n\ttestOutput := NewTestOutput(func(msg *Message) {\n\t\tif msg.Meta[0] == '1' {\n\t\t\treqCounter++\n\t\t} else {\n\t\t\trespCounter++\n\t\t}\n\t\twg.Done()\n\t})\n\thttpOutput := NewHTTPOutput(\"http://\"+replayAddr, &HTTPOutputConfig{})\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{testOutput, httpOutput},\n\t}\n\n\temitter := NewEmitter()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\taddr := \"http://\" + originAddr\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\twg.Add(3) // reqCounter + replayCounter + respCounter\n\t\tresp, err := http.Get(addr)\n\t\tif err != nil {\n\t\t\twg.Add(-3)\n\t\t}\n\t\tresp.Body.Close()\n\t}\n\n\twg.Wait()\n\tb.ReportMetric(float64(reqCounter), \"requests\")\n\tb.ReportMetric(float64(respCounter), \"responses\")\n\tb.ReportMetric(float64(replayCounter), \"replayed\")\n\temitter.Close()\n}\n" }, { "path": "input_tcp.go", "content": "package goreplay\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"io\"\n\t\"log\"\n\t\"net\"\n)\n\n// TCPInput used for internal communication\ntype TCPInput struct {\n\tdata chan *Message\n\tlistener net.Listener\n\taddress string\n\tconfig *TCPInputConfig\n\tstop chan bool // Channel used only to indicate goroutine should shutdown\n}\n\n// TCPInputConfig represents configuration of a TCP input plugin\ntype TCPInputConfig struct {\n\tSecure bool `json:\"input-tcp-secure\"`\n\tCertificatePath string `json:\"input-tcp-certificate\"`\n\tKeyPath string `json:\"input-tcp-certificate-key\"`\n}\n\n// NewTCPInput constructor for TCPInput, accepts address with port\nfunc NewTCPInput(address string, config *TCPInputConfig) (i *TCPInput) {\n\ti = new(TCPInput)\n\ti.data = make(chan *Message, 1000)\n\ti.address = address\n\ti.config = config\n\ti.stop = make(chan bool)\n\n\ti.listen(address)\n\n\treturn\n}\n\n// PluginRead returns data and details read from plugin\nfunc (i *TCPInput) PluginRead() (msg *Message, err error) {\n\tselect {\n\tcase <-i.stop:\n\t\treturn nil, ErrorStopped\n\tcase msg = <-i.data:\n\t\treturn msg, nil\n\t}\n\n}\n\n// Close closes the plugin\nfunc (i *TCPInput) Close() error {\n\tclose(i.stop)\n\ti.listener.Close()\n\treturn nil\n}\n\nfunc (i *TCPInput) listen(address string) {\n\tif i.config.Secure {\n\t\tcer, err := tls.LoadX509KeyPair(i.config.CertificatePath, i.config.KeyPath)\n\t\tif err != nil {\n\t\t\tlog.Fatalln(\"error while loading --input-tcp TLS certificate:\", err)\n\t\t}\n\n\t\tconfig := &tls.Config{Certificates: []tls.Certificate{cer}}\n\t\tlistener, err := tls.Listen(\"tcp\", address, config)\n\t\tif err != nil {\n\t\t\tlog.Fatalln(\"[INPUT-TCP] failed to start INPUT-TCP listener:\", err)\n\t\t}\n\t\ti.listener = listener\n\t} else {\n\t\tlistener, err := net.Listen(\"tcp\", address)\n\t\tif err != nil {\n\t\t\tlog.Fatalln(\"failed to start INPUT-TCP listener:\", err)\n\t\t}\n\t\ti.listener = listener\n\t}\n\tgo func() {\n\t\tfor {\n\t\t\tconn, err := i.listener.Accept()\n\t\t\tif err == nil {\n\t\t\t\tgo i.handleConnection(conn)\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif isTemporaryNetworkError(err) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif operr, ok := err.(*net.OpError); ok && operr.Err.Error() != \"use of closed network connection\" {\n\t\t\t\tDebug(0, fmt.Sprintf(\"[INPUT-TCP] listener closed, err: %q\", err))\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\t}()\n}\n\nvar payloadSeparatorAsBytes = []byte(payloadSeparator)\n\nfunc (i *TCPInput) handleConnection(conn net.Conn) {\n\tdefer conn.Close()\n\n\treader := bufio.NewReader(conn)\n\tvar buffer bytes.Buffer\n\n\tfor {\n\t\tline, err := reader.ReadBytes('\\n')\n\t\tif err != nil {\n\t\t\tif isTemporaryNetworkError(err) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif err != io.EOF {\n\t\t\t\tDebug(0, fmt.Sprintf(\"[INPUT-TCP] connection error: %q\", err))\n\t\t\t}\n\t\t\tbreak\n\t\t}\n\n\t\tif bytes.Equal(payloadSeparatorAsBytes[1:], line) {\n\t\t\t// unread the '\\n' before monkeys\n\t\t\tbuffer.UnreadByte()\n\t\t\tvar msg Message\n\t\t\tmsg.Meta, msg.Data = payloadMetaWithBody(buffer.Bytes())\n\t\t\ti.data <- &msg\n\t\t\tbuffer.Reset()\n\t\t} else {\n\t\t\tbuffer.Write(line)\n\t\t}\n\t}\n}\n\nfunc (i *TCPInput) String() string {\n\treturn \"TCP input: \" + i.address\n}\n\nfunc isTemporaryNetworkError(err error) bool {\n\tif nerr, ok := err.(net.Error); ok && nerr.Temporary() {\n\t\treturn true\n\t}\n\tif operr, ok := err.(*net.OpError); ok && operr.Temporary() {\n\t\treturn true\n\t}\n\treturn false\n}\n" }, { "path": "input_tcp_test.go", "content": "package goreplay\n\nimport (\n\t\"bytes\"\n\t\"crypto/rand\"\n\t\"crypto/rsa\"\n\t\"crypto/tls\"\n\t\"crypto/x509\"\n\t\"encoding/pem\"\n\t\"io/ioutil\"\n\t\"log\"\n\t\"math/big\"\n\t\"net\"\n\t\"os\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestTCPInput(t *testing.T) {\n\twg := new(sync.WaitGroup)\n\n\tinput := NewTCPInput(\"127.0.0.1:0\", &TCPInputConfig{})\n\toutput := NewTestOutput(func(*Message) {\n\t\twg.Done()\n\t})\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output},\n\t}\n\tplugins.All = append(plugins.All, input, output)\n\n\temitter := NewEmitter()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\ttcpAddr, err := net.ResolveTCPAddr(\"tcp\", input.listener.Addr().String())\n\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tconn, err := net.DialTCP(\"tcp\", nil, tcpAddr)\n\tif err != nil {\n\t\tlog.Fatal(err)\n\t}\n\n\tmsg := []byte(\"1 1 1\\nGET / HTTP/1.1\\r\\n\\r\\n\")\n\n\tfor i := 0; i < 100; i++ {\n\t\twg.Add(1)\n\t\tif _, err = conn.Write(msg); err == nil {\n\t\t\t_, err = conn.Write(payloadSeparatorAsBytes)\n\t\t}\n\t\tif err != nil {\n\t\t\tt.Error(err)\n\t\t\treturn\n\t\t}\n\t}\n\twg.Wait()\n\temitter.Close()\n}\n\nfunc genCertificate(template *x509.Certificate) ([]byte, []byte) {\n\tpriv, _ := rsa.GenerateKey(rand.Reader, 2048)\n\n\tserialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)\n\tserialNumber, _ := rand.Int(rand.Reader, serialNumberLimit)\n\ttemplate.SerialNumber = serialNumber\n\ttemplate.BasicConstraintsValid = true\n\ttemplate.NotBefore = time.Now()\n\ttemplate.NotAfter = time.Now().Add(time.Hour)\n\n\tderBytes, _ := x509.CreateCertificate(rand.Reader, template, template, &priv.PublicKey, priv)\n\n\tvar certPem, keyPem bytes.Buffer\n\tpem.Encode(&certPem, &pem.Block{Type: \"CERTIFICATE\", Bytes: derBytes})\n\tpem.Encode(&keyPem, &pem.Block{Type: \"RSA PRIVATE KEY\", Bytes: x509.MarshalPKCS1PrivateKey(priv)})\n\n\treturn certPem.Bytes(), keyPem.Bytes()\n}\n\nfunc TestTCPInputSecure(t *testing.T) {\n\tserverCertPem, serverPrivPem := genCertificate(&x509.Certificate{\n\t\tDNSNames: []string{\"localhost\"},\n\t\tIPAddresses: []net.IP{net.ParseIP(\"127.0.0.1\"), net.ParseIP(\"::\")},\n\t})\n\n\tserverCertPemFile, _ := ioutil.TempFile(\"\", \"server.crt\")\n\tserverCertPemFile.Write(serverCertPem)\n\tserverCertPemFile.Close()\n\n\tserverPrivPemFile, _ := ioutil.TempFile(\"\", \"server.key\")\n\tserverPrivPemFile.Write(serverPrivPem)\n\tserverPrivPemFile.Close()\n\n\tdefer func() {\n\t\tos.Remove(serverPrivPemFile.Name())\n\t\tos.Remove(serverCertPemFile.Name())\n\t}()\n\n\twg := new(sync.WaitGroup)\n\n\tinput := NewTCPInput(\"127.0.0.1:0\", &TCPInputConfig{\n\t\tSecure: true,\n\t\tCertificatePath: serverCertPemFile.Name(),\n\t\tKeyPath: serverPrivPemFile.Name(),\n\t})\n\toutput := NewTestOutput(func(*Message) {\n\t\twg.Done()\n\t})\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output},\n\t}\n\tplugins.All = append(plugins.All, input, output)\n\n\temitter := NewEmitter()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\tconf := &tls.Config{\n\t\tInsecureSkipVerify: true,\n\t}\n\n\tconn, err := tls.Dial(\"tcp\", input.listener.Addr().String(), conf)\n\tif err != nil {\n\t\tt.Fatal(err)\n\t}\n\tdefer conn.Close()\n\n\tmsg := []byte(\"1 1 1\\nGET / HTTP/1.1\\r\\n\\r\\n\")\n\n\tfor i := 0; i < 100; i++ {\n\t\twg.Add(1)\n\t\tconn.Write(msg)\n\t\tconn.Write([]byte(payloadSeparator))\n\t}\n\n\twg.Wait()\n\temitter.Close()\n}\n" }, { "path": "internal/byteutils/byteutils.go", "content": "// Package byteutils provides helpers for working with byte slices\npackage byteutils\n\nimport (\n\t\"unsafe\"\n)\n\n// Cut elements from slice for a given range\nfunc Cut(a []byte, from, to int) []byte {\n\tcopy(a[from:], a[to:])\n\ta = a[:len(a)-to+from]\n\n\treturn a\n}\n\n// Insert new slice at specified position\nfunc Insert(a []byte, i int, b []byte) []byte {\n\ta = append(a, make([]byte, len(b))...)\n\tcopy(a[i+len(b):], a[i:])\n\tcopy(a[i:i+len(b)], b)\n\n\treturn a\n}\n\n// Replace function unlike bytes.Replace allows you to specify range\nfunc Replace(a []byte, from, to int, new []byte) []byte {\n\tlenDiff := len(new) - (to - from)\n\n\tif lenDiff > 0 {\n\t\t// Extend if new segment bigger\n\t\ta = append(a, make([]byte, lenDiff)...)\n\t\tcopy(a[to+lenDiff:], a[to:])\n\t\tcopy(a[from:from+len(new)], new)\n\n\t\treturn a\n\t}\n\n\tif lenDiff < 0 {\n\t\tcopy(a[from:], new)\n\t\tcopy(a[from+len(new):], a[to:])\n\t\treturn a[:len(a)+lenDiff]\n\t}\n\n\t// same size\n\tcopy(a[from:], new)\n\treturn a\n}\n\n// SliceToString preferred for large body payload (zero allocation and faster)\nfunc SliceToString(buf []byte) string {\n\treturn *(*string)(unsafe.Pointer(&buf))\n}\n" }, { "path": "internal/byteutils/byteutils_test.go", "content": "package byteutils\n\nimport (\n\t\"bytes\"\n\t\"testing\"\n)\n\nfunc TestCut(t *testing.T) {\n\tif !bytes.Equal(Cut([]byte(\"123456\"), 2, 4), []byte(\"1256\")) {\n\t\tt.Error(\"Should properly cut\")\n\t}\n}\n\nfunc TestInsert(t *testing.T) {\n\tif !bytes.Equal(Insert([]byte(\"123456\"), 2, []byte(\"abcd\")), []byte(\"12abcd3456\")) {\n\t\tt.Error(\"Should insert into middle of slice\")\n\t}\n}\n\nfunc TestReplace(t *testing.T) {\n\tif !bytes.Equal(Replace([]byte(\"123456\"), 2, 4, []byte(\"ab\")), []byte(\"12ab56\")) {\n\t\tt.Error(\"Should replace when same length\")\n\t}\n\n\tif !bytes.Equal(Replace([]byte(\"123456\"), 2, 4, []byte(\"abcd\")), []byte(\"12abcd56\")) {\n\t\tt.Error(\"Should replace when replacement length bigger\")\n\t}\n\n\tif !bytes.Equal(Replace([]byte(\"123456\"), 2, 5, []byte(\"ab\")), []byte(\"12ab6\")) {\n\t\tt.Error(\"Should replace when replacement length bigger\")\n\t}\n}\n\nfunc BenchmarkStringtoSlice(b *testing.B) {\n\tvar s string\n\tvar buf [1 << 20]byte\n\tfor i := 0; i < b.N; i++ {\n\t\ts = SliceToString(buf[:])\n\t}\n\t_ = s // avoid gc to optimize away the loop body\n}\n" }, { "path": "internal/capture/af_packet.go", "content": "//go:build !linux\n\npackage capture\n\nimport (\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/google/gopacket\"\n)\n\nfunc newAfpacketHandle(device string, snaplen int, block_size int, num_blocks int,\n\tuseVLAN bool, timeout time.Duration) (*afpacketHandle, error) {\n\treturn nil, fmt.Errorf(\"Not implemented\")\n}\n\nfunc afpacketComputeSize(targetSizeMb int, snaplen int, pageSize int) (\n\tframeSize int, blockSize int, numBlocks int, err error) {\n\treturn 0, 0, 0, fmt.Errorf(\"Not implemented\")\n}\n\ntype afpacketHandle struct{}\n\n// ReadPacketData satisfies PacketDataSource interface\nfunc (h *afpacketHandle) ReadPacketData() (data []byte, ci gopacket.CaptureInfo, err error) {\n\treturn nil, gopacket.CaptureInfo{}, fmt.Errorf(\"Not implemented\")\n}\n\n// SetBPFFilter translates a BPF filter string into BPF RawInstruction and applies them.\nfunc (h *afpacketHandle) SetBPFFilter(filter string, snaplen int) (err error) {\n\treturn fmt.Errorf(\"Not implemented\")\n}\n" }, { "path": "internal/capture/af_packet_linux.go", "content": "//go:build linux\n\npackage capture\n\nimport (\n\t\"fmt\"\n\t\"time\"\n\n\t\"github.com/google/gopacket\"\n\t\"github.com/google/gopacket/afpacket\"\n\t\"github.com/google/gopacket/layers\"\n\t\"github.com/google/gopacket/pcap\"\n\t\"golang.org/x/net/bpf\"\n\n\t_ \"github.com/google/gopacket/layers\"\n)\n\ntype afpacketHandle struct {\n\tTPacket *afpacket.TPacket\n}\n\nfunc newAfpacketHandle(device string, snaplen int, block_size int, num_blocks int,\n\tuseVLAN bool, timeout time.Duration) (*afpacketHandle, error) {\n\n\th := &afpacketHandle{}\n\tvar err error\n\n\tif device == \"any\" {\n\t\th.TPacket, err = afpacket.NewTPacket(\n\t\t\tafpacket.OptFrameSize(snaplen),\n\t\t\tafpacket.OptBlockSize(block_size),\n\t\t\tafpacket.OptNumBlocks(num_blocks),\n\t\t\tafpacket.OptAddVLANHeader(false),\n\t\t\tafpacket.OptPollTimeout(timeout),\n\t\t\tafpacket.SocketRaw,\n\t\t\tafpacket.TPacketVersion3)\n\t} else {\n\t\th.TPacket, err = afpacket.NewTPacket(\n\t\t\tafpacket.OptInterface(device),\n\t\t\tafpacket.OptFrameSize(snaplen),\n\t\t\tafpacket.OptBlockSize(block_size),\n\t\t\tafpacket.OptNumBlocks(num_blocks),\n\t\t\tafpacket.OptAddVLANHeader(false),\n\t\t\tafpacket.OptPollTimeout(timeout),\n\t\t\tafpacket.SocketRaw,\n\t\t\tafpacket.TPacketVersion3)\n\t}\n\treturn h, err\n}\n\n// ReadPacketData satisfies PacketDataSource interface\nfunc (h *afpacketHandle) ReadPacketData() (data []byte, ci gopacket.CaptureInfo, err error) {\n\treturn h.TPacket.ReadPacketData()\n}\n\n// SetBPFFilter translates a BPF filter string into BPF RawInstruction and applies them.\nfunc (h *afpacketHandle) SetBPFFilter(filter string, snaplen int) (err error) {\n\tpcapBPF, err := pcap.CompileBPFFilter(layers.LinkTypeEthernet, snaplen, filter)\n\tif err != nil {\n\t\treturn err\n\t}\n\tbpfIns := []bpf.RawInstruction{}\n\tfor _, ins := range pcapBPF {\n\t\tbpfIns2 := bpf.RawInstruction{\n\t\t\tOp: ins.Code,\n\t\t\tJt: ins.Jt,\n\t\t\tJf: ins.Jf,\n\t\t\tK: ins.K,\n\t\t}\n\t\tbpfIns = append(bpfIns, bpfIns2)\n\t}\n\tif h.TPacket.SetBPF(bpfIns); err != nil {\n\t\treturn err\n\t}\n\treturn h.TPacket.SetBPF(bpfIns)\n}\n\n// LinkType returns ethernet link type.\nfunc (h *afpacketHandle) LinkType() layers.LinkType {\n\treturn layers.LinkTypeEthernet\n}\n\n// Close will close afpacket source.\nfunc (h *afpacketHandle) Close() {\n\th.TPacket.Close()\n}\n\n// SocketStats prints received, dropped, queue-freeze packet stats.\nfunc (h *afpacketHandle) SocketStats() (as afpacket.SocketStats, asv afpacket.SocketStatsV3, err error) {\n\treturn h.TPacket.SocketStats()\n}\n\n// afpacketComputeSize computes the block_size and the num_blocks in such a way that the\n// allocated mmap buffer is close to but smaller than target_size_mb.\n// The restriction is that the block_size must be divisible by both the\n// frame size and page size.\nfunc afpacketComputeSize(targetSizeMb int, snaplen int, pageSize int) (\n\tframeSize int, blockSize int, numBlocks int, err error) {\n\n\tif snaplen < pageSize {\n\t\tframeSize = pageSize / (pageSize / snaplen)\n\t} else {\n\t\tframeSize = (snaplen/pageSize + 1) * pageSize\n\t}\n\n\t// 128 is the default from the gopacket library so just use that\n\tblockSize = frameSize * 128\n\tnumBlocks = (targetSizeMb * 1024 * 1024) / blockSize\n\n\tif numBlocks == 0 {\n\t\treturn 0, 0, 0, fmt.Errorf(\"Interface buffersize is too small\")\n\t}\n\n\treturn frameSize, blockSize, numBlocks, nil\n}\n" }, { "path": "internal/capture/capture.go", "content": "package capture\n\nimport (\n\t\"context\"\n\t\"errors\"\n\t\"expvar\"\n\t\"fmt\"\n\t\"github.com/buger/goreplay/internal/size\"\n\t\"github.com/buger/goreplay/internal/tcp\"\n\t\"github.com/buger/goreplay/proto\"\n\t\"io\"\n\t\"log\"\n\t\"net\"\n\t\"os\"\n\t\"runtime\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n\t\"time\"\n\n\t\"github.com/google/gopacket\"\n\t\"github.com/google/gopacket/layers\"\n\t\"github.com/google/gopacket/pcap\"\n\n\tmetav1 \"k8s.io/apimachinery/pkg/apis/meta/v1\"\n\t\"k8s.io/client-go/kubernetes\"\n\t\"k8s.io/client-go/rest\"\n)\n\nvar stats *expvar.Map\n\nfunc init() {\n\tstats = expvar.NewMap(\"raw\")\n\tstats.Init()\n}\n\n// PacketHandler is a function that is used to handle packets\ntype PacketHandler func(*tcp.Packet)\n\ntype PcapStatProvider interface {\n\tStats() (*pcap.Stats, error)\n}\n\ntype PcapSetFilter interface {\n\tSetBPFFilter(string) error\n}\n\n// PcapOptions options that can be set on a pcap capture handle,\n// these options take effect on inactive pcap handles\ntype PcapOptions struct {\n\tBufferTimeout time.Duration `json:\"input-raw-buffer-timeout\"`\n\tTimestampType string `json:\"input-raw-timestamp-type\"`\n\tBPFFilter string `json:\"input-raw-bpf-filter\"`\n\tBufferSize size.Size `json:\"input-raw-buffer-size\"`\n\tPromiscuous bool `json:\"input-raw-promisc\"`\n\tMonitor bool `json:\"input-raw-monitor\"`\n\tSnaplen bool `json:\"input-raw-override-snaplen\"`\n\tEngine EngineType `json:\"input-raw-engine\"`\n\tVXLANPort int `json:\"input-raw-vxlan-port\"`\n\tVXLANVNIs []int `json:\"input-raw-vxlan-vni\"`\n\tVLAN bool `json:\"input-raw-vlan\"`\n\tVLANVIDs []int `json:\"input-raw-vlan-vid\"`\n\tExpire time.Duration `json:\"input-raw-expire\"`\n\tTrackResponse bool `json:\"input-raw-track-response\"`\n\tProtocol tcp.TCPProtocol `json:\"input-raw-protocol\"`\n\tRealIPHeader string `json:\"input-raw-realip-header\"`\n\tStats bool `json:\"input-raw-stats\"`\n\tAllowIncomplete bool `json:\"input-raw-allow-incomplete\"`\n\tIgnoreInterface []string `json:\"input-raw-ignore-interface\"`\n\tTransport string\n}\n\n// Listener handle traffic capture, this is its representation.\ntype Listener struct {\n\tsync.Mutex\n\n\tconfig PcapOptions\n\n\tActivate func() error // function is used to activate the engine. it must be called before reading packets\n\tHandles map[string]packetHandle\n\tInterfaces []pcap.Interface\n\tloopIndex int\n\tReading chan bool // this channel is closed when the listener has started reading packets\n\tmessages chan *tcp.Message\n\n\tports []uint16\n\thost string // pcap file name or interface (name, hardware addr, index or ip address)\n\n\tcloseDone chan struct{}\n\tquit chan struct{}\n\tclosed bool\n}\n\ntype packetHandle struct {\n\thandler gopacket.PacketDataSource\n\tips []net.IP\n}\n\n// EngineType ...\ntype EngineType uint8\n\n// Available engines for intercepting traffic\nconst (\n\tEnginePcap EngineType = 1 << iota\n\tEnginePcapFile\n\tEngineRawSocket\n\tEngineAFPacket\n\tEngineVXLAN\n)\n\n// Set is here so that EngineType can implement flag.Var\nfunc (eng *EngineType) Set(v string) error {\n\tswitch v {\n\tcase \"\", \"libpcap\":\n\t\t*eng = EnginePcap\n\tcase \"pcap_file\":\n\t\t*eng = EnginePcapFile\n\tcase \"raw_socket\":\n\t\t*eng = EngineRawSocket\n\tcase \"af_packet\":\n\t\t*eng = EngineAFPacket\n\tcase \"vxlan\":\n\t\t*eng = EngineVXLAN\n\tdefault:\n\t\treturn fmt.Errorf(\"invalid engine %s\", v)\n\t}\n\treturn nil\n}\n\nfunc (eng *EngineType) String() (e string) {\n\tswitch *eng {\n\tcase EnginePcapFile:\n\t\te = \"pcap_file\"\n\tcase EnginePcap:\n\t\te = \"libpcap\"\n\tcase EngineRawSocket:\n\t\te = \"raw_socket\"\n\tcase EngineAFPacket:\n\t\te = \"af_packet\"\n\tcase EngineVXLAN:\n\t\te = \"vxlan\"\n\tdefault:\n\t\te = \"\"\n\t}\n\treturn e\n}\n\n// NewListener creates and initialize a new Listener. if transport or/and engine are invalid/unsupported\n// is \"tcp\" and \"pcap\", are assumed. l.Engine and l.Transport can help to get the values used.\n// if there is an error it will be associated with getting network interfaces\nfunc NewListener(host string, ports []uint16, config PcapOptions) (l *Listener, err error) {\n\tl = &Listener{}\n\n\tl.host = host\n\tif l.host == \"localhost\" {\n\t\tl.host = \"127.0.0.1\"\n\t}\n\tl.ports = ports\n\n\tl.config = config\n\tl.config.Transport = \"tcp\"\n\tl.Handles = make(map[string]packetHandle)\n\n\tl.closeDone = make(chan struct{})\n\tl.quit = make(chan struct{})\n\tl.Reading = make(chan bool)\n\tl.messages = make(chan *tcp.Message, 10000)\n\n\tif strings.HasPrefix(l.host, \"k8s://\") {\n\t\tl.config.BPFFilter = l.Filter(pcap.Interface{}, k8sIPs(l.host[6:])...)\n\t}\n\n\tswitch config.Engine {\n\tdefault:\n\t\tl.Activate = l.activatePcap\n\tcase EngineRawSocket:\n\t\tl.Activate = l.activateRawSocket\n\tcase EngineAFPacket:\n\t\tl.Activate = l.activateAFPacket\n\tcase EnginePcapFile:\n\t\tl.Activate = l.activatePcapFile\n\t\treturn\n\tcase EngineVXLAN:\n\t\tl.Activate = l.activateVxLanSocket\n\t\treturn\n\t}\n\n\terr = l.setInterfaces()\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\treturn\n}\n\n// Listen listens for packets from the handles, and call handler on every packet received\n// until the context done signal is sent or there is unrecoverable error on all handles.\n// this function must be called after activating pcap handles\nfunc (l *Listener) Listen(ctx context.Context) (err error) {\n\tl.Lock()\n\tfor key, handle := range l.Handles {\n\t\tgo l.readHandle(key, handle)\n\t}\n\tl.Unlock()\n\n\tgo func() {\n\t\tfor {\n\t\t\ttime.Sleep(time.Second)\n\n\t\t\tif l.closed {\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\t// Check for Pod IP changes\n\t\t\tif strings.HasPrefix(l.host, \"k8s://\") {\n\t\t\t\tnewFilter := l.Filter(pcap.Interface{}, k8sIPs(l.host[6:])...)\n\t\t\t\tif newFilter != l.config.BPFFilter {\n\t\t\t\t\tfmt.Println(\"k8s pods configuration changed, new filter: \", newFilter)\n\t\t\t\t\tfor _, h := range l.Handles {\n\t\t\t\t\t\tif _, ok := h.handler.(PcapSetFilter); ok {\n\t\t\t\t\t\t\th.handler.(PcapSetFilter).SetBPFFilter(newFilter)\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\n\t\t\t\t\tl.config.BPFFilter = newFilter\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tvar prevInterfaces []string\n\t\t\tfor _, in := range l.Interfaces {\n\t\t\t\tprevInterfaces = append(prevInterfaces, in.Name)\n\t\t\t}\n\t\t\tl.setInterfaces()\n\n\t\t\tfor _, in := range l.Interfaces {\n\t\t\t\tvar found bool\n\n\t\t\t\tfor _, prev := range prevInterfaces {\n\t\t\t\t\tif in.Name == prev {\n\t\t\t\t\t\tfound = true\n\t\t\t\t\t}\n\t\t\t\t}\n\n\t\t\t\tif !found {\n\t\t\t\t\tfmt.Println(\"Found new interface:\", in.Name)\n\t\t\t\t\tl.Lock()\n\t\t\t\t\tl.Activate()\n\n\t\t\t\t\tfor key, handle := range l.Handles {\n\t\t\t\t\t\tif key == in.Name {\n\t\t\t\t\t\t\tfmt.Println(\"Activating capture on:\", in.Name)\n\t\t\t\t\t\t\tgo l.readHandle(key, handle)\n\t\t\t\t\t\t\tbreak\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\tl.Unlock()\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}()\n\n\tclose(l.Reading)\n\tdone := ctx.Done()\n\tselect {\n\tcase <-done:\n\t\tclose(l.quit) // signal close on all handles\n\t\t<-l.closeDone // wait all handles to be closed\n\t\terr = ctx.Err()\n\tcase <-l.closeDone: // all handles closed voluntarily\n\t}\n\n\tl.closed = true\n\treturn\n}\n\n// ListenBackground is like listen but can run concurrently and signal error through channel\nfunc (l *Listener) ListenBackground(ctx context.Context) chan error {\n\terr := make(chan error, 1)\n\tgo func() {\n\t\tdefer close(err)\n\t\tif e := l.Listen(ctx); err != nil {\n\t\t\terr <- e\n\t\t}\n\t}()\n\treturn err\n}\n\n// Allowed format:\n//\n//\t[namespace/]pod/[pod_name]\n//\t[namespace/]deployment/[deployment_name]\n//\t[namespace/]daemonset/[daemonset_name]\n//\t[namespace/]labelSelector/[selector]\n//\t[namespace/]fieldSelector/[selector]\nfunc k8sIPs(addr string) []string {\n\tconfig, err := rest.InClusterConfig()\n\tif err != nil {\n\t\tpanic(err.Error())\n\t}\n\n\t// creates the clientset\n\tclientset, err := kubernetes.NewForConfig(config)\n\tif err != nil {\n\t\tpanic(err.Error())\n\t}\n\n\tsections := strings.Split(addr, \"/\")\n\n\tif len(sections) < 2 {\n\t\tpanic(\"Not supported k8s scheme. Allowed values: [namespace/]pod/[pod_name], [namespace/]deployment/[deployment_name], [namespace/]daemonset/[daemonset_name], [namespace/]label/[label-name]/[label-value]\")\n\t}\n\n\t// If no namespace passed, assume it is ALL\n\tswitch sections[0] {\n\tcase \"pod\", \"deployment\", \"daemonset\", \"labelSelector\", \"fieldSelector\":\n\t\tsections = append([]string{\"\"}, sections...)\n\t}\n\n\tnamespace, selectorType, selectorValue := sections[0], sections[1], sections[2]\n\n\tlabelSelector := \"\"\n\tfieldSelector := \"\"\n\n\tswitch selectorType {\n\tcase \"pod\":\n\t\tfieldSelector = \"metadata.name=\" + selectorValue\n\tcase \"deployment\":\n\t\tlabelSelector = \"app=\" + selectorValue\n\tcase \"daemonset\":\n\t\tlabelSelector = \"pod-template-generation=1,name=\" + selectorValue\n\tcase \"labelSelector\":\n\t\tlabelSelector = selectorValue\n\tcase \"fieldSelector\":\n\t\tfieldSelector = selectorValue\n\t}\n\n\tpods, err := clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: labelSelector, FieldSelector: fieldSelector})\n\tif err != nil {\n\t\tpanic(err.Error())\n\t}\n\n\tvar podIPs []string\n\tfor _, pod := range pods.Items {\n\t\tfor _, podIP := range pod.Status.PodIPs {\n\t\t\tpodIPs = append(podIPs, podIP.IP)\n\t\t}\n\t}\n\treturn podIPs\n}\n\n// Filter returns automatic filter applied by goreplay\n// to a pcap handle of a specific interface\nfunc (l *Listener) Filter(ifi pcap.Interface, hosts ...string) (filter string) {\n\t// https://www.tcpdump.org/manpages/pcap-filter.7.html\n\n\tif len(hosts) == 0 {\n\t\t// If k8s have not found any IPs\n\t\tif strings.HasPrefix(l.host, \"k8s://\") {\n\t\t\thosts = []string{}\n\t\t} else {\n\t\t\thosts = []string{l.host}\n\n\t\t\tif listenAll(l.host) || isDevice(l.host, ifi) {\n\t\t\t\thosts = interfaceAddresses(ifi)\n\t\t\t}\n\t\t}\n\t}\n\n\tfilter = portsFilter(l.config.Transport, \"dst\", l.ports)\n\n\tif len(hosts) != 0 && !l.config.Promiscuous {\n\t\tfilter = fmt.Sprintf(\"((%s) and (%s))\", filter, hostsFilter(\"dst\", hosts))\n\t} else {\n\t\tfilter = fmt.Sprintf(\"(%s)\", filter)\n\t}\n\n\tif l.config.TrackResponse {\n\t\tresponseFilter := portsFilter(l.config.Transport, \"src\", l.ports)\n\n\t\tif len(hosts) != 0 && !l.config.Promiscuous {\n\t\t\tresponseFilter = fmt.Sprintf(\"((%s) and (%s))\", responseFilter, hostsFilter(\"src\", hosts))\n\t\t} else {\n\t\t\tresponseFilter = fmt.Sprintf(\"(%s)\", responseFilter)\n\t\t}\n\n\t\tfilter = fmt.Sprintf(\"%s or %s\", filter, responseFilter)\n\t}\n\n\tif l.config.VLAN {\n\t\tif len(l.config.VLANVIDs) > 0 {\n\t\t\tfor _, vi := range l.config.VLANVIDs {\n\t\t\t\tfilter = fmt.Sprintf(\"vlan %d and \", vi) + filter\n\t\t\t}\n\t\t} else {\n\t\t\tfilter = \"vlan and \" + filter\n\t\t}\n\t}\n\n\treturn\n}\n\n// PcapHandle returns new pcap Handle from dev on success.\n// this function should be called after setting all necessary options for this listener\nfunc (l *Listener) PcapHandle(ifi pcap.Interface) (handle *pcap.Handle, err error) {\n\tvar inactive *pcap.InactiveHandle\n\tinactive, err = pcap.NewInactiveHandle(ifi.Name)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"inactive handle error: %q, interface: %q\", err, ifi.Name)\n\t}\n\tdefer inactive.CleanUp()\n\n\tif l.config.TimestampType != \"\" && l.config.TimestampType != \"go\" {\n\t\tvar ts pcap.TimestampSource\n\t\tts, err = pcap.TimestampSourceFromString(l.config.TimestampType)\n\t\tfmt.Println(\"Setting custom Timestamp Source. Supported values: `go`, \", inactive.SupportedTimestamps())\n\t\terr = inactive.SetTimestampSource(ts)\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"%q: supported timestamps: %q, interface: %q\", err, inactive.SupportedTimestamps(), ifi.Name)\n\t\t}\n\t}\n\tif l.config.Promiscuous {\n\t\tif err = inactive.SetPromisc(l.config.Promiscuous); err != nil {\n\t\t\treturn nil, fmt.Errorf(\"promiscuous mode error: %q, interface: %q\", err, ifi.Name)\n\t\t}\n\t}\n\tif l.config.Monitor {\n\t\tif err = inactive.SetRFMon(l.config.Monitor); err != nil && !errors.Is(err, pcap.CannotSetRFMon) {\n\t\t\treturn nil, fmt.Errorf(\"monitor mode error: %q, interface: %q\", err, ifi.Name)\n\t\t}\n\t}\n\n\tvar snap int\n\n\tif !l.config.Snaplen {\n\t\tinfs, _ := net.Interfaces()\n\t\tfor _, i := range infs {\n\t\t\tif i.Name == ifi.Name {\n\t\t\t\tsnap = i.MTU + 200\n\t\t\t}\n\t\t}\n\t}\n\n\tif snap == 0 {\n\t\tsnap = 64<<10 + 200\n\t}\n\n\terr = inactive.SetSnapLen(snap)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"snapshot length error: %q, interface: %q\", err, ifi.Name)\n\t}\n\tif l.config.BufferSize > 0 {\n\t\terr = inactive.SetBufferSize(int(l.config.BufferSize))\n\t\tif err != nil {\n\t\t\treturn nil, fmt.Errorf(\"handle buffer size error: %q, interface: %q\", err, ifi.Name)\n\t\t}\n\t}\n\tif l.config.BufferTimeout == 0 {\n\t\tl.config.BufferTimeout = 2000 * time.Millisecond\n\t}\n\terr = inactive.SetTimeout(l.config.BufferTimeout)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"handle buffer timeout error: %q, interface: %q\", err, ifi.Name)\n\t}\n\thandle, err = inactive.Activate()\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"PCAP Activate device error: %q, interface: %q\", err, ifi.Name)\n\t}\n\n\tbpfFilter := l.config.BPFFilter\n\tif bpfFilter == \"\" {\n\t\tbpfFilter = l.Filter(ifi)\n\t}\n\tfmt.Println(\"Interface:\", ifi.Name, \". BPF Filter:\", bpfFilter)\n\terr = handle.SetBPFFilter(bpfFilter)\n\tif err != nil {\n\t\thandle.Close()\n\t\treturn nil, fmt.Errorf(\"BPF filter error: %q%s, interface: %q\", err, bpfFilter, ifi.Name)\n\t}\n\treturn\n}\n\n// SocketHandle returns new unix ethernet handle associated with this listener settings\nfunc (l *Listener) SocketHandle(ifi pcap.Interface) (handle Socket, err error) {\n\thandle, err = NewSocket(ifi)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(\"sock raw error: %q, interface: %q\", err, ifi.Name)\n\t}\n\tif err = handle.SetPromiscuous(l.config.Promiscuous || l.config.Monitor); err != nil {\n\t\treturn nil, fmt.Errorf(\"promiscuous mode error: %q, interface: %q\", err, ifi.Name)\n\t}\n\tif l.config.BPFFilter == \"\" {\n\t\tl.config.BPFFilter = l.Filter(ifi)\n\t}\n\tfmt.Println(\"BPF Filter: \", l.config.BPFFilter)\n\tif err = handle.SetBPFFilter(l.config.BPFFilter); err != nil {\n\t\thandle.Close()\n\t\treturn nil, fmt.Errorf(\"BPF filter error: %q%s, interface: %q\", err, l.config.BPFFilter, ifi.Name)\n\t}\n\thandle.SetLoopbackIndex(int32(l.loopIndex))\n\treturn\n}\n\nfunc http1StartHint(pckt *tcp.Packet) (isRequest, isResponse bool) {\n\tif proto.HasRequestTitle(pckt.Payload) {\n\t\treturn true, false\n\t}\n\n\tif proto.HasResponseTitle(pckt.Payload) {\n\t\treturn false, true\n\t}\n\n\t// No request or response detected\n\treturn false, false\n}\n\nfunc http1EndHint(m *tcp.Message) bool {\n\tif m.MissingChunk() {\n\t\treturn false\n\t}\n\n\treq, res := http1StartHint(m.Packets()[0])\n\treturn proto.HasFullPayload(m, m.PacketData()...) && (req || res)\n}\n\nfunc (l *Listener) readHandle(key string, hndl packetHandle) {\n\truntime.LockOSThread()\n\n\tdefer l.closeHandles(key)\n\tlinkSize := 14\n\tlinkType := int(layers.LinkTypeEthernet)\n\tif _, ok := hndl.handler.(*pcap.Handle); ok {\n\t\tlinkType = int(hndl.handler.(*pcap.Handle).LinkType())\n\t\tlinkSize, ok = pcapLinkTypeLength(linkType, l.config.VLAN)\n\t\tif !ok {\n\t\t\tif os.Getenv(\"GORDEBUG\") != \"0\" {\n\t\t\t\tlog.Printf(\"can not identify link type of an interface '%s'\\n\", key)\n\t\t\t}\n\t\t\treturn // can't find the linktype size\n\t\t}\n\t}\n\n\tmessageParser := tcp.NewMessageParser(l.messages, l.ports, hndl.ips, l.config.Expire, l.config.AllowIncomplete)\n\n\tif l.config.Protocol == tcp.ProtocolHTTP {\n\t\tmessageParser.Start = http1StartHint\n\t\tmessageParser.End = http1EndHint\n\t}\n\n\ttimer := time.NewTicker(1 * time.Second)\n\n\tfor {\n\t\tselect {\n\t\tcase <-l.quit:\n\t\t\treturn\n\t\tcase <-timer.C:\n\t\t\tif h, ok := hndl.handler.(PcapStatProvider); ok {\n\t\t\t\ts, err := h.Stats()\n\t\t\t\tif err == nil {\n\t\t\t\t\tstats.Add(\"packets_received\", int64(s.PacketsReceived))\n\t\t\t\t\tstats.Add(\"packets_dropped\", int64(s.PacketsDropped))\n\t\t\t\t\tstats.Add(\"packets_if_dropped\", int64(s.PacketsIfDropped))\n\t\t\t\t}\n\t\t\t}\n\t\tdefault:\n\t\t\tdata, ci, err := hndl.handler.ReadPacketData()\n\t\t\tif err == nil {\n\t\t\t\tif l.config.TimestampType == \"go\" {\n\t\t\t\t\tci.Timestamp = time.Now()\n\t\t\t\t}\n\n\t\t\t\tmessageParser.PacketHandler(&tcp.PcapPacket{\n\t\t\t\t\tData: data,\n\t\t\t\t\tLType: linkType,\n\t\t\t\t\tLTypeLen: linkSize,\n\t\t\t\t\tCi: &ci,\n\t\t\t\t})\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif enext, ok := err.(pcap.NextError); ok && enext == pcap.NextErrorTimeoutExpired {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif eno, ok := err.(syscall.Errno); ok && eno.Temporary() {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif enet, ok := err.(*net.OpError); ok && (enet.Temporary() || enet.Timeout()) {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tif err == io.EOF || err == io.ErrClosedPipe {\n\t\t\t\tlog.Printf(\"stopped reading from %s interface with error %s\\n\", key, err)\n\t\t\t\treturn\n\t\t\t}\n\n\t\t\tlog.Printf(\"stopped reading from %s interface with error %s\\n\", key, err)\n\t\t\treturn\n\t\t}\n\t}\n}\n\nfunc (l *Listener) Messages() chan *tcp.Message {\n\treturn l.messages\n}\n\nfunc (l *Listener) closeHandles(key string) {\n\tl.Lock()\n\tdefer l.Unlock()\n\tif handle, ok := l.Handles[key]; ok {\n\t\tif c, ok := handle.handler.(io.Closer); ok {\n\t\t\tc.Close()\n\t\t}\n\n\t\tdelete(l.Handles, key)\n\t\tif len(l.Handles) == 0 {\n\t\t\tclose(l.closeDone)\n\t\t}\n\t}\n}\n\nfunc (l *Listener) activatePcap() error {\n\tvar e error\n\tvar msg string\n\tfor _, ifi := range l.Interfaces {\n\t\tif _, found := l.Handles[ifi.Name]; found {\n\t\t\tcontinue\n\t\t}\n\n\t\tvar handle *pcap.Handle\n\t\thandle, e = l.PcapHandle(ifi)\n\t\tif e != nil {\n\t\t\tmsg += (\"\\n\" + e.Error())\n\t\t\tcontinue\n\t\t}\n\t\tl.Handles[ifi.Name] = packetHandle{\n\t\t\thandler: handle,\n\t\t\tips: interfaceIPs(ifi),\n\t\t}\n\t}\n\tif len(l.Handles) == 0 {\n\t\treturn fmt.Errorf(\"pcap handles error:%s\", msg)\n\t}\n\treturn nil\n}\n\nfunc (l *Listener) activateVxLanSocket() error {\n\thandler, err := newVXLANHandler(l.config.VXLANPort, l.config.VXLANVNIs)\n\tif err != nil {\n\t\treturn err\n\t}\n\tl.Handles[\"vxlan\"] = packetHandle{\n\t\thandler: handler,\n\t}\n\n\treturn nil\n}\n\nfunc (l *Listener) activateRawSocket() error {\n\tif runtime.GOOS != \"linux\" {\n\t\treturn fmt.Errorf(\"sock_raw is not stabilized on OS other than linux\")\n\t}\n\tvar msg string\n\tvar e error\n\tfor _, ifi := range l.Interfaces {\n\t\tif _, found := l.Handles[ifi.Name]; found {\n\t\t\tcontinue\n\t\t}\n\n\t\tvar handle Socket\n\t\thandle, e = l.SocketHandle(ifi)\n\t\tif e != nil {\n\t\t\tmsg += (\"\\n\" + e.Error())\n\t\t\tcontinue\n\t\t}\n\t\tl.Handles[ifi.Name] = packetHandle{\n\t\t\thandler: handle,\n\t\t\tips: interfaceIPs(ifi),\n\t\t}\n\t}\n\tif len(l.Handles) == 0 {\n\t\treturn fmt.Errorf(\"raw socket handles error:%s\", msg)\n\t}\n\treturn nil\n}\n\nfunc (l *Listener) activatePcapFile() (err error) {\n\tvar handle *pcap.Handle\n\tvar e error\n\tif handle, e = pcap.OpenOffline(l.host); e != nil {\n\t\treturn fmt.Errorf(\"open pcap file error: %q\", e)\n\t}\n\n\ttmp := l.host\n\tl.host = \"\"\n\tl.config.BPFFilter = l.Filter(pcap.Interface{})\n\tl.host = tmp\n\n\tif e = handle.SetBPFFilter(l.config.BPFFilter); e != nil {\n\t\thandle.Close()\n\t\treturn fmt.Errorf(\"BPF filter error: %q, filter: %s\", e, l.config.BPFFilter)\n\t}\n\n\tfmt.Println(\"BPF Filter:\", l.config.BPFFilter)\n\n\tl.Handles[\"pcap_file\"] = packetHandle{\n\t\thandler: handle,\n\t}\n\treturn\n}\n\nfunc (l *Listener) activateAFPacket() error {\n\tszFrame, szBlock, numBlocks, err := afpacketComputeSize(32, 32<<10, os.Getpagesize())\n\tif err != nil {\n\t\treturn err\n\t}\n\n\tvar msg string\n\tfor _, ifi := range l.Interfaces {\n\t\tif _, found := l.Handles[ifi.Name]; found {\n\t\t\tcontinue\n\t\t}\n\n\t\thandle, err := newAfpacketHandle(ifi.Name, szFrame, szBlock, numBlocks, false, pcap.BlockForever)\n\n\t\tif err != nil {\n\t\t\tmsg += (\"\\n\" + err.Error())\n\t\t\tcontinue\n\t\t}\n\n\t\tif l.config.BPFFilter == \"\" {\n\t\t\tl.config.BPFFilter = l.Filter(ifi)\n\t\t}\n\t\tfmt.Println(\"Interface:\", ifi.Name, \". BPF Filter:\", l.config.BPFFilter)\n\t\thandle.SetBPFFilter(l.config.BPFFilter, 64<<10)\n\n\t\tl.Handles[ifi.Name] = packetHandle{\n\t\t\thandler: handle,\n\t\t\tips: interfaceIPs(ifi),\n\t\t}\n\t}\n\n\tif len(l.Handles) == 0 {\n\t\treturn fmt.Errorf(\"pcap handles error:%s\", msg)\n\t}\n\n\treturn nil\n}\n\nfunc (l *Listener) setInterfaces() (err error) {\n\tvar pifis []pcap.Interface\n\tpifis, err = pcap.FindAllDevs()\n\tifis, _ := net.Interfaces()\n\tl.Interfaces = []pcap.Interface{}\n\n\tif err != nil {\n\t\treturn\n\t}\n\n\tfor _, pi := range pifis {\n\t\tignore := false\n\t\tfor _, ig := range l.config.IgnoreInterface {\n\t\t\tif pi.Name == ig {\n\t\t\t\tignore = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\n\t\tif ignore {\n\t\t\tcontinue\n\t\t}\n\n\t\tif strings.HasPrefix(l.host, \"k8s://\") {\n\t\t\tif !strings.HasPrefix(pi.Name, \"veth\") {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\tif isDevice(l.host, pi) {\n\t\t\tl.Interfaces = []pcap.Interface{pi}\n\t\t\treturn\n\t\t}\n\n\t\tvar ni net.Interface\n\t\tfor _, i := range ifis {\n\t\t\tif i.Name == pi.Name {\n\t\t\t\tni = i\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t\taddrs, _ := i.Addrs()\n\t\t\tfor _, a := range addrs {\n\t\t\t\tfor _, pa := range pi.Addresses {\n\t\t\t\t\tif a.String() == pa.IP.String() {\n\t\t\t\t\t\tni = i\n\t\t\t\t\t\tbreak\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tif ni.Flags&net.FlagLoopback != 0 {\n\t\t\tl.loopIndex = ni.Index\n\t\t}\n\n\t\tif runtime.GOOS != \"windows\" {\n\t\t\tif len(pi.Addresses) == 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif ni.Flags&net.FlagUp == 0 {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t}\n\n\t\tl.Interfaces = append(l.Interfaces, pi)\n\t}\n\treturn\n}\n\nfunc isDevice(addr string, ifi pcap.Interface) bool {\n\t// Windows npcap loopback have no IPs\n\tif addr == \"127.0.0.1\" && ifi.Name == `\\Device\\NPF_Loopback` {\n\t\treturn true\n\t}\n\n\tif addr == ifi.Name {\n\t\treturn true\n\t}\n\n\tif strings.HasSuffix(addr, \"*\") {\n\t\tif strings.HasPrefix(ifi.Name, addr[:len(addr)-1]) {\n\t\t\treturn true\n\t\t}\n\t}\n\n\tfor _, _addr := range ifi.Addresses {\n\t\tif _addr.IP.String() == addr {\n\t\t\treturn true\n\t\t}\n\t}\n\n\treturn false\n}\n\nfunc interfaceAddresses(ifi pcap.Interface) []string {\n\tvar hosts []string\n\tfor _, addr := range ifi.Addresses {\n\t\thosts = append(hosts, addr.IP.String())\n\t}\n\treturn hosts\n}\n\nfunc interfaceIPs(ifi pcap.Interface) []net.IP {\n\tvar ips []net.IP\n\tfor _, addr := range ifi.Addresses {\n\t\tips = append(ips, addr.IP)\n\t}\n\treturn ips\n}\n\nfunc listenAll(addr string) bool {\n\tswitch addr {\n\tcase \"\", \"0.0.0.0\", \"[::]\", \"::\":\n\t\treturn true\n\t}\n\treturn false\n}\n\nfunc portsFilter(transport string, direction string, ports []uint16) string {\n\tif len(ports) == 0 || ports[0] == 0 {\n\t\treturn fmt.Sprintf(\"%s %s portrange 0-%d\", transport, direction, 1<<16-1)\n\t}\n\n\tvar filters []string\n\tfor _, port := range ports {\n\t\tfilters = append(filters, fmt.Sprintf(\"%s %s port %d\", transport, direction, port))\n\t}\n\treturn strings.Join(filters, \" or \")\n}\n\nfunc hostsFilter(direction string, hosts []string) string {\n\tvar hostsFilters []string\n\tfor _, host := range hosts {\n\t\thostsFilters = append(hostsFilters, fmt.Sprintf(\"%s host %s\", direction, host))\n\t}\n\n\treturn strings.Join(hostsFilters, \" or \")\n}\n\nfunc pcapLinkTypeLength(lType int, vlan bool) (int, bool) {\n\tswitch layers.LinkType(lType) {\n\tcase layers.LinkTypeEthernet:\n\t\tif vlan {\n\t\t\treturn 18, true\n\t\t} else {\n\t\t\treturn 14, true\n\t\t}\n\tcase layers.LinkTypeNull, layers.LinkTypeLoop:\n\t\treturn 4, true\n\tcase layers.LinkTypeRaw, 12, 14:\n\t\treturn 0, true\n\tcase layers.LinkTypeIPv4, layers.LinkTypeIPv6:\n\t\t// (TODO:) look out for IP encapsulation?\n\t\treturn 0, true\n\tcase layers.LinkTypeLinuxSLL:\n\t\treturn 16, true\n\tcase layers.LinkTypeFDDI:\n\t\treturn 13, true\n\tcase 226 /*DLT_IPNET*/ :\n\t\t// https://www.tcpdump.org/linktypes/LINKTYPE_IPNET.html\n\t\treturn 24, true\n\tdefault:\n\t\treturn 0, false\n\t}\n}\n" }, { "path": "internal/capture/capture_test.go", "content": "package capture\n\nimport (\n\t\"testing\"\n)\n\nfunc TestSetInterfaces(t *testing.T) {\n\tlistener := &Listener{\n\t\tloopIndex: 99999,\n\t}\n\tlistener.setInterfaces()\n\n\tfor _, nic := range listener.Interfaces {\n\t\tif (len(nic.Addresses)) == 0 {\n\t\t\tt.Errorf(\"nic %s was captured with 0 addresses\", nic.Name)\n\t\t}\n\t}\n\n\tif listener.loopIndex == 99999 {\n\t\tt.Errorf(\"loopback nic index was not found\")\n\t}\n}\n" }, { "path": "internal/capture/doc.go", "content": "/*\nPackage capture provides traffic sniffier using AF_PACKET, pcap or pcap file.\nit allows you to listen for traffic from any port (e.g. sniffing) because they operate on IP level.\nPorts is TCP/IP feature, same as flow control, reliable transmission and etc.\nCurrently this package implements TCP layer: flow control is managed under tcp package.\nBPF filters can also be applied.\n\nexample:\n\n// for the transport should be \"tcp\"\nlistener, err := capture.NewListener(host, port, transport, engine, trackResponse)\n\n\tif err != nil {\n\t\t// handle error\n\t}\n\nlistener.SetPcapOptions(opts)\nerr = listner.Activate()\n\n\tif err != nil {\n\t\t// handle it\n\t}\n\n\tif err := listener.Listen(context.Background(), handler); err != nil {\n\t\t // handle error\n\t}\n\n// or\nerrCh := listener.ListenBackground(context.Background(), handler) // runs in the background\nselect {\ncase err := <- errCh:\n\n\t// handle error\n\ncase <-quit:\n\n\t//\n\ncase <- l.Reading: // if we have started reading\n}\n*/\npackage capture // import github.com/buger/goreplay/capture\n" }, { "path": "internal/capture/dump.go", "content": "// https://github.com/google/gopacket/blob/403ca653c4/pcapgo/read.go\n\npackage capture\n\nimport (\n\t\"encoding/binary\"\n\t\"fmt\"\n\t\"io\"\n\t\"time\"\n\n\t\"github.com/google/gopacket\"\n\t\"github.com/google/gopacket/layers\"\n)\n\n// Writer wraps an underlying io.Writer to write packet data in PCAP\n// format. See http://wiki.wireshark.org/Development/LibpcapFileFormat\n// for information on the file format.\n//\n// For those that care, we currently write v2.4 files with nanosecond\n// or microsecond timestamp resolution and little-endian encoding.\ntype Writer struct {\n\tw io.Writer\n\ttsScaler int\n\t// Moving this into the struct seems to save an allocation for each call to writePacketHeader\n\tbuf [16]byte\n}\n\nconst magicNanoseconds = 0xA1B23C4D\nconst magicMicroseconds = 0xA1B2C3D4\nconst versionMajor = 2\nconst versionMinor = 4\n\n// NewWriterNanos returns a new writer object, for writing packet data out\n// to the given writer. If this is a new empty writer (as opposed to\n// an append), you must call WriteFileHeader before WritePacket. Packet\n// timestamps are written with nanosecond precision.\n//\n//\t// Write a new file:\n//\tf, _ := os.Create(\"/tmp/file.pcap\")\n//\tw := pcapgo.NewWriterNanos(f)\n//\tw.WriteFileHeader(65536, layers.LinkTypeEthernet) // new file, must do this.\n//\tw.WritePacket(gopacket.CaptureInfo{...}, data1)\n//\tf.Close()\n//\t// Append to existing file (must have same snaplen and linktype)\n//\tf2, _ := os.OpenFile(\"/tmp/fileNano.pcap\", os.O_APPEND, 0700)\n//\tw2 := pcapgo.NewWriter(f2)\n//\t// no need for file header, it's already written.\n//\tw2.WritePacket(gopacket.CaptureInfo{...}, data2)\n//\tf2.Close()\nfunc NewWriterNanos(w io.Writer) *Writer {\n\treturn &Writer{w: w, tsScaler: nanosPerNano}\n}\n\n// NewWriter returns a new writer object, for writing packet data out\n// to the given writer. If this is a new empty writer (as opposed to\n// an append), you must call WriteFileHeader before WritePacket.\n// Packet timestamps are written with microsecond precision.\n//\n//\t// Write a new file:\n//\tf, _ := os.Create(\"/tmp/file.pcap\")\n//\tw := pcapgo.NewWriter(f)\n//\tw.WriteFileHeader(65536, layers.LinkTypeEthernet) // new file, must do this.\n//\tw.WritePacket(gopacket.CaptureInfo{...}, data1)\n//\tf.Close()\n//\t// Append to existing file (must have same snaplen and linktype)\n//\tf2, _ := os.OpenFile(\"/tmp/file.pcap\", os.O_APPEND, 0700)\n//\tw2 := pcapgo.NewWriter(f2)\n//\t// no need for file header, it's already written.\n//\tw2.WritePacket(gopacket.CaptureInfo{...}, data2)\n//\tf2.Close()\nfunc NewWriter(w io.Writer) *Writer {\n\treturn &Writer{w: w, tsScaler: nanosPerMicro}\n}\n\n// WriteFileHeader writes a file header out to the writer.\n// This must be called exactly once per output.\nfunc (w *Writer) WriteFileHeader(snaplen uint32, linktype layers.LinkType) error {\n\tvar buf [24]byte\n\tif w.tsScaler == nanosPerMicro {\n\t\tbinary.LittleEndian.PutUint32(buf[0:4], magicMicroseconds)\n\t} else {\n\t\tbinary.LittleEndian.PutUint32(buf[0:4], magicNanoseconds)\n\t}\n\tbinary.LittleEndian.PutUint16(buf[4:6], versionMajor)\n\tbinary.LittleEndian.PutUint16(buf[6:8], versionMinor)\n\t// bytes 8:12 stay 0 (timezone = UTC)\n\t// bytes 12:16 stay 0 (sigfigs is always set to zero, according to\n\t// http://wiki.wireshark.org/Development/LibpcapFileFormat\n\tbinary.LittleEndian.PutUint32(buf[16:20], snaplen)\n\tbinary.LittleEndian.PutUint32(buf[20:24], uint32(linktype))\n\t_, err := w.w.Write(buf[:])\n\treturn err\n}\n\nconst nanosPerMicro = 1000\nconst nanosPerNano = 1\n\nfunc (w *Writer) writePacketHeader(ci gopacket.CaptureInfo) error {\n\tt := ci.Timestamp\n\tif t.IsZero() {\n\t\tt = time.Now()\n\t}\n\tsecs := t.Unix()\n\tusecs := t.Nanosecond() / w.tsScaler\n\tbinary.LittleEndian.PutUint32(w.buf[0:4], uint32(secs))\n\tbinary.LittleEndian.PutUint32(w.buf[4:8], uint32(usecs))\n\tbinary.LittleEndian.PutUint32(w.buf[8:12], uint32(ci.CaptureLength))\n\tbinary.LittleEndian.PutUint32(w.buf[12:16], uint32(ci.Length))\n\t_, err := w.w.Write(w.buf[:])\n\treturn err\n}\n\n// WritePacket writes the given packet data out to the file.\nfunc (w *Writer) WritePacket(ci gopacket.CaptureInfo, data []byte) error {\n\tif ci.CaptureLength != len(data) {\n\t\treturn fmt.Errorf(\"capture length %d does not match data length %d\", ci.CaptureLength, len(data))\n\t}\n\tif ci.CaptureLength > ci.Length {\n\t\treturn fmt.Errorf(\"invalid capture info %+v: capture length > length\", ci)\n\t}\n\tif err := w.writePacketHeader(ci); err != nil {\n\t\treturn fmt.Errorf(\"error writing packet header: %v\", err)\n\t}\n\t_, err := w.w.Write(data)\n\treturn err\n}\n" }, { "path": "internal/capture/sock_linux.go", "content": "//go:build linux && !arm64\n\npackage capture\n\nimport (\n\t\"fmt\"\n\t\"net\"\n\t\"sync\"\n\t\"time\"\n\t\"unsafe\"\n\n\t\"golang.org/x/sys/unix\"\n\n\t\"github.com/google/gopacket\"\n\t\"github.com/google/gopacket/layers\"\n\t\"github.com/google/gopacket/pcap\"\n)\n\nconst (\n\t// ETHALL htons(ETH_P_ALL)\n\tETHALL uint16 = unix.ETH_P_ALL<<8 | unix.ETH_P_ALL>>8\n\t// BLOCKSIZE ring buffer block_size\n\tBLOCKSIZE = 64 << 10\n\t// BLOCKNR ring buffer block_nr\n\tBLOCKNR = (2 << 20) / BLOCKSIZE // 2mb / 64kb\n\t// FRAMESIZE ring buffer frame_size\n\tFRAMESIZE = BLOCKSIZE\n\t// FRAMENR ring buffer frame_nr\n\tFRAMENR = BLOCKNR * BLOCKSIZE / FRAMESIZE\n\t// MAPHUGE2MB 2mb huge map\n\tMAPHUGE2MB = 21 << unix.MAP_HUGE_SHIFT\n)\n\nvar tpacket2hdrlen = tpAlign(int(unsafe.Sizeof(unix.Tpacket2Hdr{})))\n\n// SockRaw is a linux M'maped af_packet socket\ntype SockRaw struct {\n\tmu sync.Mutex\n\tfd int\n\tifindex int\n\tsnaplen int\n\tpollTimeout uintptr\n\tframe uint32 // current frame\n\tbuf []byte // points to the memory space of the ring buffer shared with the kernel.\n\tloopIndex int32 // this field must filled to avoid reading packet twice on a loopback device\n}\n\n// NewSocket returns new M'maped sock_raw on packet version 2.\nfunc NewSocket(pifi pcap.Interface) (*SockRaw, error) {\n\tvar ifi net.Interface\n\n\tinfs, _ := net.Interfaces()\n\tfound := false\n\tfor _, i := range infs {\n\t\tif i.Name == pifi.Name {\n\t\t\tifi = i\n\t\t\tfound = true\n\t\t\tbreak\n\t\t}\n\t}\n\n\tif !found {\n\t\treturn nil, fmt.Errorf(\"can't find matching interface\")\n\t}\n\n\t// sock create\n\tfd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(ETHALL))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tsock := &SockRaw{\n\t\tfd: fd,\n\t\tifindex: ifi.Index,\n\t\tsnaplen: FRAMESIZE,\n\t\tpollTimeout: ^uintptr(0),\n\t}\n\n\t// set packet version\n\terr = unix.SetsockoptInt(fd, unix.SOL_PACKET, unix.PACKET_VERSION, unix.TPACKET_V2)\n\tif err != nil {\n\t\tunix.Close(fd)\n\t\treturn nil, fmt.Errorf(\"setsockopt packet_version: %v\", err)\n\t}\n\n\t// bind to interface\n\taddr := unix.RawSockaddrLinklayer{\n\t\tFamily: unix.AF_PACKET,\n\t\tProtocol: ETHALL,\n\t\tIfindex: int32(ifi.Index),\n\t}\n\t_, _, e := unix.Syscall(\n\t\tunix.SYS_BIND,\n\t\tuintptr(fd),\n\t\tuintptr(unsafe.Pointer(&addr)),\n\t\tuintptr(unix.SizeofSockaddrLinklayer),\n\t)\n\tif e != 0 {\n\t\tunix.Close(fd)\n\t\treturn nil, e\n\t}\n\n\t// create shared-memory ring buffer\n\ttp := &unix.TpacketReq{\n\t\tBlock_size: BLOCKSIZE,\n\t\tBlock_nr: BLOCKNR,\n\t\tFrame_size: FRAMESIZE,\n\t\tFrame_nr: FRAMENR,\n\t}\n\terr = unix.SetsockoptTpacketReq(sock.fd, unix.SOL_PACKET, unix.PACKET_RX_RING, tp)\n\tif err != nil {\n\t\tunix.Close(fd)\n\t\treturn nil, fmt.Errorf(\"setsockopt packet_rx_ring: %v\", err)\n\t}\n\tsock.buf, err = unix.Mmap(\n\t\tsock.fd,\n\t\t0,\n\t\tBLOCKSIZE*BLOCKNR,\n\t\tunix.PROT_READ|unix.PROT_WRITE,\n\t\tunix.MAP_SHARED|MAPHUGE2MB,\n\t)\n\tif err != nil {\n\t\tunix.Close(fd)\n\t\treturn nil, fmt.Errorf(\"socket mmap error: %v\", err)\n\t}\n\treturn sock, nil\n}\n\n// ReadPacketData implements gopacket.PacketDataSource.\nfunc (sock *SockRaw) ReadPacketData() (buf []byte, ci gopacket.CaptureInfo, err error) {\n\tsock.mu.Lock()\n\tdefer sock.mu.Unlock()\n\tvar tpHdr *unix.Tpacket2Hdr\n\tpoll := &unix.PollFd{\n\t\tFd: int32(sock.fd),\n\t\tEvents: unix.POLLIN,\n\t}\n\tvar i int\nread:\n\ti = int(sock.frame * FRAMESIZE)\n\ttpHdr = (*unix.Tpacket2Hdr)(unsafe.Pointer(&sock.buf[i]))\n\tsock.frame = (sock.frame + 1) % FRAMENR\n\n\tif tpHdr.Status&unix.TP_STATUS_USER == 0 {\n\t\t_, _, e := unix.Syscall(unix.SYS_POLL, uintptr(unsafe.Pointer(poll)), 1, sock.pollTimeout)\n\t\tif e != 0 && e != unix.EINTR {\n\t\t\treturn buf, ci, e\n\t\t}\n\t\t// it might be some other frame with data!\n\t\tif tpHdr.Status&unix.TP_STATUS_USER == 0 {\n\t\t\tgoto read\n\t\t}\n\t}\n\ttpHdr.Status = unix.TP_STATUS_KERNEL\n\tsockAddr := (*unix.RawSockaddrLinklayer)(unsafe.Pointer(&sock.buf[i+tpacket2hdrlen]))\n\n\t// parse out repeating packets on loopback\n\tif sockAddr.Ifindex == sock.loopIndex && sock.frame%2 != 0 {\n\t\tgoto read\n\t}\n\n\tci.Length = int(tpHdr.Len)\n\tci.Timestamp = time.Unix(int64(tpHdr.Sec), int64(tpHdr.Nsec))\n\tci.InterfaceIndex = int(sockAddr.Ifindex)\n\tbuf = make([]byte, tpHdr.Snaplen)\n\tci.CaptureLength = copy(buf, sock.buf[i+int(tpHdr.Mac):])\n\n\treturn\n}\n\n// Close closes the underlying socket\nfunc (sock *SockRaw) Close() (err error) {\n\tsock.mu.Lock()\n\tdefer sock.mu.Unlock()\n\tif sock.fd != -1 {\n\t\tunix.Munmap(sock.buf)\n\t\tsock.buf = nil\n\t\terr = unix.Close(sock.fd)\n\t\tsock.fd = -1\n\t}\n\treturn\n}\n\n// SetSnapLen sets the maximum capture length to the given value.\n// for this to take effects on the kernel level SetBPFilter should be called too.\nfunc (sock *SockRaw) SetSnapLen(snap int) error {\n\tsock.mu.Lock()\n\tdefer sock.mu.Unlock()\n\tif snap < 0 {\n\t\treturn fmt.Errorf(\"expected %d snap length to be at least 0\", snap)\n\t}\n\tif snap > FRAMESIZE {\n\t\tsnap = FRAMESIZE\n\t}\n\tsock.snaplen = snap\n\treturn nil\n}\n\n// SetTimeout sets poll wait timeout for the socket.\n// negative value will block forever\nfunc (sock *SockRaw) SetTimeout(t time.Duration) error {\n\tsock.mu.Lock()\n\tdefer sock.mu.Unlock()\n\tsock.pollTimeout = uintptr(t)\n\treturn nil\n}\n\n// GetSnapLen returns the maximum capture length\nfunc (sock *SockRaw) GetSnapLen() int {\n\tsock.mu.Lock()\n\tdefer sock.mu.Unlock()\n\treturn sock.snaplen\n}\n\n// SetBPFFilter compiles and sets a BPF filter for the socket handle.\nfunc (sock *SockRaw) SetBPFFilter(expr string) error {\n\tsock.mu.Lock()\n\tdefer sock.mu.Unlock()\n\tif expr == \"\" {\n\t\treturn unix.SetsockoptInt(sock.fd, unix.SOL_SOCKET, unix.SO_DETACH_FILTER, 0)\n\t}\n\tfilter, err := pcap.CompileBPFFilter(layers.LinkTypeEthernet, sock.snaplen, expr)\n\tif err != nil {\n\t\treturn err\n\t}\n\tif len(filter) > int(^uint16(0)) {\n\t\treturn fmt.Errorf(\"filters out of range 0-%d\", ^uint16(0))\n\t}\n\tif len(filter) == 0 {\n\t\treturn unix.SetsockoptInt(sock.fd, unix.SOL_SOCKET, unix.SO_DETACH_FILTER, 0)\n\t}\n\tfprog := &unix.SockFprog{\n\t\tLen: uint16(len(filter)),\n\t\tFilter: &(*(*[]unix.SockFilter)(unsafe.Pointer(&filter)))[0],\n\t}\n\treturn unix.SetsockoptSockFprog(sock.fd, unix.SOL_SOCKET, unix.SO_ATTACH_FILTER, fprog)\n}\n\n// SetPromiscuous sets promiscuous mode to the required value. for better result capture on all interfaces instead.\n// If it is enabled, traffic not destined for the interface will also be captured.\nfunc (sock *SockRaw) SetPromiscuous(b bool) error {\n\tsock.mu.Lock()\n\tdefer sock.mu.Unlock()\n\tmreq := unix.PacketMreq{\n\t\tIfindex: int32(sock.ifindex),\n\t\tType: unix.PACKET_MR_PROMISC,\n\t}\n\n\topt := unix.PACKET_ADD_MEMBERSHIP\n\tif !b {\n\t\topt = unix.PACKET_DROP_MEMBERSHIP\n\t}\n\n\treturn unix.SetsockoptPacketMreq(sock.fd, unix.SOL_PACKET, opt, &mreq)\n}\n\n// Stats returns number of packets and dropped packets. This will be the number of packets/dropped packets since the last call to stats (not the cummulative sum!).\nfunc (sock *SockRaw) Stats() (*unix.TpacketStats, error) {\n\tsock.mu.Lock()\n\tdefer sock.mu.Unlock()\n\treturn unix.GetsockoptTpacketStats(sock.fd, unix.SOL_PACKET, unix.PACKET_STATISTICS)\n}\n\n// SetLoopbackIndex necessary to avoid reading packet twice on a loopback device\nfunc (sock *SockRaw) SetLoopbackIndex(i int32) {\n\tsock.mu.Lock()\n\tdefer sock.mu.Unlock()\n\tsock.loopIndex = i\n}\n\n// WritePacketData transmits a raw packet.\nfunc (sock *SockRaw) WritePacketData(pkt []byte) error {\n\t_, err := unix.Write(sock.fd, pkt)\n\treturn err\n}\n\nfunc tpAlign(x int) int {\n\treturn int((uint(x) + unix.TPACKET_ALIGNMENT - 1) &^ (unix.TPACKET_ALIGNMENT - 1))\n}\n" }, { "path": "internal/capture/sock_others.go", "content": "//go:build !linux || arm64 || darwin\n\npackage capture\n\nimport (\n\t\"errors\"\n\n\t\"github.com/google/gopacket/pcap\"\n)\n\n// NewSocket returns new M'maped sock_raw on packet version 2.\nfunc NewSocket(_ pcap.Interface) (Socket, error) {\n\treturn nil, errors.New(\"afpacket socket is only available on linux\")\n}\n" }, { "path": "internal/capture/socket.go", "content": "package capture\n\nimport (\n\t\"time\"\n\n\t\"github.com/google/gopacket\"\n)\n\n// Socket is any interface that defines the behaviors of Socket\ntype Socket interface {\n\tReadPacketData() ([]byte, gopacket.CaptureInfo, error)\n\tWritePacketData([]byte) error\n\tSetBPFFilter(string) error\n\tSetPromiscuous(bool) error\n\tSetSnapLen(int) error\n\tGetSnapLen() int\n\tSetTimeout(time.Duration) error\n\tSetLoopbackIndex(i int32)\n\tClose() error\n}\n" }, { "path": "internal/capture/vxlan.go", "content": "package capture\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n\t\"github.com/google/gopacket\"\n\t\"github.com/google/gopacket/layers\"\n\t\"net\"\n\t\"time\"\n)\n\nconst VxLanPacketSize = 1526 //vxlan 8 B + ethernet II 1518 B\n\ntype vxlanHandle struct {\n\tconnection *net.UDPConn\n\tpacketChannel chan gopacket.Packet\n\tvnis []int\n}\n\nfunc newVXLANHandler(port int, vnis []int) (*vxlanHandle, error) {\n\tif port == 0 {\n\t\tport = 4789\n\t}\n\n\taddr := net.UDPAddr{\n\t\tPort: port,\n\t\tIP: net.ParseIP(\"0.0.0.0\"),\n\t}\n\n\tvxlanHandle := &vxlanHandle{}\n\tcon, err := net.ListenUDP(\"udp\", &addr)\n\tif err != nil {\n\t\treturn nil, fmt.Errorf(err.Error())\n\t}\n\tvxlanHandle.connection = con\n\tvxlanHandle.packetChannel = make(chan gopacket.Packet, 1000)\n\tvxlanHandle.vnis = vnis\n\tgo vxlanHandle.reader()\n\n\treturn vxlanHandle, nil\n}\n\nfunc (v *vxlanHandle) reader() {\n\tfor {\n\t\tinputBytes := make([]byte, VxLanPacketSize)\n\t\tlength, _, err := v.connection.ReadFromUDP(inputBytes)\n\t\tif err != nil {\n\t\t\tif errors.Is(err, net.ErrClosed) {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tpacket := gopacket.NewPacket(inputBytes[:length], layers.LayerTypeVXLAN, gopacket.NoCopy)\n\t\tci := packet.Metadata()\n\t\tci.Timestamp = time.Now()\n\t\tci.CaptureLength = length\n\t\tci.Length = length\n\n\t\tif len(v.vnis) > 0 && !v.vniIsAllowed(packet) {\n\t\t\tcontinue\n\t\t}\n\n\t\tv.packetChannel <- packet\n\t}\n}\n\nfunc (v *vxlanHandle) vniIsAllowed(packet gopacket.Packet) bool {\n\tdefaultState := false\n\tif layer := packet.Layer(layers.LayerTypeVXLAN); layer != nil {\n\t\tvxlan, _ := layer.(*layers.VXLAN)\n\t\tfor _, vn := range v.vnis {\n\t\t\tif vn > 0 && int(vxlan.VNI) == vn {\n\t\t\t\treturn true\n\t\t\t}\n\n\t\t\tif vn < 0 {\n\t\t\t\tif int(vxlan.VNI) == -vn {\n\t\t\t\t\treturn false\n\t\t\t\t}\n\t\t\t\tdefaultState = true\n\t\t\t}\n\t\t}\n\t}\n\treturn defaultState\n}\n\nfunc (v *vxlanHandle) ReadPacketData() ([]byte, gopacket.CaptureInfo, error) {\n\tpacket := <-v.packetChannel\n\tlayer := packet.Layer(layers.LayerTypeVXLAN)\n\tbytes := layer.LayerPayload()\n\n\treturn bytes, packet.Metadata().CaptureInfo, nil\n}\n\nfunc (v *vxlanHandle) Close() error {\n\tif v.connection != nil {\n\t\treturn v.connection.Close()\n\t}\n\treturn nil\n}\n" }, { "path": "internal/ring/ring.go", "content": "package ring\n\nimport (\n\t\"errors\"\n\t\"runtime\"\n\t\"sync/atomic\"\n\t\"time\"\n)\n\nvar (\n\t// ErrDisposed is returned when an operation is performed on a disposed\n\t// queue.\n\tErrDisposed = errors.New(`queue: disposed`)\n\n\t// ErrTimeout is returned when an applicable queue operation times out.\n\tErrTimeout = errors.New(`queue: poll timed out`)\n\n\t// ErrEmptyQueue is returned when an non-applicable queue operation was called\n\t// due to the queue's empty item state\n\tErrEmptyQueue = errors.New(`queue: empty queue`)\n)\n\n// roundUp takes a uint64 greater than 0 and rounds it up to the next\n// power of 2.\nfunc roundUp(v uint64) uint64 {\n\tv--\n\tv |= v >> 1\n\tv |= v >> 2\n\tv |= v >> 4\n\tv |= v >> 8\n\tv |= v >> 16\n\tv |= v >> 32\n\tv++\n\treturn v\n}\n\ntype node struct {\n\tposition uint64\n\tdata interface{}\n}\n\ntype nodes []node\n\n// RingBuffer is a MPMC buffer that achieves threadsafety with CAS operations\n// only. A put on full or get on empty call will block until an item\n// is put or retrieved. Calling Dispose on the RingBuffer will unblock\n// any blocked threads with an error. This buffer is similar to the buffer\n// described here: http://www.1024cores.net/home/lock-free-algorithms/queues/bounded-mpmc-queue\n// with some minor additions.\ntype RingBuffer struct {\n\t_padding0 [8]uint64\n\tqueue uint64\n\t_padding1 [8]uint64\n\tdequeue uint64\n\t_padding2 [8]uint64\n\tmask, disposed uint64\n\t_padding3 [8]uint64\n\tnodes nodes\n}\n\nfunc (rb *RingBuffer) init(size uint64) {\n\tsize = roundUp(size)\n\trb.nodes = make(nodes, size)\n\tfor i := uint64(0); i < size; i++ {\n\t\trb.nodes[i] = node{position: i}\n\t}\n\trb.mask = size - 1 // so we don't have to do this with every put/get operation\n}\n\n// Put adds the provided item to the queue. If the queue is full, this\n// call will block until an item is added to the queue or Dispose is called\n// on the queue. An error will be returned if the queue is disposed.\nfunc (rb *RingBuffer) Put(item interface{}) error {\n\t_, err := rb.put(item, false)\n\treturn err\n}\n\n// Offer adds the provided item to the queue if there is space. If the queue\n// is full, this call will return false. An error will be returned if the\n// queue is disposed.\nfunc (rb *RingBuffer) Offer(item interface{}) (bool, error) {\n\treturn rb.put(item, true)\n}\n\nfunc (rb *RingBuffer) put(item interface{}, offer bool) (bool, error) {\n\tvar n *node\n\tpos := atomic.LoadUint64(&rb.queue)\nL:\n\tfor {\n\t\tif atomic.LoadUint64(&rb.disposed) == 1 {\n\t\t\treturn false, ErrDisposed\n\t\t}\n\n\t\tn = &rb.nodes[pos&rb.mask]\n\t\tseq := atomic.LoadUint64(&n.position)\n\t\tswitch dif := seq - pos; {\n\t\tcase dif == 0:\n\t\t\tif atomic.CompareAndSwapUint64(&rb.queue, pos, pos+1) {\n\t\t\t\tbreak L\n\t\t\t}\n\t\tcase dif < 0:\n\t\t\tpanic(`Ring buffer in a compromised state during a put operation.`)\n\t\tdefault:\n\t\t\tpos = atomic.LoadUint64(&rb.queue)\n\t\t}\n\n\t\tif offer {\n\t\t\treturn false, nil\n\t\t}\n\n\t\truntime.Gosched() // free up the cpu before the next iteration\n\t}\n\n\tn.data = item\n\tatomic.StoreUint64(&n.position, pos+1)\n\treturn true, nil\n}\n\n// Get will return the next item in the queue. This call will block\n// if the queue is empty. This call will unblock when an item is added\n// to the queue or Dispose is called on the queue. An error will be returned\n// if the queue is disposed.\nfunc (rb *RingBuffer) Get() (interface{}, error) {\n\treturn rb.Poll(0)\n}\n\n// Poll will return the next item in the queue. This call will block\n// if the queue is empty. This call will unblock when an item is added\n// to the queue, Dispose is called on the queue, or the timeout is reached. An\n// error will be returned if the queue is disposed or a timeout occurs. A\n// non-positive timeout will block indefinitely.\nfunc (rb *RingBuffer) Poll(timeout time.Duration) (interface{}, error) {\n\tvar (\n\t\tn *node\n\t\tpos = atomic.LoadUint64(&rb.dequeue)\n\t\tstart time.Time\n\t)\n\tif timeout > 0 {\n\t\tstart = time.Now()\n\t}\nL:\n\tfor {\n\t\tif atomic.LoadUint64(&rb.disposed) == 1 {\n\t\t\treturn nil, ErrDisposed\n\t\t}\n\n\t\tn = &rb.nodes[pos&rb.mask]\n\t\tseq := atomic.LoadUint64(&n.position)\n\t\tswitch dif := seq - (pos + 1); {\n\t\tcase dif == 0:\n\t\t\tif atomic.CompareAndSwapUint64(&rb.dequeue, pos, pos+1) {\n\t\t\t\tbreak L\n\t\t\t}\n\t\tcase dif < 0:\n\t\t\tpanic(`Ring buffer in compromised state during a get operation.`)\n\t\tdefault:\n\t\t\tpos = atomic.LoadUint64(&rb.dequeue)\n\t\t}\n\n\t\tif timeout > 0 && time.Since(start) >= timeout {\n\t\t\treturn nil, ErrTimeout\n\t\t}\n\n\t\tif timeout < 0 {\n\t\t\treturn nil, ErrTimeout\n\t\t}\n\n\t\truntime.Gosched() // free up the cpu before the next iteration\n\t}\n\tdata := n.data\n\tn.data = nil\n\tatomic.StoreUint64(&n.position, pos+rb.mask+1)\n\treturn data, nil\n}\n\n// Len returns the number of items in the queue.\nfunc (rb *RingBuffer) Len() uint64 {\n\treturn atomic.LoadUint64(&rb.queue) - atomic.LoadUint64(&rb.dequeue)\n}\n\n// Cap returns the capacity of this ring buffer.\nfunc (rb *RingBuffer) Cap() uint64 {\n\treturn uint64(len(rb.nodes))\n}\n\n// Dispose will dispose of this queue and free any blocked threads\n// in the Put and/or Get methods. Calling those methods on a disposed\n// queue will return an error.\nfunc (rb *RingBuffer) Dispose() {\n\tatomic.CompareAndSwapUint64(&rb.disposed, 0, 1)\n}\n\n// IsDisposed will return a bool indicating if this queue has been\n// disposed.\nfunc (rb *RingBuffer) IsDisposed() bool {\n\treturn atomic.LoadUint64(&rb.disposed) == 1\n}\n\n// NewRingBuffer will allocate, initialize, and return a ring buffer\n// with the specified size.\nfunc NewRingBuffer(size uint64) *RingBuffer {\n\trb := &RingBuffer{}\n\trb.init(size)\n\treturn rb\n}\n" }, { "path": "internal/simpletime/time.go", "content": "package simpletime\n\nimport (\n\t\"time\"\n)\n\nvar Now time.Time\n\nfunc init() {\n\tgo func() {\n\t\tfor {\n\t\t\t// Accurate enough\n\t\t\tNow = time.Now()\n\t\t\ttime.Sleep(100 * time.Millisecond)\n\t\t}\n\t}()\n}\n" }, { "path": "internal/size/size.go", "content": "package size\n\nimport (\n\t\"fmt\"\n\t\"regexp\"\n\t\"strconv\"\n)\n\n// Size represents size that implements flag.Var\ntype Size int64\n\n// the following regexes follow Go semantics https://golang.org/ref/spec#Letters_and_digits\nvar (\n\trB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\\da-f_]+$`)\n\trKB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\\da-f_]+kb$`)\n\trMB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\\da-f_]+mb$`)\n\trGB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\\da-f_]+gb$`)\n\trTB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\\da-f_]+tb$`)\n)\n\n// Set parses size to integer from different bases and data units\nfunc (siz *Size) Set(size string) (err error) {\n\tif size == \"\" {\n\t\treturn\n\t}\n\tconst (\n\t\t_ = 1 << (iota * 10)\n\t\tKB\n\t\tMB\n\t\tGB\n\t\tTB\n\t)\n\n\tvar (\n\t\tlmt = len(size) - 2\n\t\ts = []byte(size)\n\t)\n\n\tvar _len int64\n\tswitch {\n\tcase rB.Match(s):\n\t\t_len, err = strconv.ParseInt(size, 0, 64)\n\tcase rKB.Match(s):\n\t\t_len, err = strconv.ParseInt(size[:lmt], 0, 64)\n\t\t_len *= KB\n\tcase rMB.Match(s):\n\t\t_len, err = strconv.ParseInt(size[:lmt], 0, 64)\n\t\t_len *= MB\n\tcase rGB.Match(s):\n\t\t_len, err = strconv.ParseInt(size[:lmt], 0, 64)\n\t\t_len *= GB\n\tcase rTB.Match(s):\n\t\t_len, err = strconv.ParseInt(size[:lmt], 0, 64)\n\t\t_len *= TB\n\tdefault:\n\t\treturn fmt.Errorf(\"invalid _len %q\", size)\n\t}\n\t*siz = Size(_len)\n\treturn\n}\n\nfunc (siz *Size) String() string {\n\treturn fmt.Sprintf(\"%d\", *siz)\n}\n" }, { "path": "internal/size/size_test.go", "content": "package size\n\nimport \"testing\"\n\nfunc TestParseDataUnit(t *testing.T) {\n\tvar d = map[string]int{\n\t\t\"42mb\": 42 << 20,\n\t\t\"4_2\": 42,\n\t\t\"00\": 0,\n\t\t\"0\": 0,\n\t\t\"0_600tb\": 384 << 40,\n\t\t\"0600Tb\": 384 << 40,\n\t\t\"0o12Mb\": 10 << 20,\n\t\t\"0b_10010001111_1kb\": 2335 << 10,\n\t\t\"1024\": 1 << 10,\n\t\t\"0b111\": 7,\n\t\t\"0x12gB\": 18 << 30,\n\t\t\"0x_67_7a_2f_cc_40_c6\": 113774485586118,\n\t\t\"121562380192901\": 121562380192901,\n\t}\n\tvar buf Size\n\tvar err error\n\tfor k, v := range d {\n\t\terr = buf.Set(k)\n\t\tif err != nil || buf != Size(v) {\n\t\t\tt.Errorf(\"Error parsing %s: %v\", k, err)\n\t\t}\n\t}\n}\n" }, { "path": "internal/tcp/doc.go", "content": "/*\nPackage tcp implements TCP transport layer protocol, it is responsible for\nparsing, reassembling tcp packets, handling communication with engine listeners(github.com/buger/goreplay/capture),\nand reporting errors and statistics of packets.\nthe packets are parsed by following TCP way(https://en.wikipedia.org/wiki/Transmission_Control_Protocol#TCP_segment_structure).\n\nexample:\n\nimport \"github.com/buger/goreplay/tcp\"\n\nmessageExpire := time.Second*5\nmaxSize := 5 << 20\n\ndebugger := func(debugLevel int, data ...interface{}){} // debugger can also be nil\nmessageHandler := func(mssg *tcp.Message){}\n\nmssgPool := tcp.NewMessageParser(maxMessageSize, messageExpire, debugger, messageHandler)\nlistener.Listen(ctx, mssgPool.Handler)\n\nyou can use pool.End or/and pool.Start to set custom session behaviors\n\ndebugLevel in debugger function indicates the priority of the logs, the bigger the number the lower\nthe priority. errors are signified by debug level 4 for errors, 5 for discarded packets, and 6 for received packets.\n*/\npackage tcp // import github.com/buger/goreplay/tcp\n" }, { "path": "internal/tcp/tcp_message.go", "content": "package tcp\n\nimport (\n\t\"encoding/binary\"\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"github.com/buger/goreplay/proto\"\n\t\"net\"\n\t\"reflect\"\n\t\"sort\"\n\t\"time\"\n\t\"unsafe\"\n)\n\n// TCPProtocol is a number to indicate type of protocol\ntype TCPProtocol uint8\n\nconst (\n\t// ProtocolHTTP ...\n\tProtocolHTTP TCPProtocol = iota\n\t// ProtocolBinary ...\n\tProtocolBinary\n)\n\n// Set is here so that TCPProtocol can implement flag.Var\nfunc (protocol *TCPProtocol) Set(v string) error {\n\tswitch v {\n\tcase \"\", \"http\":\n\t\t*protocol = ProtocolHTTP\n\tcase \"binary\":\n\t\t*protocol = ProtocolBinary\n\tdefault:\n\t\treturn fmt.Errorf(\"unsupported protocol %s\", v)\n\t}\n\treturn nil\n}\n\nfunc (protocol *TCPProtocol) String() string {\n\tswitch *protocol {\n\tcase ProtocolBinary:\n\t\treturn \"binary\"\n\tcase ProtocolHTTP:\n\t\treturn \"http\"\n\tdefault:\n\t\treturn \"\"\n\t}\n}\n\n// Stats every message carry its own stats object\ntype Stats struct {\n\tLostData int\n\tLength int // length of the data\n\tStart time.Time // first packet's timestamp\n\tEnd time.Time // last packet's timestamp\n\tSrcAddr string\n\tDstAddr string\n\tDirection Dir\n\tTimedOut bool // timeout before getting the whole message\n\tTruncated bool // last packet truncated due to max message size\n\tIPversion byte\n}\n\n// Message is the representation of a tcp message\ntype Message struct {\n\tpackets []*Packet\n\tparser *MessageParser\n\tfeedback interface{}\n\tcontinueAdjusted bool\n\tStats\n}\n\n// UUID returns the UUID of a TCP request and its response.\nfunc (m *Message) UUID() []byte {\n\tvar streamID uint64\n\tpckt := m.packets[0]\n\n\t// check if response or request have generated the ID before.\n\tif m.Direction == DirIncoming {\n\t\tstreamID = uint64(pckt.SrcPort)<<48 | uint64(pckt.DstPort)<<32 |\n\t\t\tuint64(ip2int(pckt.SrcIP))\n\t} else {\n\t\tstreamID = uint64(pckt.DstPort)<<48 | uint64(pckt.SrcPort)<<32 |\n\t\t\tuint64(ip2int(pckt.DstIP))\n\t}\n\n\tid := make([]byte, 12)\n\tbinary.BigEndian.PutUint64(id, streamID)\n\n\tif m.Direction == DirIncoming {\n\t\tbinary.BigEndian.PutUint32(id[8:], pckt.Ack)\n\t} else {\n\t\tbinary.BigEndian.PutUint32(id[8:], pckt.Seq)\n\t}\n\n\tuuidHex := make([]byte, 24)\n\thex.Encode(uuidHex[:], id[:])\n\n\treturn uuidHex\n}\n\nfunc (m *Message) add(packet *Packet) bool {\n\t// Skip duplicates\n\tfor _, p := range m.packets {\n\t\tif p.Seq == packet.Seq {\n\t\t\treturn false\n\t\t}\n\t}\n\n\t// Packets not always captured in same Seq order, and sometimes we need to prepend\n\tif len(m.packets) == 0 || packet.Seq > m.packets[len(m.packets)-1].Seq {\n\t\tm.packets = append(m.packets, packet)\n\t} else if packet.Seq < m.packets[0].Seq {\n\t\tm.packets = append([]*Packet{packet}, m.packets...)\n\t} else { // insert somewhere in the middle...\n\t\tfor i, p := range m.packets {\n\t\t\tif packet.Seq < p.Seq {\n\t\t\t\tm.packets = append(m.packets[:i], append([]*Packet{packet}, m.packets[i:]...)...)\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\n\tm.Length += len(packet.Payload)\n\tm.LostData += int(packet.Lost)\n\n\tif packet.Timestamp.After(m.End) || m.End.IsZero() {\n\t\tm.End = packet.Timestamp\n\t}\n\n\treturn true\n}\n\n// Packets returns packets of the message\nfunc (m *Message) Packets() []*Packet {\n\treturn m.packets\n}\n\nfunc (m *Message) MissingChunk() bool {\n\tnextSeq := m.packets[0].Seq\n\n\tfor _, p := range m.packets {\n\t\tif p.Seq != nextSeq {\n\t\t\treturn true\n\t\t}\n\n\t\tnextSeq += uint32(len(p.Payload))\n\t}\n\n\treturn false\n}\n\nfunc (m *Message) PacketData() [][]byte {\n\ttmp := make([][]byte, len(m.packets))\n\n\tfor i, p := range m.packets {\n\t\ttmp[i] = p.Payload\n\t}\n\n\treturn tmp\n}\n\n// Data returns data in this message\nfunc (m *Message) Data() []byte {\n\tpacketData := m.PacketData()\n\ttmp := packetData[0]\n\n\tif len(packetData) > 0 {\n\t\ttmp, _ = copySlice(tmp, len(packetData[0]), packetData[1:]...)\n\t}\n\n\t// Remove Expect header, since its replay not fully supported\n\tif state, ok := m.feedback.(*proto.HTTPState); ok {\n\t\tif state.Continue100 {\n\t\t\ttmp = proto.DeleteHeader(tmp, []byte(\"Expect\"))\n\t\t}\n\t}\n\n\treturn tmp\n}\n\n// SetProtocolState set feedback/data that can be used later, e.g with End or Start hint\nfunc (m *Message) SetProtocolState(feedback interface{}) {\n\tm.feedback = feedback\n}\n\n// ProtocolState returns feedback associated to this message\nfunc (m *Message) ProtocolState() interface{} {\n\treturn m.feedback\n}\n\n// Sort a helper to sort packets\nfunc (m *Message) Sort() {\n\tsort.SliceStable(m.packets, func(i, j int) bool { return m.packets[i].Seq < m.packets[j].Seq })\n}\n\n// Emitter message handler\ntype Emitter func(*Message)\n\n// HintEnd hints the parser to stop the session, see MessageParser.End\n// when set, it will be executed before checking FIN or RST flag\ntype HintEnd func(*Message) bool\n\n// HintStart hints the parser to start the reassembling the message, see MessageParser.Start\n// when set, it will be called after checking SYN flag\ntype HintStart func(*Packet) (IsRequest, IsOutgoing bool)\n\n// MessageParser holds data of all tcp messages in progress(still receiving/sending packets).\n// message is identified by its source port and dst port, and last 4bytes of src IP.\ntype MessageParser struct {\n\tm map[uint64]*Message\n\n\tmessageExpire time.Duration // the maximum time to wait for the final packet, minimum is 100ms\n\tallowIncompete bool\n\tEnd HintEnd\n\tStart HintStart\n\tticker *time.Ticker\n\tmessages chan *Message\n\tpackets chan *PcapPacket\n\tclose chan struct{} // to signal that we are able to close\n\tports []uint16\n\tips []net.IP\n}\n\n// NewMessageParser returns a new instance of message parser\nfunc NewMessageParser(messages chan *Message, ports []uint16, ips []net.IP, messageExpire time.Duration, allowIncompete bool) (parser *MessageParser) {\n\tparser = new(MessageParser)\n\n\tparser.messageExpire = messageExpire\n\tif parser.messageExpire == 0 {\n\t\tparser.messageExpire = time.Millisecond * 1000\n\t}\n\n\tparser.allowIncompete = allowIncompete\n\n\tparser.packets = make(chan *PcapPacket, 10000)\n\n\tif messages == nil {\n\t\tmessages = make(chan *Message, 1000)\n\t}\n\tparser.messages = messages\n\n\tparser.m = make(map[uint64]*Message)\n\tparser.ticker = time.NewTicker(time.Millisecond * 100)\n\tparser.close = make(chan struct{}, 1)\n\n\tparser.ports = ports\n\tparser.ips = ips\n\n\tgo parser.wait()\n\treturn parser\n}\n\nvar packetLen int\n\n// Packet returns packet handler\nfunc (parser *MessageParser) PacketHandler(packet *PcapPacket) {\n\tpacketLen++\n\tparser.packets <- packet\n}\n\nfunc (parser *MessageParser) wait() {\n\tvar (\n\t\tnow time.Time\n\t)\n\tfor {\n\t\tselect {\n\t\tcase pckt := <-parser.packets:\n\t\t\tparser.processPacket(parser.parsePacket(pckt))\n\t\tcase now = <-parser.ticker.C:\n\t\t\tparser.timer(now)\n\t\tcase <-parser.close:\n\t\t\tparser.ticker.Stop()\n\t\t\t// parser.Close should wait for this function to return\n\t\t\tparser.close <- struct{}{}\n\t\t\treturn\n\t\t\t// default:\n\t\t}\n\t}\n}\n\nfunc (parser *MessageParser) parsePacket(pcapPkt *PcapPacket) *Packet {\n\tpckt, err := ParsePacket(pcapPkt.Data, pcapPkt.LType, pcapPkt.LTypeLen, pcapPkt.Ci, false)\n\tif err != nil {\n\t\tif _, empty := err.(EmptyPacket); !empty {\n\t\t\tstats.Add(\"packet_error\", 1)\n\t\t}\n\t\treturn nil\n\t}\n\n\tfor _, p := range parser.ports {\n\t\tif pckt.DstPort == p && containsOrEmpty(pckt.DstIP, parser.ips) {\n\t\t\tpckt.Direction = DirIncoming\n\t\t\tbreak\n\t\t} else if pckt.SrcPort == p && containsOrEmpty(pckt.SrcIP, parser.ips) {\n\t\t\tpckt.Direction = DirOutcoming\n\t\t\tbreak\n\t\t}\n\t}\n\n\treturn pckt\n}\n\nfunc containsOrEmpty(element net.IP, ipList []net.IP) bool {\n\tif len(ipList) == 0 {\n\t\treturn true\n\t}\n\tfor _, ip := range ipList {\n\t\tif ip.Equal(element) {\n\t\t\treturn true\n\t\t}\n\t}\n\treturn false\n}\n\nfunc (parser *MessageParser) processPacket(pckt *Packet) {\n\tif pckt == nil {\n\t\treturn\n\t}\n\n\t// Trying to build unique hash, but there is small chance of collision\n\t// No matter if it is request or response, all packets in the same message have same\n\tm, ok := parser.m[pckt.MessageID()]\n\tswitch {\n\tcase ok:\n\t\tif m.Direction == DirUnknown {\n\t\t\tif in, out := parser.Start(pckt); in || out {\n\t\t\t\tif in {\n\t\t\t\t\tm.Direction = DirIncoming\n\t\t\t\t} else {\n\t\t\t\t\tm.Direction = DirOutcoming\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tparser.addPacket(m, pckt)\n\t\treturn\n\tcase pckt.Direction == DirUnknown && parser.Start != nil:\n\t\tif in, out := parser.Start(pckt); in || out {\n\t\t\tif in {\n\t\t\t\tpckt.Direction = DirIncoming\n\t\t\t} else {\n\t\t\t\tpckt.Direction = DirOutcoming\n\t\t\t}\n\t\t}\n\t}\n\n\tm = new(Message)\n\tm.Direction = pckt.Direction\n\tm.SrcAddr = pckt.SrcIP.String()\n\tm.DstAddr = pckt.DstIP.String()\n\n\tparser.m[pckt.MessageID()] = m\n\n\tm.Start = pckt.Timestamp\n\tm.parser = parser\n\tparser.addPacket(m, pckt)\n}\n\nfunc (parser *MessageParser) addPacket(m *Message, pckt *Packet) bool {\n\tif !m.add(pckt) {\n\t\treturn false\n\t}\n\n\t// If we are using protocol parsing, like HTTP, depend on its parsing func.\n\t// For the binary procols wait for message to expire\n\tif parser.End != nil {\n\t\tif parser.End(m) {\n\t\t\tparser.Emit(m)\n\t\t\treturn true\n\t\t}\n\n\t\tparser.Fix100Continue(m)\n\t}\n\n\treturn true\n}\n\nfunc (parser *MessageParser) Fix100Continue(m *Message) {\n\t// Only adjust a message once\n\tif state, ok := m.feedback.(*proto.HTTPState); ok && state.Continue100 && !m.continueAdjusted {\n\t\t// Shift Ack by given offset\n\t\t// Size of \"HTTP/1.1 100 Continue\\r\\n\\r\\n\" message\n\t\tfor _, p := range m.packets {\n\t\t\tp.messageID = 0\n\t\t\tp.Ack += 25\n\t\t}\n\n\t\t// If next section was aready approved and received, merge messages\n\t\tif next, found := parser.m[m.packets[0].MessageID()]; found {\n\t\t\tfor _, p := range next.packets {\n\t\t\t\tparser.addPacket(m, p)\n\t\t\t}\n\t\t}\n\n\t\t// Re-add (or override) again with new message and ID\n\t\tparser.m[m.packets[0].MessageID()] = m\n\t\tm.continueAdjusted = true\n\t}\n}\n\nfunc (parser *MessageParser) Read() *Message {\n\tm := <-parser.messages\n\treturn m\n}\n\nfunc (parser *MessageParser) Emit(m *Message) {\n\tstats.Add(\"message_count\", 1)\n\n\tdelete(parser.m, m.packets[0].MessageID())\n\n\tparser.messages <- m\n}\n\nfunc GetUnexportedField(field reflect.Value) interface{} {\n\treturn reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().Interface()\n}\n\nvar failMsg int\n\nfunc (parser *MessageParser) timer(now time.Time) {\n\tpacketLen = 0\n\n\tpacketQueueLen.Set(int64(len(parser.packets)))\n\tmessageQueueLen.Set(int64(len(parser.m)))\n\n\tfor _, m := range parser.m {\n\t\tif now.Sub(m.End) > parser.messageExpire {\n\t\t\tm.TimedOut = true\n\t\t\tstats.Add(\"message_timeout_count\", 1)\n\t\t\tfailMsg++\n\t\t\tif parser.End == nil || parser.allowIncompete {\n\t\t\t\tparser.Emit(m)\n\t\t\t}\n\n\t\t\tdelete(parser.m, m.packets[0].MessageID())\n\t\t}\n\t}\n}\n\nfunc (parser *MessageParser) Close() error {\n\tparser.close <- struct{}{}\n\t<-parser.close // wait for timer to be closed!\n\treturn nil\n}\n" }, { "path": "internal/tcp/tcp_packet.go", "content": "package tcp\n\nimport (\n\t\"encoding/binary\"\n\t\"expvar\"\n\t\"fmt\"\n\t\"net\"\n\t\"time\"\n\n\t\"github.com/google/gopacket\"\n)\n\nfunc copySlice(to []byte, skip int, from ...[]byte) ([]byte, int) {\n\tvar totalLen int\n\tfor _, s := range from {\n\t\ttotalLen += len(s)\n\t}\n\ttotalLen += skip\n\n\tif len(to) < totalLen {\n\t\tdiff := totalLen - len(to)\n\t\tto = append(to, make([]byte, diff)...)\n\t}\n\n\tfor _, s := range from {\n\t\tskip += copy(to[skip:], s)\n\t}\n\n\treturn to, skip\n}\n\nvar stats *expvar.Map\nvar packetQueueLen, messageQueueLen *expvar.Int\n\nfunc init() {\n\tpacketQueueLen = new(expvar.Int)\n\tmessageQueueLen = new(expvar.Int)\n\n\tstats = expvar.NewMap(\"tcp\")\n\tstats.Init()\n\tstats.Set(\"packet_queue\", packetQueueLen)\n\tstats.Set(\"message_queue\", messageQueueLen)\n}\n\ntype Dir int\n\nconst (\n\tDirUnknown = iota\n\tDirIncoming\n\tDirOutcoming\n)\n\n/*\nPacket represent data and layers of packet.\nparser extracts information from pcap Packet. functions of *Packet doesn't validate if packet is nil,\ncalllers must make sure that ParsePacket has'nt returned any error before calling any other\nfunction.\n*/\ntype Packet struct {\n\tDirection Dir\n\tmessageID uint64\n\tSrcIP, DstIP net.IP\n\tVersion uint8\n\tSrcPort, DstPort uint16\n\tAck, Seq uint32\n\tACK, SYN, FIN, RST bool\n\tLost uint32\n\tRetry int\n\tCaptureLength int\n\tTimestamp time.Time\n\tPayload []byte\n\tbuf []byte\n\n\tcreated time.Time\n\tgc bool\n}\n\ntype PcapPacket struct {\n\tData []byte\n\tLType int\n\tLTypeLen int\n\tCi *gopacket.CaptureInfo\n}\n\n// ParsePacket parse raw packets\nfunc ParsePacket(data []byte, lType, lTypeLen int, ci *gopacket.CaptureInfo, allowEmpty bool) (pckt *Packet, err error) {\n\tpckt = new(Packet)\n\tif err := pckt.parse(data, lType, lTypeLen, ci, allowEmpty); err != nil {\n\t\treturn nil, err\n\t}\n\n\treturn pckt, nil\n}\n\nfunc (pckt *Packet) parse(data []byte, lType, lTypeLen int, cp *gopacket.CaptureInfo, allowEmpty bool) error {\n\tpckt.Retry = 0\n\tpckt.messageID = 0\n\tpckt.buf = pckt.buf[:]\n\n\t// TODO: check resolution\n\tpckt.Timestamp = cp.Timestamp\n\n\tif len(data) < lTypeLen {\n\t\treturn ErrHdrLength(\"Link\")\n\t}\n\tif len(data) <= lTypeLen {\n\t\treturn ErrHdrMissing(\"IPv4 or IPv6\")\n\t}\n\n\tldata := data[lTypeLen:]\n\tvar proto byte\n\tvar netLayer, transLayer []byte\n\n\tif ldata[0]>>4 == 4 {\n\t\t// IPv4 header\n\t\tif len(ldata) < 20 {\n\t\t\treturn ErrHdrLength(\"IPv4\")\n\t\t}\n\t\tproto = ldata[9]\n\t\tihl := int(ldata[0]&0x0F) * 4\n\t\tif ihl < 20 {\n\t\t\treturn ErrHdrInvalid(\"IPv4's IHL\")\n\t\t}\n\t\tif len(ldata) < ihl {\n\t\t\treturn ErrHdrLength(\"IPv4 opts\")\n\t\t}\n\t\tnetLayer = ldata[:ihl]\n\t} else if ldata[0]>>4 == 6 {\n\t\tif len(ldata) < 40 {\n\t\t\treturn ErrHdrLength(\"IPv6\")\n\t\t}\n\t\tproto = ldata[6]\n\t\ttotalLen := 40\n\t\tfor ipv6ExtensionHdr(proto) {\n\t\t\thdr := len(ldata) - totalLen\n\t\t\tif hdr < 8 {\n\t\t\t\treturn ErrHdrExpected(\"IPv6 opts\")\n\t\t\t}\n\t\t\textLen := 8\n\t\t\tif proto != 44 {\n\t\t\t\textLen = int(ldata[totalLen+1]+1) * 8\n\t\t\t}\n\t\t\tif hdr < extLen {\n\t\t\t\treturn ErrHdrLength(\"IPv6 opts\")\n\t\t\t}\n\t\t\tproto = ldata[totalLen]\n\t\t\ttotalLen += extLen\n\t\t}\n\t\tnetLayer = ldata[:totalLen]\n\t} else {\n\t\treturn ErrHdrExpected(\"IPv4 or IPv6\")\n\t}\n\tif proto != 6 {\n\t\treturn ErrHdrExpected(\"TCP\")\n\t}\n\tif len(data) <= len(netLayer) {\n\t\treturn ErrHdrMissing(\"TCP\")\n\t}\n\tndata := ldata[len(netLayer):]\n\t// TCP header\n\tif len(ndata) < 20 {\n\t\treturn ErrHdrLength(\"TCP\")\n\t}\n\tdOf := int(ndata[12]>>4) * 4\n\tif dOf < 20 {\n\t\treturn ErrHdrInvalid(\"TCP's ndata offset\")\n\t}\n\tif len(ndata) < dOf {\n\t\treturn ErrHdrLength(\"TCP opts\")\n\t}\n\n\t// There are case when packet have padding but dOf shows its not\n\tempty := true\n\tfor i := 0; i < len(ndata[dOf:]); i++ {\n\t\tif ndata[dOf:][i] != 0 {\n\t\t\tempty = false\n\t\t\tbreak\n\t\t}\n\t}\n\n\tif !allowEmpty && empty {\n\t\treturn EmptyPacket(\"\")\n\t}\n\n\tif (netLayer[0] >> 4) == 4 {\n\t\t// IPv4 header\n\t\tpckt.Version = 4\n\t\tpckt.SrcIP = netLayer[12:16]\n\t\tpckt.DstIP = netLayer[16:20]\n\t} else {\n\t\t// IPv6 header\n\t\tpckt.Version = 6\n\t\tpckt.SrcIP = netLayer[8:24]\n\t\tpckt.DstIP = netLayer[24:40]\n\t}\n\n\ttransLayer = ndata[:dOf]\n\n\tpckt.CaptureLength = cp.CaptureLength\n\tpckt.SrcPort = binary.BigEndian.Uint16(transLayer[0:2])\n\tpckt.DstPort = binary.BigEndian.Uint16(transLayer[2:4])\n\tpckt.Seq = binary.BigEndian.Uint32(transLayer[4:8])\n\tpckt.Ack = binary.BigEndian.Uint32(transLayer[8:12])\n\tpckt.FIN = transLayer[13]&0x01 != 0\n\tpckt.SYN = transLayer[13]&0x02 != 0\n\tpckt.RST = transLayer[13]&0x04 != 0\n\tpckt.ACK = transLayer[13]&0x10 != 0\n\tpckt.Lost = uint32(cp.Length - cp.CaptureLength)\n\n\tpckt.Payload = ndata[dOf:]\n\n\treturn nil\n}\n\nfunc (pckt *Packet) MessageID() uint64 {\n\tif pckt.messageID == 0 {\n\t\t// All packets in the same message will share the same ID\n\t\tpckt.messageID = uint64(pckt.SrcPort)<<48 | uint64(pckt.DstPort)<<32 |\n\t\t\t(uint64(ip2int(pckt.SrcIP)) + uint64(ip2int(pckt.DstIP)) + uint64(pckt.Ack))\n\t}\n\n\treturn pckt.messageID\n}\n\n// Src returns the source socket of a packet\nfunc (pckt *Packet) Src() string {\n\treturn fmt.Sprintf(\"%s:%d\", pckt.SrcIP, pckt.SrcPort)\n}\n\n// Dst returns destination socket\nfunc (pckt *Packet) Dst() string {\n\treturn fmt.Sprintf(\"%s:%d\", pckt.DstIP, pckt.DstPort)\n}\n\ntype EmptyPacket string\n\nfunc (err EmptyPacket) Error() string {\n\treturn \"Empty packet\"\n}\n\n// ErrHdrLength returned on short header length\ntype ErrHdrLength string\n\nfunc (err ErrHdrLength) Error() string {\n\treturn \"short \" + string(err) + \" length\"\n}\n\n// ErrHdrMissing returned on missing header(s)\ntype ErrHdrMissing string\n\nfunc (err ErrHdrMissing) Error() string {\n\treturn \"missing \" + string(err) + \" header(s)\"\n}\n\n// ErrHdrExpected returned when header(s) are different from the one expected\ntype ErrHdrExpected string\n\nfunc (err ErrHdrExpected) Error() string {\n\treturn \"expected \" + string(err) + \" header(s)\"\n}\n\n// ErrHdrInvalid returned when header(s) are different from the one expected\ntype ErrHdrInvalid string\n\nfunc (err ErrHdrInvalid) Error() string {\n\treturn \"invalid \" + string(err) + \" value\"\n}\n\n// https://en.wikipedia.org/wiki/IPv6_packet#Extension_headers\nfunc ipv6ExtensionHdr(b byte) bool {\n\t// TODO: support all extension headers\n\treturn b == 0 || b == 43 || b == 44\n}\n\nfunc ip2int(ip net.IP) uint32 {\n\tif len(ip) == 0 {\n\t\treturn 0\n\t}\n\n\tif len(ip) == 16 {\n\t\treturn binary.BigEndian.Uint32(ip[12:16])\n\t}\n\treturn binary.BigEndian.Uint32(ip)\n}\n" }, { "path": "internal/tcp/tcp_test.go", "content": "package tcp\n\nimport (\n\t\"bytes\"\n\t\"encoding/binary\"\n\t\"github.com/buger/goreplay/proto\"\n\n\t// \"runtime\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n\n\t\"github.com/google/gopacket\"\n\t\"github.com/google/gopacket/layers\"\n)\n\nfunc generateHeader(request bool, seq uint32, length uint16) []byte {\n\thdr := make([]byte, 4+24+24)\n\tbinary.BigEndian.PutUint32(hdr, uint32(layers.ProtocolFamilyIPv4))\n\n\tip := hdr[4:]\n\tip[0] = 4<<4 | 6\n\tbinary.BigEndian.PutUint16(ip[2:4], length+24+24)\n\tip[9] = uint8(layers.IPProtocolTCP)\n\tcopy(ip[12:16], []byte{127, 0, 0, 1})\n\tcopy(ip[16:], []byte{127, 0, 0, 1})\n\n\t// set tcp header\n\ttcp := ip[24:]\n\ttcp[12] = 6 << 4\n\n\tif request {\n\t\tbinary.BigEndian.PutUint16(tcp, 5535)\n\t\tbinary.BigEndian.PutUint16(tcp[2:], 8000)\n\t} else {\n\t\tbinary.BigEndian.PutUint16(tcp, 8000)\n\t\tbinary.BigEndian.PutUint16(tcp[2:], 5535)\n\t}\n\tbinary.BigEndian.PutUint32(tcp[4:], seq)\n\treturn hdr\n}\n\nfunc GetPackets(request bool, start uint32, _len int, payload []byte) []*Packet {\n\tvar packets = make([]*Packet, _len)\n\tvar err error\n\tfor i := start; i < start+uint32(_len); i++ {\n\t\td := append(generateHeader(request, i, uint16(len(payload))), payload...)\n\t\tci := &gopacket.CaptureInfo{Length: len(d), CaptureLength: len(d), Timestamp: time.Now()}\n\n\t\tpackets[i-start], err = ParsePacket(d, int(layers.LinkTypeLoop), 4, ci, true)\n\t\tif request {\n\t\t\tpackets[i-start].Direction = DirIncoming\n\t\t} else {\n\t\t\tpackets[i-start].Direction = DirOutcoming\n\t\t}\n\t\tif err != nil {\n\t\t\tpanic(err)\n\t\t}\n\t}\n\treturn packets\n}\n\nfunc TestRequestResponseMapping(t *testing.T) {\n\tpackets := []*Packet{\n\t\t{SrcPort: 60000, DstPort: 80, Ack: 1, Seq: 1, Direction: DirIncoming, Timestamp: time.Unix(1, 0), Payload: []byte(\"GET / HTTP/1.1\\r\\n\")},\n\t\t{SrcPort: 60000, DstPort: 80, Ack: 1, Seq: 17, Direction: DirIncoming, Timestamp: time.Unix(2, 0), Payload: []byte(\"Host: localhost\\r\\n\\r\\n\")},\n\n\t\t// Seq of first response packet match Ack of first request packet\n\t\t{SrcPort: 80, DstPort: 60000, Ack: 36, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(3, 0), Payload: []byte(\"HTTP/1.1 200 OK\\r\\n\")},\n\t\t{SrcPort: 80, DstPort: 60000, Ack: 36, Seq: 18, Direction: DirOutcoming, Timestamp: time.Unix(4, 0), Payload: []byte(\"Content-Length: 0\\r\\n\\r\\n\")},\n\n\t\t// Same TCP stream\n\t\t{SrcPort: 60000, DstPort: 80, Ack: 39, Seq: 36, Direction: DirIncoming, Timestamp: time.Unix(5, 0), Payload: []byte(\"GET / HTTP/1.1\\r\\n\")},\n\t\t{SrcPort: 60000, DstPort: 80, Ack: 39, Seq: 52, Direction: DirIncoming, Timestamp: time.Unix(6, 0), Payload: []byte(\"Host: localhost\\r\\n\\r\\n\")},\n\n\t\t// Seq of first response packet match Ack of first request packet\n\t\t{SrcPort: 80, DstPort: 60000, Ack: 71, Seq: 39, Direction: DirOutcoming, Timestamp: time.Unix(7, 0), Payload: []byte(\"HTTP/1.1 200 OK\\r\\n\")},\n\t\t{SrcPort: 80, DstPort: 60000, Ack: 71, Seq: 56, Direction: DirOutcoming, Timestamp: time.Unix(8, 0), Payload: []byte(\"Content-Length: 0\\r\\n\\r\\n\")},\n\t}\n\n\tparser := NewMessageParser(nil, nil, nil, time.Second, false)\n\tparser.Start = func(pckt *Packet) (bool, bool) {\n\t\treturn proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload)\n\t}\n\tparser.End = func(m *Message) bool {\n\t\treturn proto.HasFullPayload(m, m.PacketData()...)\n\t}\n\n\tfor _, packet := range packets {\n\t\tparser.processPacket(packet)\n\t}\n\n\tmessages := []*Message{}\n\tfor i := 0; i < 4; i++ {\n\t\tm := parser.Read()\n\t\tmessages = append(messages, m)\n\t}\n\n\tassert.Equal(t, int(messages[0].Direction), int(DirIncoming))\n\tassert.Equal(t, int(messages[1].Direction), int(DirOutcoming))\n\tassert.Equal(t, int(messages[2].Direction), int(DirIncoming))\n\tassert.Equal(t, int(messages[3].Direction), int(DirOutcoming))\n\n\tassert.Equal(t, messages[0].UUID(), messages[1].UUID())\n\tassert.Equal(t, messages[2].UUID(), messages[3].UUID())\n\n\tassert.NotEqual(t, messages[0].UUID(), messages[2].UUID())\n}\n\nfunc TestMessageParserWithHint(t *testing.T) {\n\tparser := NewMessageParser(nil, nil, nil, time.Second, false)\n\tparser.Start = func(pckt *Packet) (bool, bool) {\n\t\treturn proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload)\n\t}\n\tparser.End = func(m *Message) bool {\n\t\treturn proto.HasFullPayload(m, m.PacketData()...)\n\t}\n\n\tpackets := []*Packet{\n\t\t// Seq of first response packet match Ack of first request packet\n\t\t{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(1, 0), Payload: []byte(\"HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nTransfer-Encoding: chunked\\r\\n\\r\\n7\\r\\n\")},\n\t\t{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 18, Direction: DirOutcoming, Timestamp: time.Unix(2, 0), Payload: []byte(\"\\r\\nMozilla\\r\\n9\\r\\nDeveloper\\r\")},\n\t\t{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 42, Direction: DirOutcoming, Timestamp: time.Unix(3, 0), Payload: []byte(\"\\n7\\r\\nNetwork\\r\\n0\\r\\n\\r\\n\")},\n\n\t\t{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 1, Direction: DirIncoming, Timestamp: time.Unix(4, 0), Payload: []byte(\"POST / HTTP/1.1\\r\\nContent-Type: text/plain\\r\\nContent-Length: 23\\r\\n\\r\\n\")},\n\t\t{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 66, Direction: DirIncoming, Timestamp: time.Unix(5, 0), Payload: []byte(\"MozillaDeveloper\")},\n\t\t{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 82, Direction: DirIncoming, Timestamp: time.Unix(6, 0), Payload: []byte(\"Network\")},\n\n\t\t{SrcPort: 80, DstPort: 60000, Ack: 89, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(7, 0), Payload: []byte(\"HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nContent-Length: 0\\r\\n\\r\\n\")},\n\t}\n\n\tfor _, p := range packets {\n\t\tparser.processPacket(p)\n\t}\n\n\tmessages := []*Message{}\n\tfor i := 0; i < 3; i++ {\n\t\tm := parser.Read()\n\t\tmessages = append(messages, m)\n\t}\n\n\tif !bytes.HasSuffix(messages[0].Data(), []byte(\"\\n7\\r\\nNetwork\\r\\n0\\r\\n\\r\\n\")) {\n\t\tt.Errorf(\"expected to %q to have suffix %q\", messages[0].Data(), []byte(\"\\n7\\r\\nNetwork\\r\\n0\\r\\n\\r\\n\"))\n\t}\n\n\tif !bytes.HasSuffix(messages[1].Data(), []byte(\"Network\")) {\n\t\tt.Errorf(\"expected to %q to have suffix %q\", messages[1].Data(), []byte(\"Network\"))\n\t}\n\n\tif !bytes.HasSuffix(messages[2].Data(), []byte(\"Content-Length: 0\\r\\n\\r\\n\")) {\n\t\tt.Errorf(\"expected to %q to have suffix %q\", messages[2].Data(), []byte(\"Content-Length: 0\\r\\n\\r\\n\"))\n\t}\n}\n\nfunc TestMessageParserWrongOrder(t *testing.T) {\n\tparser := NewMessageParser(nil, nil, nil, time.Second, false)\n\tparser.Start = func(pckt *Packet) (bool, bool) {\n\t\treturn proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload)\n\t}\n\tparser.End = func(m *Message) bool {\n\t\treturn proto.HasFullPayload(m, m.PacketData()...)\n\t}\n\tpackets := []*Packet{\n\t\t// Seq of first response packet match Ack of first request packet\n\t\t{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 66, Direction: DirIncoming, Timestamp: time.Unix(5, 0), Payload: []byte(\"MozillaDeveloper\")},\n\t\t{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(1, 0), Payload: []byte(\"HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nTransfer-Encoding: chunked\\r\\n\\r\\n7\\r\\n\")},\n\t\t{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 42, Direction: DirOutcoming, Timestamp: time.Unix(3, 0), Payload: []byte(\"\\n7\\r\\nNetwork\\r\\n0\\r\\n\\r\\n\")},\n\n\t\t{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 1, Direction: DirIncoming, Timestamp: time.Unix(4, 0), Payload: []byte(\"POST / HTTP/1.1\\r\\nContent-Type: text/plain\\r\\nContent-Length: 23\\r\\n\\r\\n\")},\n\t\t{SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 18, Direction: DirOutcoming, Timestamp: time.Unix(2, 0), Payload: []byte(\"\\r\\nMozilla\\r\\n9\\r\\nDeveloper\\r\")},\n\n\t\t{SrcPort: 80, DstPort: 60000, Ack: 89, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(7, 0), Payload: []byte(\"HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nContent-Length: 0\\r\\n\\r\\n\")},\n\t\t{SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 82, Direction: DirIncoming, Timestamp: time.Unix(6, 0), Payload: []byte(\"Network\")},\n\t}\n\n\tfor _, p := range packets {\n\t\tparser.processPacket(p)\n\t}\n\n\tm := parser.Read()\n\n\tif !bytes.HasSuffix(m.Data(), []byte(\"\\n7\\r\\nNetwork\\r\\n0\\r\\n\\r\\n\")) {\n\t\tt.Errorf(\"expected to %q to have suffix %q\", m.Data(), []byte(\"\\n7\\r\\nNetwork\\r\\n0\\r\\n\\r\\n\"))\n\t}\n\n\tm = parser.Read()\n\n\tif !bytes.HasSuffix(m.Data(), []byte(\"Content-Length: 0\\r\\n\\r\\n\")) {\n\t\tt.Errorf(\"expected to %q to have suffix %q\", m.Data(), []byte(\"Content-Length: 0\\r\\n\\r\\n\"))\n\t}\n\n\tm = parser.Read()\n\n\tif !bytes.HasSuffix(m.Data(), []byte(\"Network\")) {\n\t\tt.Errorf(\"expected to %q to have suffix %q\", m.Data(), []byte(\"Network\"))\n\t}\n}\n\nfunc TestMessageParserWithoutHint(t *testing.T) {\n\tvar data [63 << 10]byte\n\tpackets := GetPackets(true, 1, 10, data[:])\n\n\tp := NewMessageParser(nil, nil, nil, time.Second, false)\n\tfor _, v := range packets {\n\t\tp.processPacket(v)\n\t}\n\tm := p.Read()\n\n\tif m.Length != 63<<10*10 {\n\t\tt.Errorf(\"expected %d to equal %d\", m.Length, 63<<10*10)\n\t}\n}\n\nfunc TestMessageTimeoutReached(t *testing.T) {\n\tconst size = 63 << 11\n\tvar data [size >> 1]byte\n\tpackets := GetPackets(true, 1, 2, data[:])\n\tp := NewMessageParser(nil, nil, nil, 100*time.Millisecond, true)\n\tp.processPacket(packets[0])\n\n\ttime.Sleep(time.Millisecond * 20)\n\n\tp.processPacket(packets[1])\n\tm := p.Read()\n\tif m.Length != size {\n\t\tt.Errorf(\"expected %d to equal %d\", m.Length, size)\n\t}\n\tif !m.TimedOut {\n\t\tt.Error(\"expected message to be timeout\")\n\t}\n}\n\nfunc BenchmarkMessageUUID(b *testing.B) {\n\tpackets := GetPackets(true, 1, 5, nil)\n\n\tvar uuid []byte\n\tparser := NewMessageParser(nil, nil, nil, 10*time.Millisecond, true)\n\tfor _, p := range packets {\n\t\tparser.processPacket(p)\n\t}\n\n\tmsg := parser.Read()\n\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tuuid = msg.UUID()\n\t}\n\t_ = uuid\n}\n\nfunc BenchmarkPacketParseAndSort(b *testing.B) {\n\tm := new(Message)\n\tm.packets = make([]*Packet, 100)\n\tfor i, v := range GetPackets(true, 1, 100, nil) {\n\t\tm.packets[i] = v\n\t}\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tm.Sort()\n\t}\n}\n\nfunc BenchmarkMessageParserWithoutHint(b *testing.B) {\n\tvar chunk = []byte(\"111111111111111111111111111111\")\n\tpackets := GetPackets(true, 1, 1000, chunk)\n\tp := NewMessageParser(nil, nil, nil, 2*time.Second, false)\n\tb.ResetTimer()\n\tb.ReportMetric(float64(1000), \"packets/op\")\n\tfor i := 0; i < b.N; i++ {\n\t\tfor _, v := range packets {\n\t\t\tp.processPacket(v)\n\t\t}\n\t\tp.Read()\n\t}\n}\n\nfunc BenchmarkMessageParserWithHint(b *testing.B) {\n\tvar buf [1002][]byte\n\tvar chunk = []byte(\"1e\\r\\n111111111111111111111111111111\\r\\n\")\n\tbuf[0] = []byte(\"HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nTransfer-Encoding: chunked\\r\\n\\r\\n\")\n\tfor i := 1; i < 1000; i++ {\n\t\tbuf[i] = chunk\n\t}\n\tbuf[1001] = []byte(\"0\\r\\n\\r\\n\")\n\tpackets := make([]*Packet, len(buf))\n\tfor i := 0; i < len(buf); i++ {\n\t\tpackets[i] = GetPackets(false, 1, 1, buf[i])[0]\n\t}\n\n\tparser := NewMessageParser(nil, nil, nil, 2*time.Second, false)\n\tparser.Start = func(pckt *Packet) (bool, bool) {\n\t\treturn false, proto.HasResponseTitle(pckt.Payload)\n\t}\n\tparser.End = func(m *Message) bool {\n\t\treturn proto.HasFullPayload(m, m.PacketData()...)\n\t}\n\tb.ResetTimer()\n\tb.ReportMetric(float64(len(packets)), \"packets/op\")\n\tb.ReportMetric(float64(1000), \"chunks/op\")\n\tfor i := 0; i < b.N; i++ {\n\t\tfor j := range packets {\n\t\t\tparser.processPacket(packets[j])\n\t\t}\n\t\tparser.Read()\n\t}\n}\n\nfunc BenchmarkNewAndParsePacket(b *testing.B) {\n\tdata := append(generateHeader(true, 1024, 10), make([]byte, 10)...)\n\tb.ResetTimer()\n\tfor i := 0; i < b.N; i++ {\n\t\tParsePacket(data, int(layers.LinkTypeLoop), 4, &gopacket.CaptureInfo{}, true)\n\t}\n}\n" }, { "path": "k8s/README.md", "content": "# Native k8s integration\n\nAfter following steps below, you will be able to capture traffic inside k8s like this:\n\n```\ngor --input-raw k8s://namespace/deployment/app:80 --output-http http://replay.com\n```\n\nGoReplay will running as a daemonset (e.g. on each phisical k8s node. \nIt will also require giving required permission to have read access to K8s APIs, so it can dynamically filter traffic for a specific pods.\n\nSupported format for filtering required pods:\n\n```\nk8s://[namespace/]pod/[pod_name] - k8s://default/pod/nginx-7848d4b86f-5nxz8\nk8s://[namespace/]deployment/[deployment_name] - k8s://default/deployment/nginx\nk8s://[namespace/]daemonset/[daemonset_name] - k8s://default/daemonset/nginx\nk8s://[namespace/]labelSelector/[selector] - k8s://default/labelSelector/app=nginx\nk8s://[namespace/]fieldSelector/[selector] - k8s://default/fieldSelector/metadata.name=nginx-7848d4b86f-5nxz8\n```\n`namespace` is optional, omit to use all namespaces: `k8s://labelSelector/app=replay`\n\n## 1. Create a namespace\n`kubectl create namespace goreplay`\n\n## 2. Create the Kubernetes service account in the namespace:\n\n`kubectl create serviceaccount goreplay --namespace goreplay`\n\n## 3. Create Cluster Role which gives read-only access to the pods:\n`kubectl -n goreplay -f clusterrole.yaml apply`\n\n```yaml\nkind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: pod-reader\nrules:\n- apiGroups: [\"\"]\n resources: [\"pods\"]\n verbs: [\"get\", \"watch\", \"list\"]\n- apiGroups: [\"\"]\n resources: [\"deployments\"]\n verbs: [\"get\", \"watch\", \"list\"]\n- apiGroups: [\"\"]\n resources: [\"daemonset\"]\n verbs: [\"get\", \"watch\", \"list\"]\n```\n\n## 4. Attach role to goreplay service account\n`kubectl -n goreplay -f rolebinding.yaml apply`\n\n```yaml\nkind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: goreplay-reader-binding\nsubjects:\n- kind: ServiceAccount\n name: goreplay\n namespace: goreplay\nroleRef:\n kind: ClusterRole\n name: pod-reader\n apiGroup: \"\"\n```\n\n## 5. Start goreplay daemonset\n\n`kubectl -n goreplay -f goreplay.yaml apply`\n\nIn arguments, specify which service you want to capture. \nFollowing format supported:\n\n\n```yaml\napiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: goreplay-daemon\nspec:\n selector:\n matchLabels:\n app: goreplay\n template:\n metadata:\n labels:\n app: goreplay\n spec:\n hostNetwork: true\n serviceAccountName: goreplay\n containers:\n - name: goreplay\n image: buger/goreplay:2.0.0-rc4\n args:\n - \"--input-raw\"\n - \"k8s://deployments/nginx:80\"\n - \"--output-stdout\"\n```\n\n## 6. Create a simple http service (Optionally)\n\n`kubectl -n default -f nginx.yaml apply`\n\n```yaml\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: nginx\n labels:\n app: nginx\nspec:\n replicas: 1\n selector:\n matchLabels:\n app: nginx\n template:\n metadata:\n labels:\n app: nginx\n spec:\n containers:\n - name: nginx\n image: nginx\n ports:\n - containerPort: 80\n \n---\napiVersion: v1\nkind: Service\nmetadata:\n name: ngnix-service\nspec:\n selector:\n app: nginx\n type: NodePort\n ports:\n - protocol: TCP\n port: 80\n targetPort: 80\n\n```\n\n\n## 7. Verify installation, and debugging tips\n\nFind url for your service using `kubectl get svc` or `minikube service --url ngnix-service -n http`, and make a call to it.\n\nGet GoReplay logs, and check if it capture traffic of your service.\n`kubectl logs -n goreplay -l app=goreplay --all-containers`\n\nDescribe daemonset:\n`kubectl describe daemonset goreplay-daemon -n goreplay`\n\nGet GoReplay pod list:\n`kubectl get pods -n goreplay -l app=goreplay`\n\nGet logs for specific pod (take data from previous step):\n`kubectl logs goreplay-daemon- -n goreplay`\n\nGet related k8s events:\n`kubectl get events -n goreplay --field-selector involvedObject.name=goreplay-daemon-`\n\n## 8. Debuggin telemetry\n\nWe provde a special script which will get all require logs, and help you find installation errors. \n\n`curl -s https://raw.githubusercontent.com/buger/goreplay/refs/heads/master/k8s/collect_goreplay_telemetry.sh | bash`\n\nIf you are using microk8s or similar, you can also specific prefix for your kubectl command like this:\n\n`curl -s https://raw.githubusercontent.com/buger/goreplay/refs/heads/master/k8s/collect_goreplay_telemetry.sh | bash -s -- \"microk8s kubectl\"`\n" }, { "path": "k8s/clusterrole.yaml", "content": "kind: ClusterRole\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: pod-reader\nrules:\n- apiGroups: [\"\"]\n resources: [\"pods\"]\n verbs: [\"get\", \"watch\", \"list\"]\n- apiGroups: [\"apps\"]\n resources: [\"deployments\"]\n verbs: [\"get\", \"watch\", \"list\"]\n- apiGroups: [\"apps\"]\n resources: [\"daemonsets\"]\n verbs: [\"get\", \"watch\", \"list\"]" }, { "path": "k8s/collect_goreplay_telemetry.sh", "content": "#!/usr/bin/env bash\n#\n# collect_goreplay_telemetry.sh\n#\n# Gathers telemetry from a GoReplay DaemonSet in the 'goreplay' namespace.\n# Works on macOS and Linux, assuming 'kubectl' (or compatible) is installed.\n#\n# Usage examples:\n# ./collect_goreplay_telemetry.sh\n# ./collect_goreplay_telemetry.sh \"microk8s kubectl\"\n\nset -euo pipefail\n\n########################################\n# Determine kubectl command\n########################################\nif [[ $# -gt 0 ]]; then\n # If an argument was provided, use that as the kubectl command\n KUBECTL=\"$*\"\nelse\n # Default to 'kubectl'\n KUBECTL=\"kubectl\"\nfi\n\n########################################\n# Check that the base command exists\n########################################\n# For \"microk8s kubectl\", we only check \"microk8s\" in PATH. For \"oc\" we check \"oc\".\nBASE_CMD=\"${KUBECTL%% *}\" # everything before the first space\nif ! command -v \"${BASE_CMD}\" >/dev/null 2>&1; then\n echo \"ERROR: '${BASE_CMD}' not found in PATH. Please install or configure it first.\"\n exit 1\nfi\n\necho \"Using kubectl command: $KUBECTL\"\necho\n\n########################################\n# Helper function to print and run commands\n########################################\nrun_cmd() {\n echo \"Command: $*\"\n eval \"$*\"\n}\n\n########################################\n# 1. Print logs from ALL GoReplay pods\n########################################\necho \"==================================================\"\necho \"1. Gathering logs from all goreplay pods (all containers)...\"\necho \"==================================================\"\nrun_cmd \"$KUBECTL logs -n goreplay -l app=goreplay --all-containers\" || {\n echo \"WARNING: Failed to get logs from pods with label app=goreplay\"\n}\n\n########################################\n# 2. Describe the GoReplay DaemonSet\n########################################\necho\necho \"==================================================\"\necho \"2. Describing DaemonSet goreplay-daemon...\"\necho \"==================================================\"\nrun_cmd \"$KUBECTL describe daemonset goreplay-daemon -n goreplay\" || {\n echo \"WARNING: Failed to describe daemonset goreplay-daemon\"\n}\n\n########################################\n# 3. Get list of GoReplay pods (full output)\n########################################\necho\necho \"==================================================\"\necho \"3. Listing goreplay pods (full output)...\"\necho \"==================================================\"\n\n# Print full output (no -o name here):\nrun_cmd \"$KUBECTL get pods -n goreplay -l app=goreplay\"\n\n# Then retrieve just the names for further processing:\necho\necho \"Getting goreplay pod names for telemetry collection...\"\npods=$($KUBECTL get pods -n goreplay -l app=goreplay -o name 2>/dev/null) || {\n echo \"ERROR: Failed to list pods with label app=goreplay\"\n exit 1\n}\necho \"Found pods:\"\necho \"$pods\"\necho\n\n########################################\n# 4. For each pod, gather logs, describe, and get events\n########################################\nfor pod in $pods; do\n # pod looks like \"pod/goreplay-daemon-xyz\"\n pod_name=\"${pod##*/}\" # remove \"pod/\" prefix\n\n echo \"==================================================\"\n echo \"LOGS for pod: ${pod_name}\"\n echo \"==================================================\"\n run_cmd \"$KUBECTL logs ${pod_name} -n goreplay\" || {\n echo \"WARNING: Failed to get logs for pod ${pod_name}\"\n }\n\n echo\n echo \"--------------------------------------------------\"\n echo \"DESCRIBE for pod: ${pod_name}\"\n echo \"--------------------------------------------------\"\n run_cmd \"$KUBECTL describe pod -n goreplay ${pod_name}\" || {\n echo \"WARNING: Failed to describe pod ${pod_name}\"\n }\n\n echo\n echo \"--------------------------------------------------\"\n echo \"EVENTS for pod: ${pod_name}\"\n echo \"--------------------------------------------------\"\n run_cmd \"$KUBECTL get events -n goreplay --field-selector involvedObject.name=${pod_name}\" || {\n echo \"WARNING: Failed to get events for pod ${pod_name}\"\n }\n echo\ndone\n\necho \"==================================================\"\necho \"Telemetry collection complete.\"\necho \"==================================================\"\n" }, { "path": "k8s/goreplay.yaml", "content": "apiVersion: apps/v1\nkind: DaemonSet\nmetadata:\n name: goreplay-daemon\nspec:\n selector:\n matchLabels:\n app: goreplay\n template:\n metadata:\n labels:\n app: goreplay\n spec:\n hostNetwork: true\n serviceAccountName: goreplay\n containers:\n - name: goreplay\n image: buger/gor:v2.0.0-rc4\n args:\n - \"--input-raw k8s://deployments/nginx:80\"\n - \"--output-stdout\"\n - \"--verbose\"\n" }, { "path": "k8s/nginx.yaml", "content": "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: nginx\n labels:\n app: nginx\nspec:\n replicas: 1\n selector:\n matchLabels:\n app: nginx\n template:\n metadata:\n labels:\n app: nginx\n spec:\n containers:\n - name: nginx\n image: nginx\n ports:\n - containerPort: 80\n \n---\napiVersion: v1\nkind: Service\nmetadata:\n name: ngnix-service\nspec:\n selector:\n app: nginx\n type: NodePort\n ports:\n - protocol: TCP\n port: 80\n targetPort: 80\n " }, { "path": "k8s/rolebinding.yaml", "content": "kind: ClusterRoleBinding\napiVersion: rbac.authorization.k8s.io/v1\nmetadata:\n name: goreplay-reader-binding\nsubjects:\n- kind: ServiceAccount\n name: goreplay\n namespace: goreplay\nroleRef:\n kind: ClusterRole\n name: pod-reader\n apiGroup: \"\"\n" }, { "path": "kafka.go", "content": "package goreplay\n\nimport (\n\t\"bytes\"\n\t\"crypto/sha256\"\n\t\"crypto/sha512\"\n\t\"crypto/tls\"\n\t\"crypto/x509\"\n\t\"errors\"\n\t\"fmt\"\n\t\"github.com/buger/goreplay/proto\"\n\t\"io/ioutil\"\n\t\"log\"\n\n\t\"github.com/Shopify/sarama\"\n\t\"github.com/xdg-go/scram\"\n)\n\n// SASLKafkaConfig SASL configuration\ntype SASLKafkaConfig struct {\n\tUseSASL bool `json:\"input-kafka-use-sasl\"`\n\tMechanism string `json:\"input-kafka-mechanism\"`\n\tUsername string `json:\"input-kafka-username\"`\n\tPassword string `json:\"input-kafka-password\"`\n}\n\n// InputKafkaConfig should contains required information to\n// build producers.\ntype InputKafkaConfig struct {\n\tconsumer sarama.Consumer\n\tHost string `json:\"input-kafka-host\"`\n\tTopic string `json:\"input-kafka-topic\"`\n\tUseJSON bool `json:\"input-kafka-json-format\"`\n\tOffset string `json:\"input-kafka-offset\"`\n\tSASLConfig SASLKafkaConfig\n}\n\n// OutputKafkaConfig is the representation of kfka output configuration\ntype OutputKafkaConfig struct {\n\tproducer sarama.AsyncProducer\n\tHost string `json:\"output-kafka-host\"`\n\tTopic string `json:\"output-kafka-topic\"`\n\tUseJSON bool `json:\"output-kafka-json-format\"`\n\tSASLConfig SASLKafkaConfig\n}\n\n// KafkaTLSConfig should contains TLS certificates for connecting to secured Kafka clusters\ntype KafkaTLSConfig struct {\n\tCACert string `json:\"kafka-tls-ca-cert\"`\n\tClientCert string `json:\"kafka-tls-client-cert\"`\n\tClientKey string `json:\"kafka-tls-client-key\"`\n}\n\n// KafkaMessage should contains catched request information that should be\n// passed as Json to Apache Kafka.\ntype KafkaMessage struct {\n\tReqURL string `json:\"Req_URL\"`\n\tReqType string `json:\"Req_Type\"`\n\tReqID string `json:\"Req_ID\"`\n\tReqTs string `json:\"Req_Ts\"`\n\tReqMethod string `json:\"Req_Method\"`\n\tReqBody string `json:\"Req_Body,omitempty\"`\n\tReqHeaders map[string]string `json:\"Req_Headers,omitempty\"`\n}\n\n// NewTLSConfig loads TLS certificates\nfunc NewTLSConfig(clientCertFile, clientKeyFile, caCertFile string) (*tls.Config, error) {\n\ttlsConfig := tls.Config{}\n\n\tif clientCertFile != \"\" && clientKeyFile == \"\" {\n\t\treturn &tlsConfig, errors.New(\"Missing key of client certificate in kafka\")\n\t}\n\tif clientCertFile == \"\" && clientKeyFile != \"\" {\n\t\treturn &tlsConfig, errors.New(\"missing TLS client certificate in kafka\")\n\t}\n\t// Load client cert\n\tif (clientCertFile != \"\") && (clientKeyFile != \"\") {\n\t\tcert, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile)\n\t\tif err != nil {\n\t\t\treturn &tlsConfig, err\n\t\t}\n\t\ttlsConfig.Certificates = []tls.Certificate{cert}\n\t}\n\t// Load CA cert\n\tif caCertFile != \"\" {\n\t\tcaCert, err := ioutil.ReadFile(caCertFile)\n\t\tif err != nil {\n\t\t\treturn &tlsConfig, err\n\t\t}\n\t\tcaCertPool := x509.NewCertPool()\n\t\tcaCertPool.AppendCertsFromPEM(caCert)\n\t\ttlsConfig.RootCAs = caCertPool\n\t}\n\treturn &tlsConfig, nil\n}\n\n// NewKafkaConfig returns Kafka config with or without TLS\nfunc NewKafkaConfig(saslConfig *SASLKafkaConfig, tlsConfig *KafkaTLSConfig) *sarama.Config {\n\tconfig := sarama.NewConfig()\n\t// Configuration options go here\n\tif tlsConfig != nil && (tlsConfig.ClientCert != \"\" || tlsConfig.CACert != \"\") {\n\t\tconfig.Net.TLS.Enable = true\n\t\ttlsConfig, err := NewTLSConfig(tlsConfig.ClientCert, tlsConfig.ClientKey, tlsConfig.CACert)\n\t\tif err != nil {\n\t\t\tlog.Fatal(err)\n\t\t}\n\t\tconfig.Net.TLS.Config = tlsConfig\n\t}\n\tif saslConfig.UseSASL {\n\t\tmechanism := sarama.SASLMechanism(saslConfig.Mechanism)\n\t\tconfig.Net.SASL.Enable = saslConfig.UseSASL\n\t\tconfig.Net.SASL.Mechanism = mechanism\n\t\tconfig.Net.SASL.User = saslConfig.Username\n\t\tconfig.Net.SASL.Password = saslConfig.Password\n\t\tif mechanism == sarama.SASLTypeSCRAMSHA256 {\n\t\t\tconfig.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &XDGSCRAMClient{HashGeneratorFcn: SHA256} }\n\t\t} else if mechanism == sarama.SASLTypeSCRAMSHA512 {\n\t\t\tconfig.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &XDGSCRAMClient{HashGeneratorFcn: SHA512} }\n\t\t}\n\t}\n\treturn config\n}\n\n// Dump returns the given request in its HTTP/1.x wire\n// representation.\nfunc (m KafkaMessage) Dump() ([]byte, error) {\n\tvar b bytes.Buffer\n\n\tb.WriteString(fmt.Sprintf(\"%s %s %s\\n\", m.ReqType, m.ReqID, m.ReqTs))\n\tb.WriteString(fmt.Sprintf(\"%s %s HTTP/1.1\", m.ReqMethod, m.ReqURL))\n\tb.Write(proto.CRLF)\n\tfor key, value := range m.ReqHeaders {\n\t\tb.WriteString(fmt.Sprintf(\"%s: %s\", key, value))\n\t\tb.Write(proto.CRLF)\n\t}\n\n\tb.Write(proto.CRLF)\n\tb.WriteString(m.ReqBody)\n\n\treturn b.Bytes(), nil\n}\n\nvar (\n\t// SHA256 SASLMechanism\n\tSHA256 scram.HashGeneratorFcn = sha256.New\n\t// SHA512 SASLMechanism\n\tSHA512 scram.HashGeneratorFcn = sha512.New\n)\n\n// XDGSCRAMClient for SASL-Protocol\ntype XDGSCRAMClient struct {\n\t*scram.Client\n\t*scram.ClientConversation\n\tscram.HashGeneratorFcn\n}\n\n// Begin of XDGSCRAMClient\nfunc (x *XDGSCRAMClient) Begin(userName, password, authzID string) (err error) {\n\tx.Client, err = x.HashGeneratorFcn.NewClient(userName, password, authzID)\n\tif err != nil {\n\t\treturn err\n\t}\n\tx.ClientConversation = x.Client.NewConversation()\n\treturn nil\n}\n\n// Step of XDGSCRAMClient\nfunc (x *XDGSCRAMClient) Step(challenge string) (response string, err error) {\n\tresponse, err = x.ClientConversation.Step(challenge)\n\treturn\n}\n\n// Done of XDGSCRAMClient\nfunc (x *XDGSCRAMClient) Done() bool {\n\treturn x.ClientConversation.Done()\n}\n" }, { "path": "limiter.go", "content": "package goreplay\n\nimport (\n\t\"fmt\"\n\t\"io\"\n\t\"math/rand\"\n\t\"strconv\"\n\t\"strings\"\n\t\"time\"\n)\n\n// Limiter is a wrapper for input or output plugin which adds rate limiting\ntype Limiter struct {\n\tplugin interface{}\n\tlimit int\n\tisPercent bool\n\n\tcurrentRPS int\n\tcurrentTime int64\n}\n\nfunc parseLimitOptions(options string) (limit int, isPercent bool) {\n\tif n := strings.Index(options, \"%\"); n > 0 {\n\t\tlimit, _ = strconv.Atoi(options[:n])\n\t\tisPercent = true\n\t} else {\n\t\tlimit, _ = strconv.Atoi(options)\n\t\tisPercent = false\n\t}\n\n\treturn\n}\n\nfunc newLimiterExceptions(l *Limiter) {\n\n\tif !l.isPercent {\n\t\treturn\n\t}\n\tspeedFactor := float64(l.limit) / float64(100)\n\n\t// FileInput、KafkaInput have its own rate limiting. Unlike other inputs we not just dropping requests, we can slow down or speed up request emittion.\n\tswitch input := l.plugin.(type) {\n\tcase *FileInput:\n\t\tinput.speedFactor = speedFactor\n\tcase *KafkaInput:\n\t\tinput.speedFactor = speedFactor\n\t}\n}\n\n// NewLimiter constructor for Limiter, accepts plugin and options\n// `options` allow to sprcify relatve or absolute limiting\nfunc NewLimiter(plugin interface{}, options string) PluginReadWriter {\n\tl := new(Limiter)\n\tl.limit, l.isPercent = parseLimitOptions(options)\n\tl.plugin = plugin\n\tl.currentTime = time.Now().UnixNano()\n\n\tnewLimiterExceptions(l)\n\n\treturn l\n}\n\nfunc (l *Limiter) isLimitedExceptions() bool {\n\tif !l.isPercent {\n\t\treturn false\n\t}\n\t// Fileinput、Kafkainput have its own limiting algorithm\n\tswitch l.plugin.(type) {\n\tcase *FileInput:\n\t\treturn true\n\tcase *KafkaInput:\n\t\treturn true\n\tdefault:\n\t\treturn false\n\t}\n}\n\nfunc (l *Limiter) isLimited() bool {\n\tif l.isLimitedExceptions() {\n\t\treturn false\n\t}\n\n\tif l.isPercent {\n\t\treturn l.limit <= rand.Intn(100)\n\t}\n\n\tif (time.Now().UnixNano() - l.currentTime) > time.Second.Nanoseconds() {\n\t\tl.currentTime = time.Now().UnixNano()\n\t\tl.currentRPS = 0\n\t}\n\n\tif l.currentRPS >= l.limit {\n\t\treturn true\n\t}\n\n\tl.currentRPS++\n\n\treturn false\n}\n\n// PluginWrite writes message to this plugin\nfunc (l *Limiter) PluginWrite(msg *Message) (n int, err error) {\n\tif l.isLimited() {\n\t\treturn 0, nil\n\t}\n\tif w, ok := l.plugin.(PluginWriter); ok {\n\t\treturn w.PluginWrite(msg)\n\t}\n\t// avoid further writing\n\treturn 0, io.ErrClosedPipe\n}\n\n// PluginRead reads message from this plugin\nfunc (l *Limiter) PluginRead() (msg *Message, err error) {\n\tif r, ok := l.plugin.(PluginReader); ok {\n\t\tmsg, err = r.PluginRead()\n\t} else {\n\t\t// avoid further reading\n\t\treturn nil, io.ErrClosedPipe\n\t}\n\n\tif l.isLimited() {\n\t\treturn nil, nil\n\t}\n\n\treturn\n}\n\nfunc (l *Limiter) String() string {\n\treturn fmt.Sprintf(\"Limiting %s to: %d (isPercent: %v)\", l.plugin, l.limit, l.isPercent)\n}\n\n// Close closes the resources.\nfunc (l *Limiter) Close() error {\n\tif fi, ok := l.plugin.(io.Closer); ok {\n\t\tfi.Close()\n\t}\n\treturn nil\n}\n" }, { "path": "limiter_test.go", "content": "//go:build !race\n\npackage goreplay\n\nimport (\n\t\"sync\"\n\t\"testing\"\n)\n\nfunc TestOutputLimiter(t *testing.T) {\n\twg := new(sync.WaitGroup)\n\n\tinput := NewTestInput()\n\toutput := NewLimiter(NewTestOutput(func(*Message) {\n\t\twg.Done()\n\t}), \"10\")\n\twg.Add(10)\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output},\n\t}\n\tplugins.All = append(plugins.All, input, output)\n\n\temitter := NewEmitter()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\tfor i := 0; i < 100; i++ {\n\t\tinput.EmitGET()\n\t}\n\n\twg.Wait()\n\temitter.Close()\n}\n\nfunc TestInputLimiter(t *testing.T) {\n\twg := new(sync.WaitGroup)\n\n\tinput := NewLimiter(NewTestInput(), \"10\")\n\toutput := NewTestOutput(func(*Message) {\n\t\twg.Done()\n\t})\n\twg.Add(10)\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output},\n\t}\n\tplugins.All = append(plugins.All, input, output)\n\n\temitter := NewEmitter()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\tfor i := 0; i < 100; i++ {\n\t\tinput.(*Limiter).plugin.(*TestInput).EmitGET()\n\t}\n\n\twg.Wait()\n\temitter.Close()\n}\n\n// Should limit all requests\nfunc TestPercentLimiter1(t *testing.T) {\n\twg := new(sync.WaitGroup)\n\n\tinput := NewTestInput()\n\toutput := NewLimiter(NewTestOutput(func(*Message) {\n\t\twg.Done()\n\t}), \"0%\")\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output},\n\t}\n\tplugins.All = append(plugins.All, input, output)\n\n\temitter := NewEmitter()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\tfor i := 0; i < 100; i++ {\n\t\tinput.EmitGET()\n\t}\n\n\twg.Wait()\n}\n\n// Should not limit at all\nfunc TestPercentLimiter2(t *testing.T) {\n\twg := new(sync.WaitGroup)\n\n\tinput := NewTestInput()\n\toutput := NewLimiter(NewTestOutput(func(*Message) {\n\t\twg.Done()\n\t}), \"100%\")\n\twg.Add(100)\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output},\n\t}\n\tplugins.All = append(plugins.All, input, output)\n\n\temitter := NewEmitter()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\tfor i := 0; i < 100; i++ {\n\t\tinput.EmitGET()\n\t}\n\n\twg.Wait()\n}\n" }, { "path": "middleware/README.md", "content": "# GoReplay middleware\n\nGoReplay support protocol for writing middleware in any language, which allows you to implement custom logic like authentification or complex rewriting and filterting. See protocol description here: https://github.com/buger/goreplay/wiki/Middleware, but the basic idea that middleware process receive hex encoded data via STDIN and emits it back via STDOUT. STDERR for loggin inside middleware. Yes, that's simple.\n\nTo simplify middleware creation we provide packages for NodeJS and Go (upcoming).\n\nIf you want to get access to original and replayed responses, do not forget adding `--output-http-track-response` and `--input-raw-track-response` options.\n\n## NodeJS\n\nBefore starting, you should install the package via npm: `npm install goreplay_middleware`.\nAnd initialize middleware the following way:\n```javascript\nvar gor = require(\"goreplay_middleware\");\n// `init` will initialize STDIN listener\ngor.init();\n```\n\nBasic idea is that you write callbacks which respond to `request`, `response`, `replay`, or `message` events, which contain request meta information and actuall http paylod. Depending on your needs you may compare, override or filter incoming requests and responses.\n\nYou can respond to the incoming events using `on` function, by providing callbacks:\n```javascript\n// valid events are `request`, `response` (original response), `replay` (replayed response), and `message` (all events)\ngor.on('request', function(data) {\n // `data` contains incoming message its meta information.\n data\n\n // Raw HTTP payload of `Buffer` type\n // Example (hidden character for line endings shown on purpose):\n // GET / HTTP/1.1\\r\\n\n // User-Agent: Golang\\r\\n\n // \\r\\n\n data.http\n\n // Meta is an array size of 4, containing:\n // 1. request type - 1, 2 or 3 (which maps to `request`, `respose` and `replay`)\n // 2. uuid - request unique identifier. Request responses have the same ID as their request.\n // 3. timestamp of when request was made (for responses it is time of request start too)\n // 4. latency - time difference between request start and finish. For `request` is zero.\n data.meta\n\n // Unique request ID. It should be same for `request`, `response` and `replay` events of the same request.\n data.ID\n\n // You should return data at the end of function, even if you not changed request, if you do not want to filter it out.\n // If you just `return` nothing, request will be filtered\n return data\n})\n```\n### Mapping requests and responses\nYou can provide request ID as additional argument to `on` function, which allow you to map related requests and responses. Below is example of middleware which checks that original and replayed response have same HTTP status code.\n\n```javascript\n// Example of very basic way to compare if replayed traffic have no errors\ngor.on(\"request\", function(req) {\n gor.on(\"response\", req.ID, function(resp) {\n gor.on(\"replay\", req.ID, function(repl) {\n if (gor.httpStatus(resp.http) != gor.httpStatus(repl.http)) {\n // Note that STDERR is used for logging, and it actually will be send to `Gor` STDOUT.\n // This trick is used because SDTIN and STDOUT already used for process communication.\n // You can write logger that writes to files insead.\n console.error(`${gor.httpPath(req.http)} STATUS NOT MATCH: 'Expected ${gor.httpStatus(resp.http)}' got '${gor.httpStatus(repl.http)}'`)\n }\n return repl;\n })\n return resp;\n })\n return req\n})\n```\n\nThis middleware includes `searchResponses` helper which is used to compare value of original response with the replayed response. If authentication system or xsrf protection returns unique tokens in headers or the response, it will be helpful to rewrite your requests based on them. Because tokens are unique, and the value contained in original and replayed responses will be different. So, you need to extract value from both responses, and rewrite requests based on those mappings.\n\n`searchResponses` accepts request id, regexp pattern for searching the compared value (should include capture group), and callback which returns both original and replayed matched value.\n\nExample: \n```javascript\n // Compare HTTP headers for response and replayed response, and map values\nlet tokMap = {};\n\ngor.on(\"request\", function(req) {\n let tok = gor.httpHeader(req.http, \"Auth-Token\");\n if (tok && tokMap[tok]) {\n req.http = gor.setHttpHeader(req.http, \"Auth-Token\", tokMap[tok]) \n }\n \n gor.searchResponses(req.ID, \"X-Set-Token: (\\w+)$\", function(respTok, replTok) {\n if (respTok && replTok) tokMap[respTok] = replTok;\n })\n\n return req;\n})\n```\n\n\n### API documentation\n\nPackage expose following functions to process raw HTTP payloads:\n* `init` - initialize middleware object, start reading from STDIN.\n* `httpPath` - URL path of the request: `gor.httpPath(req.http)`\n* `httpMethod` - Http method: 'GET', 'POST', etc. `gor.httpMethod(req.http)`. \n* `setHttpPath` - update URL path: `req.http = gor.setHttpPath(req.http, newPath)`\n* `httpPathParam` - get param from URL path: `gor.httpPathParam(req.http, queryParam)`\n* `setHttpPathParam` - set URL param: `req.http = gor.setHttpPathParam(req.http, queryParam, value)` \n* `httpStatus` - response status code\n* `httpHeaders` - get all headers: `gor.httpHeaders(req.http)`\n* `httpHeader` - get HTTP header: `gor.httpHeader(req.http, \"Content-Length\")`\n* `setHttpHeader` - Set HTTP header, returns modified payload: `req.http = gor.setHttpHeader(req.http, \"X-Replayed\", \"1\")`\n* `httpBody` - get HTTP Body: `gor.httpBody(req.http)`\n* `setHttpBody` - Set HTTP Body and ensures that `Content-Length` header have proper value. Returns modified payload: `req.http = gor.setHttpBody(req.http, Buffer.from('hello!'))`.\n* `httpBodyParam` - get POST body param: `gor.httpBodyParam(req.http, param)`\n* `setHttpBodyParam` - set POST body param: `req.http = gor.setHttpBodyParam(req.http, param, value)`\n* `httpCookie` - get HTTP cookie: `gor.httpCookie(req.http, \"SESSSION_ID\")`\n* `setHttpCookie` - set HTTP cookie, returns modified payload: `req.http = gor.setHttpCookie(req.http, \"iam\", \"cuckoo\")`\n* `deleteHttpCookie` - delete HTTP cookie, returns modified payload: `req.http = gor.deleteHttpCookie(req.http, \"iam\")`\n\nAlso it is totally legit to use standard `Buffer` functions like `indexOf` for processing the HTTP payload. Just do not forget that if you modify the body, update the `Content-Length` header with a new value. And if you modify any of the headers, line endings should be `\\r\\n`. Rest is up to your imagination.\n\n\n## Masking PII Data\n\nThis middleware provides functionality to mask Personally Identifiable Information (PII) data in HTTP requests based on specified headers and JSON paths. It allows you to define a configuration object that specifies which headers and JSON fields should be masked and the type of data they represent.\n\n```javascript\nconst gor = require(\"goreplay_middleware\");\nconst faker = require(\"faker\");\n\n// Initialize the middleware\ngor.init();\n\n// Configuration for masking PII data\nconst maskConfig = {\n headers: [\n { name: \"Authorization\", type: \"token\" },\n { name: \"X-API-Key\", type: \"token\" },\n { name: \"X-User-Email\", type: \"email\" },\n { name: \"X-User-Name\", type: \"name\" },\n ],\n jsonPaths: [\n { path: \"$.user.email\", type: \"email\" },\n { path: \"$.user.name\", type: \"name\" },\n { path: \"$.user.phone\", type: \"phone\" },\n { path: \"$.user.address\", type: \"address\" },\n ],\n};\n\n// Function to mask a value based on its type\nfunction maskValue(type) {\n switch (type) {\n case \"email\":\n return faker.internet.email();\n case \"name\":\n return faker.name.findName();\n case \"phone\":\n return faker.phone.phoneNumber();\n case \"address\":\n return faker.address.streetAddress();\n case \"token\":\n return faker.random.alphaNumeric(32);\n default:\n return \"***\";\n }\n}\n\n// Middleware function to mask PII data\ngor.on(\"message\", (data) => {\n // Mask headers\n maskConfig.headers.forEach((header) => {\n const value = gor.httpHeader(data.http, header.name);\n if (value) {\n data.http = gor.setHttpHeader(data.http, header.name, maskValue(header.type));\n }\n });\n\n // Mask JSON fields\n const body = gor.httpBody(data.http);\n if (body) {\n try {\n const jsonBody = JSON.parse(body.toString());\n maskConfig.jsonPaths.forEach((field) => {\n const value = eval(`jsonBody${field.path.slice(1)}`);\n if (value) {\n eval(`jsonBody${field.path.slice(1)} = maskValue(field.type)`);\n }\n });\n data.http = gor.setHttpBody(data.http, Buffer.from(JSON.stringify(jsonBody)));\n } catch (error) {\n console.error(\"Error parsing JSON body:\", error);\n }\n }\n\n return data;\n});\n```\n\n### Configuration\n\nThe `maskConfig` object is used to configure the masking behavior. It consists of two properties:\n\n- `headers`: An array of objects representing the headers to be masked. Each object should have the following properties:\n - `name`: The name of the header.\n - `type`: The type of data the header represents (e.g., \"email\", \"name\", \"token\").\n\n- `jsonPaths`: An array of objects representing the JSON paths to be masked. Each object should have the following properties:\n - `path`: The JSON path to the field to be masked (e.g., \"$.user.email\").\n - `type`: The type of data the field represents (e.g., \"email\", \"name\", \"phone\", \"address\").\n\n### Masking Function\n\nThe `maskValue` function is responsible for generating masked values based on the data type. It uses the Faker library to generate realistic-looking masked data for different types such as email, name, phone, address, and token. You can extend this function to support additional data types or customize the masking behavior.\n\n### Middleware Function\n\nThe middleware function is triggered for each HTTP message (request or response) processed by GoReplay. It performs the following steps:\n\n1. Iterate over the specified headers in the `maskConfig` and mask their values using the `maskValue` function based on the associated data type.\n\n2. Parse the JSON body of the request (if present) and iterate over the specified JSON paths in the `maskConfig`. If a value exists at a given path, replace it with a masked value generated by the `maskValue` function based on the associated data type.\n\n3. Update the request body with the masked JSON data.\n\n4. Return the modified HTTP message.\n\nNote: The middleware uses the `eval` function to dynamically access and modify the JSON object based on the provided paths. Exercise caution when using `eval` and ensure that the paths are properly validated to prevent potential security risks.\n\nTo use this middleware, make sure to install the required dependencies (`goreplay_middleware` and `faker`), configure the `maskConfig` object according to your needs, and run the middleware with GoReplay.\n\n\n## Support\n\nFeel free to ask questions here and by sending email to [support@goreplay.org](mailto:support@goreplay.org). Commercial support is available and welcomed 🙈.\n" }, { "path": "middleware/middleware.js", "content": "// ======= GoReplay Middleware helper =============\n// Created by Leonid Bugaev in 2017\n//\n// For questions use GitHub or support@goreplay.org\n//\n// GoReplay: https://github.com/buger/goreplay\n// Middleware package: https://github.com/buger/goreplay/middleware\n\nvar middleware;\n\nfunction init() {\n var proxy = {\n ch: {},\n on: function(chan, id, cb) {\n if (!cb && id) {\n cb = id;\n } else if (cb && id) {\n chan = chan + \"#\" + id;\n }\n\n if (!proxy.ch[chan]) {\n proxy.ch[chan] = [];\n }\n\n proxy.ch[chan].push({\n created: new Date(),\n cb: cb\n });\n\n return proxy;\n },\n\n emit: function(msg, raw) {\n var chanPrefix;\n\n switch(msg.type) {\n case \"1\": chanPrefix = \"request\"; break;\n case \"2\": chanPrefix = \"response\"; break;\n case \"3\": chanPrefix = \"replay\"; break;\n }\n\n let resp = msg;\n\n [\"message\", chanPrefix, chanPrefix + \"#\" + msg.ID].forEach(function(chanID, idx){\n if (proxy.ch[chanID]) {\n proxy.ch[chanID].forEach(function(ch){\n let r = ch.cb(msg);\n if (resp) resp = r; // If one of callback decided not to send response back, do not override it in global callbacks\n })\n \n // Cleanup Individual message channels to avoid memory leaks\n if (idx == 2) {\n delete proxy.ch[chanID]\n }\n }\n })\n\n if (resp) {\n process.stdout.write(`${resp.rawMeta.toString('hex')}${Buffer.from(\"\\n\").toString(\"hex\")}${resp.http.toString('hex')}\\n`)\n }\n\n return resp\n }\n }\n\n // Clean up old messaged ID specific channels if they are older then 60s\n let gc = function(gcTime){\n let now = new Date();\n for (k in proxy.ch) {\n if (k.indexOf(\"#\") == -1) continue;\n\n proxy.ch[k] = proxy.ch[k].filter(function(ch){\n return (now - ch.created) < gcTime\n })\n\n if (proxy.ch[k].length == 0) {\n delete proxy.ch[k]\n }\n }\n }\n proxy.gc = gc\n\n setInterval(function(){\n gc(10 * 1000)\n }, 1000);\n\n const readline = require('readline');\n const rl = readline.createInterface({\n input: process.stdin\n });\n\n rl.on('line', function(line) {\n let msg = parseMessage(line)\n if (msg) {\n proxy.emit(msg, line)\n }\n });\n\n middleware = proxy;\n\n return proxy;\n}\n\n\nfunction parseMessage(msg) {\n try {\n let payload = Buffer.from(msg, \"hex\");\n let metaPos = payload.indexOf(\"\\n\");\n let meta = payload.slice(0, metaPos);\n let metaArr = meta.toString(\"ascii\").split(\" \");\n let pType = metaArr[0];\n let pID = metaArr[1];\n let raw = payload.slice(metaPos + 1, payload.length);\n\n return {\n type: pType,\n ID: pID,\n rawMeta: meta,\n meta: metaArr,\n http: raw\n }\n } catch(e) {\n fail(`Error while parsing incoming request: ${msg}`)\n }\n}\n\n// Used to compare values from original and replayed responses\n// Accepts request id, regexp pattern for searching the compared value (should include capture group), and callback which returns both original and replayed matched value.\n// Example: \n//\n// // Compare HTTP headers for response and replayed response, and map values\n// let tokMap = {};\n//\n// gor.on(\"request\", function(req) {\n// let tok = gor.httpHeader(req.http, \"Auth-Token\");\n// if (tok && tokMap[tok]) {\n// req.http = gor.setHttpHeader(req.http, \"Auth-Token\", tokMap[tok]) \n// }\n//\n// gor.searchResponses(req.ID, \"X-Set-Token: (\\w+)$\", function(respTok, replTok) {\n// tokMap[respTok] = replTok;\n// })\n//\n// return req;\n// })\n//\nfunction searchResponses(id, searchPattern, callback) {\n let re = new RegExp(searchPattern);\n\n // Using regexp require converting buffer to string\n // Before converting to string we can use initial `Buffer.indexOf` check\n let indexPattern = searchPattern.split(\"(\")[0];\n\n if (!indexPattern) {\n console.error(\"Search regexp should include capture group, pointing to the value: `prefix-(.*)`\")\n return\n }\n\n middleware.on(\"response\", id, function(resp){\n if (resp.http.indexOf(indexPattern) == -1) {\n callback()\n return resp\n }\n\n let respMatch = resp.http.toString('utf-8').match(re);\n if (!respMatch) {\n callback()\n return resp\n }\n\n middleware.on(\"replay\", id, function(repl) {\n if (repl.http.indexOf(indexPattern) == -1) {\n callback(respMatch[1]);\n return repl;\n }\n\n let replMatch = repl.http.toString('utf-8').match(re);\n\n if (!replMatch) {\n callback(respMatch[1]);\n return repl;\n }\n \n callback(respMatch[1], replMatch[1]);\n \n return repl;\n })\n\n return resp;\n })\n}\n\n\n// =========== HTTP parsing =================\n\n// Example HTTP payload record (including hidden characters):\n//\n// POST / HTTP/1.1\\r\\n\n// User-Agent: Node\\r\\n\n// Content-Length: 5\\r\\n\n// \\r\\n\n// hello\n\nfunction httpMethod(payload) {\n var pEnd = payload.indexOf(' ');\n return payload.slice(0, pEnd).toString(\"ascii\");\n}\n\nfunction httpPath(payload) {\n var pStart = payload.indexOf(' ') + 1;\n var pEnd = payload.indexOf(' ', pStart);\n return payload.slice(pStart, pEnd).toString(\"ascii\");\n}\n\nfunction setHttpPath(payload, newPath) {\n var pStart = payload.indexOf(' ') + 1;\n var pEnd = payload.indexOf(' ', pStart);\n\n return Buffer.concat([payload.slice(0, pStart), Buffer.from(newPath), payload.slice(pEnd, payload.length)])\n}\n\nfunction httpPathParam(payload, name) {\n let path = httpPath(payload);\n let re = new RegExp(name + \"=([^&$]+)\");\n let match = path.match(re);\n\n if (match) return decodeURI(match[1]);\n}\n\nfunction setHttpPathParam(payload, name, value) {\n let path = httpPath(payload);\n let re = new RegExp(name + \"=([^&$]+)\");\n let newPath = path.replace(re, name + \"=\" + encodeURI(value));\n \n // If we should add new param instead\n if (newPath == path) {\n if (newPath.indexOf(\"?\") == -1) {\n newPath += \"?\"\n } else {\n newPath += \"&\"\n }\n\n newPath += name + \"=\" + encodeURI(value);\n }\n\n return setHttpPath(payload, newPath)\n}\n\n// HTTP response have status code in same position as `path` for requests\nfunction httpStatus(payload) {\n return httpPath(payload);\n}\n\nfunction setHttpStatus(payload, newStatus) {\n return setHttpPath(payload, newStatus);\n}\n\nfunction httpHeaders(payload) {\n var httpHeaderString = payload.slice(0,payload.indexOf(\"\\r\\n\\r\\n\") + 4).toString().split(\"\\n\").slice(1);\n var headers = {};\n\n for (var item in httpHeaderString) {\n var parts = httpHeaderString[item].split(\":\");\n\n if (parts.length > 1) {\n headers[parts[0]] = parts.slice(1).join(\":\").trim(); \n }\n }\n\n return headers;\n}\n\nfunction httpHeader(payload, name) {\n var currentLine = 0;\n var i = 0;\n var header = { start: -1, end: -1, valueStart: -1 }\n var nameBuf = Buffer.from(name);\n var nameBufLower = Buffer.from(name.toLowerCase());\n\n while(c = payload[i]) {\n if (c == 13) { // new line \"\\n\"\n currentLine++;\n i++\n header.end = i\n\n if (currentLine > 0 && header.start > 0 && header.valueStart > 0) {\n if (nameBuf.compare(payload, header.start, header.valueStart - 1) == 0 ||\n nameBufLower.compare(payload, header.start, header.valueStart - 1) == 0) { // ensure that headers are not case sensitive\n header.value = payload.slice(header.valueStart, header.end - 1).toString(\"utf-8\").trim();\n header.name = payload.slice(header.start, header.valueStart - 1).toString(\"utf-8\");\n return header\n }\n }\n\n header.start = -1\n header.valueStart = -1\n continue;\n } else if (c == 10) { // \"\\r\"\n i++\n continue;\n } else if (c == 58) { // \":\" Header/value separator symbol\n if (header.valueStart == -1) {\n header.valueStart = i + 1;\n i++\n continue;\n }\n }\n\n if (header.start == -1) header.start = i;\n\n i++\n }\n\n return\n}\n\nfunction setHttpHeader(payload, name, value) {\n let header = httpHeader(payload, name);\n if (!header) {\n let headerStart = payload.indexOf(13) + 1;\n return Buffer.concat([payload.slice(0, headerStart + 1), Buffer.from(name + \": \" + value + \"\\r\\n\"), payload.slice(headerStart + 1, payload.length)])\n } else {\n return Buffer.concat([payload.slice(0, header.valueStart), Buffer.from(\" \" + value + \"\\r\\n\"), payload.slice(header.end + 1, payload.length)])\n }\n}\n\nfunction deleteHttpHeader(payload, name) {\n\tlet header = httpHeader(payload, name);\n\n if (header) {\n return Buffer.concat([payload.slice(0, header.start), payload.slice(header.end+1, payload.length)])\n }\n\n\treturn payload\n}\n\nfunction httpBody(payload) {\n let bodyIndex = payload.indexOf(\"\\r\\n\\r\\n\");\n if (-1 != bodyIndex){\n return payload.slice(bodyIndex + 4, payload.length); \n } else {\n return null;\n }\n}\n\nfunction setHttpBody(payload, newBody) {\n let p = setHttpHeader(payload, \"Content-Length\", newBody.length)\n let headerEnd = p.indexOf(\"\\r\\n\\r\\n\") + 4;\n return Buffer.concat([p.slice(0, headerEnd), newBody])\n}\n\nfunction httpBodyParam(payload, name) {\n let body = httpBody(payload);\n let re = new RegExp(name + \"=([^&$]+)\");\n if (body.indexOf(name + \"=\") != -1) {\n let param = body.toString('utf-8').match(re);\n if (param) {\n return decodeURI(param[1]);\n }\n }\n}\n\nfunction setHttpBodyParam(payload, name, value) {\n let body = httpBody(payload);\n let re = new RegExp(name + \"=([^&$]+)\");\n\n let newBody = body.toString('utf-8');\n\n if (newBody.indexOf(name + \"=\") != -1 ) {\n newBody = newBody.replace(re, name + \"=\" + encodeURI(value));\n } else {\n if (newBody.indexOf(\"=\") != -1) {\n newBody += \"&\";\n }\n newBody += name + \"=\" + value;\n }\n \n return setHttpBody(payload, Buffer.from(newBody));\n}\n\nfunction setHttpCookie(payload, name, value) {\n let h = httpHeader(payload, \"Cookie\");\n let cookie = h ? h.value : \"\";\n let cookies = cookie.split(\"; \").filter(function(v){ return v.indexOf(name + \"=\") != 0 })\n cookies.push(name + \"=\" + value)\n return setHttpHeader(payload, \"Cookie\", cookies.join(\"; \"))\n}\n\nfunction deleteHttpCookie(payload, name) {\n let h = httpHeader(payload, \"Cookie\");\n let cookie = h ? h.value : \"\";\n let cookies = cookie.split(\"; \").filter(function(v){ return v.indexOf(name + \"=\") != 0 })\n return setHttpHeader(payload, \"Cookie\", cookies.join(\"; \"))\n}\n\nfunction httpCookie(payload, name) {\n let h = httpHeader(payload, \"Cookie\");\n let cookie = h ? h.value : \"\";\n let value;\n let cookies = cookie.split(\"; \").forEach(function(v){\n if (v.indexOf(name + \"=\") == 0) {\n value = v.substr(name.length + 1);\n }\n })\n return value;\n}\n\nmodule.exports = {\n init: init,\n on: function(){ return middleware.on.apply(this, arguments) },\n parseMessage: parseMessage,\n searchResponses: searchResponses,\n httpPath: httpPath,\n httpMethod: httpMethod,\n setHttpPath: setHttpPath,\n httpPathParam: httpPathParam,\n setHttpPathParam: setHttpPathParam,\n httpStatus: httpStatus,\n setHttpStatus: setHttpStatus,\n httpHeader: httpHeader,\n setHttpHeader: setHttpHeader,\n deleteHttpHeader: deleteHttpHeader,\n httpBody: httpBody,\n setHttpBody: setHttpBody,\n httpBodyParam: httpBodyParam,\n setHttpBodyParam: setHttpBodyParam,\n httpCookie: httpCookie,\n setHttpCookie: setHttpCookie,\n deleteHttpCookie: deleteHttpCookie,\n test: testRunner,\n benchmark: testBenchmark,\n httpHeaders: httpHeaders\n}\n\n\n// =========== Tests ==============\n\nfunction testRunner(){\n [\"init\", \"filter\", \"parseMessage\", \"httpMethod\", \"httpPath\", \"setHttpHeader\", \"deleteHttpHeader\", \"httpPathParam\", \"httpHeader\", \"httpBody\", \"setHttpBody\", \"httpBodyParam\", \"httpCookie\", \"setHttpCookie\", \"deleteHttpCookie\", \"httpHeaders\"].forEach(function(t){\n console.log(`====== Start ${t} =======`)\n eval(`TEST_${t}()`)\n console.log(`====== End ${t} =======`)\n })\n}\n\nfunction testBenchmark(){\n const child_process = require('child_process');\n\n let gor = init();\n gor.on(\"message\", function(){\n });\n\n gor.on(\"request\", function(){\n });\n\n for (var i = 0; i<256; i++) {\n let req = parseMessage(Buffer.from(\"1 2 3\\nGET / HTTP/1.1\\r\\n\\r\\n\").toString('hex'));\n req.ID = +Date.now()\n gor.emit(req);\n\n gor.on(\"request\", req.ID+\"\", function(){\n gor.on(\"response\", req.ID+\"\", function(){\n })\n })\n\n if ( i % 3 == 0 ) {\n let resp = parseMessage(Buffer.from(\"2 2 3\\nHTTP/1.1 200 OK\\r\\n\\r\\n\").toString('hex'));\n resp.ID = req.ID\n gor.emit(resp);\n }\n }\n \n child_process.execSync(\"sleep 0.01\");\n\n gor.gc(1) \n \n fail(JSON.stringify(gor.ch))\n}\n\n// Just print in red color\nfunction fail(message) {\n console.error(\"\\x1b[31m[MIDDLEWARE] %s\\x1b[0m\", message)\n}\n\nfunction log(message) {\n console.error(message)\n}\n\nfunction TEST_init() {\n const child_process = require('child_process');\n\n let received = 0;\n let gor = init();\n gor.on(\"message\", function(){\n received++; // should be called 3 times for for every request\n });\n\n gor.on(\"request\", function(){\n received++; // should be called 1 time only for request\n });\n\n gor.on(\"response\", \"2\", function(){\n received++; // should be called 1 time only for specific response\n })\n\n if (Object.keys(gor.ch).length != 3) {\n return fail(\"Should create 3 channels\");\n }\n\n let req = parseMessage(Buffer.from(\"1 2 3\\nGET / HTTP/1.1\\r\\n\\r\\n\").toString('hex'));\n let resp = parseMessage(Buffer.from(\"2 2 3\\nHTTP/1.1 200 OK\\r\\n\\r\\n\").toString('hex'));\n let resp2 = parseMessage(Buffer.from(\"2 3 3\\nHTTP/1.1 200 OK\\r\\n\\r\\n\").toString('hex'));\n \n gor.emit(req);\n gor.emit(resp);\n gor.emit(resp2);\n\n child_process.execSync(\"sleep 0.01\");\n\n if (received != 5) {\n fail(`Should receive 5 messages: ${received}`);\n }\n}\n\nfunction TEST_filter() {\n const child_process = require('child_process');\n\n let gor = init();\n gor.on(\"request\", function(req){\n if (httpPath(req.http) != \"/filter\") {\n return req\n }\n });\n\n gor.on(\"request\", function(req){\n return req\n });\n\n\n let reqPass = parseMessage(Buffer.from(\"1 2 3\\nGET / HTTP/1.1\\r\\n\\r\\n\").toString('hex'));\n let reqFilter = parseMessage(Buffer.from(\"1 2 3\\nGET /filter HTTP/1.1\\r\\n\\r\\n\").toString('hex'));\n \n if (!gor.emit(reqPass)) {\n return fail(\"Should not filter request\")\n }\n\n if (gor.emit(reqFilter)) {\n return fail(\"Should filter request even if one middleware rejected it\")\n }\n\n}\n\nfunction TEST_parseMessage() {\n const exampleMessage = Buffer.from(\"1 2 3\\nGET / HTTP/1.1\\r\\n\\r\\n\").toString('hex')\n let msg = parseMessage(exampleMessage)\n let expected = { type: '1', ID: '2', meta: [\"1\", \"2\", \"3\"], http: Buffer.from(\"GET / HTTP/1.1\\r\\n\\r\\n\") }\n\n Object.keys(expected).forEach(function(k){\n if (msg[k].toString() != expected[k].toString()) {\n fail(`${k}: '${expected[k]}' != '${msg[k]}'`)\n }\n })\n}\n\nfunction TEST_httpPath() {\n const examplePayload = \"GET /test HTTP/1.1\\r\\n\\r\\n\";\n\n let payload = Buffer.from(examplePayload);\n let path = httpPath(payload);\n\n if (path != \"/test\") {\n return fail(`Path '${patj}' != '/test'`)\n }\n\n let newPayload = setHttpPath(payload, '/')\n if (newPayload.toString() != \"GET / HTTP/1.1\\r\\n\\r\\n\") {\n return fail(`Malformed payload '${newPayload}'`)\n }\n\n newPayload = setHttpPath(payload, '/bigger')\n if (newPayload.toString() != \"GET /bigger HTTP/1.1\\r\\n\\r\\n\") {\n return fail(`Malformed payload '${newPayload}'`)\n }\n}\n\nfunction TEST_httpMethod() {\n const examplePayload = \"GET /test HTTP/1.1\\r\\n\\r\\n\";\n\n let payload = Buffer.from(examplePayload);\n let method = httpMethod(payload);\n\n if (method != \"GET\") {\n return fail(`Path '${method}' != 'GET'`)\n }\n}\n\n\nfunction TEST_httpPathParam() {\n let p = Buffer.from(\"GET / HTTP/1.1\\r\\n\\r\\n\");\n\n if (httpPathParam(p, \"test\")) {\n return fail(\"Should not found param\")\n }\n\n p = setHttpPathParam(p, \"test\", \"123\");\n if (httpPath(p) != \"/?test=123\") {\n return fail(\"Should set first param: \" + httpPath(p));\n }\n\n if (httpPathParam(p, \"test\") != \"123\") {\n return fail(\"Should get first param: \" + httpPathParam(p, \"test\"));\n }\n\n p = setHttpPathParam(p, \"qwer\", \"ty\");\n if (httpPath(p) != \"/?test=123&qwer=ty\") {\n return fail(\"Should set second param: \" + httpPath(p));\n }\n\n p = setHttpPathParam(p, \"test\", \"4321\");\n if (httpPath(p) != \"/?test=4321&qwer=ty\") {\n return fail(\"Should update first param: \" + httpPath(p));\n }\n\n if (httpPathParam(p, \"test\") != \"4321\") {\n return fail(\"Should update first param: \" + httpPath(p));\n }\n}\n\nfunction TEST_httpBodyParam() {\n let p = Buffer.from(\"POST / HTTP/1.1\\r\\n\\r\\n\");\n\n if (httpBodyParam(p, \"test\")) {\n return fail(\"Should not found param\")\n }\n\n p = setHttpBodyParam(p, \"test\", \"123\");\n if (httpBody(p).toString() != \"test=123\") {\n return fail(\"Should set first param: \" + httpBody(p).toString());\n }\n\n if (httpBodyParam(p, \"test\") != \"123\") {\n return fail(\"Should get first param: \" + httpBodyParam(p, \"test\"));\n }\n\n p = setHttpBodyParam(p, \"qwer\", \"ty\");\n if (httpBody(p).toString() != \"test=123&qwer=ty\") {\n return fail(\"Should set second param: \" + httpBody(p).toString());\n }\n\n p = setHttpBodyParam(p, \"test\", \"4321\");\n if (httpBody(p).toString() != \"test=4321&qwer=ty\") {\n return fail(\"Should update first param: \" + httpBody(p).toString());\n }\n\n if (httpBodyParam(p, \"test\") != \"4321\") {\n return fail(\"Should update first param: \" + httpBody(p).toString());\n }\n}\n\nfunction TEST_httpHeader() {\n const examplePayload = \"GET / HTTP/1.1\\r\\nHost: localhost:3000\\r\\nUser-Agent: Node\\r\\nContent-Length:5\\r\\n\\r\\nhello\";\n\n let expected = {\"Host\": \"localhost:3000\", \"User-Agent\": \"Node\", \"Content-Length\": \"5\"}\n\n Object.keys(expected).forEach(function(name){\n let payload = Buffer.from(examplePayload);\n let header = httpHeader(payload, name);\n if (!header) {\n fail(`Header not found. Was looking for: ${name}`)\n }\n if (header && header.value != expected[name]) {\n fail(`${name}: '${expected[name]}' != '${header.value}'`)\n }\n })\n}\n\n\nfunction TEST_setHttpHeader() {\n const examplePayload = \"GET / HTTP/1.1\\r\\nUser-Agent: Node\\r\\nContent-Length: 5\\r\\n\\r\\nhello\";\n\n // Modify existing header\n [\"\", \"1\", \"Long test header\"].forEach(function(ua){\n let expected = `GET / HTTP/1.1\\r\\nUser-Agent: ${ua}\\r\\nContent-Length: 5\\r\\n\\r\\nhello`;\n let p = Buffer.from(examplePayload);\n p = setHttpHeader(p, \"User-Agent\", ua);\n if (p != expected) {\n console.error(`setHeader failed, expected User-Agent value: ${ua}.\\n${p}`)\n }\n })\n\n // Adding new header\n let expected = `GET / HTTP/1.1\\r\\nX-Test: test\\r\\nUser-Agent: Node\\r\\nContent-Length: 5\\r\\n\\r\\nhello`;\n let p = Buffer.from(examplePayload);\n p = setHttpHeader(p, \"X-Test\", \"test\");\n if (p != expected) {\n console.error(`setHeader failed, expected new header 'X-Test' header: ${p}`)\n }\n}\n\nfunction TEST_deleteHttpHeader() {\n const examplePayload = \"GET / HTTP/1.1\\r\\nUser-Agent: Node\\r\\nContent-Length: 5\\r\\n\\r\\nhello\";\n\n // Adding new header\n let expected = `GET / HTTP/1.1\\r\\nContent-Length: 5\\r\\n\\r\\nhello`;\n let p = Buffer.from(examplePayload);\n p = deleteHttpHeader(p, \"User-Agent\", \"test\");\n if (p != expected) {\n console.error(`setHeader failed, expected delete header 'User-Agent' header: ${p}`)\n }\n}\n\nfunction TEST_httpBody() {\n const examplePayload = \"GET / HTTP/1.1\\r\\nUser-Agent: Node\\r\\nContent-Length: 5\\r\\n\\r\\nhello\";\n let body = httpBody(Buffer.from(examplePayload));\n if (body != \"hello\") {\n fail(`'${body}' != 'hello'`)\n }\n\n const exampleInvalidPayload = \"Invalid HTTP Response by Network issue\";\n let invalidBody = httpBody(Buffer.from(exampleInvalidPayload));\n if (invalidBody != null) {\n fail(`'${invalidBody}' != 'null'`)\n }\n}\n\nfunction TEST_setHttpBody() {\n const examplePayload = \"GET / HTTP/1.1\\r\\nUser-Agent: Node\\r\\nContent-Length: 5\\r\\n\\r\\nhello\";\n let p = setHttpBody(Buffer.from(examplePayload), Buffer.from(\"hello, world!\"));\n\n if (p != \"GET / HTTP/1.1\\r\\nUser-Agent: Node\\r\\nContent-Length: 13\\r\\n\\r\\nhello, world!\") {\n fail(`Wrong body: '${p}'`)\n }\n}\n\nfunction TEST_httpCookie() {\n const examplePayload = \"GET / HTTP/1.1\\r\\nCookie: a=b; test=zxc\\r\\n\\r\\n\";\n let c = httpCookie(Buffer.from(examplePayload), \"test\");\n if (c != \"zxc\") {\n return fail(`Should get cookie: ${c}`);\n }\n\n c = httpCookie(Buffer.from(examplePayload), \"nope\");\n if (c != null) {\n return fail(`Should not find cookie: ${c}`);\n }\n}\n\nfunction TEST_setHttpCookie() {\n const examplePayload = \"GET / HTTP/1.1\\r\\nCookie: a=b; test=zxc\\r\\n\\r\\n\";\n let p = setHttpCookie(Buffer.from(examplePayload), \"test\", \"1\");\n if (p != \"GET / HTTP/1.1\\r\\nCookie: a=b; test=1\\r\\n\\r\\n\") {\n return fail(`Should update cookie: ${p}`)\n }\n\n p = setHttpCookie(Buffer.from(examplePayload), \"new\", \"one\");\n if (p != \"GET / HTTP/1.1\\r\\nCookie: a=b; test=zxc; new=one\\r\\n\\r\\n\") {\n return fail(`Should add new cookie: ${p}`)\n }\n}\n\nfunction TEST_deleteHttpCookie() {\n const examplePayload = \"GET / HTTP/1.1\\r\\nCookie: a=b; test=zxc\\r\\n\\r\\n\";\n let p = deleteHttpCookie(Buffer.from(examplePayload), \"a\");\n if (p != \"GET / HTTP/1.1\\r\\nCookie: test=zxc\\r\\n\\r\\n\") {\n return fail(`Should delete cookie: ${p}`)\n }\n}\n\n\nfunction TEST_httpHeaders() {\n const examplePayload = \"GET / HTTP/1.1\\r\\nHost: localhost:3000\\r\\nUser-Agent: Node\\r\\nContent-Length:5\\r\\n\\r\\nhello\";\n\n let expectedHeaders = {\"Host\": \"localhost:3000\", \"User-Agent\": \"Node\", \"Content-Length\": \"5\"}\n let payload = Buffer.from(examplePayload);\n let headers = httpHeaders(payload);\n\n [\"Host\", \"User-Agent\", \"Content-Length\"].forEach(function(header){\n let actual = headers[header];\n let expected = expectedHeaders[header];\n\n if (!actual) {\n fail(`${header} Header was not found`);\n }\n\n if (actual != expected) {\n fail(`${header} Header not Equal to Expected: ${expected} was ${actual}`);\n }\n\n })\n}\n" }, { "path": "middleware/package.json", "content": "{\n \"name\": \"goreplay_middleware\",\n \"version\": \"1.0.0\",\n \"description\": \"Package for writing middleware for GoReplay https://goreplay.org\",\n \"main\": \"middleware.js\",\n \"scripts\": {\n \"test\": \"node -e \\\"var gor = require('./middleware.js'); gor.test(); process.exit()\\\"\",\n \"benchmark\": \"node -e \\\"var gor = require('./middleware.js'); gor.benchmark(); process.exit()\\\"\"\n },\n \"keywords\": [\n \"middleware\",\n \"goreplay\"\n ],\n \"author\": \"Leonid Bugaev\",\n \"license\": \"LGPL-3.0\"\n}\n" }, { "path": "middleware.go", "content": "package goreplay\n\nimport (\n\t\"bufio\"\n\t\"context\"\n\t\"encoding/hex\"\n\t\"fmt\"\n\t\"io\"\n\t\"os\"\n\t\"os/exec\"\n\t\"strings\"\n\t\"sync\"\n\t\"syscall\"\n)\n\n// Middleware represents a middleware object\ntype Middleware struct {\n\tcommand string\n\tdata chan *Message\n\tStdin io.Writer\n\tStdout io.Reader\n\tcommandCancel context.CancelFunc\n\tstop chan bool // Channel used only to indicate goroutine should shutdown\n\tclosed bool\n\tmu sync.RWMutex\n}\n\n// NewMiddleware returns new middleware\nfunc NewMiddleware(command string) *Middleware {\n\tm := new(Middleware)\n\tm.command = command\n\tm.data = make(chan *Message, 1000)\n\tm.stop = make(chan bool)\n\n\tcommands := strings.Split(command, \" \")\n\tctx, cancl := context.WithCancel(context.Background())\n\tm.commandCancel = cancl\n\tcmd := exec.CommandContext(ctx, commands[0], commands[1:]...)\n\n\tm.Stdout, _ = cmd.StdoutPipe()\n\tm.Stdin, _ = cmd.StdinPipe()\n\n\tcmd.Stderr = os.Stderr\n\n\tgo m.read(m.Stdout)\n\n\tgo func() {\n\t\tdefer m.Close()\n\t\tvar err error\n\t\tif err = cmd.Start(); err == nil {\n\t\t\terr = cmd.Wait()\n\t\t}\n\t\tif err != nil {\n\t\t\tif e, ok := err.(*exec.ExitError); ok {\n\t\t\t\tstatus := e.Sys().(syscall.WaitStatus)\n\t\t\t\tif status.Signal() == syscall.SIGKILL /*killed or context canceld */ {\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t\tDebug(0, fmt.Sprintf(\"[MIDDLEWARE] command[%q] error: %q\", command, err.Error()))\n\t\t}\n\t}()\n\n\treturn m\n}\n\n// ReadFrom start a worker to read from this plugin\nfunc (m *Middleware) ReadFrom(plugin PluginReader) {\n\tDebug(2, fmt.Sprintf(\"[MIDDLEWARE] command[%q] Starting reading from %q\", m.command, plugin))\n\tgo m.copy(m.Stdin, plugin)\n}\n\nfunc (m *Middleware) copy(to io.Writer, from PluginReader) {\n\tvar buf, dst []byte\n\n\tfor {\n\t\tmsg, err := from.PluginRead()\n\t\tif err != nil {\n\t\t\treturn\n\t\t}\n\t\tif msg == nil || len(msg.Data) == 0 {\n\t\t\tcontinue\n\t\t}\n\t\tbuf = msg.Data\n\t\tif Settings.PrettifyHTTP {\n\t\t\tbuf = prettifyHTTP(msg.Data)\n\t\t}\n\t\tdstLen := (len(buf)+len(msg.Meta))*2 + 1\n\t\t// if enough space was previously allocated use it instead\n\t\tif dstLen > len(dst) {\n\t\t\tdst = make([]byte, dstLen)\n\t\t}\n\t\tn := hex.Encode(dst, msg.Meta)\n\t\tn += hex.Encode(dst[n:], buf)\n\t\tdst[n] = '\\n'\n\n\t\tn, err = to.Write(dst[:n+1])\n\t\tif err == nil {\n\t\t\tcontinue\n\t\t}\n\t\tif m.isClosed() {\n\t\t\treturn\n\t\t}\n\t}\n}\n\nfunc (m *Middleware) read(from io.Reader) {\n\treader := bufio.NewReader(from)\n\tvar line []byte\n\tvar e error\n\tfor {\n\t\tif line, e = reader.ReadBytes('\\n'); e != nil {\n\t\t\tif m.isClosed() {\n\t\t\t\treturn\n\t\t\t}\n\t\t\tcontinue\n\t\t}\n\t\tbuf := make([]byte, (len(line)-1)/2)\n\t\tif _, err := hex.Decode(buf, line[:len(line)-1]); err != nil {\n\t\t\tDebug(0, fmt.Sprintf(\"[MIDDLEWARE] command[%q] failed to decode err: %q\", m.command, err))\n\t\t\tcontinue\n\t\t}\n\t\tvar msg Message\n\t\tmsg.Meta, msg.Data = payloadMetaWithBody(buf)\n\t\tselect {\n\t\tcase <-m.stop:\n\t\t\treturn\n\t\tcase m.data <- &msg:\n\t\t}\n\t}\n\n}\n\n// PluginRead reads message from this plugin\nfunc (m *Middleware) PluginRead() (msg *Message, err error) {\n\tselect {\n\tcase <-m.stop:\n\t\treturn nil, ErrorStopped\n\tcase msg = <-m.data:\n\t}\n\n\treturn\n}\n\nfunc (m *Middleware) String() string {\n\treturn fmt.Sprintf(\"Modifying traffic using %q command\", m.command)\n}\n\nfunc (m *Middleware) isClosed() bool {\n\tm.mu.RLock()\n\tdefer m.mu.RUnlock()\n\treturn m.closed\n}\n\n// Close closes this plugin\nfunc (m *Middleware) Close() error {\n\tif m.isClosed() {\n\t\treturn nil\n\t}\n\tm.mu.Lock()\n\tdefer m.mu.Unlock()\n\tm.commandCancel()\n\tclose(m.stop)\n\tm.closed = true\n\treturn nil\n}\n" }, { "path": "middleware_test.go", "content": "package goreplay\n\nimport (\n\t\"bytes\"\n\t\"context\"\n\t\"os\"\n\t\"os/exec\"\n\t\"strings\"\n\t\"sync/atomic\"\n\t\"syscall\"\n\t\"testing\"\n)\n\nconst echoSh = \"./examples/middleware/echo.sh\"\nconst tokenModifier = \"go run ./examples/middleware/token_modifier.go\"\n\nvar withDebug = append(syscall.Environ(), \"GOR_TEST=1\")\n\nfunc initMiddleware(cmd *exec.Cmd, cancl context.CancelFunc, l PluginReader, c func(error)) *Middleware {\n\tvar m Middleware\n\tm.data = make(chan *Message, 1000)\n\tm.stop = make(chan bool)\n\tm.commandCancel = cancl\n\tm.Stdout, _ = cmd.StdoutPipe()\n\tm.Stdin, _ = cmd.StdinPipe()\n\tcmd.Stderr = os.Stderr\n\tgo m.read(m.Stdout)\n\tgo func() {\n\t\tdefer m.Close()\n\t\tvar err error\n\t\tif err = cmd.Start(); err == nil {\n\t\t\terr = cmd.Wait()\n\t\t}\n\t\tif err != nil {\n\t\t\tc(err)\n\t\t}\n\t}()\n\tm.ReadFrom(l)\n\treturn &m\n}\n\nfunc initCmd(command string, env []string) (*exec.Cmd, context.CancelFunc) {\n\tcommands := strings.Split(command, \" \")\n\tctx, cancl := context.WithCancel(context.Background())\n\tcmd := exec.CommandContext(ctx, commands[0], commands[1:]...)\n\tcmd.Env = env\n\treturn cmd, cancl\n}\n\nfunc TestMiddlewareEarlyClose(t *testing.T) {\n\tquit := make(chan struct{})\n\tin := NewTestInput()\n\tcmd, cancl := initCmd(echoSh, withDebug)\n\tmidd := initMiddleware(cmd, cancl, in, func(err error) {\n\t\tif err != nil {\n\t\t\tif e, ok := err.(*exec.ExitError); ok {\n\t\t\t\tstatus := e.Sys().(syscall.WaitStatus)\n\t\t\t\tif status.Signal() != syscall.SIGKILL {\n\t\t\t\t\tt.Errorf(\"expected error to be signal killed. got %s\", status.Signal().String())\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tquit <- struct{}{}\n\t})\n\tvar body = []byte(\"OPTIONS / HTTP/1.1\\r\\nHost: example.org\\r\\n\\r\\n\")\n\tcount := uint32(0)\n\tout := NewTestOutput(func(msg *Message) {\n\t\tif !bytes.Equal(body, msg.Data) {\n\t\t\tt.Errorf(\"expected %q to equal %q\", body, msg.Data)\n\t\t}\n\t\tatomic.AddUint32(&count, 1)\n\t\tif atomic.LoadUint32(&count) == 5 {\n\t\t\tquit <- struct{}{}\n\t\t}\n\t})\n\tpl := &InOutPlugins{}\n\tpl.Inputs = []PluginReader{midd, in}\n\tpl.Outputs = []PluginWriter{out}\n\tpl.All = []interface{}{midd, out, in}\n\te := NewEmitter()\n\tgo e.Start(pl, \"\")\n\tfor i := 0; i < 5; i++ {\n\t\tin.EmitBytes(body)\n\t}\n\t<-quit\n\tmidd.Close()\n\t<-quit\n}\n\n//func TestTokenMiddleware(t *testing.T) {\n//\tquit := make(chan struct{})\n//\tin := NewTestInput()\n//\tin.skipHeader = true\n//\tcmd, cancl := initCmd(tokenModifier, withDebug)\n//\tmidd := initMiddleware(cmd, cancl, in, func(err error) {})\n//\treq := []byte(\"1 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\\nGET /token HTTP/1.1\\r\\nHost: example.org\\r\\n\\r\\n\")\n//\tres := []byte(\"2 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\\nHTTP/1.1 200 OK\\r\\nContent-Length: 10\\r\\nContent-Type: text/plain; charset=utf-8\\r\\n\\r\\n17d823647c\")\n//\trep := []byte(\"3 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\\nHTTP/1.1 200 OK\\r\\nContent-Length: 15\\r\\nContent-Type: text/plain; charset=utf-8\\r\\n\\r\\n932079936fa4306\")\n//\tcount := uint32(0)\n//\tout := NewTestOutput(func(msg *Message) {\n//\t\tif msg.Meta[0] == '1' && !bytes.Equal(payloadID(msg.Meta), payloadID(req)) {\n//\t\t\ttoken, _, _ := proto.PathParam(msg.Data, []byte(\"token\"))\n//\t\t\tif !bytes.Equal(token, proto.Body(rep)) {\n//\t\t\t\tt.Errorf(\"expected the token %s to be equal to the replayed response's token %s\", token, proto.Body(rep))\n//\t\t\t}\n//\t\t}\n//\t\tatomic.AddUint32(&count, 1)\n//\t\tif atomic.LoadUint32(&count) == 2 {\n//\t\t\tquit <- struct{}{}\n//\t\t}\n//\t})\n//\tpl := &InOutPlugins{}\n//\tpl.Inputs = []PluginReader{midd, in}\n//\tpl.Outputs = []PluginWriter{out}\n//\tpl.All = []interface{}{midd, out, in}\n//\te := NewEmitter()\n//\tgo e.Start(pl, \"\")\n//\tin.EmitBytes(req) // emit original request\n//\tin.EmitBytes(res) // emit its response\n//\tin.EmitBytes(rep) // emit replayed response\n//\t// emit the request which should have modified token\n//\ttoken := []byte(\"1 8e091765ae902fef8a2b7d9dd96 14398188235873 100\\nGET /?token=17d823647c HTTP/1.1\\r\\nHost: example.org\\r\\n\\r\\n\")\n//\tin.EmitBytes(token)\n//\t<-quit\n//\tmidd.Close()\n//}\n\n//func TestMiddlewareWithPrettify(t *testing.T) {\n//\tSettings.PrettifyHTTP = true\n//\tquit := make(chan struct{})\n//\tin := NewTestInput()\n//\tcmd, cancl := initCmd(echoSh, withDebug)\n//\tmidd := initMiddleware(cmd, cancl, in, func(err error) {})\n//\tvar b1 = []byte(\"POST / HTTP/1.1\\r\\nHost: example.org\\r\\nTransfer-Encoding: chunked\\r\\n\\r\\n4\\r\\nWiki\\r\\n5\\r\\npedia\\r\\nE\\r\\n in\\r\\n\\r\\nchunks.\\r\\n0\\r\\n\\r\\n\")\n//\tvar b2 = []byte(\"POST / HTTP/1.1\\r\\nHost: example.org\\r\\nContent-Length: 25\\r\\n\\r\\nWikipedia in\\r\\n\\r\\nchunks.\")\n//\tout := NewTestOutput(func(msg *Message) {\n//\t\tif !bytes.Equal(proto.Body(b2), proto.Body(msg.Data)) {\n//\t\t\tt.Errorf(\"expected %q body to equal %q body\", b2, msg.Data)\n//\t\t}\n//\t\tquit <- struct{}{}\n//\t})\n//\tpl := &InOutPlugins{}\n//\tpl.Inputs = []PluginReader{midd, in}\n//\tpl.Outputs = []PluginWriter{out}\n//\tpl.All = []interface{}{midd, out, in}\n//\te := NewEmitter()\n//\tgo e.Start(pl, \"\")\n//\tin.EmitBytes(b1)\n//\t<-quit\n//\tmidd.Close()\n//\tSettings.PrettifyHTTP = false\n//}\n" }, { "path": "mkdocs.yml", "content": "site_name: My Docs\ntheme: readthedocs\n" }, { "path": "nfpm.yaml", "content": "# nfpm example config file\n#\n# check https://nfpm.goreleaser.com/configuration for detailed usage\n#\nname: \"GoReplay\"\narch: ${PLATFORM}\nplatform: \"linux\"\nversion: ${VERSION}\nsection: \"default\"\npriority: \"extra\"\nprovides:\n- goreplay\nmaintainer: \"Leonid Bugaev \"\ndescription: |\n GoReplay is the simplest and safest way to test your app using real traffic before you put it into production. \nvendor: \"GoReplay\"\nhomepage: \"https://goreplay.org\"\nlicense: \"AGPL\"\ncontents:\n- src: ./${BIN_NAME}\n dst: /usr/local/bin\n" }, { "path": "output_binary.go", "content": "//go:build !pro\n\npackage goreplay\n\nimport (\n\t\"errors\"\n\t\"time\"\n\n\t\"github.com/buger/goreplay/internal/size\"\n)\n\nvar _ PluginWriter = (*BinaryOutput)(nil)\n\n// BinaryOutputConfig struct for holding binary output configuration\ntype BinaryOutputConfig struct {\n\tWorkers int `json:\"output-binary-workers\"`\n\tTimeout time.Duration `json:\"output-binary-timeout\"`\n\tBufferSize size.Size `json:\"output-tcp-response-buffer\"`\n\tDebug bool `json:\"output-binary-debug\"`\n\tTrackResponses bool `json:\"output-binary-track-response\"`\n}\n\n// BinaryOutput plugin manage pool of workers which send request to replayed server\n// By default workers pool is dynamic and starts with 10 workers\n// You can specify fixed number of workers using `--output-tcp-workers`\ntype BinaryOutput struct {\n\taddress string\n}\n\n// NewBinaryOutput constructor for BinaryOutput\n// Initialize workers\nfunc NewBinaryOutput(address string, config *BinaryOutputConfig) PluginReadWriter {\n\treturn &BinaryOutput{address: address}\n}\n\n// PluginWrite writes a message to this plugin\nfunc (o *BinaryOutput) PluginWrite(msg *Message) (n int, err error) {\n\treturn 0, errors.New(\"binary output is only available in PRO version\")\n}\n\n// PluginRead reads a message from this plugin\nfunc (o *BinaryOutput) PluginRead() (*Message, error) {\n\treturn nil, errors.New(\"binary output is only available in PRO version\")\n}\n\nfunc (o *BinaryOutput) String() string {\n\treturn \"Binary output: \" + o.address + \" (PRO version required)\"\n}\n\n// Close closes this plugin for reading\nfunc (o *BinaryOutput) Close() error {\n\treturn nil\n}\n" }, { "path": "output_binary_pro.go", "content": "//go:build pro\n\npackage goreplay\n\nimport (\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/buger/goreplay/internal/size\"\n)\n\nvar _ PluginWriter = (*BinaryOutput)(nil)\n\n// BinaryOutputConfig struct for holding binary output configuration\ntype BinaryOutputConfig struct {\n\tWorkers int `json:\"output-binary-workers\"`\n\tTimeout time.Duration `json:\"output-binary-timeout\"`\n\tBufferSize size.Size `json:\"output-tcp-response-buffer\"`\n\tDebug bool `json:\"output-binary-debug\"`\n\tTrackResponses bool `json:\"output-binary-track-response\"`\n}\n\n// BinaryOutput plugin manage pool of workers which send request to replayed server\n// By default workers pool is dynamic and starts with 10 workers\n// You can specify fixed number of workers using `--output-tcp-workers`\ntype BinaryOutput struct {\n\t// Keep this as first element of struct because it guarantees 64bit\n\t// alignment. atomic.* functions crash on 32bit machines if operand is not\n\t// aligned at 64bit. See https://github.com/golang/go/issues/599\n\tactiveWorkers int64\n\taddress string\n\tqueue chan *Message\n\tresponses chan response\n\tneedWorker chan int\n\tquit chan struct{}\n\tconfig *BinaryOutputConfig\n\tqueueStats *GorStat\n}\n\n// NewBinaryOutput constructor for BinaryOutput\n// Initialize workers\nfunc NewBinaryOutput(address string, config *BinaryOutputConfig) PluginReadWriter {\n\to := new(BinaryOutput)\n\n\to.address = address\n\to.config = config\n\n\to.queue = make(chan *Message, 1000)\n\to.responses = make(chan response, 1000)\n\to.needWorker = make(chan int, 1)\n\to.quit = make(chan struct{})\n\n\t// Initial workers count\n\tif o.config.Workers == 0 {\n\t\to.needWorker <- initialDynamicWorkers\n\t} else {\n\t\to.needWorker <- o.config.Workers\n\t}\n\n\tgo o.workerMaster()\n\n\treturn o\n}\n\nfunc (o *BinaryOutput) workerMaster() {\n\tfor {\n\t\tnewWorkers := <-o.needWorker\n\t\tfor i := 0; i < newWorkers; i++ {\n\t\t\tgo o.startWorker()\n\t\t}\n\n\t\t// Disable dynamic scaling if workers poll fixed size\n\t\tif o.config.Workers != 0 {\n\t\t\treturn\n\t\t}\n\t}\n}\n\nfunc (o *BinaryOutput) startWorker() {\n\tclient := NewTCPClient(o.address, &TCPClientConfig{\n\t\tDebug: o.config.Debug,\n\t\tTimeout: o.config.Timeout,\n\t\tResponseBufferSize: int(o.config.BufferSize),\n\t})\n\n\tdeathCount := 0\n\n\tatomic.AddInt64(&o.activeWorkers, 1)\n\n\tfor {\n\t\tselect {\n\t\tcase msg := <-o.queue:\n\t\t\to.sendRequest(client, msg)\n\t\t\tdeathCount = 0\n\t\tcase <-time.After(time.Millisecond * 100):\n\t\t\t// When dynamic scaling enabled workers die after 2s of inactivity\n\t\t\tif o.config.Workers == 0 {\n\t\t\t\tdeathCount++\n\t\t\t} else {\n\t\t\t\tcontinue\n\t\t\t}\n\n\t\t\tif deathCount > 20 {\n\t\t\t\tworkersCount := atomic.LoadInt64(&o.activeWorkers)\n\n\t\t\t\t// At least 1 startWorker should be alive\n\t\t\t\tif workersCount != 1 {\n\t\t\t\t\tatomic.AddInt64(&o.activeWorkers, -1)\n\t\t\t\t\treturn\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\n// PluginWrite writes a message tothis plugin\nfunc (o *BinaryOutput) PluginWrite(msg *Message) (n int, err error) {\n\tif !isRequestPayload(msg.Meta) {\n\t\treturn len(msg.Data), nil\n\t}\n\n\to.queue <- msg\n\n\tif o.config.Workers == 0 {\n\t\tworkersCount := atomic.LoadInt64(&o.activeWorkers)\n\n\t\tif len(o.queue) > int(workersCount) {\n\t\t\to.needWorker <- len(o.queue)\n\t\t}\n\t}\n\n\treturn len(msg.Data) + len(msg.Meta), nil\n}\n\n// PluginRead reads a message from this plugin\nfunc (o *BinaryOutput) PluginRead() (*Message, error) {\n\tvar resp response\n\tvar msg Message\n\tselect {\n\tcase <-o.quit:\n\t\treturn nil, ErrorStopped\n\tcase resp = <-o.responses:\n\t}\n\tmsg.Data = resp.payload\n\tmsg.Meta = payloadHeader(ReplayedResponsePayload, resp.uuid, resp.startedAt, resp.roundTripTime)\n\n\treturn &msg, nil\n}\n\nfunc (o *BinaryOutput) sendRequest(client *TCPClient, msg *Message) {\n\tif !isRequestPayload(msg.Meta) {\n\t\treturn\n\t}\n\n\tuuid := payloadID(msg.Meta)\n\n\tstart := time.Now()\n\tresp, err := client.Send(msg.Data)\n\tstop := time.Now()\n\n\tif err != nil {\n\t\tDebug(1, \"Request error:\", err)\n\t}\n\n\tif o.config.TrackResponses {\n\t\to.responses <- response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}\n\t}\n}\n\nfunc (o *BinaryOutput) String() string {\n\treturn \"Binary output: \" + o.address\n}\n\n// Close closes this plugin for reading\nfunc (o *BinaryOutput) Close() error {\n\tclose(o.quit)\n\treturn nil\n}\n" }, { "path": "output_dummy.go", "content": "package goreplay\n\nimport (\n\t\"os\"\n)\n\n// DummyOutput used for debugging, prints all incoming requests\ntype DummyOutput struct {\n}\n\n// NewDummyOutput constructor for DummyOutput\nfunc NewDummyOutput() (di *DummyOutput) {\n\tdi = new(DummyOutput)\n\n\treturn\n}\n\n// PluginWrite writes message to this plugin\nfunc (i *DummyOutput) PluginWrite(msg *Message) (int, error) {\n\tvar n, nn int\n\tvar err error\n\tn, err = os.Stdout.Write(msg.Meta)\n\tnn, err = os.Stdout.Write(msg.Data)\n\tn += nn\n\tnn, err = os.Stdout.Write(payloadSeparatorAsBytes)\n\tn += nn\n\n\treturn n, err\n}\n\nfunc (i *DummyOutput) String() string {\n\treturn \"Dummy Output\"\n}\n" }, { "path": "output_file.go", "content": "package goreplay\n\nimport (\n\t\"bufio\"\n\t\"compress/gzip\"\n\t\"errors\"\n\t\"fmt\"\n\t\"github.com/buger/goreplay/internal/size\"\n\t\"io\"\n\t\"log\"\n\t\"math/rand\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"runtime/debug\"\n\t\"sort\"\n\t\"strconv\"\n\t\"strings\"\n\t\"sync\"\n\t\"time\"\n)\n\nvar letters = []rune(\"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\")\nvar instanceID string\n\nfunc init() {\n\tinstanceID = randSeq(8)\n}\n\nfunc randSeq(n int) string {\n\tb := make([]rune, n)\n\tfor i := range b {\n\t\tb[i] = letters[rand.Intn(len(letters))]\n\t}\n\treturn string(b)\n}\n\nvar dateFileNameFuncs = map[string]func(*FileOutput) string{\n\t\"%Y\": func(o *FileOutput) string { return time.Now().Format(\"2006\") },\n\t\"%m\": func(o *FileOutput) string { return time.Now().Format(\"01\") },\n\t\"%d\": func(o *FileOutput) string { return time.Now().Format(\"02\") },\n\t\"%H\": func(o *FileOutput) string { return time.Now().Format(\"15\") },\n\t\"%M\": func(o *FileOutput) string { return time.Now().Format(\"04\") },\n\t\"%S\": func(o *FileOutput) string { return time.Now().Format(\"05\") },\n\t\"%NS\": func(o *FileOutput) string { return fmt.Sprint(time.Now().Nanosecond()) },\n\t\"%r\": func(o *FileOutput) string { return string(o.currentID) },\n\t\"%t\": func(o *FileOutput) string { return string(o.payloadType) },\n\t\"%i\": func(o *FileOutput) string { return instanceID },\n}\n\n// FileOutputConfig ...\ntype FileOutputConfig struct {\n\tFlushInterval time.Duration `json:\"output-file-flush-interval\"`\n\tSizeLimit size.Size `json:\"output-file-size-limit\"`\n\tOutputFileMaxSize size.Size `json:\"output-file-max-size-limit\"`\n\tQueueLimit int `json:\"output-file-queue-limit\"`\n\tAppend bool `json:\"output-file-append\"`\n\tBufferPath string `json:\"output-file-buffer\"`\n\tonClose func(string)\n}\n\n// FileOutput output plugin\ntype FileOutput struct {\n\tsync.RWMutex\n\tpathTemplate string\n\tcurrentName string\n\tfile *os.File\n\tQueueLength int\n\twriter io.Writer\n\trequestPerFile bool\n\tcurrentID []byte\n\tpayloadType []byte\n\tclosed bool\n\tcurrentFileSize int\n\ttotalFileSize size.Size\n\n\tconfig *FileOutputConfig\n}\n\n// NewFileOutput constructor for FileOutput, accepts path\nfunc NewFileOutput(pathTemplate string, config *FileOutputConfig) *FileOutput {\n\to := new(FileOutput)\n\to.pathTemplate = pathTemplate\n\to.config = config\n\n\tif strings.Contains(pathTemplate, \"%r\") {\n\t\to.requestPerFile = true\n\t}\n\n\tif config.FlushInterval == 0 {\n\t\tconfig.FlushInterval = 100 * time.Millisecond\n\t}\n\n\tgo func() {\n\t\tfor {\n\t\t\ttime.Sleep(config.FlushInterval)\n\t\t\tif o.IsClosed() {\n\t\t\t\tbreak\n\t\t\t}\n\t\t\to.flush()\n\t\t}\n\t}()\n\n\treturn o\n}\n\nfunc getFileIndex(name string) int {\n\text := filepath.Ext(name)\n\twithoutExt := strings.TrimSuffix(name, ext)\n\n\tif idx := strings.LastIndex(withoutExt, \"_\"); idx != -1 {\n\t\tif i, err := strconv.Atoi(withoutExt[idx+1:]); err == nil {\n\t\t\treturn i\n\t\t}\n\t}\n\n\treturn -1\n}\n\nfunc setFileIndex(name string, idx int) string {\n\tidxS := strconv.Itoa(idx)\n\text := filepath.Ext(name)\n\twithoutExt := strings.TrimSuffix(name, ext)\n\n\tif i := strings.LastIndex(withoutExt, \"_\"); i != -1 {\n\t\tif _, err := strconv.Atoi(withoutExt[i+1:]); err == nil {\n\t\t\twithoutExt = withoutExt[:i]\n\t\t}\n\t}\n\n\treturn withoutExt + \"_\" + idxS + ext\n}\n\nfunc withoutIndex(s string) string {\n\tif i := strings.LastIndex(s, \"_\"); i != -1 {\n\t\treturn s[:i]\n\t}\n\n\treturn s\n}\n\ntype sortByFileIndex []string\n\nfunc (s sortByFileIndex) Len() int {\n\treturn len(s)\n}\n\nfunc (s sortByFileIndex) Swap(i, j int) {\n\ts[i], s[j] = s[j], s[i]\n}\n\nfunc (s sortByFileIndex) Less(i, j int) bool {\n\tif withoutIndex(s[i]) == withoutIndex(s[j]) {\n\t\treturn getFileIndex(s[i]) < getFileIndex(s[j])\n\t}\n\n\treturn s[i] < s[j]\n}\n\nfunc (o *FileOutput) filename() string {\n\to.RLock()\n\tdefer o.RUnlock()\n\n\tpath := o.pathTemplate\n\n\tfor name, fn := range dateFileNameFuncs {\n\t\tpath = strings.Replace(path, name, fn(o), -1)\n\t}\n\n\tif !o.config.Append {\n\t\tnextChunk := false\n\n\t\tif o.currentName == \"\" ||\n\t\t\t((o.config.QueueLimit > 0 && o.QueueLength >= o.config.QueueLimit) ||\n\t\t\t\t(o.config.SizeLimit > 0 && o.currentFileSize >= int(o.config.SizeLimit))) {\n\t\t\tnextChunk = true\n\t\t}\n\n\t\text := filepath.Ext(path)\n\t\twithoutExt := strings.TrimSuffix(path, ext)\n\n\t\tif matches, err := filepath.Glob(withoutExt + \"*\" + ext); err == nil {\n\t\t\tif len(matches) == 0 {\n\t\t\t\treturn setFileIndex(path, 0)\n\t\t\t}\n\t\t\tsort.Sort(sortByFileIndex(matches))\n\n\t\t\tlast := matches[len(matches)-1]\n\n\t\t\tfileIndex := 0\n\t\t\tif idx := getFileIndex(last); idx != -1 {\n\t\t\t\tfileIndex = idx\n\n\t\t\t\tif nextChunk {\n\t\t\t\t\tfileIndex++\n\t\t\t\t}\n\t\t\t}\n\n\t\t\treturn setFileIndex(last, fileIndex)\n\t\t}\n\t}\n\n\treturn path\n}\n\nfunc (o *FileOutput) updateName() {\n\tname := filepath.Clean(o.filename())\n\to.Lock()\n\to.currentName = name\n\to.Unlock()\n}\n\n// PluginWrite writes message to this plugin\nfunc (o *FileOutput) PluginWrite(msg *Message) (n int, err error) {\n\tif o.requestPerFile {\n\t\to.Lock()\n\t\tmeta := payloadMeta(msg.Meta)\n\t\to.currentID = meta[1]\n\t\to.payloadType = meta[0]\n\t\to.Unlock()\n\t}\n\n\to.updateName()\n\to.Lock()\n\tdefer o.Unlock()\n\n\tif o.file == nil || o.currentName != o.file.Name() {\n\t\to.closeLocked()\n\n\t\to.file, err = os.OpenFile(o.currentName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660)\n\t\to.file.Sync()\n\n\t\tif strings.HasSuffix(o.currentName, \".gz\") {\n\t\t\to.writer = gzip.NewWriter(o.file)\n\t\t} else {\n\t\t\to.writer = bufio.NewWriter(o.file)\n\t\t}\n\n\t\tif err != nil {\n\t\t\tlog.Fatal(o, \"Cannot open file %q. Error: %s\", o.currentName, err)\n\t\t}\n\n\t\to.QueueLength = 0\n\t}\n\n\tvar nn int\n\tn, err = o.writer.Write(msg.Meta)\n\tnn, err = o.writer.Write(msg.Data)\n\tn += nn\n\tnn, err = o.writer.Write(payloadSeparatorAsBytes)\n\tn += nn\n\n\to.totalFileSize += size.Size(n)\n\to.currentFileSize += n\n\to.QueueLength++\n\n\tif Settings.OutputFileConfig.OutputFileMaxSize > 0 && o.totalFileSize >= Settings.OutputFileConfig.OutputFileMaxSize {\n\t\treturn n, errors.New(\"File output reached size limit\")\n\t}\n\n\treturn n, err\n}\n\nfunc (o *FileOutput) flush() {\n\t// Don't exit on panic\n\tdefer func() {\n\t\tif r := recover(); r != nil {\n\t\t\tDebug(0, \"[OUTPUT-FILE] PANIC while file flush: \", r, o, string(debug.Stack()))\n\t\t}\n\t}()\n\n\to.Lock()\n\tdefer o.Unlock()\n\n\tif o.file != nil {\n\t\tif strings.HasSuffix(o.currentName, \".gz\") {\n\t\t\to.writer.(*gzip.Writer).Flush()\n\t\t} else {\n\t\t\to.writer.(*bufio.Writer).Flush()\n\t\t}\n\n\t\tif stat, err := o.file.Stat(); err == nil {\n\t\t\to.currentFileSize = int(stat.Size())\n\t\t} else {\n\t\t\tDebug(0, \"[OUTPUT-HTTP] error accessing file size\", err)\n\t\t}\n\t}\n}\n\nfunc (o *FileOutput) String() string {\n\treturn \"File output: \" + o.file.Name()\n}\n\nfunc (o *FileOutput) closeLocked() error {\n\tif o.file != nil {\n\t\tif strings.HasSuffix(o.currentName, \".gz\") {\n\t\t\to.writer.(*gzip.Writer).Close()\n\t\t} else {\n\t\t\to.writer.(*bufio.Writer).Flush()\n\t\t}\n\t\to.file.Close()\n\n\t\tif o.config.onClose != nil {\n\t\t\to.config.onClose(o.file.Name())\n\t\t}\n\t}\n\n\to.closed = true\n\to.currentFileSize = 0\n\n\treturn nil\n}\n\n// Close closes the output file that is being written to.\nfunc (o *FileOutput) Close() error {\n\to.Lock()\n\tdefer o.Unlock()\n\treturn o.closeLocked()\n}\n\n// IsClosed returns if the output file is closed or not.\nfunc (o *FileOutput) IsClosed() bool {\n\to.Lock()\n\tdefer o.Unlock()\n\treturn o.closed\n}\n" }, { "path": "output_file_test.go", "content": "package goreplay\n\nimport (\n\t\"fmt\"\n\t\"github.com/buger/goreplay/internal/size\"\n\t\"math/rand\"\n\t\"os\"\n\t\"reflect\"\n\t\"sort\"\n\t\"sync\"\n\t\"sync/atomic\"\n\t\"testing\"\n\t\"time\"\n)\n\nfunc TestFileOutput(t *testing.T) {\n\twg := new(sync.WaitGroup)\n\n\tinput := NewTestInput()\n\toutput := NewFileOutput(\"/tmp/test_requests.gor\", &FileOutputConfig{FlushInterval: time.Minute, Append: true})\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output},\n\t}\n\tplugins.All = append(plugins.All, input, output)\n\n\temitter := NewEmitter()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\tfor i := 0; i < 100; i++ {\n\t\twg.Add(2)\n\t\tinput.EmitGET()\n\t\tinput.EmitPOST()\n\t}\n\ttime.Sleep(100 * time.Millisecond)\n\toutput.flush()\n\temitter.Close()\n\n\tvar counter int64\n\tinput2 := NewFileInput(\"/tmp/test_requests.gor\", false, 100, 0, false)\n\toutput2 := NewTestOutput(func(*Message) {\n\t\tatomic.AddInt64(&counter, 1)\n\t\twg.Done()\n\t})\n\n\tplugins2 := &InOutPlugins{\n\t\tInputs: []PluginReader{input2},\n\t\tOutputs: []PluginWriter{output2},\n\t}\n\tplugins2.All = append(plugins2.All, input2, output2)\n\n\temitter2 := NewEmitter()\n\tgo emitter2.Start(plugins2, Settings.Middleware)\n\n\twg.Wait()\n\temitter2.Close()\n}\n\nfunc TestFileOutputWithNameCleaning(t *testing.T) {\n\toutput := &FileOutput{pathTemplate: \"./test_requests.gor\", config: &FileOutputConfig{FlushInterval: time.Minute, Append: false}}\n\texpectedFileName := \"test_requests_0.gor\"\n\toutput.updateName()\n\n\tif expectedFileName != output.currentName {\n\t\tt.Errorf(\"Expected path %s but got %s\", expectedFileName, output.currentName)\n\t}\n\n}\n\nfunc TestFileOutputPathTemplate(t *testing.T) {\n\toutput := &FileOutput{pathTemplate: \"/tmp/log-%Y-%m-%d-%S-%t\", config: &FileOutputConfig{FlushInterval: time.Minute, Append: true}}\n\tnow := time.Now()\n\toutput.payloadType = []byte(\"3\")\n\texpectedPath := fmt.Sprintf(\"/tmp/log-%s-%s-%s-%s-3\", now.Format(\"2006\"), now.Format(\"01\"), now.Format(\"02\"), now.Format(\"05\"))\n\tpath := output.filename()\n\n\tif expectedPath != path {\n\t\tt.Errorf(\"Expected path %s but got %s\", expectedPath, path)\n\t}\n}\n\nfunc TestFileOutputMultipleFiles(t *testing.T) {\n\toutput := NewFileOutput(\"/tmp/log-%Y-%m-%d-%S\", &FileOutputConfig{Append: true, FlushInterval: time.Minute})\n\n\tif output.file != nil {\n\t\tt.Error(\"Should not initialize file if no writes\")\n\t}\n\n\toutput.PluginWrite(&Message{Meta: []byte(\"1 1 1\\r\\n\"), Data: []byte(\"test\")})\n\tname1 := output.file.Name()\n\n\toutput.PluginWrite(&Message{Meta: []byte(\"1 1 1\\r\\n\"), Data: []byte(\"test\")})\n\tname2 := output.file.Name()\n\n\ttime.Sleep(time.Second)\n\toutput.updateName()\n\n\toutput.PluginWrite(&Message{Meta: []byte(\"1 1 1\\r\\n\"), Data: []byte(\"test\")})\n\tname3 := output.file.Name()\n\n\tif name2 != name1 {\n\t\tt.Error(\"Fast changes should happen in same file:\", name1, name2, name3)\n\t}\n\n\tif name3 == name1 {\n\t\tt.Error(\"File name should change:\", name1, name2, name3)\n\t}\n\n\tos.Remove(name1)\n\tos.Remove(name3)\n}\n\nfunc TestFileOutputFilePerRequest(t *testing.T) {\n\toutput := NewFileOutput(\"/tmp/log-%Y-%m-%d-%S-%r\", &FileOutputConfig{Append: true})\n\n\tif output.file != nil {\n\t\tt.Error(\"Should not initialize file if no writes\")\n\t}\n\n\toutput.PluginWrite(&Message{Meta: []byte(\"1 1 1\\r\\n\"), Data: []byte(\"test\")})\n\tname1 := output.file.Name()\n\n\toutput.PluginWrite(&Message{Meta: []byte(\"1 2 1\\r\\n\"), Data: []byte(\"test\")})\n\tname2 := output.file.Name()\n\n\ttime.Sleep(time.Second)\n\toutput.updateName()\n\n\toutput.PluginWrite(&Message{Meta: []byte(\"1 3 1\\r\\n\"), Data: []byte(\"test\")})\n\tname3 := output.file.Name()\n\n\tif name3 == name2 || name2 == name1 || name3 == name1 {\n\t\tt.Error(\"File name should change:\", name1, name2, name3)\n\t}\n\n\tos.Remove(name1)\n\tos.Remove(name2)\n\tos.Remove(name3)\n}\n\nfunc TestFileOutputCompression(t *testing.T) {\n\toutput := NewFileOutput(\"/tmp/log-%Y-%m-%d-%S.gz\", &FileOutputConfig{Append: true, FlushInterval: time.Minute})\n\n\tif output.file != nil {\n\t\tt.Error(\"Should not initialize file if no writes\")\n\t}\n\n\tfor i := 0; i < 1000; i++ {\n\t\toutput.PluginWrite(&Message{Meta: []byte(\"1 1 1\\r\\n\"), Data: []byte(\"test\")})\n\t}\n\n\tname := output.file.Name()\n\toutput.Close()\n\n\ts, _ := os.Stat(name)\n\tif s.Size() == 12*1000 {\n\t\tt.Error(\"Should be compressed file:\", s.Size())\n\t}\n\n\tos.Remove(name)\n}\n\nfunc TestGetFileIndex(t *testing.T) {\n\tvar tests = []struct {\n\t\tpath string\n\t\tindex int\n\t}{\n\t\t{\"/tmp/logs\", -1},\n\t\t{\"/tmp/logs_1\", 1},\n\t\t{\"/tmp/logs_2.gz\", 2},\n\t\t{\"/tmp/logs_0.gz\", 0},\n\t}\n\n\tfor _, c := range tests {\n\t\tif getFileIndex(c.path) != c.index {\n\t\t\tt.Error(c.path, \"should be\", c.index, \"instead\", getFileIndex(c.path))\n\t\t}\n\t}\n}\n\nfunc TestSetFileIndex(t *testing.T) {\n\tvar tests = []struct {\n\t\tpath string\n\t\tindex int\n\t\tnewPath string\n\t}{\n\t\t{\"/tmp/logs\", 0, \"/tmp/logs_0\"},\n\t\t{\"/tmp/logs.gz\", 1, \"/tmp/logs_1.gz\"},\n\t\t{\"/tmp/logs_1\", 0, \"/tmp/logs_0\"},\n\t\t{\"/tmp/logs_0\", 10, \"/tmp/logs_10\"},\n\t\t{\"/tmp/logs_0.gz\", 10, \"/tmp/logs_10.gz\"},\n\t\t{\"/tmp/logs_underscores.gz\", 10, \"/tmp/logs_underscores_10.gz\"},\n\t}\n\n\tfor _, c := range tests {\n\t\tif setFileIndex(c.path, c.index) != c.newPath {\n\t\t\tt.Error(c.path, \"should be\", c.newPath, \"instead\", setFileIndex(c.path, c.index))\n\t\t}\n\t}\n}\n\nfunc TestFileOutputAppendQueueLimitOverflow(t *testing.T) {\n\trnd := rand.Int63()\n\tname := fmt.Sprintf(\"/tmp/%d\", rnd)\n\n\toutput := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 2})\n\n\toutput.PluginWrite(&Message{Meta: []byte(\"1 1 1\\r\\n\"), Data: []byte(\"test\")})\n\tname1 := output.file.Name()\n\n\toutput.PluginWrite(&Message{Meta: []byte(\"1 1 1\\r\\n\"), Data: []byte(\"test\")})\n\tname2 := output.file.Name()\n\n\toutput.updateName()\n\n\toutput.PluginWrite(&Message{Meta: []byte(\"1 1 1\\r\\n\"), Data: []byte(\"test\")})\n\tname3 := output.file.Name()\n\n\tif name2 != name1 || name1 != fmt.Sprintf(\"/tmp/%d_0\", rnd) {\n\t\tt.Error(\"Fast changes should happen in same file:\", name1, name2, name3)\n\t}\n\n\tif name3 == name1 || name3 != fmt.Sprintf(\"/tmp/%d_1\", rnd) {\n\t\tt.Error(\"File name should change:\", name1, name2, name3)\n\t}\n\n\tos.Remove(name1)\n\tos.Remove(name3)\n}\n\nfunc TestFileOutputAppendQueueLimitNoOverflow(t *testing.T) {\n\trnd := rand.Int63()\n\tname := fmt.Sprintf(\"/tmp/%d\", rnd)\n\n\toutput := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 3})\n\n\toutput.PluginWrite(&Message{Meta: []byte(\"1 1 1\\r\\n\"), Data: []byte(\"test\")})\n\tname1 := output.file.Name()\n\n\toutput.PluginWrite(&Message{Meta: []byte(\"1 1 1\\r\\n\"), Data: []byte(\"test\")})\n\tname2 := output.file.Name()\n\n\toutput.updateName()\n\n\toutput.PluginWrite(&Message{Meta: []byte(\"1 1 1\\r\\n\"), Data: []byte(\"test\")})\n\tname3 := output.file.Name()\n\n\tif name2 != name1 || name1 != fmt.Sprintf(\"/tmp/%d_0\", rnd) {\n\t\tt.Error(\"Fast changes should happen in same file:\", name1, name2, name3)\n\t}\n\n\tif name3 != name1 || name3 != fmt.Sprintf(\"/tmp/%d_0\", rnd) {\n\t\tt.Error(\"File name should not change:\", name1, name2, name3)\n\t}\n\n\tos.Remove(name1)\n\tos.Remove(name3)\n}\n\nfunc TestFileOutputAppendQueueLimitGzips(t *testing.T) {\n\trnd := rand.Int63()\n\tname := fmt.Sprintf(\"/tmp/%d.gz\", rnd)\n\n\toutput := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 2})\n\n\toutput.PluginWrite(&Message{Meta: []byte(\"1 1 1\\r\\n\"), Data: []byte(\"test\")})\n\tname1 := output.file.Name()\n\n\toutput.PluginWrite(&Message{Meta: []byte(\"1 1 1\\r\\n\"), Data: []byte(\"test\")})\n\tname2 := output.file.Name()\n\n\toutput.updateName()\n\n\toutput.PluginWrite(&Message{Meta: []byte(\"1 1 1\\r\\n\"), Data: []byte(\"test\")})\n\tname3 := output.file.Name()\n\n\tif name2 != name1 || name1 != fmt.Sprintf(\"/tmp/%d_0.gz\", rnd) {\n\t\tt.Error(\"Fast changes should happen in same file:\", name1, name2, name3)\n\t}\n\n\tif name3 == name1 || name3 != fmt.Sprintf(\"/tmp/%d_1.gz\", rnd) {\n\t\tt.Error(\"File name should change:\", name1, name2, name3)\n\t}\n\n\tos.Remove(name1)\n\tos.Remove(name3)\n}\n\nfunc TestFileOutputSort(t *testing.T) {\n\tvar files = []string{\"2016_0\", \"2014_10\", \"2015_0\", \"2015_10\", \"2015_2\"}\n\tvar expected = []string{\"2014_10\", \"2015_0\", \"2015_2\", \"2015_10\", \"2016_0\"}\n\tsort.Sort(sortByFileIndex(files))\n\n\tif !reflect.DeepEqual(files, expected) {\n\t\tt.Error(\"Should properly sort file names using indexes\", files, expected)\n\t}\n}\n\nfunc TestFileOutputAppendSizeLimitOverflow(t *testing.T) {\n\trnd := rand.Int63()\n\tname := fmt.Sprintf(\"/tmp/%d\", rnd)\n\n\tmessage := []byte(\"1 1 1\\r\\ntest\")\n\n\tmessageSize := len(message) + len(payloadSeparator)\n\n\toutput := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, SizeLimit: size.Size(2 * messageSize)})\n\n\toutput.PluginWrite(&Message{Meta: []byte(\"1 1 1\\r\\n\"), Data: []byte(\"test\")})\n\tname1 := output.file.Name()\n\n\toutput.PluginWrite(&Message{Meta: []byte(\"1 1 1\\r\\n\"), Data: []byte(\"test\")})\n\tname2 := output.file.Name()\n\n\toutput.flush()\n\n\toutput.PluginWrite(&Message{Meta: []byte(\"1 1 1\\r\\n\"), Data: []byte(\"test\")})\n\tname3 := output.file.Name()\n\n\tif name2 != name1 || name1 != fmt.Sprintf(\"/tmp/%d_0\", rnd) {\n\t\tt.Error(\"Fast changes should happen in same file:\", name1, name2, name3)\n\t}\n\n\tif name3 == name1 || name3 != fmt.Sprintf(\"/tmp/%d_1\", rnd) {\n\t\tt.Error(\"File name should change:\", name1, name2, name3)\n\t}\n\n\tos.Remove(name1)\n\tos.Remove(name3)\n}\n" }, { "path": "output_http.go", "content": "package goreplay\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"log\"\n\t\"math\"\n\t\"net/http\"\n\t\"net/http/httputil\"\n\t\"net/url\"\n\t\"sync/atomic\"\n\t\"time\"\n\n\t\"github.com/buger/goreplay/internal/size\"\n)\n\nconst (\n\tinitialDynamicWorkers = 10\n\treadChunkSize = 64 * 1024\n\tmaxResponseSize = 1073741824\n)\n\ntype response struct {\n\tpayload []byte\n\tuuid []byte\n\tstartedAt int64\n\troundTripTime int64\n}\n\n// HTTPOutputConfig struct for holding http output configuration\ntype HTTPOutputConfig struct {\n\tTrackResponses bool `json:\"output-http-track-response\"`\n\tStats bool `json:\"output-http-stats\"`\n\tOriginalHost bool `json:\"output-http-original-host\"`\n\tRedirectLimit int `json:\"output-http-redirect-limit\"`\n\tWorkersMin int `json:\"output-http-workers-min\"`\n\tWorkersMax int `json:\"output-http-workers\"`\n\tStatsMs int `json:\"output-http-stats-ms\"`\n\tQueueLen int `json:\"output-http-queue-len\"`\n\tElasticSearch string `json:\"output-http-elasticsearch\"`\n\tTimeout time.Duration `json:\"output-http-timeout\"`\n\tWorkerTimeout time.Duration `json:\"output-http-worker-timeout\"`\n\tBufferSize size.Size `json:\"output-http-response-buffer\"`\n\tSkipVerify bool `json:\"output-http-skip-verify\"`\n\tCompatibilityMode bool `json:\"output-http-compatibility-mode\"`\n\tRequestGroup string `json:\"output-http-request-group\"`\n\tDebug bool `json:\"output-http-debug\"`\n\trawURL string\n\turl *url.URL\n}\n\nfunc (hoc *HTTPOutputConfig) Copy() *HTTPOutputConfig {\n\treturn &HTTPOutputConfig{\n\t\tTrackResponses: hoc.TrackResponses,\n\t\tStats: hoc.Stats,\n\t\tOriginalHost: hoc.OriginalHost,\n\t\tRedirectLimit: hoc.RedirectLimit,\n\t\tWorkersMin: hoc.WorkersMin,\n\t\tWorkersMax: hoc.WorkersMax,\n\t\tStatsMs: hoc.StatsMs,\n\t\tQueueLen: hoc.QueueLen,\n\t\tElasticSearch: hoc.ElasticSearch,\n\t\tTimeout: hoc.Timeout,\n\t\tWorkerTimeout: hoc.WorkerTimeout,\n\t\tBufferSize: hoc.BufferSize,\n\t\tSkipVerify: hoc.SkipVerify,\n\t\tCompatibilityMode: hoc.CompatibilityMode,\n\t\tRequestGroup: hoc.RequestGroup,\n\t\tDebug: hoc.Debug,\n\t}\n}\n\n// HTTPOutput plugin manage pool of workers which send request to replayed server\n// By default workers pool is dynamic and starts with 1 worker or workerMin workers\n// You can specify maximum number of workers using `--output-http-workers`\ntype HTTPOutput struct {\n\tactiveWorkers int64\n\tconfig *HTTPOutputConfig\n\tqueueStats *GorStat\n\telasticSearch *ESPlugin\n\tclient *HTTPClient\n\tstopWorker chan struct{}\n\tqueue chan *Message\n\tresponses chan *response\n\tstop chan bool // Channel used only to indicate goroutine should shutdown\n\tworkerSessions map[string]*httpWorker\n}\n\ntype httpWorker struct {\n\toutput *HTTPOutput\n\tclient *HTTPClient\n\tlastActivity time.Time\n\tqueue chan *Message\n\tstop chan bool\n}\n\nfunc newHTTPWorker(output *HTTPOutput, queue chan *Message) *httpWorker {\n\tclient := NewHTTPClient(output.config)\n\n\tw := &httpWorker{client: client, output: output}\n\tif queue == nil {\n\t\tw.queue = make(chan *Message, 100)\n\t} else {\n\t\tw.queue = queue\n\t}\n\tw.stop = make(chan bool)\n\n\tgo func() {\n\t\tfor {\n\t\t\tselect {\n\t\t\tcase msg := <-w.queue:\n\t\t\t\toutput.sendRequest(client, msg)\n\t\t\tcase <-w.stop:\n\t\t\t\treturn\n\t\t\t}\n\t\t}\n\t}()\n\n\treturn w\n}\n\n// NewHTTPOutput constructor for HTTPOutput\n// Initialize workers\nfunc NewHTTPOutput(address string, config *HTTPOutputConfig) PluginReadWriter {\n\to := new(HTTPOutput)\n\tvar err error\n\tnewConfig := config.Copy()\n\tnewConfig.url, err = url.Parse(address)\n\tif err != nil {\n\t\tlog.Fatal(fmt.Sprintf(\"[OUTPUT-HTTP] parse HTTP output URL error[%q]\", err))\n\t}\n\tif newConfig.url.Scheme == \"\" {\n\t\tnewConfig.url.Scheme = \"http\"\n\t}\n\tnewConfig.rawURL = newConfig.url.String()\n\tif newConfig.Timeout < time.Millisecond*100 {\n\t\tnewConfig.Timeout = time.Second\n\t}\n\tif newConfig.BufferSize <= 0 {\n\t\tnewConfig.BufferSize = 100 * 1024 // 100kb\n\t}\n\tif newConfig.WorkersMin <= 0 {\n\t\tnewConfig.WorkersMin = 1\n\t}\n\tif newConfig.WorkersMin > 1000 {\n\t\tnewConfig.WorkersMin = 1000\n\t}\n\tif newConfig.WorkersMax <= 0 {\n\t\tnewConfig.WorkersMax = math.MaxInt32 // ideally so large\n\t}\n\tif newConfig.WorkersMax < newConfig.WorkersMin {\n\t\tnewConfig.WorkersMax = newConfig.WorkersMin\n\t}\n\tif newConfig.QueueLen <= 0 {\n\t\tnewConfig.QueueLen = 1000\n\t}\n\tif newConfig.RedirectLimit < 0 {\n\t\tnewConfig.RedirectLimit = 0\n\t}\n\tif newConfig.WorkerTimeout <= 0 {\n\t\tnewConfig.WorkerTimeout = time.Second * 2\n\t}\n\to.config = newConfig\n\to.stop = make(chan bool)\n\tif o.config.Stats {\n\t\to.queueStats = NewGorStat(\"output_http\", o.config.StatsMs)\n\t}\n\n\to.queue = make(chan *Message, o.config.QueueLen)\n\tif o.config.TrackResponses {\n\t\to.responses = make(chan *response, o.config.QueueLen)\n\t}\n\t// it should not be buffered to avoid races\n\to.stopWorker = make(chan struct{})\n\n\tif o.config.ElasticSearch != \"\" {\n\t\to.elasticSearch = new(ESPlugin)\n\t\to.elasticSearch.Init(o.config.ElasticSearch)\n\t}\n\to.client = NewHTTPClient(o.config)\n\n\tif Settings.RecognizeTCPSessions {\n\t\to.workerSessions = make(map[string]*httpWorker, 100)\n\t\tgo o.sessionWorkerMaster()\n\t} else {\n\t\to.activeWorkers += int64(o.config.WorkersMin)\n\t\tfor i := 0; i < o.config.WorkersMin; i++ {\n\t\t\tgo o.startWorker()\n\t\t}\n\t\tgo o.workerMaster()\n\t}\n\n\treturn o\n}\n\nfunc (o *HTTPOutput) workerMaster() {\n\tvar timer = time.NewTimer(o.config.WorkerTimeout)\n\tdefer func() {\n\t\t// recover from panics caused by trying to send in\n\t\t// a closed chan(o.stopWorker)\n\t\trecover()\n\t}()\n\tdefer timer.Stop()\n\tfor {\n\t\tselect {\n\t\tcase <-o.stop:\n\t\t\treturn\n\t\tdefault:\n\t\t\t<-timer.C\n\t\t}\n\t\t// rollback workers\n\trollback:\n\t\tif atomic.LoadInt64(&o.activeWorkers) > int64(o.config.WorkersMin) && len(o.queue) < 1 {\n\t\t\t// close one worker\n\t\t\to.stopWorker <- struct{}{}\n\t\t\tatomic.AddInt64(&o.activeWorkers, -1)\n\t\t\tgoto rollback\n\t\t}\n\t\ttimer.Reset(o.config.WorkerTimeout)\n\t}\n}\n\nfunc (o *HTTPOutput) sessionWorkerMaster() {\n\tgc := time.Tick(time.Second)\n\n\tfor {\n\t\tselect {\n\t\tcase msg := <-o.queue:\n\t\t\tid := payloadID(msg.Meta)\n\t\t\tsessionID := string(id[0:20])\n\t\t\tworker, ok := o.workerSessions[sessionID]\n\n\t\t\tif !ok {\n\t\t\t\tatomic.AddInt64(&o.activeWorkers, 1)\n\t\t\t\tworker = newHTTPWorker(o, nil)\n\t\t\t\to.workerSessions[sessionID] = worker\n\t\t\t}\n\n\t\t\tworker.queue <- msg\n\t\t\tworker.lastActivity = time.Now()\n\t\tcase <-gc:\n\t\t\tnow := time.Now()\n\n\t\t\tfor id, w := range o.workerSessions {\n\t\t\t\tif !w.lastActivity.IsZero() && now.Sub(w.lastActivity) >= 120*time.Second {\n\t\t\t\t\tw.stop <- true\n\t\t\t\t\tdelete(o.workerSessions, id)\n\t\t\t\t\tatomic.AddInt64(&o.activeWorkers, -1)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc (o *HTTPOutput) startWorker() {\n\tfor {\n\t\tselect {\n\t\tcase <-o.stopWorker:\n\t\t\treturn\n\t\tcase msg := <-o.queue:\n\t\t\to.sendRequest(o.client, msg)\n\t\t}\n\t}\n}\n\n// PluginWrite writes message to this plugin\nfunc (o *HTTPOutput) PluginWrite(msg *Message) (n int, err error) {\n\tif !isRequestPayload(msg.Meta) {\n\t\treturn len(msg.Data), nil\n\t}\n\n\tselect {\n\tcase <-o.stop:\n\t\treturn 0, ErrorStopped\n\tcase o.queue <- msg:\n\t}\n\n\tif o.config.Stats {\n\t\to.queueStats.Write(len(o.queue))\n\t}\n\n\tif !Settings.RecognizeTCPSessions && o.config.WorkersMax != o.config.WorkersMin {\n\t\tworkersCount := int(atomic.LoadInt64(&o.activeWorkers))\n\n\t\tif len(o.queue) > workersCount {\n\t\t\textraWorkersReq := len(o.queue) - workersCount + 1\n\t\t\tmaxWorkersAvailable := o.config.WorkersMax - workersCount\n\t\t\tif extraWorkersReq > maxWorkersAvailable {\n\t\t\t\textraWorkersReq = maxWorkersAvailable\n\t\t\t}\n\t\t\tif extraWorkersReq > 0 {\n\t\t\t\tfor i := 0; i < extraWorkersReq; i++ {\n\t\t\t\t\tgo o.startWorker()\n\t\t\t\t\tatomic.AddInt64(&o.activeWorkers, 1)\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\n\treturn len(msg.Data) + len(msg.Meta), nil\n}\n\n// PluginRead reads message from this plugin\nfunc (o *HTTPOutput) PluginRead() (*Message, error) {\n\tif !o.config.TrackResponses {\n\t\treturn nil, ErrorStopped\n\t}\n\tvar resp *response\n\tvar msg Message\n\tselect {\n\tcase <-o.stop:\n\t\treturn nil, ErrorStopped\n\tcase resp = <-o.responses:\n\t\tmsg.Data = resp.payload\n\t}\n\n\tmsg.Meta = payloadHeader(ReplayedResponsePayload, resp.uuid, resp.startedAt, resp.roundTripTime)\n\n\treturn &msg, nil\n}\n\nfunc (o *HTTPOutput) sendRequest(client *HTTPClient, msg *Message) {\n\tif !isRequestPayload(msg.Meta) {\n\t\treturn\n\t}\n\n\tuuid := payloadID(msg.Meta)\n\tstart := time.Now()\n\tresp, err := client.Send(msg.Data)\n\tstop := time.Now()\n\n\tif err != nil {\n\t\tDebug(1, fmt.Sprintf(\"[HTTP-OUTPUT] error when sending: %q\", err))\n\t\treturn\n\t}\n\tif resp == nil {\n\t\treturn\n\t}\n\n\tif o.config.TrackResponses {\n\t\to.responses <- &response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()}\n\t}\n\n\tif o.elasticSearch != nil {\n\t\to.elasticSearch.ResponseAnalyze(msg.Data, resp, start, stop)\n\t}\n}\n\nfunc (o *HTTPOutput) String() string {\n\treturn \"HTTP output: \" + o.config.rawURL\n}\n\n// Close closes the data channel so that data\nfunc (o *HTTPOutput) Close() error {\n\tclose(o.stop)\n\tclose(o.stopWorker)\n\treturn nil\n}\n\n// HTTPClient holds configurations for a single HTTP client\ntype HTTPClient struct {\n\tconfig *HTTPOutputConfig\n\tClient *http.Client\n}\n\n// NewHTTPClient returns new http client with check redirects policy\nfunc NewHTTPClient(config *HTTPOutputConfig) *HTTPClient {\n\tclient := new(HTTPClient)\n\tclient.config = config\n\tvar transport *http.Transport\n\tclient.Client = &http.Client{\n\t\tTimeout: client.config.Timeout,\n\t\tCheckRedirect: func(req *http.Request, via []*http.Request) error {\n\t\t\tif len(via) >= client.config.RedirectLimit {\n\t\t\t\tDebug(1, fmt.Sprintf(\"[HTTPCLIENT] maximum output-http-redirects[%d] reached!\", client.config.RedirectLimit))\n\t\t\t\treturn http.ErrUseLastResponse\n\t\t\t}\n\t\t\tlastReq := via[len(via)-1]\n\t\t\tresp := req.Response\n\t\t\tDebug(2, fmt.Sprintf(\"[HTTPCLIENT] HTTP redirects from %q to %q with %q\", lastReq.Host, req.Host, resp.Status))\n\t\t\treturn nil\n\t\t},\n\t}\n\tif config.SkipVerify {\n\t\t// clone to avoid modifying global default RoundTripper\n\t\ttransport = http.DefaultTransport.(*http.Transport).Clone()\n\t\ttransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}\n\t\tclient.Client.Transport = transport\n\t}\n\n\treturn client\n}\n\n// Send sends an http request using client created by NewHTTPClient\nfunc (c *HTTPClient) Send(data []byte) ([]byte, error) {\n\tvar req *http.Request\n\tvar resp *http.Response\n\tvar err error\n\n\treq, err = http.ReadRequest(bufio.NewReader(bytes.NewReader(data)))\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\t// we don't send CONNECT or OPTIONS request\n\tif req.Method == http.MethodConnect {\n\t\treturn nil, nil\n\t}\n\n\tif !c.config.OriginalHost {\n\t\treq.Host = c.config.url.Host\n\t}\n\n\t// fix #862\n\tif c.config.url.Path == \"\" && c.config.url.RawQuery == \"\" {\n\t\treq.URL.Scheme = c.config.url.Scheme\n\t\treq.URL.Host = c.config.url.Host\n\t} else {\n\t\treq.URL = c.config.url\n\t}\n\n\t// force connection to not be closed, which can affect the global client\n\treq.Close = false\n\t// it's an error if this is not equal to empty string\n\treq.RequestURI = \"\"\n\n\tresp, err = c.Client.Do(req)\n\tif err != nil {\n\t\treturn nil, err\n\t}\n\tif c.config.TrackResponses {\n\t\treturn httputil.DumpResponse(resp, true)\n\t}\n\t_ = resp.Body.Close()\n\treturn nil, nil\n}\n" }, { "path": "output_http_test.go", "content": "package goreplay\n\nimport (\n\t\"io/ioutil\"\n\t\"net/http\"\n\t\"net/http/httptest\"\n\t_ \"net/http/httputil\"\n\t\"sync\"\n\t\"testing\"\n)\n\nfunc TestHTTPOutput(t *testing.T) {\n\twg := new(sync.WaitGroup)\n\n\tinput := NewTestInput()\n\n\tserver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {\n\t\tif req.Header.Get(\"User-Agent\") != \"Gor\" {\n\t\t\tt.Error(\"Wrong header\")\n\t\t}\n\n\t\tif req.Method == \"OPTIONS\" {\n\t\t\tt.Error(\"Wrong method\")\n\t\t}\n\n\t\tif req.Method == \"POST\" {\n\t\t\tdefer req.Body.Close()\n\t\t\tbody, _ := ioutil.ReadAll(req.Body)\n\n\t\t\tif string(body) != \"a=1&b=2\" {\n\t\t\t\tt.Error(\"Wrong POST body:\", string(body))\n\t\t\t}\n\t\t}\n\n\t\twg.Done()\n\t}))\n\tdefer server.Close()\n\n\theaders := HTTPHeaders{httpHeader{\"User-Agent\", \"Gor\"}}\n\tmethods := HTTPMethods{[]byte(\"GET\"), []byte(\"PUT\"), []byte(\"POST\")}\n\tSettings.ModifierConfig = HTTPModifierConfig{Headers: headers, Methods: methods}\n\n\thttpOutput := NewHTTPOutput(server.URL, &HTTPOutputConfig{TrackResponses: false})\n\toutput := NewTestOutput(func(*Message) {\n\t\twg.Done()\n\t})\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{httpOutput, output},\n\t}\n\tplugins.All = append(plugins.All, input, output, httpOutput)\n\n\temitter := NewEmitter()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\tfor i := 0; i < 10; i++ {\n\t\t// 2 http-output, 2 - test output request\n\t\twg.Add(4) // OPTIONS should be ignored\n\t\tinput.EmitPOST()\n\t\tinput.EmitOPTIONS()\n\t\tinput.EmitGET()\n\t}\n\n\twg.Wait()\n\temitter.Close()\n\n\tSettings.ModifierConfig = HTTPModifierConfig{}\n}\n\nfunc TestHTTPOutputKeepOriginalHost(t *testing.T) {\n\twg := new(sync.WaitGroup)\n\n\tinput := NewTestInput()\n\n\tserver := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {\n\t\tif req.Host != \"custom-host.com\" {\n\t\t\tt.Error(\"Wrong header\", req.Host)\n\t\t}\n\n\t\twg.Done()\n\t}))\n\tdefer server.Close()\n\n\theaders := HTTPHeaders{httpHeader{\"Host\", \"custom-host.com\"}}\n\tSettings.ModifierConfig = HTTPModifierConfig{Headers: headers}\n\n\toutput := NewHTTPOutput(server.URL, &HTTPOutputConfig{OriginalHost: true, SkipVerify: true})\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output},\n\t}\n\tplugins.All = append(plugins.All, input, output)\n\n\temitter := NewEmitter()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\twg.Add(1)\n\tinput.EmitGET()\n\n\twg.Wait()\n\temitter.Close()\n\tSettings.ModifierConfig = HTTPModifierConfig{}\n}\n\nfunc TestHTTPOutputSSL(t *testing.T) {\n\twg := new(sync.WaitGroup)\n\n\t// Origing and Replay server initialization\n\tserver := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\twg.Done()\n\t}))\n\n\tinput := NewTestInput()\n\toutput := NewHTTPOutput(server.URL, &HTTPOutputConfig{SkipVerify: true})\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output},\n\t}\n\tplugins.All = append(plugins.All, input, output)\n\n\temitter := NewEmitter()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\twg.Add(2)\n\n\tinput.EmitPOST()\n\tinput.EmitGET()\n\n\twg.Wait()\n\temitter.Close()\n}\n\nfunc TestHTTPOutputSessions(t *testing.T) {\n\twg := new(sync.WaitGroup)\n\n\tinput := NewTestInput()\n\tinput.skipHeader = true\n\n\tserver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {\n\t\twg.Done()\n\t}))\n\tdefer server.Close()\n\n\tSettings.RecognizeTCPSessions = true\n\tSettings.SplitOutput = true\n\toutput := NewHTTPOutput(server.URL, &HTTPOutputConfig{})\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output},\n\t}\n\tplugins.All = append(plugins.All, input, output)\n\temitter := NewEmitter()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\tuuid1 := []byte(\"1234567890123456789a0000\")\n\tuuid2 := []byte(\"1234567890123456789d0000\")\n\n\tfor i := 0; i < 10; i++ {\n\t\twg.Add(1) // OPTIONS should be ignored\n\t\tcopy(uuid1[20:], randByte(4))\n\t\tinput.EmitBytes([]byte(\"1 \" + string(uuid1) + \" 1\\n\" + \"GET / HTTP/1.1\\r\\n\\r\\n\"))\n\t}\n\n\tfor i := 0; i < 10; i++ {\n\t\twg.Add(1) // OPTIONS should be ignored\n\t\tcopy(uuid2[20:], randByte(4))\n\t\tinput.EmitBytes([]byte(\"1 \" + string(uuid2) + \" 1\\n\" + \"GET / HTTP/1.1\\r\\n\\r\\n\"))\n\t}\n\n\twg.Wait()\n\n\temitter.Close()\n\n\tSettings.RecognizeTCPSessions = false\n\tSettings.SplitOutput = false\n}\n\nfunc BenchmarkHTTPOutput(b *testing.B) {\n\twg := new(sync.WaitGroup)\n\n\tserver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\twg.Done()\n\t}))\n\tdefer server.Close()\n\n\tinput := NewTestInput()\n\toutput := NewHTTPOutput(server.URL, &HTTPOutputConfig{WorkersMax: 1})\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output},\n\t}\n\tplugins.All = append(plugins.All, input, output)\n\n\temitter := NewEmitter()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\tfor i := 0; i < b.N; i++ {\n\t\twg.Add(1)\n\t\tinput.EmitPOST()\n\t}\n\n\twg.Wait()\n\temitter.Close()\n}\n\nfunc BenchmarkHTTPOutputTLS(b *testing.B) {\n\twg := new(sync.WaitGroup)\n\n\tserver := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {\n\t\twg.Done()\n\t}))\n\tdefer server.Close()\n\n\tinput := NewTestInput()\n\toutput := NewHTTPOutput(server.URL, &HTTPOutputConfig{SkipVerify: true, WorkersMax: 1})\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output},\n\t}\n\tplugins.All = append(plugins.All, input, output)\n\n\temitter := NewEmitter()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\tfor i := 0; i < b.N; i++ {\n\t\twg.Add(1)\n\t\tinput.EmitPOST()\n\t}\n\n\twg.Wait()\n\temitter.Close()\n}\n" }, { "path": "output_kafka.go", "content": "package goreplay\n\nimport (\n\t\"encoding/json\"\n\t\"github.com/buger/goreplay/internal/byteutils\"\n\t\"github.com/buger/goreplay/proto\"\n\t\"log\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/Shopify/sarama\"\n\t\"github.com/Shopify/sarama/mocks\"\n)\n\n// KafkaOutput is used for sending payloads to kafka in JSON format.\ntype KafkaOutput struct {\n\tconfig *OutputKafkaConfig\n\tproducer sarama.AsyncProducer\n}\n\n// KafkaOutputFrequency in milliseconds\nconst KafkaOutputFrequency = 500\n\n// NewKafkaOutput creates instance of kafka producer client with TLS config\nfunc NewKafkaOutput(_ string, config *OutputKafkaConfig, tlsConfig *KafkaTLSConfig) PluginWriter {\n\tc := NewKafkaConfig(&config.SASLConfig, tlsConfig)\n\n\tvar producer sarama.AsyncProducer\n\n\tif mock, ok := config.producer.(*mocks.AsyncProducer); ok && mock != nil {\n\t\tproducer = config.producer\n\t} else {\n\t\tc.Producer.RequiredAcks = sarama.WaitForLocal\n\t\tc.Producer.Compression = sarama.CompressionSnappy\n\t\tc.Producer.Flush.Frequency = KafkaOutputFrequency * time.Millisecond\n\n\t\tbrokerList := strings.Split(config.Host, \",\")\n\n\t\tvar err error\n\t\tproducer, err = sarama.NewAsyncProducer(brokerList, c)\n\t\tif err != nil {\n\t\t\tlog.Fatalln(\"Failed to start Sarama(Kafka) producer:\", err)\n\t\t}\n\t}\n\n\to := &KafkaOutput{\n\t\tconfig: config,\n\t\tproducer: producer,\n\t}\n\n\t// Start infinite loop for tracking errors for kafka producer.\n\tgo o.ErrorHandler()\n\n\treturn o\n}\n\n// ErrorHandler should receive errors\nfunc (o *KafkaOutput) ErrorHandler() {\n\tfor err := range o.producer.Errors() {\n\t\tDebug(1, \"Failed to write access log entry:\", err)\n\t}\n}\n\n// PluginWrite writes a message to this plugin\nfunc (o *KafkaOutput) PluginWrite(msg *Message) (n int, err error) {\n\tvar message sarama.StringEncoder\n\n\tif !o.config.UseJSON {\n\t\tmessage = sarama.StringEncoder(byteutils.SliceToString(msg.Meta) + byteutils.SliceToString(msg.Data))\n\t} else {\n\t\tmimeHeader := proto.ParseHeaders(msg.Data)\n\t\theader := make(map[string]string)\n\t\tfor k, v := range mimeHeader {\n\t\t\theader[k] = strings.Join(v, \", \")\n\t\t}\n\n\t\tmeta := payloadMeta(msg.Meta)\n\t\treq := msg.Data\n\n\t\tkafkaMessage := KafkaMessage{\n\t\t\tReqURL: byteutils.SliceToString(proto.Path(req)),\n\t\t\tReqType: byteutils.SliceToString(meta[0]),\n\t\t\tReqID: byteutils.SliceToString(meta[1]),\n\t\t\tReqTs: byteutils.SliceToString(meta[2]),\n\t\t\tReqMethod: byteutils.SliceToString(proto.Method(req)),\n\t\t\tReqBody: byteutils.SliceToString(proto.Body(req)),\n\t\t\tReqHeaders: header,\n\t\t}\n\t\tjsonMessage, _ := json.Marshal(&kafkaMessage)\n\t\tmessage = sarama.StringEncoder(byteutils.SliceToString(jsonMessage))\n\t}\n\n\to.producer.Input() <- &sarama.ProducerMessage{\n\t\tTopic: o.config.Topic,\n\t\tValue: message,\n\t}\n\n\treturn len(message), nil\n}\n" }, { "path": "output_kafka_test.go", "content": "package goreplay\n\nimport (\n\t\"testing\"\n\n\t\"github.com/Shopify/sarama\"\n\t\"github.com/Shopify/sarama/mocks\"\n)\n\nfunc TestOutputKafkaRAW(t *testing.T) {\n\tconfig := sarama.NewConfig()\n\tconfig.Producer.Return.Successes = true\n\tproducer := mocks.NewAsyncProducer(t, config)\n\tproducer.ExpectInputAndSucceed()\n\n\toutput := NewKafkaOutput(\"\", &OutputKafkaConfig{\n\t\tproducer: producer,\n\t\tTopic: \"test\",\n\t\tUseJSON: false,\n\t}, nil)\n\n\toutput.PluginWrite(&Message{Meta: []byte(\"1 2 3\\n\"), Data: []byte(\"GET / HTTP1.1\\r\\nHeader: 1\\r\\n\\r\\n\")})\n\n\tresp := <-producer.Successes()\n\n\tdata, _ := resp.Value.Encode()\n\n\tif string(data) != \"1 2 3\\nGET / HTTP1.1\\r\\nHeader: 1\\r\\n\\r\\n\" {\n\t\tt.Errorf(\"Message not properly encoded: %q\", data)\n\t}\n}\n\nfunc TestOutputKafkaJSON(t *testing.T) {\n\tconfig := sarama.NewConfig()\n\tconfig.Producer.Return.Successes = true\n\tproducer := mocks.NewAsyncProducer(t, config)\n\tproducer.ExpectInputAndSucceed()\n\n\toutput := NewKafkaOutput(\"\", &OutputKafkaConfig{\n\t\tproducer: producer,\n\t\tTopic: \"test\",\n\t\tUseJSON: true,\n\t}, nil)\n\n\toutput.PluginWrite(&Message{Meta: []byte(\"1 2 3\\n\"), Data: []byte(\"GET / HTTP1.1\\r\\nHeader: 1\\r\\n\\r\\n\")})\n\n\tresp := <-producer.Successes()\n\n\tdata, _ := resp.Value.Encode()\n\n\tif string(data) != `{\"Req_URL\":\"\",\"Req_Type\":\"1\",\"Req_ID\":\"2\",\"Req_Ts\":\"3\",\"Req_Method\":\"GET\"}` {\n\t\tt.Error(\"Message not properly encoded: \", string(data))\n\t}\n}\n" }, { "path": "output_null.go", "content": "package goreplay\n\n// NullOutput used for debugging, prints nothing\ntype NullOutput struct {\n}\n\n// NewNullOutput constructor for NullOutput\nfunc NewNullOutput() (o *NullOutput) {\n\treturn new(NullOutput)\n}\n\n// PluginWrite writes message to this plugin\nfunc (o *NullOutput) PluginWrite(msg *Message) (int, error) {\n\treturn len(msg.Data) + len(msg.Meta), nil\n}\n\nfunc (o *NullOutput) String() string {\n\treturn \"Null Output\"\n}\n" }, { "path": "output_s3.go", "content": "//go:build !pro\n\npackage goreplay\n\nimport (\n\t\"errors\"\n\t\"fmt\"\n)\n\n// S3Output output plugin\ntype S3Output struct{}\n\n// NewS3Output constructor for FileOutput, accepts path\nfunc NewS3Output(pathTemplate string, config *FileOutputConfig) *S3Output {\n\tfmt.Println(\"S3 output is only available in the pro version\")\n\treturn &S3Output{}\n}\n\nfunc (o *S3Output) PluginWrite(msg *Message) (n int, err error) {\n\treturn 0, errors.New(\"S3 output is only available in the pro version\")\n}\n\nfunc (o *S3Output) String() string {\n\treturn \"S3 output (pro version only)\"\n}\n\nfunc (o *S3Output) Close() error {\n\treturn errors.New(\"S3 output is only available in the pro version\")\n}\n" }, { "path": "output_s3_pro.go", "content": "//go:build pro\n\npackage goreplay\n\nimport (\n\t_ \"bufio\"\n\t\"fmt\"\n\t_ \"io\"\n\t\"log\"\n\t\"math/rand\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"strings\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/aws/session\"\n\t\"github.com/aws/aws-sdk-go/service/s3\"\n\t_ \"github.com/aws/aws-sdk-go/service/s3/s3manager\"\n)\n\nvar _ PluginWriter = (*S3Output)(nil)\n\n// S3Output output plugin\ntype S3Output struct {\n\tpathTemplate string\n\n\tbuffer *FileOutput\n\tsession *session.Session\n\tconfig *FileOutputConfig\n\tcloseC chan struct{}\n}\n\n// NewS3Output constructor for FileOutput, accepts path\nfunc NewS3Output(pathTemplate string, config *FileOutputConfig) *S3Output {\n\to := new(S3Output)\n\to.pathTemplate = pathTemplate\n\to.config = config\n\to.config.onClose = o.onBufferUpdate\n\n\tif config.BufferPath == \"\" {\n\t\tconfig.BufferPath = \"/tmp\"\n\t}\n\n\trnd := rand.Int63()\n\tbuffer_name := fmt.Sprintf(\"gor_output_s3_%d_buf_\", rnd)\n\n\tpathParts := strings.Split(pathTemplate, \"/\")\n\tbuffer_name += pathParts[len(pathParts)-1]\n\n\tif strings.HasSuffix(o.pathTemplate, \".gz\") {\n\t\tbuffer_name += \".gz\"\n\t}\n\n\tbuffer_path := filepath.Join(config.BufferPath, buffer_name)\n\n\to.buffer = NewFileOutput(buffer_path, config)\n\to.connect()\n\n\treturn o\n}\n\nfunc (o *S3Output) connect() {\n\tif o.session == nil {\n\t\to.session = session.Must(session.NewSession(awsConfig()))\n\t\tlog.Println(\"[S3 Output] S3 connection succesfully initialized\")\n\t}\n}\n\nfunc (o *S3Output) PluginWrite(msg *Message) (n int, err error) {\n\treturn o.buffer.PluginWrite(msg)\n}\n\nfunc (o *S3Output) String() string {\n\treturn \"S3 output: \" + o.pathTemplate\n}\n\nfunc (o *S3Output) Close() error {\n\treturn o.buffer.Close()\n}\n\nfunc (o *S3Output) keyPath(idx int) (bucket, key string) {\n\tbucket, key = parseS3Url(o.pathTemplate)\n\n\tfor name, fn := range dateFileNameFuncs {\n\t\tkey = strings.Replace(key, name, fn(o.buffer), -1)\n\t}\n\n\tkey = setFileIndex(key, idx)\n\n\treturn\n}\n\nfunc (o *S3Output) onBufferUpdate(path string) {\n\tsvc := s3.New(o.session)\n\tidx := getFileIndex(path)\n\tbucket, key := o.keyPath(idx)\n\n\tfile, _ := os.Open(path)\n\t// reader := bufio.NewReader(file)\n\n\t_, err := svc.PutObject(&s3.PutObjectInput{\n\t\tBody: file,\n\t\tBucket: aws.String(bucket),\n\t\tKey: aws.String(key),\n\t})\n\tif err != nil {\n\t\tlog.Printf(\"[S3 Output] Failed to upload data to %s/%s, %s\\n\", bucket, key, err)\n\t\tos.Remove(path)\n\t\treturn\n\t}\n\n\tos.Remove(path)\n\n\tif o.closeC != nil {\n\t\to.closeC <- struct{}{}\n\t}\n}\n" }, { "path": "output_tcp.go", "content": "package goreplay\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"fmt\"\n\t\"hash/fnv\"\n\t\"net\"\n\t\"time\"\n)\n\n// TCPOutput used for sending raw tcp payloads\n// Currently used for internal communication between listener and replay server\n// Can be used for transferring binary payloads like protocol buffers\ntype TCPOutput struct {\n\taddress string\n\tlimit int\n\tbuf []chan *Message\n\tbufStats *GorStat\n\tconfig *TCPOutputConfig\n\tworkerIndex uint32\n\n\tclose bool\n}\n\n// TCPOutputConfig tcp output configuration\ntype TCPOutputConfig struct {\n\tSecure bool `json:\"output-tcp-secure\"`\n\tSticky bool `json:\"output-tcp-sticky\"`\n\tSkipVerify bool `json:\"output-tcp-skip-verify\"`\n\tWorkers int `json:\"output-tcp-workers\"`\n\n\tGetInitMessage func() *Message `json:\"-\"`\n\tWriteBeforeMessage func(conn net.Conn, msg *Message) error `json:\"-\"`\n}\n\n// NewTCPOutput constructor for TCPOutput\n// Initialize X workers which hold keep-alive connection\nfunc NewTCPOutput(address string, config *TCPOutputConfig) PluginWriter {\n\to := new(TCPOutput)\n\n\to.address = address\n\to.config = config\n\n\tif Settings.OutputTCPStats {\n\t\to.bufStats = NewGorStat(\"output_tcp\", 5000)\n\t}\n\n\t// create X buffers and send the buffer index to the worker\n\to.buf = make([]chan *Message, o.config.Workers)\n\tfor i := 0; i < o.config.Workers; i++ {\n\t\to.buf[i] = make(chan *Message, 100)\n\t\tgo o.worker(i)\n\t}\n\n\treturn o\n}\n\nfunc (o *TCPOutput) worker(bufferIndex int) {\n\tretries := 0\n\tconn, err := o.connect(o.address)\n\tfor {\n\t\tif o.close {\n\t\t\treturn\n\t\t}\n\n\t\tif err == nil {\n\t\t\tbreak\n\t\t}\n\n\t\tDebug(1, fmt.Sprintf(\"Can't connect to aggregator instance, reconnecting in 1 second. Retries:%d\", retries))\n\t\ttime.Sleep(1 * time.Second)\n\n\t\tconn, err = o.connect(o.address)\n\t\tretries++\n\t}\n\n\tif retries > 0 {\n\t\tDebug(2, fmt.Sprintf(\"Connected to aggregator instance after %d retries\", retries))\n\t}\n\n\tdefer conn.Close()\n\n\tif o.config.GetInitMessage != nil {\n\t\tmsg := o.config.GetInitMessage()\n\t\t_ = o.writeToConnection(conn, msg)\n\t}\n\n\tfor {\n\t\tmsg := <-o.buf[bufferIndex]\n\t\terr = o.writeToConnection(conn, msg)\n\t\tif err != nil {\n\t\t\tDebug(2, \"INFO: TCP output connection closed, reconnecting\")\n\t\t\tgo o.worker(bufferIndex)\n\t\t\to.buf[bufferIndex] <- msg\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nfunc (o *TCPOutput) writeToConnection(conn net.Conn, msg *Message) (err error) {\n\tif o.config.WriteBeforeMessage != nil {\n\t\terr = o.config.WriteBeforeMessage(conn, msg)\n\t}\n\n\tif err == nil {\n\t\tif _, err = conn.Write(msg.Meta); err == nil {\n\t\t\tif _, err = conn.Write(msg.Data); err == nil {\n\t\t\t\t_, err = conn.Write(payloadSeparatorAsBytes)\n\t\t\t}\n\t\t}\n\t}\n\n\treturn err\n}\n\nfunc (o *TCPOutput) getBufferIndex(msg *Message) int {\n\tif !o.config.Sticky {\n\t\to.workerIndex++\n\t\treturn int(o.workerIndex) % o.config.Workers\n\t}\n\n\thasher := fnv.New32a()\n\thasher.Write(payloadID(msg.Meta))\n\treturn int(hasher.Sum32()) % o.config.Workers\n}\n\n// PluginWrite writes message to this plugin\nfunc (o *TCPOutput) PluginWrite(msg *Message) (n int, err error) {\n\tif !isOriginPayload(msg.Meta) {\n\t\treturn len(msg.Data), nil\n\t}\n\n\tbufferIndex := o.getBufferIndex(msg)\n\to.buf[bufferIndex] <- msg\n\n\tif Settings.OutputTCPStats {\n\t\to.bufStats.Write(len(o.buf[bufferIndex]))\n\t}\n\n\treturn len(msg.Data) + len(msg.Meta), nil\n}\n\nfunc (o *TCPOutput) connect(address string) (conn net.Conn, err error) {\n\tif o.config.Secure {\n\t\tvar d tls.Dialer\n\t\td.Config = &tls.Config{InsecureSkipVerify: o.config.SkipVerify}\n\t\tctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)\n\t\tdefer cancel()\n\t\tconn, err = d.DialContext(ctx, \"tcp\", address)\n\t} else {\n\t\tvar d net.Dialer\n\t\tctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)\n\t\tdefer cancel()\n\t\tconn, err = d.DialContext(ctx, \"tcp\", address)\n\t}\n\n\treturn\n}\n\nfunc (o *TCPOutput) String() string {\n\treturn fmt.Sprintf(\"TCP output %s, limit: %d\", o.address, o.limit)\n}\n\nfunc (o *TCPOutput) Close() {\n\to.close = true\n}\n" }, { "path": "output_tcp_test.go", "content": "package goreplay\n\nimport (\n\t\"bufio\"\n\t\"log\"\n\t\"net\"\n\t\"strings\"\n\t\"sync\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestTCPOutput(t *testing.T) {\n\twg := new(sync.WaitGroup)\n\n\tlistener := startTCP(func(data []byte) {\n\t\twg.Done()\n\t})\n\toutput := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{Workers: 10})\n\trunTCPOutput(wg, output, 10, false)\n}\n\nfunc startTCP(cb func([]byte)) net.Listener {\n\tlistener, err := net.Listen(\"tcp\", \"127.0.0.1:0\")\n\n\tif err != nil {\n\t\tlog.Fatal(\"Can't start:\", err)\n\t}\n\n\tgo func() {\n\t\tfor {\n\t\t\tconn, _ := listener.Accept()\n\n\t\t\tgo func(conn net.Conn) {\n\t\t\t\tdefer conn.Close()\n\t\t\t\treader := bufio.NewReader(conn)\n\t\t\t\tscanner := bufio.NewScanner(reader)\n\t\t\t\tscanner.Split(payloadScanner)\n\n\t\t\t\tfor scanner.Scan() {\n\t\t\t\t\tcb(scanner.Bytes())\n\t\t\t\t}\n\t\t\t}(conn)\n\t\t}\n\t}()\n\n\treturn listener\n}\n\nfunc BenchmarkTCPOutput(b *testing.B) {\n\twg := new(sync.WaitGroup)\n\n\tlistener := startTCP(func(data []byte) {\n\t\twg.Done()\n\t})\n\tinput := NewTestInput()\n\tinput.data = make(chan []byte, b.N)\n\tfor i := 0; i < b.N; i++ {\n\t\tinput.EmitGET()\n\t}\n\twg.Add(b.N)\n\toutput := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{Workers: 10})\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output},\n\t}\n\n\temitter := NewEmitter()\n\t// avoid counting above initialization\n\tb.ResetTimer()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\twg.Wait()\n\temitter.Close()\n}\n\nfunc TestStickyDisable(t *testing.T) {\n\ttcpOutput := TCPOutput{config: &TCPOutputConfig{Sticky: false, Workers: 10}}\n\n\tfor i := 0; i < 10; i++ {\n\t\tindex := tcpOutput.getBufferIndex(getTestBytes())\n\t\tif index != (i+1)%10 {\n\t\t\tt.Errorf(\"Sticky is disable. Got: %d want %d\", index, (i+1)%10)\n\t\t}\n\t}\n}\n\nfunc TestBufferDistribution(t *testing.T) {\n\tnumberOfWorkers := 10\n\tnumberOfMessages := 10000\n\tpercentDistributionErrorRange := 20\n\n\tbuffer := make([]int, numberOfWorkers)\n\ttcpOutput := TCPOutput{config: &TCPOutputConfig{Sticky: true, Workers: 10}}\n\tfor i := 0; i < numberOfMessages; i++ {\n\t\tbuffer[tcpOutput.getBufferIndex(getTestBytes())]++\n\t}\n\n\texpectedDistribution := numberOfMessages / numberOfWorkers\n\tlowerDistribution := expectedDistribution - (expectedDistribution * percentDistributionErrorRange / 100)\n\tupperDistribution := expectedDistribution + (expectedDistribution * percentDistributionErrorRange / 100)\n\tfor i := 0; i < numberOfWorkers; i++ {\n\t\tif buffer[i] < lowerDistribution {\n\t\t\tt.Errorf(\"Under expected distribution. Got %d expected lower distribution %d\", buffer[i], lowerDistribution)\n\t\t}\n\t\tif buffer[i] > upperDistribution {\n\t\t\tt.Errorf(\"Under expected distribution. Got %d expected upper distribution %d\", buffer[i], upperDistribution)\n\t\t}\n\t}\n}\n\nfunc getTestBytes() *Message {\n\treturn &Message{\n\t\tMeta: payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1),\n\t\tData: []byte(\"GET / HTTP/1.1\\r\\nHost: www.w3.org\\r\\nUser-Agent: Go 1.1 package http\\r\\nAccept-Encoding: gzip\\r\\n\\r\\n\"),\n\t}\n}\n\nfunc TestTCPOutputGetInitMessage(t *testing.T) {\n\twg := new(sync.WaitGroup)\n\n\tvar dataList [][]byte\n\tlistener := startTCP(func(data []byte) {\n\t\tdataList = append(dataList, data)\n\t\twg.Done()\n\t})\n\tgetInitMessage := func() *Message {\n\t\treturn &Message{\n\t\t\tMeta: []byte{},\n\t\t\tData: []byte(\"test1\"),\n\t\t}\n\t}\n\toutput := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{Workers: 1, GetInitMessage: getInitMessage})\n\n\trunTCPOutput(wg, output, 1, true)\n\n\tif assert.Equal(t, 2, len(dataList)) {\n\t\tassert.Equal(t, \"test1\", string(dataList[0]))\n\t}\n}\n\nfunc TestTCPOutputGetInitMessageAndWriteBeforeMessage(t *testing.T) {\n\twg := new(sync.WaitGroup)\n\n\tvar dataList [][]byte\n\tlistener := startTCP(func(data []byte) {\n\t\tdataList = append(dataList, data)\n\t\twg.Done()\n\t})\n\tgetInitMessage := func() *Message {\n\t\treturn &Message{\n\t\t\tMeta: []byte{},\n\t\t\tData: []byte(\"test2\"),\n\t\t}\n\t}\n\twriteBeforeMessage := func(conn net.Conn, _ *Message) error {\n\t\t_, err := conn.Write([]byte(\"before\"))\n\t\treturn err\n\t}\n\toutput := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{Workers: 1, GetInitMessage: getInitMessage, WriteBeforeMessage: writeBeforeMessage})\n\n\trunTCPOutput(wg, output, 1, true)\n\n\tif assert.Equal(t, 2, len(dataList)) {\n\t\tassert.Equal(t, \"beforetest2\", string(dataList[0]))\n\t\tassert.True(t, strings.HasPrefix(string(dataList[1]), \"before\"))\n\t}\n}\n\nfunc runTCPOutput(wg *sync.WaitGroup, output PluginWriter, repeat int, initMessage bool) {\n\tinput := NewTestInput()\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output},\n\t}\n\n\temitter := NewEmitter()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\tif initMessage {\n\t\twg.Add(1)\n\t}\n\tfor i := 0; i < repeat; i++ {\n\t\twg.Add(1)\n\t\tinput.EmitGET()\n\t}\n\n\twg.Wait()\n\temitter.Close()\n}\n" }, { "path": "output_ws.go", "content": "package goreplay\n\nimport (\n\t\"context\"\n\t\"crypto/tls\"\n\t\"encoding/base64\"\n\t\"fmt\"\n\t\"hash/fnv\"\n\t\"log\"\n\t\"net/http\"\n\t\"net/url\"\n\t\"strings\"\n\t\"time\"\n\n\t\"github.com/gorilla/websocket\"\n)\n\n// WebSocketOutput used for sending raw tcp payloads\n// Can be used for transferring binary payloads like protocol buffers\ntype WebSocketOutput struct {\n\taddress string\n\tlimit int\n\tbuf []chan *Message\n\tbufStats *GorStat\n\tconfig *WebSocketOutputConfig\n\tworkerIndex uint32\n\theaders http.Header\n\n\tclose bool\n}\n\n// WebSocketOutputConfig WebSocket output configuration\ntype WebSocketOutputConfig struct {\n\tSticky bool `json:\"output-ws-sticky\"`\n\tSkipVerify bool `json:\"output-ws-skip-verify\"`\n\tWorkers int `json:\"output-ws-workers\"`\n\n\tHeaders map[string][]string `json:\"output-ws-headers\"`\n}\n\n// NewWebSocketOutput constructor for WebSocketOutput\n// Initialize X workers which hold keep-alive connection\nfunc NewWebSocketOutput(address string, config *WebSocketOutputConfig) PluginWriter {\n\to := new(WebSocketOutput)\n\n\tu, err := url.Parse(address)\n\tif err != nil {\n\t\tlog.Fatal(fmt.Sprintf(\"[OUTPUT-WS] parse WS output URL error[%q]\", err))\n\t}\n\n\to.config = config\n\to.headers = http.Header{\n\t\t\"Authorization\": []string{\"Basic \" + base64.StdEncoding.EncodeToString([]byte(u.User.String()))},\n\t}\n\tfor k, values := range config.Headers {\n\t\tfor _, v := range values {\n\t\t\to.headers.Add(k, v)\n\t\t}\n\t}\n\n\tu.User = nil // must be after creating the headers\n\to.address = u.String()\n\n\tif Settings.OutputWebSocketStats {\n\t\to.bufStats = NewGorStat(\"output_ws\", 5000)\n\t}\n\n\t// create X buffers and send the buffer index to the worker\n\to.buf = make([]chan *Message, o.config.Workers)\n\tfor i := 0; i < o.config.Workers; i++ {\n\t\to.buf[i] = make(chan *Message, 100)\n\t\tgo o.worker(i)\n\t}\n\n\treturn o\n}\n\nfunc (o *WebSocketOutput) worker(bufferIndex int) {\n\tretries := 0\n\tconn, err := o.connect(o.address)\n\tfor {\n\t\tif o.close {\n\t\t\treturn\n\t\t}\n\n\t\tif err == nil {\n\t\t\tbreak\n\t\t}\n\n\t\tDebug(1, fmt.Sprintf(\"Can't connect to aggregator instance, reconnecting in 1 second. Retries:%d\", retries))\n\t\ttime.Sleep(1 * time.Second)\n\n\t\tconn, err = o.connect(o.address)\n\t\tretries++\n\t}\n\n\tif retries > 0 {\n\t\tDebug(2, fmt.Sprintf(\"Connected to aggregator instance after %d retries\", retries))\n\t}\n\n\tdefer conn.Close()\n\n\tfor {\n\t\tmsg := <-o.buf[bufferIndex]\n\t\terr = conn.WriteMessage(websocket.BinaryMessage, append(msg.Meta, msg.Data...))\n\t\tif err != nil {\n\t\t\tDebug(2, \"INFO: WebSocket output connection closed, reconnecting \"+err.Error())\n\t\t\tgo o.worker(bufferIndex)\n\t\t\to.buf[bufferIndex] <- msg\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nfunc (o *WebSocketOutput) getBufferIndex(msg *Message) int {\n\tif !o.config.Sticky {\n\t\to.workerIndex++\n\t\treturn int(o.workerIndex) % o.config.Workers\n\t}\n\n\thasher := fnv.New32a()\n\thasher.Write(payloadID(msg.Meta))\n\treturn int(hasher.Sum32()) % o.config.Workers\n}\n\n// PluginWrite writes message to this plugin\nfunc (o *WebSocketOutput) PluginWrite(msg *Message) (n int, err error) {\n\tif !isOriginPayload(msg.Meta) {\n\t\treturn len(msg.Data), nil\n\t}\n\n\tbufferIndex := o.getBufferIndex(msg)\n\to.buf[bufferIndex] <- msg\n\n\tif Settings.OutputTCPStats {\n\t\to.bufStats.Write(len(o.buf[bufferIndex]))\n\t}\n\n\treturn len(msg.Data) + len(msg.Meta), nil\n}\n\nfunc (o *WebSocketOutput) connect(address string) (conn *websocket.Conn, err error) {\n\tctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)\n\tdefer cancel()\n\n\td := websocket.DefaultDialer\n\tif strings.HasPrefix(address, \"wss://\") {\n\t\td.TLSClientConfig = &tls.Config{InsecureSkipVerify: o.config.SkipVerify}\n\t}\n\n\tconn, _, err = d.DialContext(ctx, address, o.headers)\n\treturn\n}\n\nfunc (o *WebSocketOutput) String() string {\n\treturn fmt.Sprintf(\"WebSocket output %s, limit: %d\", o.address, o.limit)\n}\n\n// Close closes the output\nfunc (o *WebSocketOutput) Close() {\n\to.close = true\n}\n" }, { "path": "output_ws_test.go", "content": "package goreplay\n\nimport (\n\t\"log\"\n\t\"net/http\"\n\t\"sync\"\n\t\"testing\"\n\n\t\"github.com/gorilla/websocket\"\n\t\"github.com/stretchr/testify/assert\"\n)\n\nfunc TestWebSocketOutput(t *testing.T) {\n\twg := new(sync.WaitGroup)\n\n\tvar gotHeader http.Header\n\twsAddr := startWebsocket(func(data []byte) {\n\t\twg.Done()\n\t}, func(header http.Header) {\n\t\tgotHeader = header\n\t})\n\tinput := NewTestInput()\n\theaders := map[string][]string{\n\t\t\"key1\": {\"value1\"},\n\t\t\"key2\": {\"value2\"},\n\t}\n\toutput := NewWebSocketOutput(wsAddr, &WebSocketOutputConfig{Workers: 1, Headers: headers})\n\n\tplugins := &InOutPlugins{\n\t\tInputs: []PluginReader{input},\n\t\tOutputs: []PluginWriter{output},\n\t}\n\n\temitter := NewEmitter()\n\tgo emitter.Start(plugins, Settings.Middleware)\n\n\tfor i := 0; i < 10; i++ {\n\t\twg.Add(1)\n\t\tinput.EmitGET()\n\t}\n\n\twg.Wait()\n\temitter.Close()\n\n\tif assert.NotNil(t, gotHeader) {\n\t\tassert.Equal(t, \"Basic dXNlcjE=\", gotHeader.Get(\"Authorization\"))\n\t\tfor k, values := range headers {\n\t\t\tassert.Equal(t, 1, len(values))\n\t\t\tassert.Equal(t, values[0], gotHeader.Get(k))\n\t\t}\n\t}\n}\n\nfunc startWebsocket(cb func([]byte), headercb func(http.Header)) string {\n\tupgrader := websocket.Upgrader{}\n\n\thttp.HandleFunc(\"/test\", func(w http.ResponseWriter, r *http.Request) {\n\t\theadercb(r.Header)\n\t\tc, err := upgrader.Upgrade(w, r, nil)\n\t\tif err != nil {\n\t\t\tlog.Print(\"upgrade:\", err)\n\t\t\treturn\n\t\t}\n\n\t\tgo func(conn *websocket.Conn) {\n\t\t\tdefer conn.Close()\n\t\t\tfor {\n\t\t\t\t_, msg, _ := conn.ReadMessage()\n\t\t\t\tcb(msg)\n\t\t\t}\n\t\t}(c)\n\t})\n\n\tgo func() {\n\t\terr := http.ListenAndServe(\"localhost:8081\", nil)\n\t\tif err != nil {\n\t\t\tlog.Fatal(\"Can't start:\", err)\n\t\t}\n\t}()\n\n\treturn \"ws://user1@localhost:8081/test\"\n}\n" }, { "path": "plugins.go", "content": "package goreplay\n\nimport (\n\t\"reflect\"\n\t\"strings\"\n)\n\n// Message represents data across plugins\ntype Message struct {\n\tMeta []byte // metadata\n\tData []byte // actual data\n}\n\n// PluginReader is an interface for input plugins\ntype PluginReader interface {\n\tPluginRead() (msg *Message, err error)\n}\n\n// PluginWriter is an interface for output plugins\ntype PluginWriter interface {\n\tPluginWrite(msg *Message) (n int, err error)\n}\n\n// PluginReadWriter is an interface for plugins that support reading and writing\ntype PluginReadWriter interface {\n\tPluginReader\n\tPluginWriter\n}\n\n// InOutPlugins struct for holding references to plugins\ntype InOutPlugins struct {\n\tInputs []PluginReader\n\tOutputs []PluginWriter\n\tAll []interface{}\n}\n\n// extractLimitOptions detects if plugin get called with limiter support\n// Returns address and limit\nfunc extractLimitOptions(options string) (string, string) {\n\tsplit := strings.Split(options, \"|\")\n\n\tif len(split) > 1 {\n\t\treturn split[0], split[1]\n\t}\n\n\treturn split[0], \"\"\n}\n\n// Automatically detects type of plugin and initialize it\n//\n// See this article if curious about reflect stuff below: http://blog.burntsushi.net/type-parametric-functions-golang\nfunc (plugins *InOutPlugins) registerPlugin(constructor interface{}, options ...interface{}) {\n\tvar path, limit string\n\tvc := reflect.ValueOf(constructor)\n\n\t// Pre-processing options to make it work with reflect\n\tvo := []reflect.Value{}\n\tfor _, oi := range options {\n\t\tvo = append(vo, reflect.ValueOf(oi))\n\t}\n\n\tif len(vo) > 0 {\n\t\t// Removing limit options from path\n\t\tpath, limit = extractLimitOptions(vo[0].String())\n\n\t\t// Writing value back without limiter \"|\" options\n\t\tvo[0] = reflect.ValueOf(path)\n\t}\n\n\t// Calling our constructor with list of given options\n\tplugin := vc.Call(vo)[0].Interface()\n\n\tif limit != \"\" {\n\t\tplugin = NewLimiter(plugin, limit)\n\t}\n\n\t// Some of the output can be Readers as well because return responses\n\tif r, ok := plugin.(PluginReader); ok {\n\t\tplugins.Inputs = append(plugins.Inputs, r)\n\t}\n\n\tif w, ok := plugin.(PluginWriter); ok {\n\t\tplugins.Outputs = append(plugins.Outputs, w)\n\t}\n\tplugins.All = append(plugins.All, plugin)\n}\n\n// NewPlugins specify and initialize all available plugins\nfunc NewPlugins() *InOutPlugins {\n\tplugins := new(InOutPlugins)\n\n\tfor _, options := range Settings.InputDummy {\n\t\tplugins.registerPlugin(NewDummyInput, options)\n\t}\n\n\tfor range Settings.OutputDummy {\n\t\tplugins.registerPlugin(NewDummyOutput)\n\t}\n\n\tif Settings.OutputStdout {\n\t\tplugins.registerPlugin(NewDummyOutput)\n\t}\n\n\tif Settings.OutputNull {\n\t\tplugins.registerPlugin(NewNullOutput)\n\t}\n\n\tfor _, options := range Settings.InputRAW {\n\t\tplugins.registerPlugin(NewRAWInput, options, Settings.InputRAWConfig)\n\t}\n\n\tfor _, options := range Settings.InputTCP {\n\t\tplugins.registerPlugin(NewTCPInput, options, &Settings.InputTCPConfig)\n\t}\n\n\tfor _, options := range Settings.OutputTCP {\n\t\tplugins.registerPlugin(NewTCPOutput, options, &Settings.OutputTCPConfig)\n\t}\n\n\tfor _, options := range Settings.OutputWebSocket {\n\t\tplugins.registerPlugin(NewWebSocketOutput, options, &Settings.OutputWebSocketConfig)\n\t}\n\n\tfor _, options := range Settings.InputFile {\n\t\tplugins.registerPlugin(NewFileInput, options, Settings.InputFileLoop, Settings.InputFileReadDepth, Settings.InputFileMaxWait, Settings.InputFileDryRun)\n\t}\n\n\tfor _, path := range Settings.OutputFile {\n\t\tif strings.HasPrefix(path, \"s3://\") {\n\t\t\tplugins.registerPlugin(NewS3Output, path, &Settings.OutputFileConfig)\n\t\t} else {\n\t\t\tplugins.registerPlugin(NewFileOutput, path, &Settings.OutputFileConfig)\n\t\t}\n\t}\n\n\tfor _, options := range Settings.InputHTTP {\n\t\tplugins.registerPlugin(NewHTTPInput, options)\n\t}\n\n\t// If we explicitly set Host header http output should not rewrite it\n\t// Fix: https://github.com/buger/gor/issues/174\n\tfor _, header := range Settings.ModifierConfig.Headers {\n\t\tif header.Name == \"Host\" {\n\t\t\tSettings.OutputHTTPConfig.OriginalHost = true\n\t\t\tbreak\n\t\t}\n\t}\n\n\tfor _, options := range Settings.OutputHTTP {\n\t\tplugins.registerPlugin(NewHTTPOutput, options, &Settings.OutputHTTPConfig)\n\t}\n\n\tfor _, options := range Settings.OutputBinary {\n\t\tplugins.registerPlugin(NewBinaryOutput, options, &Settings.OutputBinaryConfig)\n\t}\n\n\tif Settings.OutputKafkaConfig.Host != \"\" && Settings.OutputKafkaConfig.Topic != \"\" {\n\t\tplugins.registerPlugin(NewKafkaOutput, \"\", &Settings.OutputKafkaConfig, &Settings.KafkaTLSConfig)\n\t}\n\n\tif Settings.InputKafkaConfig.Host != \"\" && Settings.InputKafkaConfig.Topic != \"\" {\n\t\tplugins.registerPlugin(NewKafkaInput, Settings.InputKafkaConfig.Offset, &Settings.InputKafkaConfig, &Settings.KafkaTLSConfig)\n\t}\n\n\treturn plugins\n}\n" }, { "path": "plugins_test.go", "content": "package goreplay\n\nimport (\n\t\"testing\"\n)\n\nfunc TestPluginsRegistration(t *testing.T) {\n\tSettings.InputDummy = []string{\"[]\"}\n\tSettings.OutputDummy = []string{\"[]\"}\n\tSettings.OutputHTTP = []string{\"www.example.com|10\"}\n\tSettings.InputFile = []string{\"/dev/null\"}\n\n\tplugins := NewPlugins()\n\n\tif len(plugins.Inputs) != 3 {\n\t\tt.Errorf(\"Should be 3 inputs got %d\", len(plugins.Inputs))\n\t}\n\n\tif _, ok := plugins.Inputs[0].(*DummyInput); !ok {\n\t\tt.Errorf(\"First input should be DummyInput\")\n\t}\n\n\tif _, ok := plugins.Inputs[1].(*FileInput); !ok {\n\t\tt.Errorf(\"Second input should be FileInput\")\n\t}\n\n\tif len(plugins.Outputs) != 2 {\n\t\tt.Errorf(\"Should be 2 output %d\", len(plugins.Outputs))\n\t}\n\n\tif _, ok := plugins.Outputs[0].(*DummyOutput); !ok {\n\t\tt.Errorf(\"First output should be DummyOutput\")\n\t}\n\n\tif l, ok := plugins.Outputs[1].(*Limiter); ok {\n\t\tif _, ok := l.plugin.(*HTTPOutput); !ok {\n\t\t\tt.Errorf(\"HTTPOutput should be wrapped in limiter\")\n\t\t}\n\t} else {\n\t\tt.Errorf(\"Second output should be Limiter\")\n\t}\n\n}\n" }, { "path": "pro.go", "content": "//go:build pro\n\npackage goreplay\n\n// PRO this value indicates if goreplay is running in PRO mode..\n// it must not be modified explicitly in production\nvar PRO = true\n\n// SettingsHook is intentionally left as a no-op\nvar SettingsHook = func(*AppSettings) {}\n" }, { "path": "proto/fuzz.go", "content": "//go:build gofuzz\n\npackage proto\n\nfunc Fuzz(data []byte) int {\n\n\tParseHeaders(data)\n\n\treturn 1\n}\n" }, { "path": "proto/proto.go", "content": "/*\nPackage proto provides byte-level interaction with HTTP request payload.\n\nExample of HTTP payload for future references, new line symbols escaped:\n\n\tPOST /upload HTTP/1.1\\r\\n\n\tUser-Agent: Gor\\r\\n\n\tContent-Length: 11\\r\\n\n\t\\r\\n\n\tHello world\n\n\tGET /index.html HTTP/1.1\\r\\n\n\tUser-Agent: Gor\\r\\n\n\t\\r\\n\n\t\\r\\n\n*/\npackage proto\n\nimport (\n\t\"bufio\"\n\t\"bytes\"\n\t\"github.com/buger/goreplay/internal/byteutils\"\n\n\t_ \"fmt\"\n\t\"net/http\"\n\t\"net/textproto\"\n\t\"strings\"\n)\n\n// CRLF In HTTP newline defined by 2 bytes (for both windows and *nix support)\nvar CRLF = []byte(\"\\r\\n\")\n\n// EmptyLine acts as separator: end of Headers or Body (in some cases)\nvar EmptyLine = []byte(\"\\r\\n\\r\\n\")\n\n// HeaderDelim Separator for Header line. Header looks like: `HeaderName: value`\nvar HeaderDelim = []byte(\": \")\n\n// MIMEHeadersEndPos finds end of the Headers section, which should end with empty line.\nfunc MIMEHeadersEndPos(payload []byte) int {\n\tpos := bytes.Index(payload, EmptyLine)\n\tif pos < 0 {\n\t\treturn -1\n\t}\n\treturn pos + 4\n}\n\n// MIMEHeadersStartPos finds start of Headers section\n// It just finds position of second line (first contains location and method).\nfunc MIMEHeadersStartPos(payload []byte) int {\n\tpos := bytes.Index(payload, CRLF)\n\tif pos < 0 {\n\t\treturn -1\n\t}\n\treturn pos + 2 // Find first line end\n}\n\n// header return value and positions of header/value start/end.\n// If not found, value will be blank, and headerStart will be -1\n// Do not support multi-line headers.\nfunc header(payload []byte, name []byte) (value []byte, headerStart, headerEnd, valueStart, valueEnd int) {\n\tif HasTitle(payload) {\n\t\theaderStart = MIMEHeadersStartPos(payload)\n\t\tif headerStart < 0 {\n\t\t\treturn\n\t\t}\n\t} else {\n\t\theaderStart = 0\n\t}\n\n\tvar colonIndex int\n\tfor headerStart < len(payload) {\n\t\theaderEnd = bytes.IndexByte(payload[headerStart:], '\\n')\n\t\tif headerEnd == -1 {\n\t\t\tbreak\n\t\t}\n\t\theaderEnd += headerStart\n\t\tcolonIndex = bytes.IndexByte(payload[headerStart:headerEnd], ':')\n\t\tif colonIndex == -1 {\n\t\t\t// Malformed header, skip, most likely packet with partial headers\n\t\t\theaderStart = headerEnd + 1\n\t\t\tcontinue\n\t\t}\n\t\tcolonIndex += headerStart\n\n\t\tif bytes.EqualFold(payload[headerStart:colonIndex], name) {\n\t\t\tvalueStart = colonIndex + 1\n\t\t\tvalueEnd = headerEnd - 2\n\t\t\tbreak\n\t\t}\n\t\theaderStart = headerEnd + 1 // move to the next header\n\t}\n\tif valueStart == 0 {\n\t\theaderStart = -1\n\t\theaderEnd = -1\n\t\tvalueEnd = -1\n\t\tvalueStart = -1\n\t\treturn\n\t}\n\n\t// ignore empty space after ':'\n\tfor valueStart < valueEnd {\n\t\tif payload[valueStart] < 0x21 {\n\t\t\tvalueStart++\n\t\t} else {\n\t\t\tbreak\n\t\t}\n\t}\n\n\t// ignore empty space at end of header value\n\tfor valueEnd > valueStart {\n\t\tif payload[valueEnd] < 0x21 {\n\t\t\tvalueEnd--\n\t\t} else {\n\t\t\tbreak\n\t\t}\n\t}\n\tvalue = payload[valueStart : valueEnd+1]\n\n\treturn\n}\n\n// ParseHeaders Parsing headers from the payload\nfunc ParseHeaders(p []byte) textproto.MIMEHeader {\n\t// trimming off the title of the request\n\tif HasTitle(p) {\n\t\theaderStart := MIMEHeadersStartPos(p)\n\t\tif headerStart > len(p)-1 {\n\t\t\treturn nil\n\t\t}\n\t\tp = p[headerStart:]\n\t}\n\theaderEnd := MIMEHeadersEndPos(p)\n\tif headerEnd > 1 {\n\t\tp = p[:headerEnd]\n\t}\n\treturn GetHeaders(p)\n}\n\n// GetHeaders returns mime headers from the payload\nfunc GetHeaders(p []byte) textproto.MIMEHeader {\n\treader := textproto.NewReader(bufio.NewReader(bytes.NewReader(p)))\n\tmime, err := reader.ReadMIMEHeader()\n\tif err != nil {\n\t\treturn nil\n\t}\n\treturn mime\n}\n\n// Header returns header value, if header not found, value will be blank\nfunc Header(payload, name []byte) []byte {\n\tval, _, _, _, _ := header(payload, name)\n\n\treturn val\n}\n\n// SetHeader sets header value. If header not found it creates new one.\n// Returns modified request payload\nfunc SetHeader(payload, name, value []byte) []byte {\n\t_, hs, _, vs, ve := header(payload, name)\n\n\tif hs != -1 {\n\t\t// If header found we just replace its value\n\t\treturn byteutils.Replace(payload, vs, ve+1, value)\n\t}\n\n\treturn AddHeader(payload, name, value)\n}\n\n// AddHeader takes http payload and appends new header to the start of headers section\n// Returns modified request payload\nfunc AddHeader(payload, name, value []byte) []byte {\n\tmimeStart := MIMEHeadersStartPos(payload)\n\tif mimeStart < 1 {\n\t\treturn payload\n\t}\n\theader := make([]byte, len(name)+2+len(value)+2)\n\tcopy(header[0:], name)\n\tcopy(header[len(name):], HeaderDelim)\n\tcopy(header[len(name)+2:], value)\n\tcopy(header[len(header)-2:], CRLF)\n\n\treturn byteutils.Insert(payload, mimeStart, header)\n}\n\n// DeleteHeader takes http payload and removes header name from headers section\n// Returns modified request payload\nfunc DeleteHeader(payload, name []byte) []byte {\n\t_, hs, he, _, _ := header(payload, name)\n\tif hs != -1 {\n\t\treturn byteutils.Cut(payload, hs, he+1)\n\t}\n\treturn payload\n}\n\n// Body returns request/response body\nfunc Body(payload []byte) []byte {\n\tpos := MIMEHeadersEndPos(payload)\n\tif pos == -1 || len(payload) <= pos {\n\t\treturn nil\n\t}\n\treturn payload[pos:]\n}\n\n// Path takes payload and returns request path: Split(firstLine, ' ')[1]\nfunc Path(payload []byte) []byte {\n\tif !HasRequestTitle(payload) {\n\t\treturn nil\n\t}\n\tstart := bytes.IndexByte(payload, ' ') + 1\n\tend := bytes.IndexByte(payload[start:], ' ')\n\n\treturn payload[start : start+end]\n}\n\n// SetPath takes payload, sets new path and returns modified payload\nfunc SetPath(payload, path []byte) []byte {\n\tif !HasTitle(payload) {\n\t\treturn nil\n\t}\n\tstart := bytes.IndexByte(payload, ' ') + 1\n\tend := bytes.IndexByte(payload[start:], ' ')\n\n\treturn byteutils.Replace(payload, start, start+end, path)\n}\n\n// PathParam returns URL query attribute by given name, if no found: valueStart will be -1\nfunc PathParam(payload, name []byte) (value []byte, valueStart, valueEnd int) {\n\tpath := Path(payload)\n\n\tparamStart := -1\n\tif paramStart = bytes.Index(path, append([]byte{'&'}, append(name, '=')...)); paramStart == -1 {\n\t\tif paramStart = bytes.Index(path, append([]byte{'?'}, append(name, '=')...)); paramStart == -1 {\n\t\t\treturn []byte(\"\"), -1, -1\n\t\t}\n\t}\n\n\tvalueStart = paramStart + len(name) + 2\n\tparamEnd := bytes.IndexByte(path[valueStart:], '&')\n\n\t// Param can end with '&' (another param), or end of line\n\tif paramEnd == -1 { // It is final param\n\t\tparamEnd = len(path)\n\t} else {\n\t\tparamEnd += valueStart\n\t}\n\treturn path[valueStart:paramEnd], valueStart, paramEnd\n}\n\n// SetPathParam takes payload and updates path Query attribute\n// If query param not found, it will append new\n// Returns modified payload\nfunc SetPathParam(payload, name, value []byte) []byte {\n\tpath := Path(payload)\n\t_, vs, ve := PathParam(payload, name)\n\n\tif vs != -1 { // If param found, replace its value and set new Path\n\t\tnewPath := make([]byte, len(path))\n\t\tcopy(newPath, path)\n\t\tnewPath = byteutils.Replace(newPath, vs, ve, value)\n\n\t\treturn SetPath(payload, newPath)\n\t}\n\n\t// if param not found append to end of url\n\t// Adding 2 because of '?' or '&' at start, and '=' in middle\n\tnewParam := make([]byte, len(name)+len(value)+2)\n\n\tif bytes.IndexByte(path, '?') == -1 {\n\t\tnewParam[0] = '?'\n\t} else {\n\t\tnewParam[0] = '&'\n\t}\n\n\t// Copy \"param=value\" into buffer, after it looks like \"?param=value\"\n\tcopy(newParam[1:], name)\n\tnewParam[1+len(name)] = '='\n\tcopy(newParam[2+len(name):], value)\n\n\t// Append param to the end of path\n\tnewPath := make([]byte, len(path)+len(newParam))\n\tcopy(newPath, path)\n\tcopy(newPath[len(path):], newParam)\n\n\treturn SetPath(payload, newPath)\n}\n\n// SetHost updates Host header for HTTP/1.1 or updates host in path for HTTP/1.0 or Proxy requests\n// Returns modified payload\nfunc SetHost(payload, url, host []byte) []byte {\n\t// If this is HTTP 1.0 traffic or proxy traffic it may include host right into path variable, so instead of setting Host header we rewrite Path\n\t// Fix for https://github.com/buger/gor/issues/156\n\tif path := Path(payload); bytes.HasPrefix(path, []byte(\"http\")) {\n\t\thostStart := bytes.IndexByte(path, ':') // : position \"https?:\"\n\t\thostStart += 3 // Skip 1 ':' and 2 '\\'\n\t\thostEnd := hostStart + bytes.IndexByte(path[hostStart:], '/')\n\n\t\tnewPath := make([]byte, len(path))\n\t\tcopy(newPath, path)\n\t\tnewPath = byteutils.Replace(newPath, 0, hostEnd, url)\n\n\t\treturn SetPath(payload, newPath)\n\t}\n\n\treturn SetHeader(payload, []byte(\"Host\"), host)\n}\n\n// Method returns HTTP method\nfunc Method(payload []byte) []byte {\n\tend := bytes.IndexByte(payload, ' ')\n\tif end == -1 {\n\t\treturn nil\n\t}\n\n\treturn payload[:end]\n}\n\n// Status returns response status.\n// It happens to be in same position as request payload path\nfunc Status(payload []byte) []byte {\n\tif !HasResponseTitle(payload) {\n\t\treturn nil\n\t}\n\tstart := bytes.IndexByte(payload, ' ') + 1\n\t// status code are in range 100-600\n\treturn payload[start : start+3]\n}\n\n// Methods holds the http methods ordered in ascending order\nvar Methods = [...]string{\n\thttp.MethodConnect, http.MethodDelete, http.MethodGet,\n\thttp.MethodHead, http.MethodOptions, http.MethodPatch,\n\thttp.MethodPost, http.MethodPut, http.MethodTrace,\n}\n\nconst (\n\t//MinRequestCount GET / HTTP/1.1\\r\\n\n\tMinRequestCount = 16\n\t// MinResponseCount HTTP/1.1 200\\r\\n\n\tMinResponseCount = 14\n\t// VersionLen HTTP/1.1\n\tVersionLen = 8\n)\n\n// HasResponseTitle reports whether this payload has an HTTP/1 response title\nfunc HasResponseTitle(payload []byte) bool {\n\ts := byteutils.SliceToString(payload)\n\tif len(s) < MinResponseCount {\n\t\treturn false\n\t}\n\ttitleLen := bytes.Index(payload, CRLF)\n\tif titleLen == -1 {\n\t\treturn false\n\t}\n\tmajor, minor, ok := http.ParseHTTPVersion(s[0:VersionLen])\n\tif !(ok && major == 1 && (minor == 0 || minor == 1)) {\n\t\treturn false\n\t}\n\tif s[VersionLen] != ' ' {\n\t\treturn false\n\t}\n\tstatus, ok := atoI(payload[VersionLen+1:VersionLen+4], 10)\n\tif !ok {\n\t\treturn false\n\t}\n\t// only validate status codes mentioned in rfc2616.\n\tif http.StatusText(status) == \"\" {\n\t\treturn false\n\t}\n\t// handle cases from #875\n\treturn payload[VersionLen+4] == ' ' || payload[VersionLen+4] == '\\r'\n}\n\n// HasRequestTitle reports whether this payload has an HTTP/1 request title\nfunc HasRequestTitle(payload []byte) bool {\n\ts := byteutils.SliceToString(payload)\n\tif len(s) < MinRequestCount {\n\t\treturn false\n\t}\n\ttitleLen := bytes.Index(payload, CRLF)\n\tif titleLen == -1 {\n\t\treturn false\n\t}\n\tif strings.Count(s[:titleLen], \" \") != 2 {\n\t\treturn false\n\t}\n\tmethod := string(Method(payload))\n\tvar methodFound bool\n\tfor _, m := range Methods {\n\t\tif methodFound = method == m; methodFound {\n\t\t\tbreak\n\t\t}\n\t}\n\tif !methodFound {\n\t\treturn false\n\t}\n\tpath := strings.Index(s[len(method)+1:], \" \")\n\tif path == -1 {\n\t\treturn false\n\t}\n\tmajor, minor, ok := http.ParseHTTPVersion(s[path+len(method)+2 : titleLen])\n\treturn ok && major == 1 && (minor == 0 || minor == 1)\n}\n\n// HasTitle reports if this payload has an http/1 title\nfunc HasTitle(payload []byte) bool {\n\treturn HasRequestTitle(payload) || HasResponseTitle(payload)\n}\n\n// CheckChunked checks HTTP/1 chunked data integrity(https://tools.ietf.org/html/rfc7230#section-4.1)\n// and returns the length of total valid scanned chunks(including chunk size, extensions and CRLFs) and\n// full is true if all chunks was scanned.\nfunc CheckChunked(bufs ...[]byte) (chunkEnd int, full bool) {\n\tvar buf []byte\n\tif len(bufs) > 0 {\n\t\tbuf = bufs[0]\n\t}\n\tfor chunkEnd < len(buf) {\n\t\tsz := bytes.IndexByte(buf[chunkEnd:], '\\r')\n\t\tif sz < 1 {\n\t\t\tbreak\n\t\t}\n\t\t// don't parse chunk extensions https://github.com/golang/go/issues/13135.\n\t\t// chunks extensions are no longer a thing, but we do check if the byte\n\t\t// following the parsed hex number is ';'\n\t\tsz += chunkEnd\n\t\tchkLen, ok := atoI(buf[chunkEnd:sz], 16)\n\t\tif !ok && bytes.IndexByte(buf[chunkEnd:sz], ';') < 1 {\n\t\t\tbreak\n\t\t}\n\t\tsz++ // + '\\n'\n\t\t// total length = SIZE + CRLF + OCTETS + CRLF\n\t\tallChunk := sz + chkLen + 2\n\t\tif allChunk >= len(buf) ||\n\t\t\tbuf[sz]&buf[allChunk] != '\\n' ||\n\t\t\tbuf[allChunk-1] != '\\r' {\n\t\t\tbreak\n\t\t}\n\t\tchunkEnd = allChunk + 1\n\t\tif chkLen == 0 {\n\t\t\tfull = true\n\t\t\tbreak\n\t\t}\n\t}\n\treturn\n}\n\n// ProtocolStateSetter is an interface used to provide protocol state for future use\ntype ProtocolStateSetter interface {\n\tSetProtocolState(interface{})\n\tProtocolState() interface{}\n}\n\ntype HTTPState struct {\n\tBody int // body index\n\tHeaderStart int\n\tHeaderEnd int\n\tHeaderParsed bool // we checked necessary headers\n\tHasFullPayload bool // all chunks has been parsed\n\tIsChunked bool // Transfer-Encoding: chunked\n\tBodyLen int // Content-Length's value\n\tHasTrailer bool // Trailer header?\n\tContinue100 bool\n}\n\n// HasFullPayload checks if this message has full or valid payloads and returns true.\n// Message param is optional but recommended on cases where 'data' is storing\n// partial-to-full stream of bytes(packets).\nfunc HasFullPayload(m ProtocolStateSetter, payloads ...[]byte) bool {\n\tvar state *HTTPState\n\tif m != nil {\n\t\tstate, _ = m.ProtocolState().(*HTTPState)\n\t}\n\tif state == nil {\n\t\tstate = new(HTTPState)\n\t\tif m != nil {\n\t\t\tm.SetProtocolState(state)\n\t\t}\n\t}\n\n\t// Http Packets can only start with a few things, check if this is one of them\n\tif len(payloads) == 0 {\n\t\treturn false\n\t}\n\tif !HasRequestTitle(payloads[0]) && !HasResponseTitle(payloads[0]) {\n\t\treturn false\n\t}\n\n\tif state.HeaderStart < 1 {\n\t\tfor _, data := range payloads {\n\t\t\tstate.HeaderStart = MIMEHeadersStartPos(data)\n\t\t\tif state.HeaderStart < 0 {\n\t\t\t\treturn false\n\t\t\t} else {\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\n\tif state.Body < 1 || state.HeaderEnd < 1 {\n\t\tvar pos int\n\t\tfor _, data := range payloads {\n\t\t\tendPos := MIMEHeadersEndPos(data)\n\t\t\tif endPos < 0 {\n\t\t\t\tpos += len(data)\n\t\t\t} else {\n\t\t\t\tpos += endPos\n\t\t\t\tstate.HeaderEnd = pos\n\t\t\t}\n\n\t\t\tif endPos > 0 {\n\t\t\t\tstate.Body = pos\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\n\tif state.HeaderEnd < 1 {\n\t\treturn false\n\t}\n\n\tif !state.HeaderParsed {\n\t\tvar pos int\n\t\tfor _, data := range payloads {\n\t\t\tchunked := Header(data, []byte(\"Transfer-Encoding\"))\n\n\t\t\tif len(chunked) > 0 && bytes.Index(data, []byte(\"chunked\")) > 0 {\n\t\t\t\tstate.IsChunked = true\n\t\t\t\t// trailers are generally not allowed in non-chunks body\n\t\t\t\tstate.HasTrailer = len(Header(data, []byte(\"Trailer\"))) > 0\n\t\t\t} else {\n\t\t\t\tcontentLen := Header(data, []byte(\"Content-Length\"))\n\t\t\t\tstate.BodyLen, _ = atoI(contentLen, 10)\n\t\t\t}\n\n\t\t\tpos += len(data)\n\n\t\t\tif string(Header(data, []byte(\"Expect\"))) == \"100-continue\" {\n\t\t\t\tstate.Continue100 = true\n\t\t\t}\n\n\t\t\tif state.BodyLen > 0 || pos >= state.Body {\n\t\t\t\tstate.HeaderParsed = true\n\t\t\t\tbreak\n\t\t\t}\n\t\t}\n\t}\n\n\tbodyLen := 0\n\tfor _, data := range payloads {\n\t\tbodyLen += len(data)\n\t}\n\tbodyLen -= state.Body\n\n\tif state.IsChunked {\n\t\t// check chunks\n\t\tif bodyLen < 1 {\n\t\t\treturn false\n\t\t}\n\n\t\t// check trailer headers\n\t\tif state.HasTrailer {\n\t\t\tif bytes.HasSuffix(payloads[len(payloads)-1], []byte(\"\\r\\n\\r\\n\")) {\n\t\t\t\treturn true\n\t\t\t}\n\t\t} else {\n\t\t\tif bytes.HasSuffix(payloads[len(payloads)-1], []byte(\"0\\r\\n\\r\\n\")) {\n\t\t\t\tstate.HasFullPayload = true\n\t\t\t\treturn true\n\t\t\t}\n\t\t}\n\n\t\treturn false\n\t}\n\n\t// check for content-length header\n\treturn state.BodyLen == bodyLen\n}\n\n// this works with positive integers\nfunc atoI(s []byte, base int) (num int, ok bool) {\n\tvar v int\n\tok = true\n\tfor i := 0; i < len(s); i++ {\n\t\tif s[i] > 127 {\n\t\t\tok = false\n\t\t\tbreak\n\t\t}\n\t\tv = int(hexTable[s[i]])\n\t\tif v >= base || (v == 0 && s[i] != '0') {\n\t\t\tok = false\n\t\t\tbreak\n\t\t}\n\t\tnum = (num * base) + v\n\t}\n\treturn\n}\n\nvar hexTable = [128]byte{\n\t'0': 0,\n\t'1': 1,\n\t'2': 2,\n\t'3': 3,\n\t'4': 4,\n\t'5': 5,\n\t'6': 6,\n\t'7': 7,\n\t'8': 8,\n\t'9': 9,\n\t'A': 10,\n\t'a': 10,\n\t'B': 11,\n\t'b': 11,\n\t'C': 12,\n\t'c': 12,\n\t'D': 13,\n\t'd': 13,\n\t'E': 14,\n\t'e': 14,\n\t'F': 15,\n\t'f': 15,\n}\n" }, { "path": "proto/proto_test.go", "content": "package proto\n\nimport (\n\t\"bytes\"\n\t\"net/textproto\"\n\t\"reflect\"\n\t\"testing\"\n)\n\nfunc TestHeader(t *testing.T) {\n\tvar payload, val []byte\n\tvar headerStart int\n\n\t// Value with space at start\n\tpayload = []byte(\"POST /post HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\n\tif val = Header(payload, []byte(\"Content-Length\")); !bytes.Equal(val, []byte(\"7\")) {\n\t\tt.Error(\"Should find header value\")\n\t}\n\n\t// Value with space at end\n\tpayload = []byte(\"POST /post HTTP/1.1\\r\\nContent-Length: 7 \\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\n\tif val = Header(payload, []byte(\"Content-Length\")); !bytes.Equal(val, []byte(\"7\")) {\n\t\tt.Error(\"Should find header value without space after 7\")\n\t}\n\n\t// Value without space at start\n\tpayload = []byte(\"POST /post HTTP/1.1\\r\\nContent-Length:7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\n\tif val = Header(payload, []byte(\"Content-Length\")); !bytes.Equal(val, []byte(\"7\")) {\n\t\tt.Error(\"Should find header value without space after :\")\n\t}\n\n\t// Value is empty\n\tpayload = []byte(\"GET /p HTTP/1.1\\r\\nCookie:\\r\\nHost: www.w3.org\\r\\n\\r\\n\")\n\n\tif val = Header(payload, []byte(\"Cookie\")); len(val) > 0 {\n\t\tt.Error(\"Should return empty value\")\n\t}\n\n\t// Header not found\n\tif _, headerStart, _, _, _ = header(payload, []byte(\"Not-Found\")); headerStart != -1 {\n\t\tt.Error(\"Should not found header\")\n\t}\n\n\t// Lower case headers\n\tpayload = []byte(\"POST /post HTTP/1.1\\r\\ncontent-length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\n\tif val = Header(payload, []byte(\"Content-Length\")); !bytes.Equal(val, []byte(\"7\")) {\n\t\tt.Error(\"Should find lower case 2 word header\")\n\t}\n\n\tpayload = []byte(\"POST /post HTTP/1.1\\r\\ncontent-length: 7\\r\\nhost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\n\tif val = Header(payload, []byte(\"host\")); !bytes.Equal(val, []byte(\"www.w3.org\")) {\n\t\tt.Error(\"Should find lower case 1 word header\")\n\t}\n\n\tpayload = []byte(\"GT\\r\\nContent-Length: 10\\r\\n\\r\\n\")\n\n\tif val = Header(payload, []byte(\"Content-Length\")); !bytes.Equal(val, []byte(\"10\")) {\n\t\tt.Error(\"Should find in partial payload\")\n\t}\n}\n\nfunc TestMIMEHeadersEndPos(t *testing.T) {\n\thead := []byte(\"POST /post HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\n\")\n\tpayload := []byte(\"POST /post HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\n\tend := MIMEHeadersEndPos(payload)\n\n\tif !bytes.Equal(payload[:end], head) {\n\t\tt.Error(\"Wrong headers end position:\", end, head, payload[:end])\n\t}\n}\n\nfunc TestMIMEHeadersStartPos(t *testing.T) {\n\theaders := []byte(\"Content-Length: 7\\r\\nHost: www.w3.org\")\n\tpayload := []byte(\"POST /post HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\n\tstart := MIMEHeadersStartPos(payload)\n\tend := MIMEHeadersEndPos(payload) - 4\n\n\tif !bytes.Equal(payload[start:end], headers) {\n\t\tt.Error(\"Wrong headers end position:\", start, end, payload[start:end])\n\t}\n}\n\nfunc TestSetHeader(t *testing.T) {\n\tvar payload, payloadAfter []byte\n\n\tpayload = []byte(\"POST /post HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\tpayloadAfter = []byte(\"POST /post HTTP/1.1\\r\\nContent-Length: 14\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\n\tif payload = SetHeader(payload, []byte(\"Content-Length\"), []byte(\"14\")); !bytes.Equal(payload, payloadAfter) {\n\t\tt.Error(\"Should update header if it exists\", string(payload))\n\t}\n\n\tpayload = []byte(\"POST /post HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\tpayloadAfter = []byte(\"POST /post HTTP/1.1\\r\\nUser-Agent: Gor\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\n\tif payload = SetHeader(payload, []byte(\"User-Agent\"), []byte(\"Gor\")); !bytes.Equal(payload, payloadAfter) {\n\t\tt.Error(\"Should add header if not found\", string(payload))\n\t}\n\tinvalidPayload := []byte(\"POST /post HTTP/1.1\")\n\tif invalidPayload = SetHeader(invalidPayload, []byte(\"User-Agent\"), []byte(\"Gor\")); !bytes.Equal(invalidPayload, []byte(\"POST /post HTTP/1.1\")) {\n\t\tt.Error(\"Should not modify payload if request is invalid\", string(payload))\n\t}\n}\n\nfunc TestDeleteHeader(t *testing.T) {\n\tvar payload, payloadAfter []byte\n\n\tpayload = []byte(\"POST /post HTTP/1.1\\r\\nUser-Agent: Gor\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\tpayloadAfter = []byte(\"POST /post HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\n\tif payload = DeleteHeader(payload, []byte(\"User-Agent\")); !bytes.Equal(payload, payloadAfter) {\n\t\tt.Error(\"Should delete header if found\", string(payload), string(payloadAfter))\n\t}\n\n\t//Whitespace at end of User-Agent\n\tpayload = []byte(\"POST /post HTTP/1.1\\r\\nUser-Agent: Gor \\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\tpayloadAfter = []byte(\"POST /post HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\n\tif payload = DeleteHeader(payload, []byte(\"User-Agent\")); !bytes.Equal(payload, payloadAfter) {\n\t\tt.Error(\"Should delete header if found\", string(payload), string(payloadAfter))\n\t}\n}\n\nfunc TestParseHeaders(t *testing.T) {\n\tpayload := [][]byte{[]byte(\"POST /post HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.or\"), []byte(\"g\\r\\nUser-Ag\"), []byte(\"ent:Chrome\\r\\n\\r\\n\"), []byte(\"Fake-Header: asda\")}\n\n\theaders := ParseHeaders(bytes.Join(payload, nil))\n\n\texpected := textproto.MIMEHeader{\n\t\t\"Content-Length\": []string{\"7\"},\n\t\t\"Host\": []string{\"www.w3.org\"},\n\t\t\"User-Agent\": []string{\"Chrome\"},\n\t}\n\n\tif !reflect.DeepEqual(headers, expected) {\n\t\tt.Error(\"Headers do not properly parsed\", headers)\n\t}\n\n\t// Response with Reason phrase\n\tpayload = [][]byte{[]byte(\"HTTP/1.1 200 OK\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\nUser-Agent:Chrome\\r\\n\\r\\nbody\")}\n\n\theaders = ParseHeaders(bytes.Join(payload, nil))\n\n\tif !reflect.DeepEqual(headers, expected) {\n\t\tt.Error(\"Headers do not properly parsed\", headers)\n\t}\n\n\t// Response without Reason phrase\n\tpayload = [][]byte{[]byte(\"HTTP/1.1 200\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\nUser-Agent:Chrome\\r\\n\\r\\nbody\")}\n\n\theaders = ParseHeaders(bytes.Join(payload, nil))\n\n\tif !reflect.DeepEqual(headers, expected) {\n\t\tt.Error(\"Headers do not properly parsed\", headers)\n\t}\n}\n\n// See https://github.com/dvyukov/go-fuzz and fuzz.go\nfunc TestFuzzCrashers(t *testing.T) {\n\tvar crashers = []string{\n\t\t\"\\n:00\\n\",\n\t}\n\n\tfor _, f := range crashers {\n\t\tParseHeaders([]byte(f))\n\t}\n}\n\nfunc TestParseHeadersWithComplexUserAgent(t *testing.T) {\n\t// User-Agent could contain inside ':'\n\t// Parser should wait for \\r\\n\n\tpayload := [][]byte{[]byte(\"POST /post HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.or\"), []byte(\"g\\r\\nUser-Ag\"), []byte(\"ent:Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko\\r\\n\\r\\n\"), []byte(\"Fake-Header: asda\")}\n\n\theaders := ParseHeaders(bytes.Join(payload, nil))\n\n\texpected := map[string]string{\n\t\t\"User-Agent\": \"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko\",\n\t}\n\n\tif expected[\"User-Agent\"] != headers[\"User-Agent\"][0] {\n\t\tt.Errorf(\"Header 'User-Agent' expected '%s' and parsed: '%s'\", expected[\"User-Agent\"], headers[\"User-Agent\"])\n\t}\n}\n\nfunc TestParseHeadersWithOrigin(t *testing.T) {\n\t// User-Agent could contain inside ':'\n\t// Parser should wait for \\r\\n\n\tpayload := [][]byte{[]byte(\"POST /post HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.or\"), []byte(\"g\\r\\nReferrer: http://127.0.0.1:3000\\r\\nOrigi\"), []byte(\"n: https://www.example.com\\r\\nUser-Ag\"), []byte(\"ent:Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko\\r\\n\\r\\n\"), []byte(\"in:https://www.example.com\\r\\n\\r\\n\"), []byte(\"Fake-Header: asda\")}\n\n\theaders := ParseHeaders(bytes.Join(payload, nil))\n\n\texpected := map[string]string{\n\t\t\"Origin\": \"https://www.example.com\",\n\t\t\"User-Agent\": \"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko\",\n\t\t\"Referrer\": \"http://127.0.0.1:3000\",\n\t}\n\n\tif expected[\"Referrer\"] != headers[\"Referrer\"][0] {\n\t\tt.Errorf(\"Header 'Referrer' expected '%s' and parsed: '%s'\", expected[\"Referrer\"], headers[\"Referrer\"])\n\t}\n\n\tif expected[\"Origin\"] != headers[\"Origin\"][0] {\n\t\tt.Errorf(\"Header 'Origin' expected '%s' and parsed: '%s'\", expected[\"Origin\"], headers[\"Origin\"])\n\t}\n\n\tif expected[\"User-Agent\"] != headers[\"User-Agent\"][0] {\n\t\tt.Errorf(\"Header 'User-Agent' expected '%s' and parsed: '%s'\", expected[\"User-Agent\"], headers[\"User-Agent\"])\n\t}\n}\n\nfunc TestPath(t *testing.T) {\n\tvar path, payload []byte\n\n\tpayload = []byte(\"POST /post HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\n\tif path = Path(payload); !bytes.Equal(path, []byte(\"/post\")) {\n\t\tt.Error(\"Should find path\", string(path))\n\t}\n\n\tpayload = []byte(\"GET /get\\r\\n\\r\\nHost: www.w3.org\\r\\n\\r\\n\")\n\n\tif path = Path(payload); !bytes.Equal(path, nil) {\n\t\tt.Error(\"1Should not find path\", string(path))\n\t}\n\n\tpayload = []byte(\"GET /get\\n\")\n\n\tif path = Path(payload); !bytes.Equal(path, nil) {\n\t\tt.Error(\"2Should not find path\", string(path))\n\t}\n\n\tpayload = []byte(\"GET /get\")\n\n\tif path = Path(payload); !bytes.Equal(path, nil) {\n\t\tt.Error(\"3Should not find path\", string(path))\n\t}\n}\n\nfunc TestStatus(t *testing.T) {\n\tvar status, payload []byte\n\n\tpayload = []byte(\"HTTP/1.1 200 OK\\r\\n\")\n\tif status = Status(payload); !bytes.Equal(status, []byte(\"200\")) {\n\t\tt.Error(\"Should find status 200 but:\", string(status))\n\t}\n\n\tpayload = []byte(\"HTTP/1.1 200\\r\\n\")\n\tif status = Status(payload); !bytes.Equal(status, []byte(\"200\")) {\n\t\tt.Error(\"1Should find status 200 but:\", string(status))\n\t}\n\n\tpayload = []byte(\"HTTP/1.1 404 Not Found\\r\\n\")\n\tif status = Status(payload); !bytes.Equal(status, []byte(\"404\")) {\n\t\tt.Error(\"2Should find status 404 but:\", string(status))\n\t}\n}\n\nfunc TestSetPath(t *testing.T) {\n\tvar payload, payloadAfter []byte\n\n\tpayload = []byte(\"POST /post HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\tpayloadAfter = []byte(\"POST /new_path HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\n\tif payload = SetPath(payload, []byte(\"/new_path\")); !bytes.Equal(payload, payloadAfter) {\n\t\tt.Error(\"Should replace path\", string(payload))\n\t}\n\n}\n\nfunc TestPathParam(t *testing.T) {\n\tvar payload []byte\n\n\tpayload = []byte(\"POST /post?param=test&user_id=1&d_type=1&type=2&d_type=3 HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\n\tif val, _, _ := PathParam(payload, []byte(\"param\")); !bytes.Equal(val, []byte(\"test\")) {\n\t\tt.Error(\"Should detect attribute\", string(val))\n\t}\n\n\tif val, _, _ := PathParam(payload, []byte(\"user_id\")); !bytes.Equal(val, []byte(\"1\")) {\n\t\tt.Error(\"Should detect attribute\", string(val))\n\t}\n\n\tif val, _, _ := PathParam(payload, []byte(\"type\")); !bytes.Equal(val, []byte(\"2\")) {\n\t\tt.Error(\"Should detect attribute\", string(val))\n\t}\n\n\tif val, _, _ := PathParam(payload, []byte(\"d_type\")); !bytes.Equal(val, []byte(\"1\")) {\n\t\t// this function is not designed for cases with duplicate param keys\n\t\tt.Error(\"Should detect attribute\", string(val))\n\t}\n}\n\nfunc TestSetPathParam(t *testing.T) {\n\tvar payload, payloadAfter []byte\n\n\tpayload = []byte(\"POST /post?param=test&user_id=1 HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\tpayloadAfter = []byte(\"POST /post?param=new&user_id=1 HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\n\tif payload = SetPathParam(payload, []byte(\"param\"), []byte(\"new\")); !bytes.Equal(payload, payloadAfter) {\n\t\tt.Error(\"Should replace existing value\", string(payload))\n\t}\n\n\tpayload = []byte(\"POST /post?param=test&user_id=1 HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\tpayloadAfter = []byte(\"POST /post?param=test&user_id=2 HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\n\tif payload = SetPathParam(payload, []byte(\"user_id\"), []byte(\"2\")); !bytes.Equal(payload, payloadAfter) {\n\t\tt.Error(\"Should replace existing value\", string(payload))\n\t}\n\n\tpayload = []byte(\"POST /post HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\tpayloadAfter = []byte(\"POST /post?param=test HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\n\tif payload = SetPathParam(payload, []byte(\"param\"), []byte(\"test\")); !bytes.Equal(payload, payloadAfter) {\n\t\tt.Error(\"Should set param if url have no params\", string(payload))\n\t}\n\n\tpayload = []byte(\"POST /post?param=test HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\tpayloadAfter = []byte(\"POST /post?param=test&user_id=1 HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\n\tif payload = SetPathParam(payload, []byte(\"user_id\"), []byte(\"1\")); !bytes.Equal(payload, payloadAfter) {\n\t\tt.Error(\"Should set param at the end if url params\", string(payload))\n\t}\n}\n\nfunc TestSetHostHTTP10(t *testing.T) {\n\tvar payload, payloadAfter []byte\n\n\tpayload = []byte(\"POST http://example.com/post HTTP/1.0\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\tpayloadAfter = []byte(\"POST http://new.com/post HTTP/1.0\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n\n\tif payload = SetHost(payload, []byte(\"http://new.com\"), []byte(\"new.com\")); !bytes.Equal(payload, payloadAfter) {\n\t\tt.Error(\"Should replace host\", string(payload))\n\t}\n\n\tpayload = []byte(\"POST /post HTTP/1.0\\r\\nContent-Length: 7\\r\\nHost: example.com\\r\\n\\r\\na=1&b=2\")\n\tpayloadAfter = []byte(\"POST /post HTTP/1.0\\r\\nContent-Length: 7\\r\\nHost: new.com\\r\\n\\r\\na=1&b=2\")\n\n\tif payload = SetHost(payload, nil, []byte(\"new.com\")); !bytes.Equal(payload, payloadAfter) {\n\t\tt.Error(\"Should replace host\", string(payload))\n\t}\n\n\tpayload = []byte(\"POST /post HTTP/1.0\\r\\nContent-Length: 7\\r\\n\\r\\na=1&b=2\")\n\n\tif payload = SetHost(payload, nil, []byte(\"new.com\")); !bytes.Equal(payload, payload) {\n\t\tt.Error(\"Should replace host\", string(payload))\n\t}\n}\n\nfunc TestHasResponseTitle(t *testing.T) {\n\tvar m = map[string]bool{\n\t\t\"HTTP\": false,\n\t\t\"\": false,\n\t\t\"HTTP/1.1 100 Continue\": false,\n\t\t\"HTTP/1.1 100 Continue\\r\\n\": true,\n\t\t\"HTTP/1.1 \\r\\n\": false,\n\t\t\"HTTP/4.0 100Continue\\r\\n\": false,\n\t\t\"HTTP/1.0 100Continue\\r\\n\": false,\n\t\t\"HTTP/1.0 10r Continue\\r\\n\": false,\n\t\t\"HTTP/1.1 200\\r\\n\": true,\n\t\t\"HTTP/1.1 200\\r\\nServer: Tengine\\r\\nContent-Length: 0\\r\\nConnection: close\\r\\n\\r\\n\": true,\n\t}\n\tfor k, v := range m {\n\t\tif HasResponseTitle([]byte(k)) != v {\n\t\t\tt.Errorf(\"%q should yield %v\", k, v)\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nfunc TestHasRequestTitle(t *testing.T) {\n\tvar m = map[string]bool{\n\t\t\"POST /post HTTP/1.0\\r\\n\": true,\n\t\t\"\": false,\n\t\t\"POST /post HTTP/1.\\r\\n\": false,\n\t\t\"POS /post HTTP/1.1\\r\\n\": false,\n\t\t\"GET / HTTP/1.1\\r\\n\": true,\n\t\t\"GET / HTTP/1.1\\r\": false,\n\t\t\"GET / HTTP/1.400\\r\\n\": false,\n\t}\n\tfor k, v := range m {\n\t\tif HasRequestTitle([]byte(k)) != v {\n\t\t\tt.Errorf(\"%q should yield %v\", k, v)\n\t\t\tbreak\n\t\t}\n\t}\n}\n\nfunc TestCheckChunks(t *testing.T) {\n\tvar m = \"4\\r\\nWiki\\r\\n5\\r\\npedia\\r\\nE\\r\\n in\\r\\n\\r\\nchunks.\\r\\n0\\r\\n\\r\\n\"\n\tchunkEnd, _ := CheckChunked([]byte(m))\n\texpected := len(m)\n\tif chunkEnd != expected {\n\t\tt.Errorf(\"expected %d to equal %d\", chunkEnd, expected)\n\t}\n\n\tm = \"7\\r\\nMozia\\r\\n9\\r\\nDeveloper\\r\\n7\\r\\nNetwork\\r\\n0\\r\\n\\r\\n\"\n\tchunkEnd, _ = CheckChunked([]byte(m))\n\tif chunkEnd != 0 {\n\t\tt.Errorf(\"expected %d to equal %d\", chunkEnd, 0)\n\t}\n\n\t// with trailers\n\tm = \"4\\r\\nWiki\\r\\n5\\r\\npedia\\r\\nE\\r\\n in\\r\\n\\r\\nchunks.\\r\\n0\\r\\n\\r\\nEXpires\"\n\tchunkEnd, _ = CheckChunked([]byte(m))\n\texpected = len(m) - 7\n\tif chunkEnd != expected {\n\t\tt.Errorf(\"expected %d to equal %d\", chunkEnd, expected)\n\t}\n\n\t// last chunk inside the body\n\t// with trailers\n\tm = \"4\\r\\nWiki\\r\\n5\\r\\npedia\\r\\nE\\r\\n in\\r\\n\\r\\nchunks.\\r\\n3\\r\\n0\\r\\n\\r\\n0\\r\\n\\r\\nEXpires\"\n\tchunkEnd, _ = CheckChunked([]byte(m))\n\texpected = len(m) - 7\n\tif chunkEnd != expected {\n\t\tt.Errorf(\"expected %d to equal %d\", chunkEnd, expected)\n\t}\n\n\t// checks with chucks-extensions\n\tm = \"4\\r\\nWiki\\r\\n5\\r\\npedia\\r\\nE; name='quoted string'\\r\\n in\\r\\n\\r\\nchunks.\\r\\n3\\r\\n0\\r\\n\\r\\n0\\r\\n\\r\\nEXpires\"\n\tchunkEnd, _ = CheckChunked([]byte(m))\n\texpected = len(m) - 7\n\tif chunkEnd != expected {\n\t\tt.Errorf(\"expected %d to equal %d\", chunkEnd, expected)\n\t}\n}\n\nfunc TestHasFullPayload(t *testing.T) {\n\tvar m string\n\tvar got, expected bool\n\n\tgot = HasFullPayload(nil,\n\t\t[]byte(\"HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\n\"),\n\t\t[]byte(\"Transfer-Encoding: chunked\\r\\n\\r\\n\"),\n\t\t[]byte(\"7\\r\\nMozilla\\r\\n9\\r\\nDeveloper\\r\\n\"),\n\t\t[]byte(\"7\\r\\nNetwork\\r\\n0\\r\\n\\r\\n\"))\n\texpected = true\n\tif got != expected {\n\t\tt.Errorf(\"expected %v to equal %v\", got, expected)\n\t}\n\n\t// check chunks with trailers\n\tm = \"HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nTransfer-Encoding: chunked\\r\\nTrailer: Expires\\r\\n\\r\\n7\\r\\nMozilla\\r\\n9\\r\\nDeveloper\\r\\n7\\r\\nNetwork\\r\\n0\\r\\n\\r\\nExpires: Wed, 21 Oct 2015 07:28:00 GMT\\r\\n\\r\\n\"\n\tgot = HasFullPayload(nil, []byte(m))\n\texpected = true\n\tif got != expected {\n\t\tt.Errorf(\"expected %v to equal %v\", got, expected)\n\t}\n\n\t// check with missing trailers\n\tm = \"HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nTransfer-Encoding: chunked\\r\\nTrailer: Expires\\r\\n\\r\\n7\\r\\nMozilla\\r\\n9\\r\\nDeveloper\\r\\n7\\r\\nNetwork\\r\\n0\\r\\n\\r\\nExpires: Wed, 21 Oct 2015 07:28:00\"\n\tgot = HasFullPayload(nil, []byte(m))\n\texpected = false\n\tif got != expected {\n\t\tt.Errorf(\"expected %v to equal %v\", got, expected)\n\t}\n\n\t// check with content-length\n\tm = \"HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nContent-Length: 23\\r\\n\\r\\nMozillaDeveloperNetwork\"\n\tgot = HasFullPayload(nil, []byte(m))\n\texpected = true\n\tif got != expected {\n\t\tt.Errorf(\"expected %v to equal %v\", got, expected)\n\t}\n\n\t// check missing total length\n\tm = \"HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nContent-Length: 23\\r\\n\\r\\nMozillaDeveloperNet\"\n\tgot = HasFullPayload(nil, []byte(m))\n\texpected = false\n\tif got != expected {\n\t\tt.Errorf(\"expected %v to equal %v\", got, expected)\n\t}\n\n\t// check with no body\n\tm = \"HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\n\\r\\n\"\n\tgot = HasFullPayload(nil, []byte(m))\n\texpected = true\n\tif got != expected {\n\t\tt.Errorf(\"expected %v to equal %v\", got, expected)\n\t}\n\n\t// check with trailer and no header\n\tm = \"Content-Type: text/plain\\r\\nContent-Length: 23\\r\\n\\r\\nMozillaDeveloperNetwork\"\n\tgot = HasFullPayload(nil, []byte(m))\n\texpected = false\n\tif got != expected {\n\t\tt.Errorf(\"expected %v to equal %v\", got, expected)\n\t}\n}\n\nfunc BenchmarkHasFullPayload(b *testing.B) {\n\tdata := []byte(\"HTTP/1.1 200 OK\\r\\nContent-Type: text/plain\\r\\nTransfer-Encoding: chunked\\r\\n\\r\\n1e\\r\\n111111111111111111111111111111\\r\\n0\\r\\n\\r\\n\")\n\tfor i := 0; i < b.N; i++ {\n\t\tif !HasFullPayload(nil, data) {\n\t\t\tb.Fail()\n\t\t}\n\t}\n}\n" }, { "path": "protocol.go", "content": "package goreplay\n\nimport (\n\t\"bytes\"\n\t\"crypto/rand\"\n\t\"encoding/hex\"\n\t\"fmt\"\n)\n\n// These constants help to indicate the type of payload\nconst (\n\tRequestPayload = '1'\n\tResponsePayload = '2'\n\tReplayedResponsePayload = '3'\n)\n\nfunc randByte(len int) []byte {\n\tb := make([]byte, len/2)\n\trand.Read(b)\n\n\th := make([]byte, len)\n\thex.Encode(h, b)\n\n\treturn h\n}\n\nfunc uuid() []byte {\n\treturn randByte(24)\n}\n\nvar payloadSeparator = \"\\n🐵🙈🙉\\n\"\n\nfunc payloadScanner(data []byte, atEOF bool) (advance int, token []byte, err error) {\n\tif atEOF && len(data) == 0 {\n\t\treturn 0, nil, nil\n\t}\n\n\tif i := bytes.Index(data, []byte(payloadSeparator)); i >= 0 {\n\t\t// We have a full newline-terminated line.\n\t\treturn i + len([]byte(payloadSeparator)), data[0:i], nil\n\t}\n\n\tif atEOF {\n\t\treturn len(data), data, nil\n\t}\n\treturn 0, nil, nil\n}\n\n// Timing is request start or round-trip time, depending on payloadType\nfunc payloadHeader(payloadType byte, uuid []byte, timing int64, latency int64) (header []byte) {\n\t//Example:\n\t// 3 f45590522cd1838b4a0d5c5aab80b77929dea3b3 13923489726487326 1231\\n\n\treturn []byte(fmt.Sprintf(\"%c %s %d %d\\n\", payloadType, uuid, timing, latency))\n}\n\nfunc payloadBody(payload []byte) []byte {\n\theaderSize := bytes.IndexByte(payload, '\\n')\n\treturn payload[headerSize+1:]\n}\n\nfunc payloadMeta(payload []byte) [][]byte {\n\theaderSize := bytes.IndexByte(payload, '\\n')\n\tif headerSize < 0 {\n\t\treturn nil\n\t}\n\treturn bytes.Split(payload[:headerSize], []byte{' '})\n}\n\nfunc payloadMetaWithBody(payload []byte) (meta, body []byte) {\n\tif i := bytes.IndexByte(payload, '\\n'); i > 0 && len(payload) > i+1 {\n\t\tmeta = payload[:i+1]\n\t\tbody = payload[i+1:]\n\t\treturn\n\t}\n\t// we assume the message did not have meta data\n\treturn nil, payload\n}\n\nfunc payloadID(payload []byte) (id []byte) {\n\tmeta := payloadMeta(payload)\n\n\tif len(meta) < 2 {\n\t\treturn\n\t}\n\treturn meta[1]\n}\n\nfunc isOriginPayload(payload []byte) bool {\n\treturn payload[0] == RequestPayload || payload[0] == ResponsePayload\n}\n\nfunc isRequestPayload(payload []byte) bool {\n\treturn payload[0] == RequestPayload\n}\n" }, { "path": "s3/index.html", "content": "\n\n\n Gor PRO\n\n \n\n\n
\n

Gor PRO releases

\n

See releases page on GitHub for changelog

\n\n

v0.16.2

\n \n\n\n

v0.16.1

\n \n\n\n

v0.16.0

\n \n\n

v0.15.1

\n \n\n

v0.15.0

\n \n\n

v0.14.1

\n \n\n\n" }, { "path": "s3_reader.go", "content": "package goreplay\n\nimport (\n\t\"bytes\"\n\t\"log\"\n\t\"os\"\n\t\"strconv\"\n\t\"strings\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/aws/session\"\n\t\"github.com/aws/aws-sdk-go/service/s3\"\n)\n\n// S3ReadCloser ...\ntype S3ReadCloser struct {\n\tbucket string\n\tkey string\n\toffset int\n\ttotalSize int\n\treadBytes int\n\tsess *session.Session\n\tbuf *bytes.Buffer\n}\n\nfunc awsConfig() *aws.Config {\n\tregion := os.Getenv(\"AWS_DEFAULT_REGION\")\n\tif region == \"\" {\n\t\tregion = os.Getenv(\"AWS_REGION\")\n\t\tif region == \"\" {\n\t\t\tregion = \"us-east-1\"\n\t\t}\n\t}\n\n\tconfig := &aws.Config{Region: aws.String(region)}\n\n\tif endpoint := os.Getenv(\"AWS_ENDPOINT_URL\"); endpoint != \"\" {\n\t\tconfig.Endpoint = aws.String(endpoint)\n\t\tlog.Println(\"Custom endpoint:\", endpoint)\n\t}\n\n\tlog.Println(\"Connecting to S3. Region: \" + region)\n\n\tconfig.CredentialsChainVerboseErrors = aws.Bool(true)\n\n\tif os.Getenv(\"AWS_DEBUG\") != \"\" {\n\t\tconfig.LogLevel = aws.LogLevel(aws.LogDebugWithHTTPBody)\n\t}\n\n\treturn config\n}\n\n// NewS3ReadCloser returns new instance of S3 read closer\nfunc NewS3ReadCloser(path string) *S3ReadCloser {\n\tif !PRO {\n\t\tlog.Fatal(\"Using S3 input and output require PRO license\")\n\t\treturn nil\n\t}\n\n\tbucket, key := parseS3Url(path)\n\tsess := session.Must(session.NewSession(awsConfig()))\n\n\tlog.Println(\"[S3 Input] S3 connection successfully initialized\", path)\n\n\treturn &S3ReadCloser{\n\t\tbucket: bucket,\n\t\tkey: key,\n\t\tsess: sess,\n\t\tbuf: &bytes.Buffer{},\n\t}\n}\n\n// Read reads buffer from s3 session\nfunc (s *S3ReadCloser) Read(b []byte) (n int, e error) {\n\tif s.readBytes == 0 || s.readBytes+len(b) > s.offset {\n\t\tsvc := s3.New(s.sess)\n\n\t\tobjectRange := \"bytes=\" + strconv.Itoa(s.offset)\n\t\ts.offset += 1000000 // Reading in chunks of 1 mb\n\t\tobjectRange += \"-\" + strconv.Itoa(s.offset-1)\n\n\t\tparams := &s3.GetObjectInput{\n\t\t\tBucket: aws.String(s.bucket),\n\t\t\tKey: aws.String(s.key),\n\t\t\tRange: aws.String(objectRange),\n\t\t}\n\t\tresp, err := svc.GetObject(params)\n\n\t\tif err != nil {\n\t\t\tlog.Println(\"[S3 Input] Error during getting file\", s.bucket, s.key, err)\n\t\t} else {\n\t\t\ts.totalSize, _ = strconv.Atoi(strings.Split(*resp.ContentRange, \"/\")[1])\n\t\t\ts.buf.ReadFrom(resp.Body)\n\t\t}\n\t}\n\n\ts.readBytes += len(b)\n\n\treturn s.buf.Read(b)\n}\n\n// Close is here to make S3ReadCloser satisfy ReadCloser interface\nfunc (s *S3ReadCloser) Close() error {\n\treturn nil\n}\n" }, { "path": "s3_test.go", "content": "//go:build pro\n\npackage goreplay\n\nimport (\n\t\"fmt\"\n\t\"math/rand\"\n\t\"os\"\n\t\"path/filepath\"\n\t\"testing\"\n\t\"time\"\n\n\t\"github.com/aws/aws-sdk-go/aws\"\n\t\"github.com/aws/aws-sdk-go/service/s3\"\n)\n\nfunc TestS3Output(t *testing.T) {\n\tbucket := aws.String(\"test-gor\")\n\trnd := rand.Int63()\n\tpath := fmt.Sprintf(\"s3://test-gor/%d/requests.gz\", rnd)\n\n\toutput := NewS3Output(path, &FileOutputConfig{queueLimit: 2})\n\n\tsvc := s3.New(output.session)\n\n\toutput.Write([]byte(\"1 1 1\\ntest\"))\n\toutput.Write([]byte(\"1 1 1\\ntest\"))\n\toutput.buffer.updateName()\n\toutput.Write([]byte(\"1 1 1\\ntest\"))\n\toutput.Write([]byte(\"1 1 1\\ntest\"))\n\toutput.buffer.updateName()\n\toutput.Write([]byte(\"1 1 1\\ntest\"))\n\n\ttime.Sleep(time.Second)\n\n\tparams := &s3.ListObjectsInput{\n\t\tBucket: bucket,\n\t\tPrefix: aws.String(fmt.Sprintf(\"%d\", rnd)),\n\t}\n\n\tresp, _ := svc.ListObjects(params)\n\tif len(resp.Contents) != 2 {\n\t\tt.Error(\"Should create 2 objects\", len(resp.Contents))\n\t} else {\n\t\tif *resp.Contents[0].Key != fmt.Sprintf(\"%d/requests_0.gz\", rnd) ||\n\t\t\t*resp.Contents[1].Key != fmt.Sprintf(\"%d/requests_1.gz\", rnd) {\n\t\t\tt.Error(\"Should assign proper names\", resp.Contents)\n\t\t}\n\t}\n\n\tfor _, c := range resp.Contents {\n\t\tsvc.DeleteObject(&s3.DeleteObjectInput{Bucket: bucket, Key: c.Key})\n\t}\n\n\tmatches, _ := filepath.Glob(fmt.Sprintf(\"/tmp/gor_output_s3_*\"))\n\tfor _, m := range matches {\n\t\tos.Remove(m)\n\t}\n}\n\nfunc TestS3OutputQueueLimit(t *testing.T) {\n\tbucket := aws.String(\"test-gor\")\n\trnd := rand.Int63()\n\tpath := fmt.Sprintf(\"s3://test-gor/%d/requests.gz\", rnd)\n\n\toutput := NewS3Output(path, &FileOutputConfig{queueLimit: 100})\n\toutput.closeCh = make(chan struct{}, 3)\n\n\tsvc := s3.New(output.session)\n\n\tfor i := 0; i < 3; i++ {\n\t\tfor i := 0; i < 100; i++ {\n\t\t\toutput.Write([]byte(\"1 1 1\\ntest\"))\n\t\t}\n\t\toutput.buffer.updateName()\n\t}\n\toutput.buffer.updateName()\n\toutput.Write([]byte(\"1 1 1\\ntest\"))\n\n\tfor i := 0; i < 3; i++ {\n\t\t<-output.closeCh\n\t}\n\n\tparams := &s3.ListObjectsInput{\n\t\tBucket: bucket,\n\t\tPrefix: aws.String(fmt.Sprintf(\"%d\", rnd)),\n\t}\n\n\tresp, _ := svc.ListObjects(params)\n\tif len(resp.Contents) != 3 {\n\t\tt.Error(\"Should create 3 object\", len(resp.Contents))\n\t} else {\n\t\tif *resp.Contents[0].Key != fmt.Sprintf(\"%d/requests_0.gz\", rnd) ||\n\t\t\t*resp.Contents[1].Key != fmt.Sprintf(\"%d/requests_1.gz\", rnd) {\n\t\t\tt.Error(\"Should assign proper names\", resp.Contents)\n\t\t}\n\t}\n\n\tfor _, c := range resp.Contents {\n\t\tsvc.DeleteObject(&s3.DeleteObjectInput{Bucket: bucket, Key: c.Key})\n\t}\n\n\tmatches, _ := filepath.Glob(fmt.Sprintf(\"/tmp/gor_output_s3_*\"))\n\tfor _, m := range matches {\n\t\tos.Remove(m)\n\t}\n}\n\nfunc TestInputFileFromS3(t *testing.T) {\n\trnd := rand.Int63()\n\tpath := fmt.Sprintf(\"s3://test-gor-eu/%d/requests.gz\", rnd)\n\n\toutput := NewS3Output(path, &FileOutputConfig{queueLimit: 5000})\n\toutput.closeCh = make(chan struct{}, 10)\n\n\tfor i := 0; i <= 20000; i++ {\n\t\toutput.Write([]byte(\"1 1 1\\ntest\"))\n\n\t\tif i%5000 == 0 {\n\t\t\toutput.buffer.updateName()\n\t\t}\n\t}\n\n\toutput.Write([]byte(\"1 1 1\\ntest\"))\n\n\tfor i := 0; i < 2; i++ {\n\t\t<-output.closeCh\n\t}\n\n\tinput := NewFileInput(fmt.Sprintf(\"s3://test-gor-eu/%d\", rnd), false, 100, 0, false)\n\n\tbuf := make([]byte, 1000)\n\tfor i := 0; i <= 19999; i++ {\n\t\tinput.Read(buf)\n\t}\n\n\t// Cleanup artifacts\n\tbucket := aws.String(\"test-gor\")\n\tsvc := s3.New(output.session)\n\tparams := &s3.ListObjectsInput{\n\t\tBucket: bucket,\n\t\tPrefix: aws.String(fmt.Sprintf(\"%d\", rnd)),\n\t}\n\n\tresp, _ := svc.ListObjects(params)\n\n\tfor _, c := range resp.Contents {\n\t\tsvc.DeleteObject(&s3.DeleteObjectInput{Bucket: bucket, Key: c.Key})\n\t}\n}\n" }, { "path": "settings.go", "content": "package goreplay\n\nimport (\n\t\"flag\"\n\t\"fmt\"\n\t\"os\"\n\t\"strconv\"\n\t\"sync\"\n\t\"time\"\n\n\t\"github.com/buger/goreplay/internal/size\"\n)\n\n// DEMO indicates that goreplay is running in demo mode\nvar DEMO string\n\n// MultiOption allows to specify multiple flags with same name and collects all values into array\ntype MultiOption struct {\n\ta *[]string\n}\n\nfunc (h *MultiOption) String() string {\n\tif h.a == nil {\n\t\treturn \"\"\n\t}\n\treturn fmt.Sprint(*h.a)\n}\n\n// Set gets called multiple times for each flag with same name\nfunc (h *MultiOption) Set(value string) error {\n\tif h.a == nil {\n\t\treturn nil\n\t}\n\n\t*h.a = append(*h.a, value)\n\treturn nil\n}\n\n// MultiOption allows to specify multiple flags with same name and collects all values into array\ntype MultiIntOption struct {\n\ta *[]int\n}\n\nfunc (h *MultiIntOption) String() string {\n\tif h.a == nil {\n\t\treturn \"\"\n\t}\n\n\treturn fmt.Sprint(*h.a)\n}\n\n// Set gets called multiple times for each flag with same name\nfunc (h *MultiIntOption) Set(value string) error {\n\tif h.a == nil {\n\t\treturn nil\n\t}\n\n\tval, _ := strconv.Atoi(value)\n\t*h.a = append(*h.a, val)\n\treturn nil\n}\n\n// AppSettings is the struct of main configuration\ntype AppSettings struct {\n\tVerbose int `json:\"verbose\"`\n\tStats bool `json:\"stats\"`\n\tExitAfter time.Duration `json:\"exit-after\"`\n\n\tSplitOutput bool `json:\"split-output\"`\n\tRecognizeTCPSessions bool `json:\"recognize-tcp-sessions\"`\n\tPprof string `json:\"http-pprof\"`\n\n\tCopyBufferSize size.Size `json:\"copy-buffer-size\"`\n\n\tInputDummy []string `json:\"input-dummy\"`\n\tOutputDummy []string\n\tOutputStdout bool `json:\"output-stdout\"`\n\tOutputNull bool `json:\"output-null\"`\n\n\tInputTCP []string `json:\"input-tcp\"`\n\tInputTCPConfig TCPInputConfig\n\tOutputTCP []string `json:\"output-tcp\"`\n\tOutputTCPConfig TCPOutputConfig\n\tOutputTCPStats bool `json:\"output-tcp-stats\"`\n\n\tOutputWebSocket []string `json:\"output-ws\"`\n\tOutputWebSocketConfig WebSocketOutputConfig\n\tOutputWebSocketStats bool `json:\"output-ws-stats\"`\n\n\tInputFile []string `json:\"input-file\"`\n\tInputFileLoop bool `json:\"input-file-loop\"`\n\tInputFileReadDepth int `json:\"input-file-read-depth\"`\n\tInputFileDryRun bool `json:\"input-file-dry-run\"`\n\tInputFileMaxWait time.Duration `json:\"input-file-max-wait\"`\n\tOutputFile []string `json:\"output-file\"`\n\tOutputFileConfig FileOutputConfig\n\n\tInputRAW []string `json:\"input_raw\"`\n\tInputRAWConfig RAWInputConfig\n\n\tMiddleware string `json:\"middleware\"`\n\n\tInputHTTP []string\n\tOutputHTTP []string `json:\"output-http\"`\n\tPrettifyHTTP bool `json:\"prettify-http\"`\n\n\tOutputHTTPConfig HTTPOutputConfig\n\n\tOutputBinary []string `json:\"output-binary\"`\n\tOutputBinaryConfig BinaryOutputConfig\n\n\tModifierConfig HTTPModifierConfig\n\n\tInputKafkaConfig InputKafkaConfig\n\tOutputKafkaConfig OutputKafkaConfig\n\tKafkaTLSConfig KafkaTLSConfig\n}\n\n// Settings holds Gor configuration\nvar Settings AppSettings\n\nfunc usage() {\n\tfmt.Printf(\"Gor is a simple http traffic replication tool written in Go. Its main goal is to replay traffic from production servers to staging and dev environments.\\nProject page: https://github.com/buger/gor\\nAuthor: leonsbox@gmail.com\\nCurrent Version: v%s\\n\\n\", VERSION)\n\tflag.PrintDefaults()\n\tos.Exit(2)\n}\n\nfunc init() {\n\tflag.Usage = usage\n\tflag.StringVar(&Settings.Pprof, \"http-pprof\", \"\", \"Enable profiling. Starts http server on specified port, exposing special /debug/pprof endpoint. Example: `:8181`\")\n\tflag.IntVar(&Settings.Verbose, \"verbose\", 0, \"set the level of verbosity, if greater than zero then it will turn on debug output\")\n\tflag.BoolVar(&Settings.Stats, \"stats\", false, \"Turn on queue stats output\")\n\n\tif DEMO == \"\" {\n\t\tflag.DurationVar(&Settings.ExitAfter, \"exit-after\", 0, \"exit after specified duration\")\n\t} else {\n\t\tSettings.ExitAfter = 5 * time.Minute\n\t}\n\n\tflag.BoolVar(&Settings.SplitOutput, \"split-output\", false, \"By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.\")\n\tflag.BoolVar(&Settings.RecognizeTCPSessions, \"recognize-tcp-sessions\", false, \"[PRO] If turned on http output will create separate worker for each TCP session. Splitting output will session based as well.\")\n\n\tflag.Var(&MultiOption{&Settings.InputDummy}, \"input-dummy\", \"Used for testing outputs. Emits 'Get /' request every 1s\")\n\tflag.BoolVar(&Settings.OutputStdout, \"output-stdout\", false, \"Used for testing inputs. Just prints to console data coming from inputs.\")\n\tflag.BoolVar(&Settings.OutputNull, \"output-null\", false, \"Used for testing inputs. Drops all requests.\")\n\n\tflag.Var(&MultiOption{&Settings.InputTCP}, \"input-tcp\", \"Used for internal communication between Gor instances. Example: \\n\\t# Receive requests from other Gor instances on 28020 port, and redirect output to staging\\n\\tgor --input-tcp :28020 --output-http staging.com\")\n\tflag.BoolVar(&Settings.InputTCPConfig.Secure, \"input-tcp-secure\", false, \"Turn on TLS security. Do not forget to specify certificate and key files.\")\n\tflag.StringVar(&Settings.InputTCPConfig.CertificatePath, \"input-tcp-certificate\", \"\", \"Path to PEM encoded certificate file. Used when TLS turned on.\")\n\tflag.StringVar(&Settings.InputTCPConfig.KeyPath, \"input-tcp-certificate-key\", \"\", \"Path to PEM encoded certificate key file. Used when TLS turned on.\")\n\n\tflag.Var(&MultiOption{&Settings.OutputTCP}, \"output-tcp\", \"Used for internal communication between Gor instances. Example: \\n\\t# Listen for requests on 80 port and forward them to other Gor instance on 28020 port\\n\\tgor --input-raw :80 --output-tcp replay.local:28020\")\n\tflag.BoolVar(&Settings.OutputTCPConfig.Secure, \"output-tcp-secure\", false, \"Use TLS secure connection. --input-file on another end should have TLS turned on as well.\")\n\tflag.BoolVar(&Settings.OutputTCPConfig.SkipVerify, \"output-tcp-skip-verify\", false, \"Don't verify hostname on TLS secure connection.\")\n\tflag.BoolVar(&Settings.OutputTCPConfig.Sticky, \"output-tcp-sticky\", false, \"Use Sticky connection. Request/Response with same ID will be sent to the same connection.\")\n\tflag.IntVar(&Settings.OutputTCPConfig.Workers, \"output-tcp-workers\", 10, \"Number of parallel tcp connections, default is 10\")\n\tflag.BoolVar(&Settings.OutputTCPStats, \"output-tcp-stats\", false, \"Report TCP output queue stats to console every 5 seconds.\")\n\n\tflag.Var(&MultiOption{&Settings.OutputWebSocket}, \"output-ws\", \"Just like output tcp, just with WebSocket. Example: \\n\\t# Listen for requests on 80 port and forward them to other Gor instance on 28020 port\\n\\tgor --input-raw :80 --output-ws wss://replay.local:28020/endpoint\")\n\tflag.BoolVar(&Settings.OutputWebSocketConfig.SkipVerify, \"output-ws-skip-verify\", false, \"Don't verify hostname on TLS secure connection.\")\n\tflag.BoolVar(&Settings.OutputWebSocketConfig.Sticky, \"output-ws-sticky\", false, \"Use Sticky connection. Request/Response with same ID will be sent to the same connection.\")\n\tflag.IntVar(&Settings.OutputWebSocketConfig.Workers, \"output-ws-workers\", 10, \"Number of parallel ws connections, default is 10\")\n\tflag.BoolVar(&Settings.OutputWebSocketStats, \"output-ws-stats\", false, \"Report WebSocket output queue stats to console every 5 seconds.\")\n\n\tflag.Var(&MultiOption{&Settings.InputFile}, \"input-file\", \"Read requests from file: \\n\\tgor --input-file ./requests.gor --output-http staging.com\")\n\tflag.BoolVar(&Settings.InputFileLoop, \"input-file-loop\", false, \"Loop input files, useful for performance testing.\")\n\tflag.IntVar(&Settings.InputFileReadDepth, \"input-file-read-depth\", 100, \"GoReplay tries to read and cache multiple records, in advance. In parallel it also perform sorting of requests, if they came out of order. Since it needs hold this buffer in memory, bigger values can cause worse performance\")\n\tflag.BoolVar(&Settings.InputFileDryRun, \"input-file-dry-run\", false, \"Simulate reading from the data source without replaying it. You will get information about expected replay time, number of found records etc.\")\n\tflag.DurationVar(&Settings.InputFileMaxWait, \"input-file-max-wait\", 0, \"Set the maximum time between requests. Can help in situations when you have too long periods between request, and you want to skip them. Example: --input-raw-max-wait 1s\")\n\n\tflag.Var(&MultiOption{&Settings.OutputFile}, \"output-file\", \"Write incoming requests to file: \\n\\tgor --input-raw :80 --output-file ./requests.gor\")\n\tflag.DurationVar(&Settings.OutputFileConfig.FlushInterval, \"output-file-flush-interval\", time.Second, \"Interval for forcing buffer flush to the file, default: 1s.\")\n\tflag.BoolVar(&Settings.OutputFileConfig.Append, \"output-file-append\", false, \"The flushed chunk is appended to existence file or not. \")\n\tflag.Var(&Settings.OutputFileConfig.SizeLimit, \"output-file-size-limit\", \"Size of each chunk. Default: 32mb\")\n\tflag.IntVar(&Settings.OutputFileConfig.QueueLimit, \"output-file-queue-limit\", 256, \"The length of the chunk queue. Default: 256\")\n\tflag.Var(&Settings.OutputFileConfig.OutputFileMaxSize, \"output-file-max-size-limit\", \"Max size of output file, Default: 1TB\")\n\n\tflag.StringVar(&Settings.OutputFileConfig.BufferPath, \"output-file-buffer\", \"/tmp\", \"The path for temporary storing current buffer: \\n\\tgor --input-raw :80 --output-file s3://mybucket/logs/%Y-%m-%d.gz --output-file-buffer /mnt/logs\")\n\n\tflag.BoolVar(&Settings.PrettifyHTTP, \"prettify-http\", false, \"If enabled, will automatically decode requests and responses with: Content-Encoding: gzip and Transfer-Encoding: chunked. Useful for debugging, in conjunction with --output-stdout\")\n\n\tflag.Var(&Settings.CopyBufferSize, \"copy-buffer-size\", \"Set the buffer size for an individual request (default 5MB)\")\n\n\t// input raw flags\n\tflag.Var(&MultiOption{&Settings.InputRAW}, \"input-raw\", \"Capture traffic from given port (use RAW sockets and require *sudo* access):\\n\\t# Capture traffic from 8080 port\\n\\tgor --input-raw :8080 --output-http staging.com\")\n\tflag.BoolVar(&Settings.InputRAWConfig.TrackResponse, \"input-raw-track-response\", false, \"If turned on Gor will track responses in addition to requests, and they will be available to middleware and file output.\")\n\tflag.IntVar(&Settings.InputRAWConfig.VXLANPort, \"input-raw-vxlan-port\", 4789, \"VXLAN port. Can be used only when engine set to `vxlan`. Default: 4789\")\n\tflag.Var(&MultiIntOption{&Settings.InputRAWConfig.VXLANVNIs}, \"input-raw-vxlan-vni\", \"VXLAN VNI to capture. By default capture all VNIs. Ignore VNI by setting them with minus sign, example: `--input-raw-vxlan-vni -2`\")\n\tflag.BoolVar(&Settings.InputRAWConfig.VLAN, \"input-raw-vlan\", false, \"Enable VLAN (802.1Q) support\")\n\tflag.Var(&MultiIntOption{&Settings.InputRAWConfig.VLANVIDs}, \"input-raw-vlan-vid\", \"VLAN VID to capture. By default capture all VIDs\")\n\tflag.Var(&Settings.InputRAWConfig.Engine, \"input-raw-engine\", \"Intercept traffic using `libpcap` (default), `raw_socket`, `pcap_file`, `vxlan`\")\n\tflag.Var(&Settings.InputRAWConfig.Protocol, \"input-raw-protocol\", \"Specify application protocol of intercepted traffic. Possible values: http, binary\")\n\tflag.StringVar(&Settings.InputRAWConfig.RealIPHeader, \"input-raw-realip-header\", \"\", \"If not blank, injects header with given name and real IP value to the request payload. Usually this header should be named: X-Real-IP\")\n\tflag.DurationVar(&Settings.InputRAWConfig.Expire, \"input-raw-expire\", time.Second*2, \"How much it should wait for the last TCP packet, till consider that TCP message complete.\")\n\tflag.StringVar(&Settings.InputRAWConfig.BPFFilter, \"input-raw-bpf-filter\", \"\", \"BPF filter to write custom expressions. Can be useful in case of non standard network interfaces like tunneling or SPAN port. Example: --input-raw-bpf-filter 'dst port 80'\")\n\tflag.StringVar(&Settings.InputRAWConfig.TimestampType, \"input-raw-timestamp-type\", \"\", \"Possible values: PCAP_TSTAMP_HOST, PCAP_TSTAMP_HOST_LOWPREC, PCAP_TSTAMP_HOST_HIPREC, PCAP_TSTAMP_ADAPTER, PCAP_TSTAMP_ADAPTER_UNSYNCED. This values not supported on all systems, GoReplay will tell you available values of you put wrong one.\")\n\tflag.BoolVar(&Settings.InputRAWConfig.Snaplen, \"input-raw-override-snaplen\", false, \"Override the capture snaplen to be 64k. Required for some Virtualized environments\")\n\tflag.DurationVar(&Settings.InputRAWConfig.BufferTimeout, \"input-raw-buffer-timeout\", 0, \"set the pcap timeout. for immediate mode don't set this flag\")\n\tflag.Var(&Settings.InputRAWConfig.BufferSize, \"input-raw-buffer-size\", \"Controls size of the OS buffer which holds packets until they dispatched. Default value depends by system: in Linux around 2MB. If you see big package drop, increase this value.\")\n\tflag.BoolVar(&Settings.InputRAWConfig.Promiscuous, \"input-raw-promisc\", false, \"enable promiscuous mode\")\n\tflag.BoolVar(&Settings.InputRAWConfig.Monitor, \"input-raw-monitor\", false, \"enable RF monitor mode\")\n\tflag.BoolVar(&Settings.InputRAWConfig.Stats, \"input-raw-stats\", false, \"enable stats generator on raw TCP messages\")\n\tflag.BoolVar(&Settings.InputRAWConfig.AllowIncomplete, \"input-raw-allow-incomplete\", false, \"If turned on Gor will record HTTP messages with missing packets\")\n\tflag.Var(&MultiOption{&Settings.InputRAWConfig.IgnoreInterface}, \"input-raw-ignore-interface\", \"In case if you want listen for all interfaces except a few ones. Can be used in k8s environment. Example: --input-raw-ignore-interface cbr0 --input-raw-ignore-interface eth0 --input-raw-ignore-interface localhost\")\n\n\tflag.StringVar(&Settings.Middleware, \"middleware\", \"\", \"Used for modifying traffic using external command\")\n\n\tflag.Var(&MultiOption{&Settings.OutputHTTP}, \"output-http\", \"Forwards incoming requests to given http address.\\n\\t# Redirect all incoming requests to staging.com address \\n\\tgor --input-raw :80 --output-http http://staging.com\")\n\n\t/* outputHTTPConfig */\n\tflag.Var(&Settings.OutputHTTPConfig.BufferSize, \"output-http-response-buffer\", \"HTTP response buffer size, all data after this size will be discarded.\")\n\tflag.IntVar(&Settings.OutputHTTPConfig.WorkersMin, \"output-http-workers-min\", 0, \"Gor uses dynamic worker scaling. Enter a number to set a minimum number of workers. default = 1.\")\n\tflag.IntVar(&Settings.OutputHTTPConfig.WorkersMax, \"output-http-workers\", 0, \"Gor uses dynamic worker scaling. Enter a number to set a maximum number of workers. default = 0 = unlimited.\")\n\tflag.IntVar(&Settings.OutputHTTPConfig.QueueLen, \"output-http-queue-len\", 1000, \"Number of requests that can be queued for output, if all workers are busy. default = 1000\")\n\tflag.BoolVar(&Settings.OutputHTTPConfig.SkipVerify, \"output-http-skip-verify\", false, \"Don't verify hostname on TLS secure connection.\")\n\tflag.DurationVar(&Settings.OutputHTTPConfig.WorkerTimeout, \"output-http-worker-timeout\", 2*time.Second, \"Duration to rollback idle workers.\")\n\n\tflag.IntVar(&Settings.OutputHTTPConfig.RedirectLimit, \"output-http-redirects\", 0, \"Enable how often redirects should be followed.\")\n\tflag.DurationVar(&Settings.OutputHTTPConfig.Timeout, \"output-http-timeout\", 5*time.Second, \"Specify HTTP request/response timeout. By default 5s. Example: --output-http-timeout 30s\")\n\tflag.BoolVar(&Settings.OutputHTTPConfig.TrackResponses, \"output-http-track-response\", false, \"If turned on, HTTP output responses will be set to all outputs like stdout, file and etc.\")\n\n\tflag.BoolVar(&Settings.OutputHTTPConfig.Stats, \"output-http-stats\", false, \"Report http output queue stats to console every N milliseconds. See output-http-stats-ms\")\n\tflag.IntVar(&Settings.OutputHTTPConfig.StatsMs, \"output-http-stats-ms\", 5000, \"Report http output queue stats to console every N milliseconds. default: 5000\")\n\tflag.BoolVar(&Settings.OutputHTTPConfig.OriginalHost, \"http-original-host\", false, \"Normally gor replaces the Host http header with the host supplied with --output-http. This option disables that behavior, preserving the original Host header.\")\n\tflag.StringVar(&Settings.OutputHTTPConfig.ElasticSearch, \"output-http-elasticsearch\", \"\", \"Send request and response stats to ElasticSearch:\\n\\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'\")\n\t/* outputHTTPConfig */\n\n\tflag.Var(&MultiOption{&Settings.OutputBinary}, \"output-binary\", \"Forwards incoming binary payloads to given address.\\n\\t# Redirect all incoming requests to staging.com address \\n\\tgor --input-raw :80 --input-raw-protocol binary --output-binary staging.com:80\")\n\n\t/* outputBinaryConfig */\n\tflag.Var(&Settings.OutputBinaryConfig.BufferSize, \"output-tcp-response-buffer\", \"TCP response buffer size, all data after this size will be discarded.\")\n\tflag.IntVar(&Settings.OutputBinaryConfig.Workers, \"output-binary-workers\", 0, \"Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers.\")\n\tflag.DurationVar(&Settings.OutputBinaryConfig.Timeout, \"output-binary-timeout\", 0, \"Specify HTTP request/response timeout. By default 5s. Example: --output-binary-timeout 30s\")\n\tflag.BoolVar(&Settings.OutputBinaryConfig.TrackResponses, \"output-binary-track-response\", false, \"If turned on, Binary output responses will be set to all outputs like stdout, file and etc.\")\n\n\tflag.BoolVar(&Settings.OutputBinaryConfig.Debug, \"output-binary-debug\", false, \"Enables binary debug output.\")\n\t/* outputBinaryConfig */\n\n\tflag.StringVar(&Settings.OutputKafkaConfig.Host, \"output-kafka-host\", \"\", \"Read request and response stats from Kafka:\\n\\tgor --input-raw :8080 --output-kafka-host '192.168.0.1:9092,192.168.0.2:9092'\")\n\tflag.StringVar(&Settings.OutputKafkaConfig.Topic, \"output-kafka-topic\", \"\", \"Read request and response stats from Kafka:\\n\\tgor --input-raw :8080 --output-kafka-topic 'kafka-log'\")\n\tflag.BoolVar(&Settings.OutputKafkaConfig.UseJSON, \"output-kafka-json-format\", false, \"If turned on, it will serialize messages from GoReplay text format to JSON.\")\n\tflag.BoolVar(&Settings.OutputKafkaConfig.SASLConfig.UseSASL, \"output-kafka-use-sasl\", false, \"--output-kafka-use-sasl true\")\n\tflag.StringVar(&Settings.OutputKafkaConfig.SASLConfig.Mechanism, \"output-kafka-mechanism\", \"\", \"mechanism\\n\\tgor --input-raw :8080 --output-kafka-mechanism 'SCRAM-SHA-512'\")\n\tflag.StringVar(&Settings.OutputKafkaConfig.SASLConfig.Username, \"output-kafka-username\", \"\", \"username\\n\\tgor --input-raw :8080 --output-kafka-username 'username'\")\n\tflag.StringVar(&Settings.OutputKafkaConfig.SASLConfig.Password, \"output-kafka-password\", \"\", \"password\\n\\tgor --input-raw :8080 --output-kafka-password 'password'\")\n\n\tflag.StringVar(&Settings.InputKafkaConfig.Host, \"input-kafka-host\", \"\", \"Send request and response stats to Kafka:\\n\\tgor --output-stdout --input-kafka-host '192.168.0.1:9092,192.168.0.2:9092'\")\n\tflag.StringVar(&Settings.InputKafkaConfig.Topic, \"input-kafka-topic\", \"\", \"Send request and response stats to Kafka:\\n\\tgor --output-stdout --input-kafka-topic 'kafka-log'\")\n\tflag.BoolVar(&Settings.InputKafkaConfig.UseJSON, \"input-kafka-json-format\", false, \"If turned on, it will assume that messages coming in JSON format rather than GoReplay text format.\")\n\tflag.BoolVar(&Settings.InputKafkaConfig.SASLConfig.UseSASL, \"input-kafka-use-sasl\", false, \"use-sasl\\n\\t--use-sasl true\")\n\tflag.StringVar(&Settings.InputKafkaConfig.SASLConfig.Mechanism, \"input-kafka-mechanism\", \"\", \"mechanism\\n\\tgor --input-raw :8080 --output-kafka-mechanism 'SCRAM-SHA-512'\")\n\tflag.StringVar(&Settings.InputKafkaConfig.SASLConfig.Username, \"input-kafka-username\", \"\", \"username\\n\\tgor --input-raw :8080 --output-kafka-username 'username'\")\n\tflag.StringVar(&Settings.InputKafkaConfig.SASLConfig.Password, \"input-kafka-password\", \"\", \"password\\n\\tgor --input-raw :8080 --output-kafka-password 'password'\")\n\tflag.StringVar(&Settings.InputKafkaConfig.Offset, \"input-kafka-offset\", \"-1\", \"Specify offset in Kafka partitions start to consume\\n\\t-1: Starts from newest, -2: Starts from oldest\\nAnd supported for showdown or speedup for emitting!\\n\\tgor --input-kafka-offset \\\"-2|200%\\\"\")\n\n\tflag.StringVar(&Settings.KafkaTLSConfig.CACert, \"kafka-tls-ca-cert\", \"\", \"CA certificate for Kafka TLS Config:\\n\\tgor --input-raw :3000 --output-kafka-host '192.168.0.1:9092' --output-kafka-topic 'topic' --kafka-tls-ca-cert cacert.cer.pem --kafka-tls-client-cert client.cer.pem --kafka-tls-client-key client.key.pem\")\n\tflag.StringVar(&Settings.KafkaTLSConfig.ClientCert, \"kafka-tls-client-cert\", \"\", \"Client certificate for Kafka TLS Config (mandatory with to kafka-tls-ca-cert and kafka-tls-client-key)\")\n\tflag.StringVar(&Settings.KafkaTLSConfig.ClientKey, \"kafka-tls-client-key\", \"\", \"Client Key for Kafka TLS Config (mandatory with to kafka-tls-client-cert and kafka-tls-client-key)\")\n\n\tflag.Var(&Settings.ModifierConfig.Headers, \"http-set-header\", \"Inject additional headers to http request:\\n\\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'\")\n\tflag.Var(&Settings.ModifierConfig.HeaderRewrite, \"http-rewrite-header\", \"Rewrite the request header based on a mapping:\\n\\tgor --input-raw :8080 --output-http staging.com --http-rewrite-header Host: (.*).example.com,$1.beta.example.com\")\n\tflag.Var(&Settings.ModifierConfig.Params, \"http-set-param\", \"Set request url param, if param already exists it will be overwritten:\\n\\tgor --input-raw :8080 --output-http staging.com --http-set-param api_key=1\")\n\tflag.Var(&Settings.ModifierConfig.Methods, \"http-allow-method\", \"Whitelist of HTTP methods to replay. Anything else will be dropped:\\n\\tgor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS\")\n\tflag.Var(&Settings.ModifierConfig.URLRegexp, \"http-allow-url\", \"A regexp to match requests against. Filter get matched against full url with domain. Anything else will be dropped:\\n\\t gor --input-raw :8080 --output-http staging.com --http-allow-url ^www.\")\n\tflag.Var(&Settings.ModifierConfig.URLNegativeRegexp, \"http-disallow-url\", \"A regexp to match requests against. Filter get matched against full url with domain. Anything else will be forwarded:\\n\\t gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.\")\n\tflag.Var(&Settings.ModifierConfig.URLRewrite, \"http-rewrite-url\", \"Rewrite the request url based on a mapping:\\n\\tgor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\\\/]+)/ping:/v2/user/$1/ping\")\n\tflag.Var(&Settings.ModifierConfig.HeaderFilters, \"http-allow-header\", \"A regexp to match a specific header against. Requests with non-matching headers will be dropped:\\n\\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1\")\n\tflag.Var(&Settings.ModifierConfig.HeaderNegativeFilters, \"http-disallow-header\", \"A regexp to match a specific header against. Requests with matching headers will be dropped:\\n\\t gor --input-raw :8080 --output-http staging.com --http-disallow-header \\\"User-Agent: Replayed by Gor\\\"\")\n\tflag.Var(&Settings.ModifierConfig.HeaderBasicAuthFilters, \"http-basic-auth-filter\", \"A regexp to match the decoded basic auth string against. Requests with non-matching headers will be dropped:\\n\\t gor --input-raw :8080 --output-http staging.com --http-basic-auth-filter \\\"^customer[0-9].*\\\"\")\n\tflag.Var(&Settings.ModifierConfig.HeaderHashFilters, \"http-header-limiter\", \"Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\\n\\t gor --input-raw :8080 --output-http staging.com --http-header-limiter user-id:25%\")\n\tflag.Var(&Settings.ModifierConfig.ParamHashFilters, \"http-param-limiter\", \"Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\\n\\t gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%\")\n\n\t// default values, using for tests\n\tSettings.OutputFileConfig.SizeLimit = 33554432\n\tSettings.OutputFileConfig.OutputFileMaxSize = 1099511627776\n\tSettings.CopyBufferSize = 5242880\n\n}\n\nfunc CheckSettings() {\n\tSettingsHook(&Settings)\n\n\tif Settings.OutputFileConfig.SizeLimit < 1 {\n\t\tSettings.OutputFileConfig.SizeLimit.Set(\"32mb\")\n\t}\n\tif Settings.OutputFileConfig.OutputFileMaxSize < 1 {\n\t\tSettings.OutputFileConfig.OutputFileMaxSize.Set(\"1tb\")\n\t}\n\tif Settings.CopyBufferSize < 1 {\n\t\tSettings.CopyBufferSize.Set(\"5mb\")\n\t}\n}\n\nvar previousDebugTime = time.Now()\nvar debugMutex sync.Mutex\n\n// Debug take an effect only if --verbose greater than 0 is specified\nfunc Debug(level int, args ...interface{}) {\n\tif Settings.Verbose >= level {\n\t\tdebugMutex.Lock()\n\t\tdefer debugMutex.Unlock()\n\t\tnow := time.Now()\n\t\tdiff := now.Sub(previousDebugTime)\n\t\tpreviousDebugTime = now\n\t\tfmt.Fprintf(os.Stderr, \"[DEBUG][elapsed %s]: \", diff)\n\t\tfmt.Fprintln(os.Stderr, args...)\n\t}\n}\n" }, { "path": "settings_test.go", "content": "package goreplay\n\nimport (\n\t\"encoding/json\"\n\t\"testing\"\n)\n\nfunc TestAppSettings(t *testing.T) {\n\ta := AppSettings{}\n\t_, err := json.Marshal(&a)\n\tif err != nil {\n\t\tt.Error(err)\n\t}\n}\n" }, { "path": "sidenav.css", "content": ".wy-affix {\n position: fixed;\n top: 1.618em;\n}\n\n.wy-menu a:hover {\n text-decoration: none;\n}\n\n.wy-menu-vertical header,\n.wy-menu-vertical p.caption {\n height: 32px;\n display: inline-block;\n line-height: 32px;\n padding: 0 1.618em;\n margin-bottom: 0;\n margin-top: 14px;\n display: block;\n font-weight: 700;\n text-transform: uppercase;\n font-size: 85%;\n color: #ccc;\n white-space: nowrap;\n}\n\n.wy-menu-vertical span {\n color: #666;\n}\n\n.wy-menu-vertical ul {\n margin-bottom: 0;\n}\n\n.wy-menu-vertical li.divide-top {\n border-top: solid 1px #404040;\n}\n\n.wy-menu-vertical li.divide-bottom {\n border-bottom: solid 1px #404040;\n}\n\n.wy-menu-vertical li.current {\n background-color: #e5e5e5;\n}\n\n.wy-menu-vertical li.current a {\n color: rgba(0, 93, 255, 0.7);\n border-right: none;\n}\n\n.wy-menu-vertical li.current a:hover {\n color: rgba(0, 93, 255, 0.9);\n}\n\n.wy-menu-vertical li code,\n.wy-menu-vertical li .rst-content tt,\n.rst-content .wy-menu-vertical li tt {\n border: none;\n background: inherit;\n color: inherit;\n padding-left: 0;\n padding-right: 0\n}\n\n.wy-menu-vertical li span.toctree-expand {\n display: block;\n float: left;\n margin-left: -1.2em;\n font-size: .8em;\n line-height: 1.6em;\n color: #999;\n}\n\n.wy-menu-vertical li.on a,\n.wy-menu-vertical li.current>a {\n color: rgba(0, 93, 255, 0.9);\n font-weight: 700;\n position: relative;\n background: #fafafa;\n border: none;\n}\n\n/*.wy-menu-vertical li.on a:hover span.toctree-expand,\n.wy-menu-vertical li.current>a:hover span.toctree-expand {\n color: gray;\n}*/\n\n.wy-menu-vertical li.on a span.toctree-expand,\n.wy-menu-vertical li.current>a span.toctree-expand {\n display: block;\n font-size: .8em;\n line-height: 1.6em;\n color: #333;\n}\n\n.wy-menu-vertical li.toctree-l1.current li.toctree-l2>ul,\n.wy-menu-vertical li.toctree-l2.current li.toctree-l3>ul {\n display: none;\n}\n\n.wy-menu-vertical li.toctree-l1.current li.toctree-l2.current>ul,\n.wy-menu-vertical li.toctree-l2.current li.toctree-l3.current>ul {\n display: block;\n}\n\n.wy-menu-vertical li.toctree-l2.current li.toctree-l3>a {\n display: block;\n padding: .4045em 4.045em;\n}\n\n.wy-menu-vertical li.toctree-l2 a:hover span.toctree-expand {\n color: #999;\n}\n\n.wy-menu-vertical li.toctree-l2 span.toctree-expand {\n color: #999;\n}\n\n.wy-menu-vertical li.toctree-l3 {\n background-color: #eee;\n font-size: .9em;\n}\n\n.wy-menu-vertical li.toctree-l3.current>a {\n padding: .4045em 4.045em;\n}\n\n.wy-menu-vertical li.toctree-l3.current li.toctree-l4>a {\n display: block;\n padding: .4045em 5.663em;\n border-top: none;\n border-bottom: none;\n}\n\n.wy-menu-vertical li.toctree-l3 a:hover span.toctree-expand {\n color: rgba(0, 93, 255, 0.9);\n}\n\n.wy-menu-vertical li.toctree-l3 span.toctree-expand {\n color: #999;\n}\n\n.wy-menu-vertical li.toctree-l4 {\n font-size: .9em;\n}\n\n.wy-menu-vertical li.current ul {\n display: block;\n}\n\n.wy-menu-vertical .local-toc li ul {\n display: block;\n}\n\n.wy-menu-vertical li ul li a {\n margin-bottom: 0;\n color: rgba(0, 93, 255, 0.7);\n font-weight: 400;\n}\n\n.wy-menu-vertical a {\n display: inline-block;\n line-height: 18px;\n padding: .4045em 1.618em;\n display: block;\n position: relative;\n font-size: 90%;\n color: rgba(0, 93, 255, 0.7);\n}\n\n.wy-menu-vertical li.on a:hover,\n.wy-menu-vertical li.current>a:hover {\n background-color: #fafafa;\n}\n\n.wy-menu-vertical a:hover {\n color: rgba(0, 93, 255, 0.9);\n cursor: pointer;\n background-color: #fafafa;\n}\n\n.wy-menu-vertical a:hover span.toctree-expand {\n color: rgba(0, 93, 255, 0.5);\n}\n\n.wy-menu-vertical a:active span.toctree-expand {\n color: rgba(0, 93, 255, 0.7);\n}\n\n/* Search */\n\n.wy-side-nav-search {\n z-index: 200;\n background-color: #fafafa;\n border-bottom: #333;\n text-align: center;\n padding: .809em;\n display: block;\n color: #333;\n margin-bottom: .809em\n}\n\n.wy-side-nav-search input[type=text] {\n width: 100%;\n color: #333;\n border-radius: 3px;\n outline: 0;\n padding: 10px;\n background-color: #fff;\n border: solid 1px #6d6d6d;\n box-shadow: none\n}\n\n.wy-side-nav-search img {\n display: block;\n margin: auto auto .809em;\n height: 45px;\n width: 45px;\n background-color: #2980B9;\n padding: 5px;\n border-radius: 100%\n}\n\n.wy-side-nav-search>a,\n.wy-side-nav-search .wy-dropdown>a {\n color: #333;\n font-size: 100%;\n font-weight: 700;\n display: inline-block;\n padding: 4px 6px;\n margin-bottom: .809em\n}\n\n.wy-side-nav-search>a:hover,\n.wy-side-nav-search .wy-dropdown>a:hover {\n background: rgba(255,255,255,0.1)\n}\n\n.wy-side-nav-search>a img.logo,\n.wy-side-nav-search .wy-dropdown>a img.logo {\n display: block;\n margin: 0 auto;\n height: auto;\n width: auto;\n border-radius: 0;\n max-width: 100%;\n background: transparent\n}\n\n.wy-side-nav-search>a.icon img.logo,\n.wy-side-nav-search .wy-dropdown>a.icon img.logo {\n margin-top: .85em\n}\n\n.wy-nav .wy-menu-vertical header {\n color: #2980B9\n}\n\n.wy-nav .wy-menu-vertical a {\n color: #b3b3b3\n}\n\n.wy-nav .wy-menu-vertical a:hover {\n background-color: #2980B9;\n color: #fff\n}\n\n[data-menu-wrap] {\n -webkit-transition: all .2s ease-in;\n -moz-transition: all .2s ease-in;\n transition: all .2s ease-in;\n position: absolute;\n opacity: 1;\n width: 100%;\n opacity: 0\n}\n\n[data-menu-wrap].move-center {\n left: 0;\n right: auto;\n opacity: 1\n}\n\n[data-menu-wrap].move-left {\n right: auto;\n left: -100%;\n opacity: 0\n}\n\n[data-menu-wrap].move-right {\n right: -100%;\n left: auto;\n opacity: 0\n}\n\n.wy-body-for-nav {\n background: left repeat-y #fcfcfc;\n background-image: url(data:image/png;\n base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyRpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMy1jMDExIDY2LjE0NTY2MSwgMjAxMi8wMi8wNi0xNDo1NjoyNyAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENTNiAoTWFjaW50b3NoKSIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDoxOERBMTRGRDBFMUUxMUUzODUwMkJCOThDMEVFNURFMCIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDoxOERBMTRGRTBFMUUxMUUzODUwMkJCOThDMEVFNURFMCI+IDx4bXBNTTpEZXJpdmVkRnJvbSBzdFJlZjppbnN0YW5jZUlEPSJ4bXAuaWlkOjE4REExNEZCMEUxRTExRTM4NTAyQkI5OEMwRUU1REUwIiBzdFJlZjpkb2N1bWVudElEPSJ4bXAuZGlkOjE4REExNEZDMEUxRTExRTM4NTAyQkI5OEMwRUU1REUwIi8+IDwvcmRmOkRlc2NyaXB0aW9uPiA8L3JkZjpSREY+IDwveDp4bXBtZXRhPiA8P3hwYWNrZXQgZW5kPSJyIj8+EwrlwAAAAA5JREFUeNpiMDU0BAgwAAE2AJgB9BnaAAAAAElFTkSuQmCC);\n background-size: 300px 1px\n}\n\n.wy-grid-for-nav {\n position: absolute;\n width: 100%;\n height: 100%\n}\n\n.wy-nav-side {\n position: fixed;\n top: 0;\n bottom: 0;\n left: 0;\n padding-bottom: 2em;\n width: 300px;\n overflow-x: hidden;\n overflow-y: scroll;\n min-height: 100%;\n background-color: #fafafa;\n z-index: 200;\n border-right: 2px solid #eee;\n}\n\n.wy-nav-top {\n display: none;\n background-color: #333;\n color: #fff;\n padding: .4045em .809em;\n position: relative;\n line-height: 50px;\n text-align: center;\n font-size: 100%;\n *zoom: 1\n}\n\n.wy-nav-top:before,\n.wy-nav-top:after {\n display: table;\n content: \"\"\n}\n\n.wy-nav-top:after {\n clear: both\n}\n\n.wy-nav-top a {\n color: #fff;\n font-weight: 700\n}\n\n.wy-nav-top img {\n margin-right: 12px;\n height: 45px;\n width: 45px;\n background-color: #2980B9;\n padding: 5px;\n border-radius: 100%\n}\n\n.wy-nav-top i {\n font-size: 30px;\n line-height: 50px;\n float: left;\n cursor: pointer\n}\n\n.wy-nav-content-wrap {\n margin-left: 300px;\n background: #fcfcfc;\n min-height: 100%\n}\n\n.wy-nav-content {\n padding: 1.618em 3.236em;\n height: 100%;\n max-width: 1100px;\n margin: auto\n}\n\n.wy-body-mask {\n position: fixed;\n width: 100%;\n height: 100%;\n background: rgba(0,0,0,0.2);\n display: none;\n z-index: 499\n}\n\n.wy-body-mask.on {\n display: block\n}\n\n@media screen and (max-width: 768px) {\n .wy-body-for-nav {\n background: #fcfcfc\n }\n\n .wy-nav-top {\n display: block\n }\n\n .wy-nav-side {\n left: -300px\n }\n\n .wy-nav-side.shift {\n width: 85%;\n left: 0\n }\n\n .wy-nav-content-wrap {\n margin-left: 0\n }\n\n .wy-nav-content-wrap .wy-nav-content {\n padding: 1.618em\n }\n\n .wy-nav-content-wrap.shift {\n position: fixed;\n min-width: 100%;\n left: 85%;\n top: 0;\n height: 100%;\n overflow: hidden\n }\n\n}\n\n@media screen and (min-width: 1400px) {\n .wy-nav-content {\n margin: 0;\n background: #fcfcfc\n }\n}" }, { "path": "site/.gitignore", "content": "_site\n.sass-cache\n.jekyll-metadata\n" }, { "path": "site/Gemfile", "content": "source \"https://rubygems.org\"\nruby RUBY_VERSION\n\n# Hello! This is where you manage which Jekyll version is used to run.\n# When you want to use a different version, change it below, save the\n# file and run `bundle install`. Run Jekyll with `bundle exec`, like so:\n#\n# bundle exec jekyll serve\n#\n# This will help ensure the proper Jekyll version is running.\n# Happy Jekylling!\ngem \"jekyll\", \"3.3.1\"\n\n# This is the default theme for new Jekyll sites. You may change this to anything you like.\ngem \"minima\", \"~> 2.0\"\n\n# If you want to use GitHub Pages, remove the \"gem \"jekyll\"\" above and\n# uncomment the line below. To upgrade, run `bundle update github-pages`.\n# gem \"github-pages\", group: :jekyll_plugins\n\n# If you have any plugins, put them here!\ngroup :jekyll_plugins do\n gem \"jekyll-feed\", \"~> 0.6\"\nend\n" }, { "path": "site/_config.yml", "content": "# Welcome to Jekyll!\n#\n# This config file is meant for settings that affect your whole blog, values\n# which you are expected to set up once and rarely edit after that. If you find\n# yourself editing this file very often, consider using Jekyll's data files\n# feature for the data you need to update frequently.\n#\n# For technical reasons, this file is *NOT* reloaded automatically when you use\n# 'bundle exec jekyll serve'. If you change this file, please restart the server process.\n\n# Site settings\n# These are used to personalize your new site. If you look in the HTML files,\n# you will see them accessed via {{ site.title }}, {{ site.email }}, and so on.\n# You can create any custom variable you would like, and they will be accessible\n# in the templates via {{ site.myvariable }}.\ntitle: Your awesome title\nemail: your-email@domain.com\ndescription: > # this means to ignore newlines until \"baseurl:\"\n Write an awesome description for your new site here. You can edit this\n line in _config.yml. It will appear in your document head meta (for\n Google search results) and in your feed.xml site description.\nbaseurl: \"\" # the subpath of your site, e.g. /blog\nurl: \"\" # the base hostname & protocol for your site, e.g. http://example.com\ntwitter_username: jekyllrb\ngithub_username: jekyll\n\n# Build settings\nmarkdown: kramdown\ntheme: minima\ngems:\n - jekyll-feed\nexclude:\n - Gemfile\n - Gemfile.lock\n" }, { "path": "site/_posts/2017-01-06-welcome-to-jekyll.markdown", "content": "---\nlayout: post\ntitle: \"Welcome to Jekyll!\"\ndate: 2017-01-06 11:19:34 +0300\ncategories: jekyll update\n---\nYou’ll find this post in your `_posts` directory. Go ahead and edit it and re-build the site to see your changes. You can rebuild the site in many different ways, but the most common way is to run `jekyll serve`, which launches a web server and auto-regenerates your site when a file is updated.\n\nTo add new posts, simply add a file in the `_posts` directory that follows the convention `YYYY-MM-DD-name-of-post.ext` and includes the necessary front matter. Take a look at the source for this post to get an idea about how it works.\n\nJekyll also offers powerful support for code snippets:\n\n{% highlight ruby %}\ndef print_hi(name)\n puts \"Hi, #{name}\"\nend\nprint_hi('Tom')\n#=> prints 'Hi, Tom' to STDOUT.\n{% endhighlight %}\n\nCheck out the [Jekyll docs][jekyll-docs] for more info on how to get the most out of Jekyll. File all bugs/feature requests at [Jekyll’s GitHub repo][jekyll-gh]. If you have questions, you can ask them on [Jekyll Talk][jekyll-talk].\n\n[jekyll-docs]: http://jekyllrb.com/docs/home\n[jekyll-gh]: https://github.com/jekyll/jekyll\n[jekyll-talk]: https://talk.jekyllrb.com/\n" }, { "path": "site/about.md", "content": "---\nlayout: page\ntitle: About\npermalink: /about/\n---\n\nThis is the base Jekyll theme. You can find out more info about customizing your Jekyll theme, as well as basic Jekyll usage documentation at [jekyllrb.com](http://jekyllrb.com/)\n\nYou can find the source code for the Jekyll new theme at:\n{% include icon-github.html username=\"jekyll\" %} /\n[minima](https://github.com/jekyll/minima)\n\nYou can find the source code for Jekyll at\n{% include icon-github.html username=\"jekyll\" %} /\n[jekyll](https://github.com/jekyll/jekyll)\n" }, { "path": "site/index.md", "content": "---\n# You don't need to edit this file, it's empty on purpose.\n# Edit theme's home layout instead if you wanna make some changes\n# See: https://jekyllrb.com/docs/themes/#overriding-theme-defaults\nlayout: home\n---\n" }, { "path": "snapcraft.yaml", "content": "name: goreplay\nversion: '1.0'\nsummary: GoReplay is an open-source tool for capturing and replaying live HTTP traffic \ndescription: |\n GoReplay is an open-source tool for capturing and replaying \n live HTTP traffic into a test environment in order to continuously \n test your system with real data. It can be used to increase confidence \n in code deployments, configuration changes and infrastructure changes. \ngrade: stable\nconfinement: strict\nbase: core18\nparts:\n goreplay:\n plugin: go\n source: https://github.com/buger/goreplay.git\n go-importpath: github.com/buger/goreplay\n build-packages:\n - build-essential\n - libpcap-dev\n stage-packages:\n - libpcap0.8\n\napps:\n goreplay:\n command: bin/goreplay\n daemon: simple\n restart-condition: on-abnormal\n plugs:\n - home\n - network\n - network-bind\n - network-control\n - network-observe\n - netlink-connector\n - netlink-audit\n - bluetooth-control\n - firewall-control\n - x11\n \n" }, { "path": "tcp_client.go", "content": "package goreplay\n\nimport (\n\t\"crypto/tls\"\n\t\"io\"\n\t\"net\"\n\t\"runtime/debug\"\n\t\"syscall\"\n\t\"time\"\n)\n\n// TCPClientConfig client configuration\ntype TCPClientConfig struct {\n\tDebug bool\n\tConnectionTimeout time.Duration\n\tTimeout time.Duration\n\tResponseBufferSize int\n\tSecure bool\n}\n\n// TCPClient client connection properties\ntype TCPClient struct {\n\tbaseURL string\n\taddr string\n\tconn net.Conn\n\trespBuf []byte\n\tconfig *TCPClientConfig\n\tredirectsCount int\n}\n\n// NewTCPClient returns new TCPClient\nfunc NewTCPClient(addr string, config *TCPClientConfig) *TCPClient {\n\tif config.Timeout.Nanoseconds() == 0 {\n\t\tconfig.Timeout = 5 * time.Second\n\t}\n\n\tconfig.ConnectionTimeout = config.Timeout\n\n\tif config.ResponseBufferSize == 0 {\n\t\tconfig.ResponseBufferSize = 100 * 1024 // 100kb\n\t}\n\n\tclient := &TCPClient{config: config, addr: addr}\n\tclient.respBuf = make([]byte, config.ResponseBufferSize)\n\n\treturn client\n}\n\n// Connect creates a tcp connection of the client\nfunc (c *TCPClient) Connect() (err error) {\n\tc.Disconnect()\n\n\tc.conn, err = net.DialTimeout(\"tcp\", c.addr, c.config.ConnectionTimeout)\n\n\tif c.config.Secure {\n\t\ttlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true})\n\n\t\tif err = tlsConn.Handshake(); err != nil {\n\t\t\treturn\n\t\t}\n\n\t\tc.conn = tlsConn\n\t}\n\n\treturn\n}\n\n// Disconnect closes the client connection\nfunc (c *TCPClient) Disconnect() {\n\tif c.conn != nil {\n\t\tc.conn.Close()\n\t\tc.conn = nil\n\t\tDebug(1, \"[TCPClient] Disconnected: \", c.baseURL)\n\t}\n}\n\nfunc (c *TCPClient) isAlive() bool {\n\tone := make([]byte, 1)\n\n\t// Ready 1 byte from socket without timeout to check if it not closed\n\tc.conn.SetReadDeadline(time.Now().Add(time.Millisecond))\n\t_, err := c.conn.Read(one)\n\n\tif err == nil {\n\t\treturn true\n\t} else if err == io.EOF {\n\t\tDebug(1, \"[TCPClient] connection closed, reconnecting\")\n\t\treturn false\n\t} else if err == syscall.EPIPE {\n\t\tDebug(1, \"Detected broken pipe.\", err)\n\t\treturn false\n\t}\n\n\treturn true\n}\n\n// Send sends data over created tcp connection\nfunc (c *TCPClient) Send(data []byte) (response []byte, err error) {\n\t// Don't exit on panic\n\tdefer func() {\n\t\tif r := recover(); r != nil {\n\t\t\tDebug(1, \"[TCPClient]\", r, string(data))\n\n\t\t\tif _, ok := r.(error); !ok {\n\t\t\t\tDebug(1, \"[TCPClient] Failed to send request: \", string(data))\n\t\t\t\tDebug(1, \"PANIC: pkg:\", r, debug.Stack())\n\t\t\t}\n\t\t}\n\t}()\n\n\tif c.conn == nil || !c.isAlive() {\n\t\tDebug(1, \"[TCPClient] Connecting:\", c.baseURL)\n\t\tif err = c.Connect(); err != nil {\n\t\t\tDebug(1, \"[TCPClient] Connection error:\", err)\n\t\t\treturn\n\t\t}\n\t}\n\n\ttimeout := time.Now().Add(c.config.Timeout)\n\n\tc.conn.SetWriteDeadline(timeout)\n\n\tif c.config.Debug {\n\t\tDebug(1, \"[TCPClient] Sending:\", string(data))\n\t}\n\n\tif _, err = c.conn.Write(data); err != nil {\n\t\tDebug(1, \"[TCPClient] Write error:\", err, c.baseURL)\n\t\treturn\n\t}\n\n\tvar readBytes, n int\n\tvar currentChunk []byte\n\ttimeout = time.Now().Add(c.config.Timeout)\n\n\tfor {\n\t\tc.conn.SetReadDeadline(timeout)\n\n\t\tif readBytes < len(c.respBuf) {\n\t\t\tn, err = c.conn.Read(c.respBuf[readBytes:])\n\t\t\treadBytes += n\n\n\t\t\tif err != nil {\n\t\t\t\tif err == io.EOF {\n\t\t\t\t\terr = nil\n\t\t\t\t}\n\t\t\t\tbreak\n\t\t\t}\n\t\t} else {\n\t\t\tif currentChunk == nil {\n\t\t\t\tcurrentChunk = make([]byte, readChunkSize)\n\t\t\t}\n\n\t\t\tn, err = c.conn.Read(currentChunk)\n\n\t\t\tif err == io.EOF {\n\t\t\t\tbreak\n\t\t\t} else if err != nil {\n\t\t\t\tDebug(1, \"[TCPClient] Read the whole body error:\", err, c.baseURL)\n\t\t\t\tbreak\n\t\t\t}\n\n\t\t\treadBytes += int(n)\n\t\t}\n\n\t\tif readBytes >= maxResponseSize {\n\t\t\tDebug(1, \"[TCPClient] Body is more than the max size\", maxResponseSize,\n\t\t\t\tc.baseURL)\n\t\t\tbreak\n\t\t}\n\n\t\t// For following chunks expect less timeout\n\t\ttimeout = time.Now().Add(c.config.Timeout / 5)\n\t}\n\n\tif err != nil {\n\t\tDebug(1, \"[TCPClient] Response read error\", err, c.conn, readBytes)\n\t\treturn\n\t}\n\n\tif readBytes > len(c.respBuf) {\n\t\treadBytes = len(c.respBuf)\n\t}\n\n\tpayload := make([]byte, readBytes)\n\tcopy(payload, c.respBuf[:readBytes])\n\n\tif c.config.Debug {\n\t\tDebug(1, \"[TCPClient] Received:\", string(payload))\n\t}\n\n\treturn payload, err\n}\n" }, { "path": "test_input.go", "content": "package goreplay\n\nimport (\n\t\"encoding/base64\"\n\t\"errors\"\n\t\"math/rand\"\n\t\"time\"\n)\n\n// ErrorStopped is the error returned when the go routines reading the input is stopped.\nvar ErrorStopped = errors.New(\"reading stopped\")\n\n// TestInput used for testing purpose, it allows emitting requests on demand\ntype TestInput struct {\n\tdata chan []byte\n\tskipHeader bool\n\tstop chan bool // Channel used only to indicate goroutine should shutdown\n}\n\n// NewTestInput constructor for TestInput\nfunc NewTestInput() (i *TestInput) {\n\ti = new(TestInput)\n\ti.data = make(chan []byte, 100)\n\ti.stop = make(chan bool)\n\treturn\n}\n\n// PluginRead reads message from this plugin\nfunc (i *TestInput) PluginRead() (*Message, error) {\n\tvar msg Message\n\tselect {\n\tcase buf := <-i.data:\n\t\tmsg.Data = buf\n\t\tif !i.skipHeader {\n\t\t\tmsg.Meta = payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1)\n\t\t} else {\n\t\t\tmsg.Meta, msg.Data = payloadMetaWithBody(msg.Data)\n\t\t}\n\n\t\treturn &msg, nil\n\tcase <-i.stop:\n\t\treturn nil, ErrorStopped\n\t}\n}\n\n// Close closes this plugin\nfunc (i *TestInput) Close() error {\n\tclose(i.stop)\n\treturn nil\n}\n\n// EmitBytes sends data\nfunc (i *TestInput) EmitBytes(data []byte) {\n\ti.data <- data\n}\n\n// EmitGET emits GET request without headers\nfunc (i *TestInput) EmitGET() {\n\ti.data <- []byte(\"GET / HTTP/1.1\\r\\n\\r\\n\")\n}\n\n// EmitPOST emits POST request with Content-Length\nfunc (i *TestInput) EmitPOST() {\n\ti.data <- []byte(\"POST /pub/WWW/ HTTP/1.1\\r\\nContent-Length: 7\\r\\nHost: www.w3.org\\r\\n\\r\\na=1&b=2\")\n}\n\n// EmitChunkedPOST emits POST request with `Transfer-Encoding: chunked` and chunked body\nfunc (i *TestInput) EmitChunkedPOST() {\n\ti.data <- []byte(\"POST /pub/WWW/ HTTP/1.1\\r\\nHost: www.w3.org\\r\\nTransfer-Encoding: chunked\\r\\n\\r\\n4\\r\\nWiki\\r\\n5\\r\\npedia\\r\\ne\\r\\n in\\r\\n\\r\\nchunks.\\r\\n0\\r\\n\\r\\n\")\n}\n\n// EmitLargePOST emits POST request with large payload (5mb)\nfunc (i *TestInput) EmitLargePOST() {\n\tsize := 5 * 1024 * 1024 // 5 MB\n\trb := make([]byte, size)\n\trand.Read(rb)\n\n\trs := base64.URLEncoding.EncodeToString(rb)\n\n\ti.data <- []byte(\"POST / HTTP/1.1\\r\\nHost: www.w3.org\\nContent-Length:5242880\\r\\n\\r\\n\" + rs)\n}\n\n// EmitSizedPOST emit a POST with a payload set to a supplied size\nfunc (i *TestInput) EmitSizedPOST(payloadSize int) {\n\trb := make([]byte, payloadSize)\n\trand.Read(rb)\n\n\trs := base64.URLEncoding.EncodeToString(rb)\n\n\ti.data <- []byte(\"POST / HTTP/1.1\\r\\nHost: www.w3.org\\nContent-Length:5242880\\r\\n\\r\\n\" + rs)\n}\n\n// EmitOPTIONS emits OPTIONS request, similar to GET\nfunc (i *TestInput) EmitOPTIONS() {\n\ti.data <- []byte(\"OPTIONS / HTTP/1.1\\r\\nHost: www.w3.org\\r\\n\\r\\n\")\n}\n\nfunc (i *TestInput) String() string {\n\treturn \"Test Input\"\n}\n" }, { "path": "test_output.go", "content": "package goreplay\n\ntype writeCallback func(*Message)\n\n// TestOutput used in testing to intercept any output into callback\ntype TestOutput struct {\n\tcb writeCallback\n}\n\n// NewTestOutput constructor for TestOutput, accepts callback which get called on each incoming Write\nfunc NewTestOutput(cb writeCallback) PluginWriter {\n\ti := new(TestOutput)\n\ti.cb = cb\n\n\treturn i\n}\n\n// PluginWrite write message to this plugin\nfunc (i *TestOutput) PluginWrite(msg *Message) (int, error) {\n\ti.cb(msg)\n\n\treturn len(msg.Data) + len(msg.Meta), nil\n}\n\nfunc (i *TestOutput) String() string {\n\treturn \"Test Output\"\n}\n" }, { "path": "version.go", "content": "package goreplay\n\n// VERSION the current version of goreplay\nvar VERSION = \"2.0.0\"\n" } ]