Repository: probelabs/goreplay Branch: master Commit: 251e45abd242 Files: 162 Total size: 655.8 KB Directory structure: gitextract_1d_7z0pv/ ├── .clabot ├── .deepsource.toml ├── .dockerignore ├── .github/ │ ├── dependabot.yaml │ └── workflows/ │ ├── ci-docker.yaml │ ├── ci-test.yaml │ ├── codesee-arch-diagram.yml │ ├── probe.yaml │ └── semgrep.yml ├── .gitignore ├── .gitmodules ├── .request ├── COMM-LICENSE ├── Dockerfile ├── Dockerfile.dev ├── ELASTICSEARCH.md ├── LICENSE.txt ├── Makefile ├── Procfile ├── README.md ├── ce.go ├── circle.yml ├── docs/ │ ├── CNAME │ ├── Capturing-and-replaying-traffic.md │ ├── Compilation.md │ ├── Development-Setup.md │ ├── Distributed-configuration.md │ ├── Exporting-to-ElasticSearch.md │ ├── FAQ.md │ ├── Middleware.md │ ├── Rate-limiting.md │ ├── Replaying-HTTP-traffic.md │ ├── Request-filtering.md │ ├── Request-rewriting.md │ ├── Running-as-non-root-user.md │ ├── Saving-and-Replaying-from-file.md │ ├── Troubleshooting.md │ ├── _Footer.md │ ├── _config.yml │ ├── commercial/ │ │ ├── collaboration.md │ │ ├── faq.md │ │ └── support.md │ ├── css/ │ │ ├── breadcrumbs.css │ │ ├── code.css │ │ ├── fabric.css │ │ ├── goreplay.css │ │ └── sidenav.css │ ├── getting-started/ │ │ ├── basics.md │ │ └── tutorial.md │ ├── index.md │ ├── js/ │ │ ├── base.js │ │ └── turbolinks.js │ └── pro/ │ ├── recording-and-replaying-keep-alive-tcp-sessions.md │ └── replaying-binary-protocols.md ├── elasticsearch.go ├── elasticsearch_test.go ├── emitter.go ├── emitter_test.go ├── examples/ │ └── middleware/ │ ├── echo.clj │ ├── echo.java │ ├── echo.js │ ├── echo.py │ ├── echo.rb │ ├── echo.sh │ └── token_modifier.go ├── go.mod ├── go.sum ├── gor_stat.go ├── homebrew/ │ └── gor.rb ├── http_modifier.go ├── http_modifier_settings.go ├── http_modifier_settings_test.go ├── http_modifier_test.go ├── http_prettifier.go ├── http_prettifier_test.go ├── input_dummy.go ├── input_file.go ├── input_file_test.go ├── input_http.go ├── input_http_test.go ├── input_kafka.go ├── input_kafka_test.go ├── input_raw.go ├── input_raw_test.go ├── input_tcp.go ├── input_tcp_test.go ├── internal/ │ ├── byteutils/ │ │ ├── byteutils.go │ │ └── byteutils_test.go │ ├── capture/ │ │ ├── af_packet.go │ │ ├── af_packet_linux.go │ │ ├── capture.go │ │ ├── capture_test.go │ │ ├── doc.go │ │ ├── dump.go │ │ ├── sock_linux.go │ │ ├── sock_others.go │ │ ├── socket.go │ │ └── vxlan.go │ ├── ring/ │ │ └── ring.go │ ├── simpletime/ │ │ └── time.go │ ├── size/ │ │ ├── size.go │ │ └── size_test.go │ └── tcp/ │ ├── doc.go │ ├── tcp_message.go │ ├── tcp_packet.go │ └── tcp_test.go ├── k8s/ │ ├── README.md │ ├── clusterrole.yaml │ ├── collect_goreplay_telemetry.sh │ ├── goreplay.yaml │ ├── nginx.yaml │ └── rolebinding.yaml ├── kafka.go ├── limiter.go ├── limiter_test.go ├── middleware/ │ ├── README.md │ ├── middleware.js │ └── package.json ├── middleware.go ├── middleware_test.go ├── mkdocs.yml ├── nfpm.yaml ├── output_binary.go ├── output_binary_pro.go ├── output_dummy.go ├── output_file.go ├── output_file_test.go ├── output_http.go ├── output_http_test.go ├── output_kafka.go ├── output_kafka_test.go ├── output_null.go ├── output_s3.go ├── output_s3_pro.go ├── output_tcp.go ├── output_tcp_test.go ├── output_ws.go ├── output_ws_test.go ├── plugins.go ├── plugins_test.go ├── pro.go ├── proto/ │ ├── fuzz.go │ ├── proto.go │ └── proto_test.go ├── protocol.go ├── s3/ │ └── index.html ├── s3_reader.go ├── s3_test.go ├── settings.go ├── settings_test.go ├── sidenav.css ├── site/ │ ├── .gitignore │ ├── Gemfile │ ├── _config.yml │ ├── _posts/ │ │ └── 2017-01-06-welcome-to-jekyll.markdown │ ├── about.md │ └── index.md ├── snapcraft.yaml ├── tcp_client.go ├── test_input.go ├── test_output.go └── version.go ================================================ FILE CONTENTS ================================================ ================================================ FILE: .clabot ================================================ { "contributors": ["buger"] } ================================================ FILE: .deepsource.toml ================================================ version = 1 exclude_patterns = [ "vendor/**" ] [[analyzers]] name = "go" enabled = true [analyzers.meta] import_paths = ["github.com/ankitdobhal/goreplay"] [[analyzers]] name = "docker" enabled = true [[analyzers]] name = "ruby" enabled = true [[analyzers]] name = "javascript" enabled = true [analyzers.meta] environment = ["nodejs"] ================================================ FILE: .dockerignore ================================================ *.tar.gz gor gor.test ================================================ FILE: .github/dependabot.yaml ================================================ version: 2 updates: # Maintain dependencies for GitHub Actions - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" pull-request-branch-name: # Separate sections of the branch name with a hyphen separator: "-" - package-ecosystem: "gomod" directory: "/" schedule: interval: "weekly" - package-ecosystem: "docker" directory: "/" schedule: interval: "weekly" ================================================ FILE: .github/workflows/ci-docker.yaml ================================================ name: ci on: release: types: [published] jobs: docker-build-and-push: runs-on: ubuntu-latest steps: - name: "Checkout repository" uses: actions/checkout@v3 - name: "Set up Docker Buildx" uses: docker/setup-buildx-action@v2 - name: "Cache Docker layers" uses: actions/cache@v3.0.5 with: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx- - name: "Login to Container Registry" uses: docker/login-action@v1 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: "Build and push docker image" uses: docker/build-push-action@v3 with: context: . push: true tags: ${{ github.repository }}:${{ github.event.release.tag_name }} build-args: RELEASE_VERSION=${{ github.event.release.tag_name }} cache-from: type=local,src=/tmp/.buildx-cache cache-to: type=local,mode=max,dest=/tmp/.buildx-cache ================================================ FILE: .github/workflows/ci-test.yaml ================================================ name: test on: [push, pull_request] jobs: test: strategy: matrix: go-version: [1.18.x, 1.19.x] # two latest minor versions runs-on: ubuntu-latest steps: - name: update package index run: sudo apt-get update - name: install libpcap run: sudo apt-get install libpcap-dev -y - name: install Go uses: actions/setup-go@v2 with: go-version: ${{ matrix.go-version }} - name: checkout code uses: actions/checkout@v3 - uses: actions/cache@v3.0.5 with: path: | ~/go/pkg/mod # Module download cache ~/.cache/go-build # Build cache (Linux) key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go- - name: test run: sudo go test ./... -v -timeout 120s ================================================ FILE: .github/workflows/codesee-arch-diagram.yml ================================================ # This workflow was added by CodeSee. Learn more at https://codesee.io/ # This is v2.0 of this workflow file on: push: branches: - master pull_request_target: types: [opened, synchronize, reopened] name: CodeSee permissions: read-all jobs: codesee: runs-on: ubuntu-latest continue-on-error: true name: Analyze the repo with CodeSee steps: - uses: Codesee-io/codesee-action@v2 with: codesee-token: ${{ secrets.CODESEE_ARCH_DIAG_API_TOKEN }} codesee-url: https://app.codesee.io ================================================ FILE: .github/workflows/probe.yaml ================================================ name: AI Comment Handler on: pull_request: types: [opened] #[opened , labeled] issue_comment: types: [created] issues: types: [opened] #[opened, labeled] # Define permissions needed for the workflow permissions: issues: write pull-requests: write contents: read jobs: trigger_probe_chat: # Uncomment if you want to run on on specific lables, in this example `probe` # if: | # (github.event_name == 'pull_request' && github.event.action == 'opened') || # (github.event_name == 'issues' && github.event.action == 'opened') || # (github.event_name == 'issue_comment' && github.event.action == 'created') || # ((github.event_name == 'pull_request' || github.event_name == 'issues') && # github.event.action == 'labeled' && github.event.label.name == 'probe') # Use the reusable workflow from your repository (replace and ) uses: buger/probe/.github/workflows/probe.yml@main # Pass required inputs with: command_prefix: "/probe" # Or '/ai', '/ask', etc. # Optionally override the default npx command if the secret isn't set # default_probe_chat_command: 'node path/to/custom/script.js' # Pass ALL secrets from this repository to the reusable workflow # This includes GITHUB_TOKEN, PROBE_CHAT_COMMAND (if set), ANTHROPIC_API_KEY, etc. secrets: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} ANTHROPIC_API_URL: ${{ secrets.ANTHROPIC_API_URL }} ================================================ FILE: .github/workflows/semgrep.yml ================================================ on: pull_request: {} push: branches: - main - master paths: - .github/workflows/semgrep.yml schedule: # random HH:MM to avoid a load spike on GitHub Actions at 00:00 - cron: 53 23 * * * name: Semgrep jobs: semgrep: name: Scan runs-on: ubuntu-20.04 env: SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }} container: image: returntocorp/semgrep steps: - uses: actions/checkout@v3 - run: semgrep ci ================================================ FILE: .gitignore ================================================ vendor *.swp *.gor *.rpm *.dep *.deb *.pkg *.exe *.pprof *.out hey *.bin lib/ output/ *.gz *.zip .aider* *.class *.test .idea *.iml gor *.mprof *.pcap .DS_Store goreplay corpus crashers suppressions dist ================================================ FILE: .gitmodules ================================================ ================================================ FILE: .request ================================================ POST /post HTTP/1.1 Content-Length: 7 Host: www.w3.org a=1&b=2 ================================================ FILE: COMM-LICENSE ================================================ END-USER LICENSE AGREEMENT ------------------------------------------------------------------------------ IMPORTANT: THIS SOFTWARE END-USER LICENSE AGREEMENT ("EULA") IS A LEGAL AGREEMENT (“Agreement”) BETWEEN YOU (THE CUSTOMER, EITHER AS AN INDIVIDUAL OR, IF PURCHASED OR OTHERWISE ACQUIRED BY OR FOR AN ENTITY, AS AN ENTITY), hereafter "Customer", AND Replay Software LLC ("Licensor"). READ IT CAREFULLY BEFORE COMPLETING THE INSTALLATION PROCESS AND USING GOREPLAY PRO AND RELATED SOFTWARE COMPONENTS (“SOFTWARE”). IT PROVIDES A LICENSE TO USE THE SOFTWARE AND CONTAINS WARRANTY INFORMATION AND LIABILITY DISCLAIMERS. BY INSTALLING AND USING THE SOFTWARE, YOU ARE CONFIRMING YOUR ACCEPTANCE OF THE SOFTWARE AND AGREEING TO BECOME BOUND BY THE TERMS OF THIS AGREEMENT. ------------------------------------------------------------------------------ In order to use the Software under this Agreement, you must receive a “Source URL” at the time of purchase, in accordance with the scope of use and other terms specified for each type of Software and as set forth in this Section 1 of this Agreement. 1. License Grant 1.1 General Use. This Agreement grants you a non-exclusive, non-transferable, limited license to the use rights for the Software, without the right to grant sublicenses, subject to the terms and conditions in this Agreement. The Software is licensed, not sold. 1.2 Unlimited Organization License. If you purchased an Organization License (included with the GoReplay Pro Software), you may install the Software on an unlimited number of Hosts. “Host” means any physical or virtual machine which is controlled by you. You may concurrently run the software on an unlimited number of Hosts. 1.3 Appliance License. If you purchased an Appliance License, you may distribute the Software in any applications, frameworks, or elements (collectively referred to as an “Application” or “Applications”) that you develop using the Software in accordance with this EULA, provided that such distribution does not violate the restrictions set forth in section 3 of this EULA. You must not remove, obscure or interfere with any copyright, acknowledgment, attribution, trademark, warning or disclaimer statement affixed to, incorporated in or otherwise applied in connection with the Software. You are required to ensure that the Software is not reused by or with any applications other than those with which you distribute it as permitted herein. For example, if You install the Software on a customer’s server, that customer is not permitted to use the Software independently of your Application. You must inform Licensor of your knowledge of any infringing use of the Software by any of your customers. You are liable for compliance by those third parties with the terms and conditions of this EULA. You will not owe Licensor any royalties for your distribution of the Software in accordance with this EULA. 1.4 Archive Copies. You are entitled to make a reasonable amount of copies of the Software for archival purposes. Each copy must reproduce all copyright and other proprietary rights notices on or in the Software Product. 1.5 Electronic Delivery. All Software and license documentation shall be delivered by electronic means unless otherwise specified on the applicable invoice or at the time of purchase. Software shall be deemed delivered when it is made available for download by you (“Delivery”). 2. Modifications. Licensor shall provide you with source code so that you can create Modifications of the original software. “Modification” means: (a) any addition to or deletion from the contents of a file included in the original Software or previous Modifications created by You, or (b) any new file that contains any part of the original Software or previous Modifications. While you retain all rights to any original work authored by you as part of the Modifications, We continue to own all copyright and other intellectual property rights in the Software. 3. Restricted Uses. 3.1 You shall not (and shall not allow any third party to): (a) decompile, disassemble, or otherwise reverse engineer the Software or attempt to reconstruct or discover any source code, underlying ideas, algorithms, file formats or programming interfaces of the Software by any means whatsoever (except and only to the extent that applicable law prohibits or restricts reverse engineering restrictions); (b) distribute, sell, sublicense, rent, lease or use the Software for time sharing, hosting, service provider or like purposes, except as expressly permitted under this Agreement; (c) redistribute the Software or Modifications other than by including the Software or a portion thereof within your own product, which must have substantially different functionality than the Software or Modifications and must not allow any third party to use the Software or Modifications, or any portions thereof, for software development or application development purposes; (d) redistribute the Software as part of a product, "appliance" or "virtual server"; (e) redistribute the Software on any server which is not directly under your control; (f) remove any product identification, proprietary, copyright or other notices contained in the Software; (g) modify any part of the Software, create a derivative work of any part of the Software (except as permitted in Section 4), or incorporate the Software, except to the extent expressly authorized in writing by Licensor; (h) publicly disseminate performance information or analysis (including, without limitation, benchmarks) from any source relating to the Software; (i) utilize any equipment, device, software, or other means designed to circumvent or remove any form of Source URL or copy protection used by Licensor in connection with the Software, or use the Software together with any authorization code, Source URL, serial number, or other copy protection device not supplied by Licensor; (j) use the Software to develop a product which is competitive with any Licensor product offerings; or (k) use unauthorized Source URLS or keycode(s) or distribute or publish Source URLs or keycode(s), except as may be expressly permitted by Licensor in writing. If your unique Source URL is ever published, Licensor reserves the right to terminate your access without notice. 3.2 UNDER NO CIRCUMSTANCES MAY YOU USE THE SOFTWARE AS PART OF A PRODUCT OR SERVICE THAT PROVIDES SIMILAR FUNCTIONALITY TO THE SOFTWARE ITSELF. The Open Source version of the Software (“LGPL Version”) is licensed under the terms of the GNU Lesser General Public License versions 3.0 (“LGPL”) and not under this EULA. 4. Ownership. Notwithstanding anything to the contrary contained herein, except for the limited license rights expressly provided herein, Licensor and its suppliers have and will retain all rights, title and interest (including, without limitation, all patent, copyright, trademark, trade secret and other intellectual property rights) in and to the Software and all copies, modifications and derivative works thereof (including any changes which incorporate any of your ideas, feedback or suggestions). You acknowledge that you are obtaining only a limited license right to the Software, and that irrespective of any use of the words “purchase”, “sale” or like terms hereunder no ownership rights are being conveyed to you under this Agreement or otherwise. 5. Fees and Payment. The Software license fees will be due and payable in full as set forth in the applicable invoice or at the time of purchase. If the Software does not function properly within two weeks of purchase, please contact us within those two weeks for a refund. You shall be responsible for all taxes, withholdings, duties and levies arising from the order (excluding taxes based on the net income of Licensor). 6. Support, Maintenance and Services. Subject to the terms and conditions of this Agreement, as set forth in your invoice, and as set forth on the GoReplay Pro support page (https://github.com/buger/goreplay/wiki/Pro-Support), support and maintenance services may be included with the purchase of your license subscription. 7. Term of Agreement. 7.1 Term. This Agreement is effective as of the Delivery of the Software and expires at such time as all license and service subscriptions hereunder have expired in accordance with their own terms (the “Term”). For clarification, the term of your license under this Agreement may be perpetual, limited for Evaluation Version, or designated as a fixed-term license in the Invoice, and shall be specified at your time of purchase. Either party may terminate this Agreement (including all related Invoices) if the other party: (a) fails to cure any material breach of this Agreement within thirty (30) days after written notice of such breach, provided that Licensor may terminate this Agreement immediately upon any breach of Section 3 or if you exceed any other restrictions contained in Section 1, unless otherwise specified in this agreement; (b) ceases operation without a successor; or (c) seeks protection under any bankruptcy, receivership, trust deed, creditors arrangement, composition or comparable proceeding, or if any such proceeding is instituted against such party (and not dismissed within sixty (60) days)). Termination is not an exclusive remedy and the exercise by either party of any remedy under this Agreement will be without prejudice to any other remedies it may have under this Agreement, by law, or otherwise. 7.2 Termination. Upon any termination of this Agreement, you shall cease any and all use of any Software and destroy all copies thereof. 7.3 Expiration of License. Upon the expiration of any term under this Agreement, (a) all Software updates and services pursuant to the license shall cease, (b) you may only continue to run existing installations of the Software, (c) you may not install the Software on any additional Hosts, and (d) any new installation of the Software shall require the purchase of a new license subscription from Licensor. 8. Disclaimer of Warranties. The Software is provided "as is," with all faults, defects and errors, and without warranty of any kind. Licensor does not warrant that the Software will be free of bugs, errors, viruses or other defects, and Licensor shall have no liability of any kind for the use of or inability to use the Software, the Software content or any associated service, and you acknowledge that it is not technically practicable for Licensor to do so. To the maximum extent permitted by applicable law, Licensor disclaims all warranties, express, implied, arising by law or otherwise, regarding the Software, the Software content and their respective performance or suitability for your intended use, including without limitation any implied warranty of merchantability, fitness for a particular purpose. Notwithstanding the foregoing, Licensor represents and warrants that it either owns the entire right to, title to, interest in, or has the right to license, the Software, and your proper use of the same will not violate any intellectual property or proprietary rights of another. 9. Limitation of Liability. In no event will either party be liable for any direct, indirect, consequential, incidental, special, exemplary, or punitive damages or liabilities whatsoever arising from or relating to the Software, the Software content or this Agreement, whether based on contract, tort (including negligence), strict liability or other theory, even if such party has been advised of the possibility of such damages. Except for indemnification obligations in Section 13.5, in no event will Licensor liability exceed the Software license price as indicated in the invoice. The existence of more than one claim will not enlarge or extend this limit. 10. Remedies. Your exclusive remedy and Licensor’ entire liability for breach of this Agreement shall be limited, at Licensor’ sole and exclusive discretion, to (a) replacement of any defective software or documentation; or (b) refund of the license fee paid to Licensor, payable in accordance with Licensor' refund policy. 11. Acknowledgements. 11.1 Consent to the Use of Data. You agree that Licensor and its affiliates may collect and use technical information gathered as part of the product support services to be used in an anonymized manner. Licensor may use this information solely to improve products and services and will not disclose this information in a form that personally identifies you. 11.2 Verification. We or a certified auditor acting on our behalf, may, upon its reasonable request and at its expense, audit you with respect to the use of the Software, subject to your reasonable information security and confidentiality protections. Such audit may be conducted by mail, electronic means or through an in-person visit to your place of business. Any such in-person audit shall be conducted during regular business hours at your facilities and shall not unreasonably interfere with your business activities. We shall not remove, copy, or redistribute any electronic material during the course of an audit. If an audit reveals that you are using the Software in a way that is in material violation of the terms of the EULA, then you shall pay our reasonable costs of conducting the audit. In the case of a material violation, you agree to pay Us any amounts owing that are attributable to the unauthorized use. In the alternative, We reserve the right, at our sole option, to terminate the licenses for the Software. 11.3 Government End Users. If the Software and related documentation are supplied to or purchased by or on behalf of the United States Government, then the Software is deemed to be "commercial software" as that term is used in the Federal Acquisition Regulation system. Rights of the United States shall not exceed the minimum rights set forth in FAR 52.227-19 for "restricted computer software". All other terms and conditions of this Agreement apply. 12. Third Party Software. Examples included in Software may provide links to third party libraries or code (collectively “Third Party Software”) to implement various functions. Third Party Software does not comprise part of the Software. In some cases, access to Third Party Software may be included along with the Software delivery as a convenience for demonstration purposes. Such source code and libraries may be included in the “…/examples” source tree delivered with the Software and do not comprise the Software. Licensee acknowledges (1) that some part of Third Party Software may require additional licensing of copyright and patents from the owners of such, and (2) that distribution of any of the Software referencing or including any portion of a Third Party Software may require appropriate licensing from such third parties. 13. Miscellaneous 13.1 Entire Agreement. This Agreement sets forth our entire agreement with respect to the Software and the subject matter hereof and supersedes all prior and contemporaneous understandings and agreements whether written or oral. 13.2 Amendment. Licensor reserves the right, in its sole discretion, to amend this Agreement from time. Amendments to this Agreement can be located at: https://github.com/buger/goreplay/blob/master/COMM-LICENSE. 13.3 Assignment. You may not assign this Agreement or any of its rights under this Agreement without the prior written consent of Licensor and any attempted assignment without such consent shall be void. 13.4 Export Compliance. Each party agrees to comply with all applicable laws and regulations, including laws, regulations, orders or other restrictions on export, re-export or redistribution of software. 13.5 Indemnification. Licensor agrees to defend, indemnify, and hold harmless You from and against any lawsuits, claims, losses, damages, fines and expenses (including attorneys' fees and costs) arising out of a claim brought by a third-party alleging that the Software directly infringes such third-party’s intellectual property rights; provided that Licensor shall have no indemnity obligation for claims arising out of your modification to the Software. You agree to defend, indemnify, and hold harmless Licensor from and against any lawsuits, claims, losses, damages, fines and expenses (including attorneys' fees and costs) brought by a third-party arising out of your modifications to the Software or breach of this Agreement. 13.6 Governing Law. This Agreement is governed by the laws of the State of Oregon and the United States without regard to conflicts of laws provisions thereof, and without regard to the United Nations Convention on the International Sale of Goods or the Uniform Computer Information Transactions Act, as currently enacted by any jurisdiction or as may be codified or amended from time to time by any jurisdiction. The jurisdiction and venue for actions related to the subject matter hereof shall be the state of Oregon and United States federal courts located in Portland, Oregon, and both parties hereby submit to the personal jurisdiction of such courts. 13.7 Attorneys’ Fees and Costs. The prevailing party in any action to enforce this Agreement will be entitled to recover its attorneys’ fees and costs in connection with such action. 13.8 Severability. If any provision of this Agreement is held by a court of competent jurisdiction to be invalid, illegal, or unenforceable, the remainder of this Agreement will remain in full force and effect. 13.9 Waiver. Failure or neglect by either party to enforce at any time any of the provisions of this licence Agreement shall not be construed or deemed to be a waiver of that party's rights under this Agreement. 13.10 Headings. The headings of sections and paragraphs of this Agreement are for convenience of reference only and are not intended to restrict, affect or be of any weight in the interpretation or construction of the provisions of such sections or paragraphs. 14. Contact Information. If you have any questions about this EULA, or if you want to contact Licensor for any reason, please direct correspondence to hello@goreplay.org ================================================ FILE: Dockerfile ================================================ FROM alpine:3.16 as builder ARG RELEASE_VERSION RUN apk add --no-cache ca-certificates openssl RUN wget https://github.com/buger/goreplay/releases/download/${RELEASE_VERSION}/gor_${RELEASE_VERSION}_x64.tar.gz -O gor.tar.gz RUN tar xzf gor.tar.gz FROM scratch COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ COPY --from=builder /gor . ENTRYPOINT ["./gor"] ================================================ FILE: Dockerfile.dev ================================================ ARG BASE_IMAGE FROM ${BASE_IMAGE} RUN apk add --no-cache \ gcc \ g++ \ make \ linux-headers \ bison \ flex \ git \ wget RUN wget http://www.tcpdump.org/release/libpcap-1.10.0.tar.gz && tar xzf libpcap-1.10.0.tar.gz && cd libpcap-1.10.0 && ./configure && make install WORKDIR /go/src/github.com/buger/goreplay/ ADD . /go/src/github.com/buger/goreplay/ RUN go get golang.org/x/lint/golint RUN go get ================================================ FILE: ELASTICSEARCH.md ================================================ gor & elasticsearch =================== Prerequisites ------------- - elasticsearch - kibana (Get it here: http://www.elasticsearch.org/overview/kibana/) - gor elasticsearch ------------- The default elasticsearch configuration is just fine for most workloads. You won't need clustering, sharding or something like that. In this example we're installing it on our gor replay server which gives us the elasticsearch listener on _http://localhost:9200_ kibana ------ Kibana (elasticsearch analytics web-ui) is just as simple. Download it, extract it and serve it via a simple webserver. (Could be nginx or apache) You could also use a shell, ```cd``` into the kibana directory and start a little quick and dirty python webserver with: ``` python -m SimpleHTTPServer 8000 ``` In this example we're also choosing the gor replay server as our kibana host. If you choose a different server you'll have to point kibana to your elasticsearch host. gor --- Start your gor replay server with elasticsearch option: ``` ./gor --input-raw :8000 --output-http http://staging.com --output-http-elasticsearch localhost:9200/gor ``` (You don't have to create the index upfront. That will be done for you automatically) Now visit your kibana url, load the predefined dashboard from the gist https://gist.github.com/gottwald/b2c875037f24719a9616 and watch the data rush in. Troubleshooting --------------- The replay process may complain about __too many open files__. That's because your typical linux shell has a small open files soft limit at 1024. You can easily raise that when you do this before starting your _gor replay_ process: ``` ulimit -n 64000 ``` Please be aware, this is not a permanent setting. It's just valid for the following jobs you start from that shell. We reached the 1024 limit in our tests with a ubuntu box replaying about 9000 requests per minute. (We had very slow responses there, should be way more with fast responses) ================================================ FILE: LICENSE.txt ================================================ Copyright (c) 2011-present Leonid Bugaev Portions of this software are licensed as follows: * All content residing under the "doc/" directory of this repository is licensed under "Creative Commons: CC BY-SA 4.0 license". * The file "pro.go" and all files ending with the "_pro.go" suffix are released under the commercial license specified in the "COMM-LICENSE" file. * Content outside of the above mentioned directories or restrictions above is available under the "LGPLv3" license as defined below. GoReplay is an Open Source project licensed under the terms of the LGPLv3 license. Please see for license text. As a special exception to the GNU Lesser General Public License version 3 ("LGPL3"), the copyright holders of this Library give you permission to convey to a third party a Combined Work that links statically or dynamically to this Library without providing any Minimal Corresponding Source or Minimal Application Code as set out in 4d or providing the installation information set out in section 4e, provided that you comply with the other provisions of LGPL3 and provided that you meet, for the Application the terms and conditions of the license(s) which apply to the Application. TLDR: You are free to use Gor subpackages like `byteutils` or `proto` in your commercial projects. GoReplay Pro has a commercial-friendly license allowing private forks and modifications of GoReplay. Please see https://goreplay.org/pro.html for more detail. You can find the commercial license terms in COMM-LICENSE. ================================================ FILE: Makefile ================================================ SOURCE = $(shell ls -1 *.go | grep -v _test.go) PROJECT_NAME := goreplay SOURCE_PATH = /go/src/github.com/buger/goreplay/ PORT = 8000 FADDR = :8000 DIST_PATH = dist CONTAINER_AMD=gor-amd64 CONTAINER_ARM=gor-arm64 RUN = docker run --rm -v `pwd`:$(SOURCE_PATH) -e AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) -e AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) -p 0.0.0.0:$(PORT):$(PORT) -t -i $(CONTAINER_AMD) BENCHMARK = BenchmarkRAWInput TEST = TestRawListenerBench BIN_NAME = gor VERSION := DEV-$(shell date +%s) CUSTOM_TAGS := --tags "ngo$(if $(CUSTOM_BUILD_TAGS), $(CUSTOM_BUILD_TAGS),)" LDFLAGS = -ldflags "-X main.VERSION=$(VERSION) -extldflags \"-static\" -X main.DEMO=$(DEMO)" MAC_LDFLAGS = -ldflags "-X main.VERSION=$(VERSION) -X main.DEMO=$(DEMO)" DOCKER_FPM_CMD := docker run --rm -t -v `pwd`:/src -w /src fleetdm/fpm FPM_COMMON= \ --name $(PROJECT_NAME) \ --description "GoReplay is an open-source network monitoring tool which can record your live traffic, and use it for shadowing, load testing, monitoring and detailed analysis." \ -v $(VERSION) \ --vendor "Leonid Bugaev" \ -m "" \ --url "https://goreplay.org" \ -s dir release: clean release-linux-amd64 release-linux-arm64 release-mac-amd64 release-mac-arm64 release-windows .PHONY: vendor vendor: go mod vendor release-bin-linux-amd64: vendor docker run --platform linux/amd64 --rm -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=amd64 -i $(CONTAINER_AMD) go build -mod=vendor -o $(BIN_NAME) $(CUSTOM_TAGS) $(LDFLAGS) ./cmd/gor/ release-bin-linux-arm64: vendor docker run --platform linux/arm64 --rm -v `pwd`:$(SOURCE_PATH) -t --env GOOS=linux --env GOARCH=arm64 -i $(CONTAINER_ARM) go build -mod=vendor -o $(BIN_NAME) $(CUSTOM_TAGS) $(LDFLAGS) ./cmd/gor/ release-bin-mac-amd64: vendor GOOS=darwin go build -mod=vendor -o $(BIN_NAME) $(CUSTOM_TAGS) $(MAC_LDFLAGS) ./cmd/gor/ release-bin-mac-arm64: vendor GOOS=darwin GOARCH=arm64 CGO_ENABLED=1 go build -mod=vendor -o $(BIN_NAME) $(CUSTOM_TAGS) $(MAC_LDFLAGS) release-bin-windows: vendor docker run -it --rm -v `pwd`:$(SOURCE_PATH) -w $(SOURCE_PATH) -e CGO_ENABLED=1 docker.elastic.co/beats-dev/golang-crossbuild:1.19.2-main --build-cmd "make VERSION=$(VERSION) CUSTOM_BUILD_TAGS=$(CUSTOM_BUILD_TAGS) build" -p "windows/amd64" ./cmd/gor/ mv $(BIN_NAME) "$(BIN_NAME).exe" release-linux-amd64: dist release-bin-linux-amd64 tar -czf $(DIST_PATH)/gor_$(VERSION)_linux_amd64.tar.gz $(BIN_NAME) $(DOCKER_FPM_CMD) $(FPM_COMMON) -f -t deb -a arm64 -p ./$(DIST_PATH) ./gor=/usr/local/bin $(DOCKER_FPM_CMD) $(FPM_COMMON) -f -t rpm -a arm64 -p ./$(DIST_PATH) ./gor=/usr/local/bin rm -rf $(BIN_NAME) release-linux-arm64: dist release-bin-linux-arm64 tar -czf $(DIST_PATH)/gor_$(VERSION)_linux_arm64.tar.gz $(BIN_NAME) $(DOCKER_FPM_CMD) $(FPM_COMMON) -f -t deb -a arm64 -p ./$(DIST_PATH) ./gor=/usr/local/bin $(DOCKER_FPM_CMD) $(FPM_COMMON) -f -t rpm -a arm64 -p ./$(DIST_PATH) ./gor=/usr/local/bin rm -rf $(BIN_NAME) release-mac-amd64: dist release-bin-mac-amd64 tar -czf $(DIST_PATH)/gor_$(VERSION)_darwin_amd64.tar.gz $(BIN_NAME) fpm $(FPM_COMMON) -f -t osxpkg -a amd64 -p ./$(DIST_PATH) ./gor=/usr/local/bin mv ./$(DIST_PATH)/$(PROJECT_NAME)-$(VERSION).pkg ./$(DIST_PATH)/$(PROJECT_NAME)-$(VERSION)-amd64.pkg rm -rf $(BIN_NAME) release-mac-arm64: dist release-bin-mac-arm64 tar -czf $(DIST_PATH)/gor_$(VERSION)_darwin_arm64.tar.gz $(BIN_NAME) fpm $(FPM_COMMON) -f -t osxpkg -a arm64 -p ./$(DIST_PATH) ./gor=/usr/local/bin mv ./$(DIST_PATH)/$(PROJECT_NAME)-$(VERSION).pkg ./$(DIST_PATH)/$(PROJECT_NAME)-$(VERSION)-arm64.pkg rm -rf $(BIN_NAME) release-windows: dist release-bin-windows zip $(DIST_PATH)/gor-$(VERSION)_windows.zip "$(BIN_NAME).exe" rm -rf "$(BIN_NAME).exe" clean: rm -rf $(DIST_PATH) build: go build -mod=vendor -o $(BIN_NAME) $(CUSTOM_TAGS) $(LDFLAGS) install: go install $(CUSTOM_TAGS) $(MAC_LDFLAGS) build-env: build-amd64-env build-arm64-env build-amd64-env: docker buildx build --build-arg BASE_IMAGE=golang:1.22-alpine --platform linux/amd64 -t $(CONTAINER_AMD) -f Dockerfile.dev --load . build-arm64-env: docker buildx build --build-arg BASE_IMAGE=arm64v8/golang:1.22-alpine --platform linux/arm64 -t $(CONTAINER_ARM) -f Dockerfile.dev --load . build-docker: docker build -t gor-dev -f Dockerfile . profile: go build && ./$(BIN_NAME) --output-http="http://localhost:9000" --input-dummy 0 --input-raw :9000 --input-http :9000 --memprofile=./mem.out --cpuprofile=./cpu.out --stats --output-http-stats --output-http-timeout 100ms lint: $(RUN) golint $(PKG) race: $(RUN) go test ./... $(ARGS) -v -race -timeout 15s test: $(RUN) go test ./. -timeout 120s $(CUSTOM_TAGS) $(LDFLAGS) $(ARGS) -v test_all: $(RUN) go test ./... -timeout 120s $(CUSTOM_TAGS) $(LDFLAGS) $(ARGS) -v testone: $(RUN) go test ./. -timeout 60s $(CUSTOM_TAGS) $(LDFLAGS) -run $(TEST) $(ARGS) -v cover: $(RUN) go test $(ARGS) -race -v -timeout 15s -coverprofile=coverage.out go tool cover -html=coverage.out fmt: $(RUN) gofmt -w -s ./.. vet: $(RUN) go vet bench: $(RUN) go test $(CUSTOM_TAGS) $(LDFLAGS) -v -run NOT_EXISTING -bench $(BENCHMARK) -benchtime 5s profile_test: $(RUN) go test $(CUSTOM_TAGS) $(LDFLAGS) -run $(TEST) ./capture/. $(ARGS) -memprofile mem.mprof -cpuprofile cpu.out $(RUN) go test $(CUSTOM_TAGS) $(LDFLAGS) -run $(TEST) ./capture/. $(ARGS) -c # Used mainly for debugging, because docker container do not have access to parent machine ports run: $(RUN) go run $(CUSTOM_TAGS) $(LDFLAGS) $(SOURCE) --input-dummy=0 --output-http="http://localhost:9000" --input-raw-track-response --input-raw 127.0.0.1:9000 --verbose 0 --middleware "./examples/middleware/echo.sh" --output-file requests.gor run-2: $(RUN) go run $(CUSTOM_TAGS) $(LDFLAGS) $(SOURCE) --input-raw :8000 --input-raw-bpf-filter "dst port 8000" --output-stdout --output-http "http://localhost:8000" --input-dummy=0 run-3: sudo -E go run $(CUSTOM_TAGS) $(SOURCE) --input-tcp :27001 --output-stdout run-arg: sudo -E go run $(CUSTOM_TAGS) $(SOURCE) $(ARGS) file-server: go run $(CUSTOM_TAGS) $(SOURCE) file-server $(FADDR) readpcap: go run $(CUSTOM_TAGS) $(SOURCE) --input-raw $(FILE) --input-raw-track-response --input-raw-engine pcap_file --output-stdout record: $(RUN) go run $(CUSTOM_TAGS) $(SOURCE) --input-dummy=0 --output-file=requests.gor --verbose --debug replay: $(RUN) go run $(CUSTOM_TAGS) $(SOURCE) --input-file=requests.bin --output-tcp=:9000 --verbose -h bash: $(RUN) /bin/bash dist: mkdir -p $(DIST_PATH) ================================================ FILE: Procfile ================================================ web: python -m SimpleHTTPServer 8000 replayed_web: python -m SimpleHTTPServer 8001 listener: sudo -E go run ./bin/gor.go --input-raw :8000 --output-tcp :8002 --verbose replay: go run ./bin/gor.go --input-tcp :8002 --output-http localhost:8001 --verbose ================================================ FILE: README.md ================================================ [![GitHub release](https://img.shields.io/github/release/buger/gor.svg?maxAge=3600)](https://github.com/buger/goreplay/releases) [![codebeat](https://codebeat.co/badges/6427d589-a78e-416c-a546-d299b4089893)](https://codebeat.co/projects/github-com-buger-gor) [![Go Report Card](https://goreportcard.com/badge/github.com/buger/gor)](https://goreportcard.com/report/github.com/buger/gor) [![Join the chat at https://gitter.im/buger/gor](https://badges.gitter.im/buger/gor.svg)](https://gitter.im/buger/gor?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Reviewed by Hound](https://img.shields.io/badge/Reviewed_by-Hound-8E64B0.svg)](https://houndci.com) ![Go Replay](http://i.imgur.com/ZG2ki5n.png) ## https://goreplay.org/ GoReplay is an open-source network monitoring tool which can record your live traffic and use it for shadowing, load testing, monitoring and detailed analysis. ## About As your application grows, the effort required to test it also grows exponentially. GoReplay offers you the simple idea of reusing your existing traffic for testing, which makes it incredibly powerful. Our state of art technique allows you to analyze and record your application traffic without affecting it. This eliminates the risks that come with putting a third party component in the critical path. GoReplay increases your confidence in code deployments, configuration and infrastructure changes. GoReplay offers a unique approach for shadowing. Instead of being a proxy, GoReplay listens in the background for traffic on your network interfaces, requiring no changes in your production infrastructure, other than running GoReplay daemon on the same machine as your service. ![Diagram](https://i.imgur.com/IN2xfDm.png) Check [latest documentation](http://github.com/buger/goreplay/wiki). ## Installation Download the latest binary from https://github.com/buger/goreplay/releases or [compile by yourself](https://github.com/buger/goreplay/wiki/Compilation). ## Getting started The most basic setup will be `sudo ./gor --input-raw :8000 --output-stdout` which acts like tcpdump. If you already have a test environment, you can start replaying by running: `sudo ./gor --input-raw :8000 --output-http http://staging.env`. See our [documentation](https://github.com/buger/goreplay/wiki/) and the [Getting Started](https://github.com/buger/goreplay/wiki/Getting-Started) page for more info. ## Newsletter Subscribe to our [newsletter](https://www.getdrip.com/forms/89690474/submissions/new) to stay informed about the latest features and changes to the Gor project. ## Want to Upgrade? We have created a [GoReplay PRO](https://goreplay.org/pro.html) extension which provides additional features such as support for binary protocols like Thrift or ProtocolBuffers, saving and replaying from cloud storage, TCP session replication, etc. The PRO version also includes a commercial-friendly license, dedicated support, and it also allows you to support high-quality open source development. ## Problems? If you have a problem, please review the [FAQ](https://github.com/buger/goreplay/wiki/FAQ) and [Troubleshooting](https://github.com/buger/goreplay/wiki/Troubleshooting) wiki pages. Searching the [issues](https://github.com/buger/goreplay/issues) for your problem is also a good idea. All bug-reports and suggestions should go through Github Issues or our [Google Group](https://groups.google.com/forum/#!forum/gor-users) (you can just send email to gor-users@googlegroups.com). If you have a private question feel free to send email to support@gortool.com. ## Contributing 1. Fork it 2. Create your feature branch (git checkout -b my-new-feature) 3. Commit your changes (git commit -am 'Added some feature') 4. Push to the branch (git push origin my-new-feature) 5. Create new Pull Request ## Companies using Gor * [GOV.UK](https://www.gov.uk) - UK Government Digital Service * [theguardian.com](http://theguardian.com) - Most popular online newspaper in the UK * [TomTom](http://www.tomtom.com/) - Global leader in navigation, traffic and map products, GPS Sport Watches and fleet management solutions. * [3SCALE](http://www.3scale.net/) - API infrastructure to manage your APIs for internal or external users * [Optionlab](http://www.opinionlab.com) - Optimize customer experience and drive engagement across multiple channels * [TubeMogul](http://tubemogul.com) - Software for Brand Advertising * [Videology](http://www.videologygroup.com/) - Video advertising platform * [ForeksMobile](http://foreksmobile.com/) - One of the leading financial application development company in Turkey * [Granify](http://granify.com) - AI backed SaaS solution that enables online retailers to maximise their sales * And many more! If you are using Gor, we are happy to add you to the list and share your story, just write to: hello@goreplay.org ## Author Leonid Bugaev, [@buger](https://twitter.com/buger), https://leonsbox.com ================================================ FILE: ce.go ================================================ //go:build !pro package goreplay import ( "fmt" ) // PRO this value indicates if goreplay is running in PRO mode. var PRO = false func SettingsHook(settings *AppSettings) { if settings.RecognizeTCPSessions { settings.RecognizeTCPSessions = false fmt.Println("[ERROR] TCP session recognition is not supported in the open-source version of GoReplay") } } ================================================ FILE: circle.yml ================================================ dependencies: pre: - sudo apt-get install libpcap-dev -y test: override: - sudo bash -l -c "export GOPATH='/home/ubuntu/.go_workspace:/usr/local/go_workspace:/home/ubuntu/.go_project' && GORACE='halt_on_error=1' /usr/local/go/bin/go test ./... -v -timeout 120s -race" ================================================ FILE: docs/CNAME ================================================ docs.goreplay.org ================================================ FILE: docs/Capturing-and-replaying-traffic.md ================================================ Think about Gor more like a network analyzer or tcpdump on steroids, it is not a proxy and does not affect your app anyhow. You specify application port, and it will capture and replay incoming data. Simplest setup will be: ```bash # Run on servers where you want to catch traffic. You can run it on every `web` machine. sudo gor --input-raw :80 --output-http http://staging.com ``` It will record and replay traffic from the same machine. However, it is possible to use [[Aggregator-forwarder setup]], when Gor on your web machines forward traffic to Gor aggregator instance running on the separate server. > You may notice that it require `sudo`: to analyze network Gor need permissions which available only to root users. However, it is possible to configure Gor [beign run for non-root users](Running as a non-root user). ### Forwarding to multiple addresses You can forward traffic to multiple endpoints. ``` gor --input-tcp :28020 --output-http "http://staging.com" --output-http "http://dev.com" ``` ### Splitting traffic By default, it will send same traffic to all outputs, but you have options to equally split it (round-robin) using `--split-output` option. ``` gor --input-raw :80 --output-http "http://staging.com" --output-http "http://dev.com" --split-output true ``` ### Tracking responses By default `input-raw` does not intercept responses, only requests. You can turn response tracking using `--input-raw-track-response` option. When enable you will be able to access response information in middleware and `output-file`. ### Traffic interception engine By default, Gor will use `libpcap` for intercepting traffic, it should work in most cases. If you have any troubles with it, you may try alternative engine: `raw_socket`. ``` sudo gor --input-raw :80 --input-raw-engine "raw_socket" --output-http "http://staging.com" ``` You can read more about [[Replaying HTTP traffic]]. You can use VXLAN or traffic mirroring from AWS to capture the traffic. The 4789 UDP port will be opened and that works as you are launched GoReplay on the source machine. ``` gor --input-raw :80 --input-raw-engine vxlan -output-stdout ``` ### Tracking original IP addresses You can use `--input-raw-realip-header` option to specify header name: If not blank, injects header with given name and real IP value to the request payload. Usually, this header should be named: `X-Real-IP`, but you can specify any name. `gor --input-raw :80 --input-raw-realip-header "X-Real-IP" ...` *** Also you may want to know about [[Rate limiting]], [[Request rewriting]] and [[Request filtering]] ================================================ FILE: docs/Compilation.md ================================================ We provide pre-compiled binaries for Mac and Linux, but you are free to compile Gor by yourself. Gor is written using Go, so first you need to download it from here https://golang.org/, use the latest stable version. The only Gor dependency is [libpcap](https://github.com/the-tcpdump-group/libpcap), which is the interface to various kernel packet capture mechanisms, and https://github.com/google/gopacket, which is a Go wrapper around libpcap. Latest libpcap version can be obtained at http://www.tcpdump.org/release/. Libpcap itself depend on `flex` and `bison` packages, many operating systems already have them installed. ```bash # Fetch libpcap dependencies. Depending on your OS, instead of `apt` you will use `yum` or `rpm`, or `brew` on Mac. sudo apt-get install flex bison -y # Download latest stable release, compile and install it wget http://www.tcpdump.org/release/libpcap-1.7.4.tar.gz && tar xzf libpcap-1.7.4.tar.gz cd libpcap-1.7.4 ./configure && make install # Lets fetch Gor source code mkdir $HOME/gocode # See more information about GOPATH https://github.com/golang/go/wiki/GOPATH export GOPATH=$HOME/gocode # Fetch code from the Github go get github.com/buger/gor # Compile from source cd $HOME/gocode/src/github.com/buger/gor go build LDFLAGS = -ldflags "-extldflags \"-static\"" ``` After you finished, you should see `gor` binary in current directory. ================================================ FILE: docs/Development-Setup.md ================================================ ## STEP 1: Install Docker For local development we recommend to use Docker. If you don’t have it you can read how to install it here: https://docs.docker.com/engine/getstarted/step_one/#step-3-verify-your-installation ## STEP 2: Download repository `git clone git@github.com:buger/goreplay.git` ## STEP 3: Setup container ``` cd ./goreplay make build ``` ## Testing To run tests execute next command: ``` make test ``` You can copy the command that is produced and modify it. For example, if you need to run one test copy the command and add `-run TestName`, e.g.: ``` docker run -v `pwd`:/go/src/github.com/buger/gor/ -p 0.0.0.0:8000:8000 -t -i gor:go go test ./. -run TestEmitterFiltered -timeout 60s -ldflags "-X main.VERSION=DEV-1482398347 -extldflags \"-static\"" -v ``` ## Building To get a binary file run ``` make release-bin ``` ================================================ FILE: docs/Distributed-configuration.md ================================================ Sometimes it makes sense to use separate Gor instance for replaying traffic and performing things like load testing, so your production machines do not spend precious resources. It is possible to configure Gor on your web machines forward traffic to Gor aggregator instance running on the separate server. ```bash # Run on servers where you want to catch traffic. You can run it on each `web` machine. sudo gor --input-raw :80 --output-tcp replay.local:28020 # Replay server (replay.local). gor --input-tcp replay.local:28020 --output-http http://staging.com ``` If you have multiple replay machines you can split traffic among them using `--split-output` option: it will equally split all incoming traffic to all outputs using round robin algorithm. ``` gor --input-raw :80 --split-output --output-tcp replay1.local:28020 --output-tcp replay2.local:28020 ``` [GoReplay PRO](https://goreplay.org/pro.html) support accurate recording and replaying of tcp sessions, and when `--recognize-tcp-sessions` option is passed, instead of round-robin it will use a smarter algorithm which ensures that same sessions will be sent to the same replay instance. In case if you are planning a large load testing, you may consider use separate master instance which will control Gor slaves which actually replay traffic. For example: ``` # This command will read multiple log files, replay them on 10x speed and loop them if needed for 30 seconds, and will distributed traffic (tcp session aware) among multiple workers gor --input-file logs_from_multiple_machines.*|1000% --input-file-loop --exit-after 30s --recognize-tcp-sessions --split-output --output-tcp worker1.local --output-tcp worker2.local:27017 --output-tcp worker3.local:27017 ... --output-tcp workerN.local:27017 # worker gor --input-tcp :27017 --ouput-http load_test.target ``` ================================================ FILE: docs/Exporting-to-ElasticSearch.md ================================================ Gor can export requests and replayed response data to ElasticSearch: ``` ./gor --input-raw :8000 --output-http http://staging.com --output-http-elasticsearch localhost:9200/gor ``` You don't have to create the index upfront. That will be done for you automatically. ### Format Following structure represents ES format: ``` type ESRequestResponse struct { ReqURL string `json:"Req_URL"` ReqMethod string `json:"Req_Method"` ReqUserAgent string `json:"Req_User-Agent"` ReqAcceptLanguage string `json:"Req_Accept-Language,omitempty"` ReqAccept string `json:"Req_Accept,omitempty"` ReqAcceptEncoding string `json:"Req_Accept-Encoding,omitempty"` ReqIfModifiedSince string `json:"Req_If-Modified-Since,omitempty"` ReqConnection string `json:"Req_Connection,omitempty"` ReqCookies string `json:"Req_Cookies,omitempty"` RespStatus string `json:"Resp_Status"` RespStatusCode string `json:"Resp_Status-Code"` RespProto string `json:"Resp_Proto,omitempty"` RespContentLength string `json:"Resp_Content-Length,omitempty"` RespContentType string `json:"Resp_Content-Type,omitempty"` RespTransferEncoding string `json:"Resp_Transfer-Encoding,omitempty"` RespContentEncoding string `json:"Resp_Content-Encoding,omitempty"` RespExpires string `json:"Resp_Expires,omitempty"` RespCacheControl string `json:"Resp_Cache-Control,omitempty"` RespVary string `json:"Resp_Vary,omitempty"` RespSetCookie string `json:"Resp_Set-Cookie,omitempty"` Rtt int64 `json:"RTT"` Timestamp time.Time } ``` ================================================ FILE: docs/FAQ.md ================================================ ### What OS are supported? Gor will run everywhere where [libpcap](http://www.tcpdump.org/) works, and it works on most of the platforms. However, currently, we test it on Linux and Mac. See more about [[Compilation]]. ### Why does the `--input-raw` requires sudo or root access? Listener works by sniffing traffic from a given port. It's accessible only by using sudo or root access. But it is possible to [[Running as non root user]]. ### How do you deal with user session to replay the traffic correctly? You can rewrite session related headers/params to match your staging environment. If you require custom logic (e.g random token based auth) follow this discussion: https://github.com/buger/gor/issues/154 ### Can I use Gor to intercept SSL traffic? Basic idea is that SSL was made to protect itself from traffic interception. There 2 options: 1. Move SSL handling to proxy like Nginx or Amazon ELB. And allow Gor to listen on upstreams. 2. Use `--input-http` so you can duplicate request payload directly from your app to Gor, but it will require your app modifications. More can be find here: https://github.com/buger/gor/issues/85 ### Is there a limit for size of HTTP request when using output-http? Due to the fact that Gor can't guarantee interception of all packets, for large payloads > 200kb there is chance of missing some packets and corrupting body. Treat it as a feature and chance to test broken bodies handling :) The only way to guarantee delivery is using `--input-http`, but you will miss some features. ### I'm getting 'too many open files' error Typical Linux shell has a small open files soft limit at 1024. You can easily raise that when you do this before starting your gor replay process: ulimit -n 64000 More about ulimit: http://www.thecodingmachine.com/solving-the-too-many-open-files-exception-in-red5-or-any-other-application/ ### The CPU average across my load-balanced targets is higher than the source If you are replaying traffic from multiple listeners to a load-balanced target and you use sticky sessions, you may observe that the target servers have a higher CPU load than the listener servers. This may be because the sticky session cookie of the original load balancer is not honored by the target load balancer thus resulting in requests that would normally hit the same target server hitting different servers on the backend thus reducing some caching benefits gained via the load balancing. Try running just one listener against one replay target and see if the CPU utilization comparison is more accurate. Also see [[Troubleshooting]]. ================================================ FILE: docs/Middleware.md ================================================ #### Overview Middleware is a program that accepts request and response payload at STDIN and emits modified requests at STDOUT. You can implement any custom logic like stripping private data, advanced rewriting, support for oAuth and etc. Check examples [included into our repo](https://github.com/buger/gor/tree/master/examples/middleware). ``` Original request +--------------+ +-------------+----------STDIN---------->+ | | Gor input | | Middleware | +-------------+----------STDIN---------->+ | Original response (1) +------+---+---+ | ^ +-------------+ Modified request v | | Gor output +<---------STDOUT-----------------+ | +-----+-------+ | | | | Replayed response | +------------------STDIN----------------->----+ ``` (1): Original responses will only be sent to the middleware if the `--input-raw-track-response` option is specified. Middleware can be written in any language, see `examples/middleware` folder for examples. Middleware program should accept the fact that all communication with Gor is asynchronous, there is no guarantee that original request and response messages will come one after each other. Your app should take care of the state if logic depends on original or replayed response, see `examples/middleware/token_modifier.go` as example. Simple bash echo middleware (returns same request) will look like this: ```bash while read line; do echo $line done ``` Middleware can be enabled using `--middleware` option, by specifying path to executable file: ``` gor --input-raw :80 --middleware "/opt/middleware_executable" --output-http "http://staging.server" ``` #### Communication protocol All messages should be hex encoded, new line character specifieds the end of the message, eg. new message per line. Decoded payload consist of 2 parts: header and HTTP payload, separated by new line character. Example request payload: ``` 1 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 GET /a HTTP/1.1 Host: 127.0.0.1 ``` Example response payload (note: you will only receive this if you specify `--input-raw-track-response`) ``` 2 8e091765ae902fef8a2b7d9dd960e9d52222bd8c 1439818823587996305 2782013 HTTP/1.1 200 OK Date: Mon, 17 Aug 2015 13:40:23 GMT Content-Length: 0 Content-Type: text/plain; charset=utf-8 ``` Header contains request meta information separated by spaces. First value is payload type, possible values: `1` - request, `2` - original response, `3` - replayed response. Next goes request id: unique among all requests (sha1 of time and Ack), but remain same for original and replayed response, so you can create associations between request and responses. The third argument is the time when request/response was initiated/received. Forth argument is populated only for responses and means latency. HTTP payload is unmodified HTTP requests/responses intercepted from network. You can read more about request format [here](http://www.jmarshall.com/easy/http/), [here](https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol) and [here](http://www.w3.org/Protocols/rfc2616/rfc2616.html). You can operate with payload as you want, add headers, change path, and etc. Basically you just editing a string, just ensure that it is RCF compliant. At the end modified (or untouched) request should be emitted back to STDOUT, keeping original header, and hex-encoded. If you want to filter request, just not send it. Emitting responses back is required, even if you did not touch them. #### Advanced example Imagine that you have auth system that randomly generate access tokens, which used later for accessing secure content. Since there is no pre-defined token value, naive approach without middleware (or if middleware use only request payloads) will fail, because replayed server have own tokens, not synced with origin. To fix this, our middleware should take in account responses of replayed and origin server, store `originalToken -> replayedToken` aliases and rewrite all requests using this token to use replayed alias. See [examples/middleware/token_modifier.go](https://github.com/buger/gor/tree/master/examples/middleware/token_modifier.go) and [middleware_test.go#TestTokenMiddleware](https://github.com/buger/gor/tree/master/middleware_test.go) as example of described scheme. *** You may also read about [[Request filtering]], [[Rate limiting]] and [[Request rewriting]]. ================================================ FILE: docs/Rate-limiting.md ================================================ Rate limiting can be useful if you only want to forward parts of incoming traffic, for example, to not overload your test environment. There are two strategies: dropping random requests or dropping fractions of requests based on Header or URL param value. ### Dropping random requests Every input and output support random rate limiting. There are two limiting algorithms: absolute or percentage based. **Absolute**: If for current second it reached specified requests limit - disregard the rest, on next second counter reset. **Percentage**: For input-file it will slowdown or speedup request execution, for the rest it will use the random generator to decide if request pass or not based on the chance you specified. You can specify your desired limit using the "|" operator after the server address, see examples below. #### Limiting replay using absolute number ``` # staging.server will not get more than ten requests per second gor --input-tcp :28020 --output-http "http://staging.com|10" ``` #### Limiting listener using percentage based limiter ``` # replay server will not get more than 10% of requests # useful for high-load environments gor --input-raw :80 --output-tcp "replay.local:28020|10%" ``` ### Consistent limiting based on Header or URL param value If you have unique user id (like API key) stored in header or URL you can consistently forward specified percent of traffic only for the fraction of this users. Basic formula looks like this: `FNV32-1A_hashing(value) % 100 >= chance`. Examples: ``` # Limit based on header value gor --input-raw :80 --output-tcp "replay.local:28020|10%" --http-header-limiter "X-API-KEY: 10%" # Limit based on header value gor --input-raw :80 --output-tcp "replay.local:28020|10%" --http-param-limiter "api_key: 10%" ``` When limiting based on header or param only percentage based limiting supported. ================================================ FILE: docs/Replaying-HTTP-traffic.md ================================================ Gor can replay HTTP traffic using `--output-http` option: ```bash sudo ./gor --input-raw :8000 --output-http="http://staging.env" ``` You can [filter](Request filtering), [rate limit](Rate limiting) and [rewrite](Request rewriting) requests on the fly. ### HTTP output workers By default Gor creates a dynamic pool of workers: it starts with 10 and creates more HTTP output workers when the HTTP output queue length is greater than 10. The number of workers created (N) is equal to the queue length at the time which it is checked and found to have a length greater than 10. The queue length is checked every time a message is written to the HTTP output queue. No more workers will be spawned until that request to spawn N workers is satisfied. If a dynamic worker cannot process a message at that time, it will sleep for 100 milliseconds. If a dynamic worker cannot process a message for 2 seconds it dies. You may specify fixed number of workers using `--output-http-workers=20` option. ### Following redirects By default Gor will ignore all redirects since they are handled by clients using your app, but in scenarios where your replayed environment introduces new redirects, you can enable them like this: ``` gor --input-tcp replay.local:28020 --output-http http://staging.com --output-http-redirects 2 ``` The given example will follow up to 2 redirects per request. ### HTTP timeouts By default http timeout for both request and response is 5 seconds. You can override it like this: ``` gor --input-tcp replay.local:28020 --output-http http://staging.com --output-http-timeout 30s ``` ### Response buffer By default, to reduce memory consumption, internal HTTP client will fetch max 200kb of the response body (used if you use middleware), by you can increase limit using `--output-http-response-buffer` option (accepts number of bytes). ### Basic Auth If your development or staging environment is protected by Basic Authentication then those credentials can be injected in during the replay: ``` gor --input-raw :80 --output-http "http://user:pass@staging.com" ``` Note: This will overwrite any Authorization headers in the original request. ### Multiple domains support If you app accepts traffic from multiple domains, and you want to keep original headers, there is specific `--http-original-host` with tells Gor do not touch Host header at all. *** You may also read about [[Saving and Replaying from file]] ================================================ FILE: docs/Request-filtering.md ================================================ Filtering is useful when you need to capture only specific part of traffic, like API requests. It is possible to filter by URL, HTTP header or HTTP method. #### Allow url regexp ``` # only forward requests being sent to the /api endpoint gor --input-raw :8080 --output-http staging.com --http-allow-url /api ``` #### Disallow url regexp ``` # only forward requests NOT being sent to the /api... endpoint gor --input-raw :8080 --output-http staging.com --http-disallow-url /api ``` #### Filter based on regexp of header ``` # only forward requests with an api version of 1.0x gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^1\.0\d # only forward requests NOT containing User-Agent header value "Replayed by Gor" gor --input-raw :8080 --output-http staging.com --http-disallow-header "User-Agent: Replayed by Gor" ``` #### Filter based on HTTP method Requests not matching a specified whitelist can be filtered out. For example to strip non-nullipotent requests: ``` gor --input-raw :80 --output-http "http://staging.server" \ --http-allow-method GET \ --http-allow-method OPTIONS ``` ----- You may also read about [[Request rewriting]], [[Rate limiting]] and [[Middleware]] ================================================ FILE: docs/Request-rewriting.md ================================================ Gor supports rewriting of URLs, URL params and headers, see below. Rewriting may be useful if you test environment does not have the same data as your production, and you want to perform all actions in the context of `test` user: for example rewrite all API tokens to some test value. Other possible use cases are toggling features on/off using custom headers or rewriting URL's if they changed in the new environment. For more complex logic you can use [Middleware](middleware.md). #### Rewrite URL based on a mapping `--http-rewrite-url` expects value in ":" format: ":" is a dilimiter. In `` section you may use captured regexp group values. This works similar to `replace` method in Javascript or `gsub` in Ruby. ``` # Rewrites all `/v1/user//ping` requests to `/v2/user//ping` gor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping ``` #### Set URL param Set request url param, if param already exists it will be overwritten. ``` gor --input-raw :8080 --output-http staging.com --http-set-param api_key=1 ``` #### Set Header Set request header, if header already exists it will be overwritten. May be useful if you need to identify requests generated by Gor or enable feature flagged functionality in an application: ``` gor --input-raw :80 --output-http "http://staging.server" \ --http-set-header "User-Agent: Replayed by Gor" \ --http-set-header "Enable-Feature-X: true" ``` #### Host header Host header gets special treatment. By default Host get set to the value specified in --output-http. If you manually set --http-set-header "Host: anonther.com", Gor will not override Host value. If you app accepts traffic from multiple domains, and you want to keep original headers, there is specific `--http-original-host` with tells Gor do not touch Host header at all. *** You may also read about [[Request filtering]], [[Rate limiting]] and [[Middleware]] ================================================ FILE: docs/Running-as-non-root-user.md ================================================ You can enable Gor for non-root users in a secure method by using the following commands ``` # Following commands assume that you put `gor` binary to /usr/local/bin add gor addgroup gor chgrp gor /usr/local/bin/gor chmod 0750 /usr/local/bin/gor setcap "cap_net_raw,cap_net_admin+eip" /usr/local/bin/gor ``` As a brief explanation of the above. * We create a group called gor. * We then add the user you want to the new group so they will be able to use gor without sudo * We then change the user/group of gor binary the new group. * We then make sure the permissions are set on gor binary so that members of the group can execute it but other normal users cannot. * We then use `setcap` to give the CAP_NET_RAW and CAP_NET_ADMIN privilege to the executable when it runs. This is so that Gor can open its raw socket which is not normally permitted unless you are root. ================================================ FILE: docs/Saving-and-Replaying-from-file.md ================================================ You can save requests to file, and replay them later. While replaying it will preserve the original time differences between requests. If you apply [percentage based limiting](Rate Limiting) timing between requests will be reduced or increased appropriately: this approach opens possibilities like load testing, see below. ```bash # write to file gor --input-raw :80 --output-file requests.log # read from file gor --input-file requests.gor --output-http "http://staging.com" ``` By default Gor writes files in chunks. This configurable using `--output-file-append` option: the flushed chunk is appended to existence file or not. The default is **false**. By default, `--output-file` flushes each chunk to a different path. ```bash gor ... --output-file %Y%m%d.log # append false 20140608_0.log 20140608_1.log 20140609_0.log 20140609_1.log ``` This makes parallel file processing easy. But if you want to disable this behavior, you can disable it by adding `--output-file-append` option: ```bash gor ... --output-file %Y%m%d.log --output-file-append # append true 20140608.log 20140609.log ``` If you run gor multiple times, and it finds existing files, it will continue from last known index. ### Chunk size You can set chunk limits using `--output-file-size-limit` and `--output-file-queue-limit` options. The length of the chunk queue and the size of each chunk, respectively. The default values are 256 and 32mb, respectively. The suffixes “k” (KB), “m” (MB), and “g” (GB) can be used for `output-file-size-limit`. If you want to have only size constraint, you can set `--output-file-queue-limit` to 0, and vice versa. ```bash gor --input-raw :80 --output-file %Y-%m-%d.gz --output-file-size-limit 256m --output-file-queue-limit 0 ``` ### Using date variables in file names For example, you can tell to create new file each hour: `--output-file /mnt/logs/requests-%Y-%m-%d-%H.log` It will create new file for each hour: requests-2016-06-01-12.log, requests-2016-06-01-13.log, ... The time format used as part of the file name. The following characters are replaced with actual values when the file is created: * `%Y`: year including the century (at least 4 digits) * `%m`: month of the year (01..12) * `%d`: Day of the month (01..31) * `%H`: Hour of the day, 24-hour clock (00..23) * `%M`: Minute of the hour (00..59) * `%S`: Second of the minute (00..60) The default format is `%Y%m%d%H`, which creates one file per hour. ### GZIP compression To read or write GZIP compressed files ensure that file extension ends with ".gz": `--output-file log.gz` ### Replaying from multiple files `--input-file` accepts file pattern, for example: `--input-file logs-2016-05-*`: it will replay all the files, sorting them in lexicographical order. ### Buffered file output Gor has memory buffer when it writes to file, and continuously flush changes to the file. Flushing to file happens if the buffer is filled, forced flush every 1 second, or if Gor is closed. You can change it using `--output-file-flush-interval` option. It most cases it should not be touched. ### File format HTTP requests stored as it is, plain text: headers and bodies. Requests separated by `\n🐵🙈🙉\n` line (using such sequence for uniqueness and fun). Before each request goes single line with meta information containing payload type (1 - request, 2 - response, 3 - replayed response), unique request ID (request and response have the same) and timestamp when request was made. An example of 2 requests: ``` 1 d7123dasd913jfd21312dasdhas31 127345969\n GET / HTTP/1.1\r\n \r\n \n 🐵🙈🙉 \n POST /upload HTTP/1.1\r\n Content-Length: 7\r\n Host: www.w3.org\r\n \r\n a=1&b=2 ``` Note that technically \r and \n symbols are invisible, and indicate new lines. I made them visible in example just to show how it looks on byte level. Making it text friendly allows writing simple parsers and use console tools like `grep` to do an analysis. You can even edit them manually, but be sure that your file editor does not change line endings. ## Performance testing Currently, this functionality supported only by `input-file` and only when using percentage based limiter. Unlike default limiter for `input-file` instead of dropping requests it will slowdown or speedup request emitting. Note that **limiter is applied to input**: ``` # Replay from file on 2x speed gor --input-file "requests.gor|200%" --output-http "staging.com" ``` Use `--stats --output-http-stats` to see latency stats. ### Looping files for replaying indefinitely You can loop the same set of files, so when the last one replays all the requests, it will not stop, and will start from first one again. Having the only small amount of requests you can do extensive performance testing. Pass `--input-file-loop` to make it work. *** You may also read about [[Capturing and replaying traffic]] and [[Rate limiting]] ================================================ FILE: docs/Troubleshooting.md ================================================ Gor can report stats on the `output-tcp` and `output-http` request queues. Stats are reported to the console every 5 seconds in the form `latest,mean,max,count,count/second` by using the `--output-http-stats` and `--output-tcp-stats` options. Examples: ``` 2014/04/23 21:17:50 output_tcp:latest,mean,max,count,count/second 2014/04/23 21:17:50 output_tcp:0,0,0,0,0 2014/04/23 21:17:55 output_tcp:1,1,2,68,13 2014/04/23 21:18:00 output_tcp:1,1,2,92,18 2014/04/23 21:18:05 output_tcp:1,1,2,119,23 ``` ``` 2014/04/23 21:19:46 output_http:latest,mean,max,count,count/second 2014/04/23 21:19:46 output_http:0,0,0,0,0 2014/04/23 21:19:51 output_http:0,0,0,0,0 2014/04/23 21:19:56 output_http:0,0,0,0,0 2014/04/23 21:20:01 output_http:1,0,1,50,10 2014/04/23 21:20:06 output_http:1,1,4,72,14 2014/04/23 21:20:11 output_http:1,0,1,179,35 2014/04/23 21:20:16 output_http:1,0,1,148,29 2014/04/23 21:20:21 output_http:1,1,2,91,18 2014/04/23 21:20:26 output_http:1,1,2,150,30 2014/04/23 21:18:15 output_http:100,99,100,70,14 2014/04/23 21:18:21 output_http:100,99,100,55,11 ``` ### How can I tell if I have bottlenecks? Key areas that sometimes experience bottlenecks are the output-tcp and output-http functions which have internal queues for requests. Each queue has an upper limit of 100. Enable stats reporting to see if any queues are experiencing bottleneck behavior. #### Output HTTP bottlenecks When running a Gor replay the output-http feature may bottleneck if: * the replay has inadequate bandwidth. If the replay is receiving or sending more messages than its network adapter can handle the output-http-stats may report that the output-http queue is filling up. See if there is a way to upgrade the replay's bandwidth. * with `--output-http-workers` set to anything other than `-1` the `-output-http` target is unable to respond to messages in a timely manner. The http output workers which take messages off the output-http queue, process the request, and ensure that the request did not result in an error may not be able to keep up with the number of incoming requests. If the replay is not using dynamic worker scaling (`--output-http-workers=-1`) The optimal number of output-http-workers can be determined with the formula `output-workers = (Average number of requests per second)/(Average target response time per second)`. #### Output TCP bottlenecks When using the Gor listener the output-tcp feature may bottleneck if: * the replay is unable to accept and process more requests than the listener is able generate. Prior to troubleshooting the output-tcp bottleneck, ensure that the replay target is not experiencing any bottlenecks. * the replay target has inadequate bandwidth to handle all its incoming requests. If a replay target's incoming bandwidth is maxed out the output-tcp-stats may report that the output-tcp queue is filling up. See if there is a way to upgrade the replay's bandwidth. #### Tuning To achieve the top most performance you should tune the source server system limits: net.ipv4.tcp_max_tw_buckets = 65536 net.ipv4.tcp_tw_recycle = 1 net.ipv4.tcp_tw_reuse = 0 net.ipv4.tcp_max_syn_backlog = 131072 net.ipv4.tcp_syn_retries = 3 net.ipv4.tcp_synack_retries = 3 net.ipv4.tcp_retries1 = 3 net.ipv4.tcp_retries2 = 8 net.ipv4.tcp_rmem = 16384 174760 349520 net.ipv4.tcp_wmem = 16384 131072 262144 net.ipv4.tcp_mem = 262144 524288 1048576 net.ipv4.tcp_max_orphans = 65536 net.ipv4.tcp_fin_timeout = 10 net.ipv4.tcp_low_latency = 1 net.ipv4.tcp_syncookies = 0 *** ### Gor is crashing with following stacktrace ``` fatal error: unexpected signal during runtime execution [signal 0xb code=0x1 addr=0x63 pc=0x7ffcdfdf8b2c] runtime stack: runtime.throw(0xad8380, 0x2a) /usr/local/go/src/runtime/panic.go:547 +0x90 runtime.sigpanic() /usr/local/go/src/runtime/sigpanic_unix.go:12 +0x5a goroutine 103 [syscall, locked to thread]: runtime.cgocall(0x7b35a0, 0xc82121f1e8, 0x0) /usr/local/go/src/runtime/cgocall.go:123 +0x11b fp=0xc82121f188 sp=0xc82121f158 net._C2func_getaddrinfo(0x7ffcec0008c0, 0x0, 0xc821b221e0, 0xc8217b2b18, 0x0, 0x0, 0x0) ??:0 +0x55 fp=0xc82121f1e8 sp=0xc82121f188 net.cgoLookupIPCNAME(0x7fffb17208ab, 0x12, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb17200) ``` There is a chance that you hit Go bug. The crash comes from the CGO version of DNS resolver. By default Go based version used, but ins some cases [it switches to CGO based](https://golang.org/pkg/net/#hdr-Name_Resolution). It is possible to force Go based DNS resolver using GODEBUG environment variable: `sudo GODEBUG="netdns=go" ./gor --input-raw :80 --output-http staging.env` Also, see [[FAQ]] ================================================ FILE: docs/_Footer.md ================================================ [Website](https://goreplay.org) | [PRO version](https://goreplay.org/pro.html) | [[Getting started]] | [[FAQ]] | [Join newsletter](https://www.getdrip.com/forms/89690474/submissions/new) ================================================ FILE: docs/_config.yml ================================================ theme: jekyll-theme-cayman ================================================ FILE: docs/commercial/collaboration.md ================================================ Collaboration is difficult with commercial closed source but I do want to keep as much of the OSS ethos as possible available to customers who want to fix it themselves. ## Legal In order to unambiguously own and sell Gor commercial products, I must have the copyright associated with the entire codebase. Any code you create which is merged must be owned by me. That's not me trying to be a jerk, that's just the way it works. ## Application If you wish access to the product repository so you can send a PR, just open a new Gor issue and include the following info: 1. the email address that bought the license, a max of one collaborator per license 1. the following statement "I assign all rights, including copyright, to any future Gor work by myself to Leonid Bugaev" You should be granted access to the private repo soon after. ## Notes 1. You should **never** work on the master branch. Only I may merge changes. 1. I may revoke access for any reason at any time. Access is not guaranteed with purchase. ================================================ FILE: docs/commercial/faq.md ================================================ ### What are GoReplay PRO and GoReplay Enterprise? [GoReplay PRO](https://goreplay.org/pro.html) and GoReplay Enterprise are extensions to GoReplay which add more functionality and provide additional support options for customers. ### Is there a trial version? There's no free trial but we do offer a 14 day period with full refund if it does not work for you. ### What is the license? See [COMM-LICENSE](https://github.com/buger/gor/blob/master/COMM-LICENSE) in the root of the GoReplay repo. ### How does PRO licensing work? Every organization running GoReplay Pro on its own servers must have a license. There's no limit to the amount of servers or environments used by that organization. ### How does Enterprise licensing work? Every organization running Gor Enterprise on its own servers must have a license. There's **no limit** to the amount of servers or environments used by that organization. ### What happens if my subscription lapses? You must have an active subscription to run GoReplay Pro or Enterprise. After a one week grace period, you'll lose access to binaries and priority support. You won't get any more updates or bug fixes. ### How do I buy GoReplay Enterprise? Send email to [support@gortool.com](mailto:support@gortool.com) with your info. A PDF quote will be emailed to you with the price. Reply to that email with your purchase order or just "Sounds good" and we will send an invoice which can be paid with a credit card, ACH bank transfer or a paper check. ### Can I upgrade from GoReplay Pro? Yes! Current subscribers can upgrade by [requesting a quote](mailto:support@gortool.com). **Please note that you are an existing Pro subscriber.** We will add a one-time discount on your first invoice to reflect any remaining Pro subscription credit. If you purchased GoReplay Pro 6 months ago for $950, you'll get a $475 discount. ### Can I distribute GoReplay PRO or Enterprise to my customers? This is a common requirement for "on-site installs" or "appliances" sold to large corporations. The standard license is appropriate for SaaS usage as it does not allow distribution. GoReplay PRO and Enterprise have an Appliance license option which **does** allow you to distribute them. The Appliance license is $9,500/yr for Pro and $19,500/yr for Enterprise. It allows you to distribute the Pro or Enterprise binaries as part of your application and each of your customers to run GoReplay Pro or Enterprise. Email [support@gortool.com](mailto:support@gortool.com) to purchase. ### Can you transfer a license? Licenses are **not** transferable to another company. We will transfer the license from a user-specific email to a group email address (e.g. john_smith@example.com -> tech@example.com) but only for **the same domain**. It is strongly recommended that you buy the license using a group email address so the license is not attached to any one employee's email address. ### What does the license require me to do? Your purchase gets you a unique access URL for downloading the Pro and/or Enterprise binaries. The license agreement requires you to keep this access URL private. If we find your access URL is ever publicized: 1. We'll send you a warning email with details. You need to remove the content and send a new email address so we can generate a new access URL for you. The old access URL will stop working immediately so you'll need to update your apps. 2. If your access URL is publicized a second time, we reserve the right to permanently remove access. ### Can I get a refund? Yes, up to two weeks after purchase. Let us know the reason and maybe we can help but either way it's not a problem. Email [support@gortool.com](mailto:support@gortool.com). ### How do I update my credit card info? If you purchased GoReplay Enterprise, there's nothing to do. Each annual invoice is paid separately. If you purchased GoReplay PRO, log into [Gumroad](https://gumroad.com) with your email address, click the Billing tab and enter your new card. I can't provide support for the Gumroad website and don't have the ability to edit customer info - if you can't log in or change your credit card, you can always let your current subscription expire and purchase a new subscription. ================================================ FILE: docs/commercial/support.md ================================================ Gor offers only community support. Gor Pro and Enterprise offer priority support via email. ## Priority Support Covers 1 incident per quarter, with a max response time of 2 working days. Scope is limited to Gor and Gor Pro and Enterprise features and APIs, not the application or infrastructure. For support, email **support** AT **gortool.com**. Please email using the same domain as the original license email or explain your connection to the licensed company. More aggressive support contracts (phone, quicker response time) are available separately, email with your needs. ## Onboarding Enterprise customers may request a one hour video chat session with @buger to discuss their application(s), requirements and how best to leverage the various Gor features. Contact support to set up your session. ================================================ FILE: docs/css/breadcrumbs.css ================================================ .wy-breadcrumbs li { display: inline-block; } .wy-breadcrumbs li.wy-breadcrumbs-aside { float: right; } .wy-breadcrumbs li a { display: inline-block; padding: 5px; } .wy-breadcrumbs li a:first-child { padding-left: 0; } .wy-breadcrumbs-extra { margin-bottom: 0; color: #b3b3b3; font-size: 80%; display: inline-block; } @media screen and (max-width: 480px) { .wy-breadcrumbs-extra { display: none; } .wy-breadcrumbs li.wy-breadcrumbs-aside { display: none; } } @media print { .wy-breadcrumbs li.wy-breadcrumbs-aside { display: none; } } ================================================ FILE: docs/css/code.css ================================================ .codeblock-example { border: 1px solid #e1e4e5; border-bottom: none; padding: 24px; padding-top: 48px; font-weight: 500; background: #fff; position: relative; } .codeblock-example:after { content: "Example"; position: absolute; top: 0; left: 0; background: #9B59B6; color: #fff; padding: 6px 12px; } .codeblock-example.prettyprint-example-only { border: 1px solid #e1e4e5; margin-bottom: 24px; } .codeblock, pre.literal-block, .rst-content .literal-block, .rst-content pre.literal-block, div[class^='highlight'] { border: 1px solid #e1e4e5; padding: 0; overflow-x: auto; background: #fff; margin: 1px 0 24px; } .codeblock div[class^='highlight'], pre.literal-block div[class^='highlight'], .rst-content .literal-block div[class^='highlight'], div[class^='highlight'] div[class^='highlight'] { border: none; background: none; margin: 0; } div[class^='highlight'] td.code { width: 100%; } .linenodiv pre { border-right: solid 1px #e6e9ea; margin: 0; padding: 12px; font-family: "Source Code Pro","Andale Mono WT","Andale Mono","Lucida Console","Lucida Sans Typewriter","DejaVu Sans Mono","Bitstream Vera Sans Mono","Liberation Mono","Nimbus Mono L",Monaco,"Courier New",Courier,monospace; font-size: 12px; line-height: 1.5; color: #d9d9d9; } div[class^='highlight'] pre { white-space: pre; margin: 0; padding: 12px; font-family: "Source Code Pro","Andale Mono WT","Andale Mono","Lucida Console","Lucida Sans Typewriter","DejaVu Sans Mono","Bitstream Vera Sans Mono","Liberation Mono","Nimbus Mono L",Monaco,"Courier New",Courier,monospace; font-size: 12px; line-height: 1.5; display: block; overflow: auto; color: #404040; } @media print { .codeblock, pre.literal-block, .rst-content .literal-block, .rst-content pre.literal-block, div[class^='highlight'], div[class^='highlight'] pre { white-space: pre-wrap; } } .hll { background-color: #ffc; margin: 0 -12px; padding: 0 12px; display: block; } .c { color: #998; font-style: italic; } .err { color: #a61717; background-color: #e3d2d2; } .k { font-weight: 700; } .o { font-weight: 700; } .cm { color: #998; font-style: italic; } .cp { color: #999; font-weight: 700; } .c1 { color: #998; font-style: italic; } .cs { color: #999; font-weight: 700; font-style: italic; } .gd { color: #000; background-color: #fdd; } .gd .x { color: #000; background-color: #faa; } .ge { font-style: italic; } .gr { color: #a00 } .gh { color: #999; } .gi { color: #000; background-color: #dfd; } .gi .x { color: #000; background-color: #afa; } .go { color: #888; } .gp { color: #555; } .gs { font-weight: 700; } .gu { color: purple; font-weight: 700; } .gt { color: #a00; } .kc { font-weight: 700; } .kd { font-weight: 700; } .kn { font-weight: 700; } .kp { font-weight: 700; } .kr { font-weight: 700; } .kt { color: #458; font-weight: 700; } .m { color: #099; } .s { color: #d14; } .n { color: #333; } .na { color: teal; } .nb { color: #0086b3; } .nc { color: #458; font-weight: 700; } .no { color: teal; } .ni { color: purple; } .ne { color: #900; font-weight: 700; } .nf { color: #900; font-weight: 700; } .nn { color: #555; } .nt { color: navy; } .nv { color: teal; } .ow { font-weight: 700; } .w { color: #bbb; } .mf { color: #099; } .mh { color: #099; } .mi { color: #099; } .mo { color: #099; } .sb { color: #d14; } .sc { color: #d14; } .sd { color: #d14; } .s2 { color: #d14 } .se { color: #d14; } .sh { color: #d14; } .si { color: #d14; } .sx { color: #d14; } .sr { color: #009926; } .s1 { color: #d14; } .ss { color: #990073; } .bp { color: #999; } .vc { color: teal; } .vg { color: teal; } .vi { color: teal; } .il { color: #099; } .gc { color: #999; background-color: #EAF2F5; } ================================================ FILE: docs/css/fabric.css ================================================ /* Taken from https://docs.fabric.io/apple/fabric/overview.html */ * { -webkit-box-sizing: border-box; -moz-box-sizing: border-box; box-sizing: border-box } article, aside, details, figcaption, figure, footer, header, hgroup, nav, section { display: block } audio, canvas, video { display: inline-block; *display: inline; *zoom: 1 } audio:not([controls]) { display: none } [hidden] { display: none } * { -webkit-box-sizing: border-box; -moz-box-sizing: border-box; box-sizing: border-box } html { font-size: 100%; -webkit-text-size-adjust: 100%; -ms-text-size-adjust: 100% } body { margin: 0 } a:hover, a:active { outline: 0 } abbr[title] { border-bottom: 1px dotted } b, strong { font-weight: 700 } blockquote { margin: 0 } dfn { font-style: italic } ins { background: #ff9; color: #000; text-decoration: none } mark { background: #ff0; color: #000; font-style: italic; font-weight: 700 } pre, code, .rst-content tt, .rst-content code, kbd, samp { font-family: monospace,serif; _font-family: "courier new",monospace; font-size: 1em } pre { white-space: pre } q { quotes: none } q:before, q:after { content: ""; content: none } small { font-size: 85% } sub, sup { font-size: 75%; line-height: 0; position: relative; vertical-align: baseline } sup { top: -.5em } sub { bottom: -.25em } ul, ol, dl { margin: 0; padding: 0; list-style: none; list-style-image: none } li { list-style: none } dd { margin: 0 } img { border: 0; -ms-interpolation-mode: bicubic; vertical-align: middle; max-width: 100% } svg:not(:root) { overflow: hidden } figure { margin: 0 } form { margin: 0 } fieldset { border: 0; margin: 0; padding: 0 } label { cursor: pointer } legend { border: 0; *margin-left: -7px; padding: 0; white-space: normal } button, input, select, textarea { font-size: 100%; margin: 0; vertical-align: baseline; *vertical-align: middle } button, input { line-height: normal } button, input[type="button"], input[type="reset"], input[type="submit"] { cursor: pointer; -webkit-appearance: button; *overflow: visible } button[disabled], input[disabled] { cursor: default } input[type="checkbox"], input[type="radio"] { box-sizing: border-box; padding: 0; *width: 13px; *height: 13px } input[type="search"] { -webkit-appearance: textfield; -moz-box-sizing: content-box; -webkit-box-sizing: content-box; box-sizing: content-box } input[type="search"]::-webkit-search-decoration, input[type="search"]::-webkit-search-cancel-button { -webkit-appearance: none } button::-moz-focus-inner, input::-moz-focus-inner { border: 0; padding: 0 } textarea { overflow: auto; vertical-align: top; resize: vertical } table { border-collapse: collapse; border-spacing: 0 } td { vertical-align: top } .chromeframe { margin: .2em 0; background: #ccc; color: #000; padding: .2em 0 } .ir { display: block; border: 0; text-indent: -999em; overflow: hidden; background-color: transparent; background-repeat: no-repeat; text-align: left; direction: ltr; *line-height: 0 } .ir br { display: none } .hidden { display: none!important; visibility: hidden } .visuallyhidden { border: 0; clip: rect(0 0 0 0); height: 1px; margin: -1px; overflow: hidden; padding: 0; position: absolute; width: 1px } .visuallyhidden.focusable:active, .visuallyhidden.focusable:focus { clip: auto; height: auto; margin: 0; overflow: visible; position: static; width: auto } .invisible { visibility: hidden } .relative { position: relative } big, small { font-size: 100% } @media print { html, body, section { background: none!important } * { box-shadow: none!important; text-shadow: none!important; filter: none!important; -ms-filter: none!important } a, .ir a:after, a[href^="javascript:"]:after, a[href^="#"]:after { content: "" } pre, blockquote { page-break-inside: avoid } thead { display: table-header-group } tr, img { page-break-inside: avoid } img { max-width: 100%!important } @page { margin: .5cm } p, h2, h3 { orphans: 3; widows: 3 } h2, h3 { page-break-after: avoid } } .fa:before, .wy-menu-vertical li span.toctree-expand:before, .wy-menu-vertical li.on a span.toctree-expand:before, .wy-menu-vertical li.current>a span.toctree-expand:before, .rst-content .admonition-title:before, .rst-content h1 .headerlink:before, .rst-content h2 .headerlink:before, .rst-content h3 .headerlink:before, .rst-content h4 .headerlink:before, .rst-content h5 .headerlink:before, .rst-content h6 .headerlink:before, .rst-content dl dt .headerlink:before, .rst-content p.caption .headerlink:before, .rst-content tt.download span:first-child:before, .rst-content code.download span:first-child:before, .icon:before, .wy-dropdown .caret:before, .wy-inline-validate.wy-inline-validate-success .wy-input-context:before, .wy-inline-validate.wy-inline-validate-danger .wy-input-context:before, .wy-inline-validate.wy-inline-validate-warning .wy-input-context:before, .wy-inline-validate.wy-inline-validate-info .wy-input-context:before, .wy-alert, .rst-content .note, .rst-content .attention, .rst-content .caution, .rst-content .danger, .rst-content .error, .rst-content .hint, .rst-content .important, .rst-content .tip, .rst-content .warning, .rst-content .seealso, .rst-content .admonition-todo, .btn, input[type="text"], input[type="password"], input[type="email"], input[type="url"], input[type="date"], input[type="month"], input[type="time"], input[type="datetime"], input[type="datetime-local"], input[type="week"], input[type="number"], input[type="search"], input[type="tel"], input[type="color"], select, textarea, .wy-menu-vertical li.on a, .wy-menu-vertical li.current>a, .wy-side-nav-search>a, .wy-side-nav-search .wy-dropdown>a, .wy-nav-top a { -webkit-font-smoothing: antialiased } .clearfix { *zoom: 1 } .clearfix:before, .clearfix:after { display: table; content: "" } .clearfix:after { clear: both } @font-face { font-family: 'FontAwesome'; src: url(../fonts/fontawesome-webfont.eot?v=4.2.0); src: url(../fonts/fontawesome-webfont.eot?#iefix&v=4.2.0) format("embedded-opentype"),url(../fonts/fontawesome-webfont.woff?v=4.2.0) format("woff"),url(../fonts/fontawesome-webfont.ttf?v=4.2.0) format("truetype"),url(../fonts/fontawesome-webfont.svg?v=4.2.0#fontawesomeregular) format("svg"); font-weight: 400; font-style: normal } .fa, .wy-menu-vertical li span.toctree-expand, .wy-menu-vertical li.on a span.toctree-expand, .wy-menu-vertical li.current>a span.toctree-expand, .rst-content .admonition-title, .rst-content h1 .headerlink, .rst-content h2 .headerlink, .rst-content h3 .headerlink, .rst-content h4 .headerlink, .rst-content h5 .headerlink, .rst-content h6 .headerlink, .rst-content dl dt .headerlink, .rst-content p.caption .headerlink, .rst-content tt.download span:first-child, .rst-content code.download span:first-child, .icon { display: inline-block; font: normal normal normal 14px/1 FontAwesome; font-size: inherit; text-rendering: auto; -webkit-font-smoothing: antialiased; -moz-osx-font-smoothing: grayscale } .fa-lg { font-size: 1.33333em; line-height: .75em; vertical-align: -15% } .fa-2x { font-size: 2em } .fa-3x { font-size: 3em } .fa-4x { font-size: 4em } .fa-5x { font-size: 5em } .fa-fw { width: 1.28571em; text-align: center } .fa-ul { padding-left: 0; margin-left: 2.14286em; list-style-type: none } .fa-ul>li { position: relative } .fa-li { position: absolute; left: -2.14286em; width: 2.14286em; top: .14286em; text-align: center } .fa-li.fa-lg { left: -1.85714em } .fa-border { padding: .2em .25em .15em; border: solid .08em #eee; border-radius: .1em } .pull-right { float: right } .pull-left { float: left } .fa.pull-left, .wy-menu-vertical li span.pull-left.toctree-expand, .wy-menu-vertical li.on a span.pull-left.toctree-expand, .wy-menu-vertical li.current>a span.pull-left.toctree-expand, .rst-content .pull-left.admonition-title, .rst-content h1 .pull-left.headerlink, .rst-content h2 .pull-left.headerlink, .rst-content h3 .pull-left.headerlink, .rst-content h4 .pull-left.headerlink, .rst-content h5 .pull-left.headerlink, .rst-content h6 .pull-left.headerlink, .rst-content dl dt .pull-left.headerlink, .rst-content p.caption .pull-left.headerlink, .rst-content tt.download span.pull-left:first-child, .rst-content code.download span.pull-left:first-child, .pull-left.icon { margin-right: .3em } .fa.pull-right, .wy-menu-vertical li span.pull-right.toctree-expand, .wy-menu-vertical li.on a span.pull-right.toctree-expand, .wy-menu-vertical li.current>a span.pull-right.toctree-expand, .rst-content .pull-right.admonition-title, .rst-content h1 .pull-right.headerlink, .rst-content h2 .pull-right.headerlink, .rst-content h3 .pull-right.headerlink, .rst-content h4 .pull-right.headerlink, .rst-content h5 .pull-right.headerlink, .rst-content h6 .pull-right.headerlink, .rst-content dl dt .pull-right.headerlink, .rst-content p.caption .pull-right.headerlink, .rst-content tt.download span.pull-right:first-child, .rst-content code.download span.pull-right:first-child, .pull-right.icon { margin-left: .3em } .fa, .wy-menu-vertical li span.toctree-expand, .wy-menu-vertical li.on a span.toctree-expand, .wy-menu-vertical li.current>a span.toctree-expand, .rst-content .admonition-title, .rst-content h1 .headerlink, .rst-content h2 .headerlink, .rst-content h3 .headerlink, .rst-content h4 .headerlink, .rst-content h5 .headerlink, .rst-content h6 .headerlink, .rst-content dl dt .headerlink, .rst-content p.caption .headerlink, .rst-content tt.download span:first-child, .rst-content code.download span:first-child, .icon, .wy-dropdown .caret, .wy-inline-validate.wy-inline-validate-success .wy-input-context, .wy-inline-validate.wy-inline-validate-danger .wy-input-context, .wy-inline-validate.wy-inline-validate-warning .wy-input-context, .wy-inline-validate.wy-inline-validate-info .wy-input-context { font-family: inherit } .fa:before, .wy-menu-vertical li span.toctree-expand:before, .wy-menu-vertical li.on a span.toctree-expand:before, .wy-menu-vertical li.current>a span.toctree-expand:before, .rst-content .admonition-title:before, .rst-content h1 .headerlink:before, .rst-content h2 .headerlink:before, .rst-content h3 .headerlink:before, .rst-content h4 .headerlink:before, .rst-content h5 .headerlink:before, .rst-content h6 .headerlink:before, .rst-content dl dt .headerlink:before, .rst-content p.caption .headerlink:before, .rst-content tt.download span:first-child:before, .rst-content code.download span:first-child:before, .icon:before, .wy-dropdown .caret:before, .wy-inline-validate.wy-inline-validate-success .wy-input-context:before, .wy-inline-validate.wy-inline-validate-danger .wy-input-context:before, .wy-inline-validate.wy-inline-validate-warning .wy-input-context:before, .wy-inline-validate.wy-inline-validate-info .wy-input-context:before { font-family: "FontAwesome"; display: inline-block; font-style: normal; font-weight: 400; line-height: 1; text-decoration: inherit } a .fa, a .wy-menu-vertical li span.toctree-expand, .wy-menu-vertical li a span.toctree-expand, .wy-menu-vertical li.on a span.toctree-expand, .wy-menu-vertical li.current>a span.toctree-expand, a .rst-content .admonition-title, .rst-content a .admonition-title, a .rst-content h1 .headerlink, .rst-content h1 a .headerlink, a .rst-content h2 .headerlink, .rst-content h2 a .headerlink, a .rst-content h3 .headerlink, .rst-content h3 a .headerlink, a .rst-content h4 .headerlink, .rst-content h4 a .headerlink, a .rst-content h5 .headerlink, .rst-content h5 a .headerlink, a .rst-content h6 .headerlink, .rst-content h6 a .headerlink, a .rst-content dl dt .headerlink, .rst-content dl dt a .headerlink, a .rst-content p.caption .headerlink, .rst-content p.caption a .headerlink, a .rst-content tt.download span:first-child, .rst-content tt.download a span:first-child, a .rst-content code.download span:first-child, .rst-content code.download a span:first-child, a .icon { display: inline-block; text-decoration: inherit } .btn .fa, .btn .wy-menu-vertical li span.toctree-expand, .wy-menu-vertical li .btn span.toctree-expand, .btn .wy-menu-vertical li.on a span.toctree-expand, .wy-menu-vertical li.on a .btn span.toctree-expand, .btn .wy-menu-vertical li.current>a span.toctree-expand, .wy-menu-vertical li.current>a .btn span.toctree-expand, .btn .rst-content .admonition-title, .rst-content .btn .admonition-title, .btn .rst-content h1 .headerlink, .rst-content h1 .btn .headerlink, .btn .rst-content h2 .headerlink, .rst-content h2 .btn .headerlink, .btn .rst-content h3 .headerlink, .rst-content h3 .btn .headerlink, .btn .rst-content h4 .headerlink, .rst-content h4 .btn .headerlink, .btn .rst-content h5 .headerlink, .rst-content h5 .btn .headerlink, .btn .rst-content h6 .headerlink, .rst-content h6 .btn .headerlink, .btn .rst-content dl dt .headerlink, .rst-content dl dt .btn .headerlink, .btn .rst-content p.caption .headerlink, .rst-content p.caption .btn .headerlink, .btn .rst-content tt.download span:first-child, .rst-content tt.download .btn span:first-child, .btn .rst-content code.download span:first-child, .rst-content code.download .btn span:first-child, .btn .icon, .nav .fa, .nav .wy-menu-vertical li span.toctree-expand, .wy-menu-vertical li .nav span.toctree-expand, .nav .wy-menu-vertical li.on a span.toctree-expand, .wy-menu-vertical li.on a .nav span.toctree-expand, .nav .wy-menu-vertical li.current>a span.toctree-expand, .wy-menu-vertical li.current>a .nav span.toctree-expand, .nav .rst-content .admonition-title, .rst-content .nav .admonition-title, .nav .rst-content h1 .headerlink, .rst-content h1 .nav .headerlink, .nav .rst-content h2 .headerlink, .rst-content h2 .nav .headerlink, .nav .rst-content h3 .headerlink, .rst-content h3 .nav .headerlink, .nav .rst-content h4 .headerlink, .rst-content h4 .nav .headerlink, .nav .rst-content h5 .headerlink, .rst-content h5 .nav .headerlink, .nav .rst-content h6 .headerlink, .rst-content h6 .nav .headerlink, .nav .rst-content dl dt .headerlink, .rst-content dl dt .nav .headerlink, .nav .rst-content p.caption .headerlink, .rst-content p.caption .nav .headerlink, .nav .rst-content tt.download span:first-child, .rst-content tt.download .nav span:first-child, .nav .rst-content code.download span:first-child, .rst-content code.download .nav span:first-child, .nav .icon { display: inline } .btn .fa.fa-large, .btn .wy-menu-vertical li span.fa-large.toctree-expand, .wy-menu-vertical li .btn span.fa-large.toctree-expand, .btn .rst-content .fa-large.admonition-title, .rst-content .btn .fa-large.admonition-title, .btn .rst-content h1 .fa-large.headerlink, .rst-content h1 .btn .fa-large.headerlink, .btn .rst-content h2 .fa-large.headerlink, .rst-content h2 .btn .fa-large.headerlink, .btn .rst-content h3 .fa-large.headerlink, .rst-content h3 .btn .fa-large.headerlink, .btn .rst-content h4 .fa-large.headerlink, .rst-content h4 .btn .fa-large.headerlink, .btn .rst-content h5 .fa-large.headerlink, .rst-content h5 .btn .fa-large.headerlink, .btn .rst-content h6 .fa-large.headerlink, .rst-content h6 .btn .fa-large.headerlink, .btn .rst-content dl dt .fa-large.headerlink, .rst-content dl dt .btn .fa-large.headerlink, .btn .rst-content p.caption .fa-large.headerlink, .rst-content p.caption .btn .fa-large.headerlink, .btn .rst-content tt.download span.fa-large:first-child, .rst-content tt.download .btn span.fa-large:first-child, .btn .rst-content code.download span.fa-large:first-child, .rst-content code.download .btn span.fa-large:first-child, .btn .fa-large.icon, .nav .fa.fa-large, .nav .wy-menu-vertical li span.fa-large.toctree-expand, .wy-menu-vertical li .nav span.fa-large.toctree-expand, .nav .rst-content .fa-large.admonition-title, .rst-content .nav .fa-large.admonition-title, .nav .rst-content h1 .fa-large.headerlink, .rst-content h1 .nav .fa-large.headerlink, .nav .rst-content h2 .fa-large.headerlink, .rst-content h2 .nav .fa-large.headerlink, .nav .rst-content h3 .fa-large.headerlink, .rst-content h3 .nav .fa-large.headerlink, .nav .rst-content h4 .fa-large.headerlink, .rst-content h4 .nav .fa-large.headerlink, .nav .rst-content h5 .fa-large.headerlink, .rst-content h5 .nav .fa-large.headerlink, .nav .rst-content h6 .fa-large.headerlink, .rst-content h6 .nav .fa-large.headerlink, .nav .rst-content dl dt .fa-large.headerlink, .rst-content dl dt .nav .fa-large.headerlink, .nav .rst-content p.caption .fa-large.headerlink, .rst-content p.caption .nav .fa-large.headerlink, .nav .rst-content tt.download span.fa-large:first-child, .rst-content tt.download .nav span.fa-large:first-child, .nav .rst-content code.download span.fa-large:first-child, .rst-content code.download .nav span.fa-large:first-child, .nav .fa-large.icon { line-height: .9em } .btn .fa.fa-spin, .btn .wy-menu-vertical li span.fa-spin.toctree-expand, .wy-menu-vertical li .btn span.fa-spin.toctree-expand, .btn .rst-content .fa-spin.admonition-title, .rst-content .btn .fa-spin.admonition-title, .btn .rst-content h1 .fa-spin.headerlink, .rst-content h1 .btn .fa-spin.headerlink, .btn .rst-content h2 .fa-spin.headerlink, .rst-content h2 .btn .fa-spin.headerlink, .btn .rst-content h3 .fa-spin.headerlink, .rst-content h3 .btn .fa-spin.headerlink, .btn .rst-content h4 .fa-spin.headerlink, .rst-content h4 .btn .fa-spin.headerlink, .btn .rst-content h5 .fa-spin.headerlink, .rst-content h5 .btn .fa-spin.headerlink, .btn .rst-content h6 .fa-spin.headerlink, .rst-content h6 .btn .fa-spin.headerlink, .btn .rst-content dl dt .fa-spin.headerlink, .rst-content dl dt .btn .fa-spin.headerlink, .btn .rst-content p.caption .fa-spin.headerlink, .rst-content p.caption .btn .fa-spin.headerlink, .btn .rst-content tt.download span.fa-spin:first-child, .rst-content tt.download .btn span.fa-spin:first-child, .btn .rst-content code.download span.fa-spin:first-child, .rst-content code.download .btn span.fa-spin:first-child, .btn .fa-spin.icon, .nav .fa.fa-spin, .nav .wy-menu-vertical li span.fa-spin.toctree-expand, .wy-menu-vertical li .nav span.fa-spin.toctree-expand, .nav .rst-content .fa-spin.admonition-title, .rst-content .nav .fa-spin.admonition-title, .nav .rst-content h1 .fa-spin.headerlink, .rst-content h1 .nav .fa-spin.headerlink, .nav .rst-content h2 .fa-spin.headerlink, .rst-content h2 .nav .fa-spin.headerlink, .nav .rst-content h3 .fa-spin.headerlink, .rst-content h3 .nav .fa-spin.headerlink, .nav .rst-content h4 .fa-spin.headerlink, .rst-content h4 .nav .fa-spin.headerlink, .nav .rst-content h5 .fa-spin.headerlink, .rst-content h5 .nav .fa-spin.headerlink, .nav .rst-content h6 .fa-spin.headerlink, .rst-content h6 .nav .fa-spin.headerlink, .nav .rst-content dl dt .fa-spin.headerlink, .rst-content dl dt .nav .fa-spin.headerlink, .nav .rst-content p.caption .fa-spin.headerlink, .rst-content p.caption .nav .fa-spin.headerlink, .nav .rst-content tt.download span.fa-spin:first-child, .rst-content tt.download .nav span.fa-spin:first-child, .nav .rst-content code.download span.fa-spin:first-child, .rst-content code.download .nav span.fa-spin:first-child, .nav .fa-spin.icon { display: inline-block } .btn.fa:before, .wy-menu-vertical li span.btn.toctree-expand:before, .rst-content .btn.admonition-title:before, .rst-content h1 .btn.headerlink:before, .rst-content h2 .btn.headerlink:before, .rst-content h3 .btn.headerlink:before, .rst-content h4 .btn.headerlink:before, .rst-content h5 .btn.headerlink:before, .rst-content h6 .btn.headerlink:before, .rst-content dl dt .btn.headerlink:before, .rst-content p.caption .btn.headerlink:before, .rst-content tt.download span.btn:first-child:before, .rst-content code.download span.btn:first-child:before, .btn.icon:before { opacity: .5; -webkit-transition: opacity .05s ease-in; -moz-transition: opacity .05s ease-in; transition: opacity .05s ease-in } .btn.fa:hover:before, .wy-menu-vertical li span.btn.toctree-expand:hover:before, .rst-content .btn.admonition-title:hover:before, .rst-content h1 .btn.headerlink:hover:before, .rst-content h2 .btn.headerlink:hover:before, .rst-content h3 .btn.headerlink:hover:before, .rst-content h4 .btn.headerlink:hover:before, .rst-content h5 .btn.headerlink:hover:before, .rst-content h6 .btn.headerlink:hover:before, .rst-content dl dt .btn.headerlink:hover:before, .rst-content p.caption .btn.headerlink:hover:before, .rst-content tt.download span.btn:first-child:hover:before, .rst-content code.download span.btn:first-child:hover:before, .btn.icon:hover:before { opacity: 1 } .btn-mini .fa:before, .btn-mini .wy-menu-vertical li span.toctree-expand:before, .wy-menu-vertical li .btn-mini span.toctree-expand:before, .btn-mini .rst-content .admonition-title:before, .rst-content .btn-mini .admonition-title:before, .btn-mini .rst-content h1 .headerlink:before, .rst-content h1 .btn-mini .headerlink:before, .btn-mini .rst-content h2 .headerlink:before, .rst-content h2 .btn-mini .headerlink:before, .btn-mini .rst-content h3 .headerlink:before, .rst-content h3 .btn-mini .headerlink:before, .btn-mini .rst-content h4 .headerlink:before, .rst-content h4 .btn-mini .headerlink:before, .btn-mini .rst-content h5 .headerlink:before, .rst-content h5 .btn-mini .headerlink:before, .btn-mini .rst-content h6 .headerlink:before, .rst-content h6 .btn-mini .headerlink:before, .btn-mini .rst-content dl dt .headerlink:before, .rst-content dl dt .btn-mini .headerlink:before, .btn-mini .rst-content p.caption .headerlink:before, .rst-content p.caption .btn-mini .headerlink:before, .btn-mini .rst-content tt.download span:first-child:before, .rst-content tt.download .btn-mini span:first-child:before, .btn-mini .rst-content code.download span:first-child:before, .rst-content code.download .btn-mini span:first-child:before, .btn-mini .icon:before { font-size: 14px; vertical-align: -15% } .wy-alert, .rst-content .note, .rst-content .attention, .rst-content .caution, .rst-content .danger, .rst-content .error, .rst-content .hint, .rst-content .important, .rst-content .tip, .rst-content .warning, .rst-content .seealso, .rst-content .admonition-todo { padding: 12px; line-height: 24px; margin-bottom: 24px; background: #e7f2fa } .wy-alert-title, .rst-content .admonition-title { color: #fff; font-weight: 700; display: block; color: #fff; background: #6ab0de; margin: -12px; padding: 6px 12px; margin-bottom: 12px } .wy-alert.wy-alert-danger, .rst-content .wy-alert-danger.note, .rst-content .wy-alert-danger.attention, .rst-content .wy-alert-danger.caution, .rst-content .danger, .rst-content .error, .rst-content .wy-alert-danger.hint, .rst-content .wy-alert-danger.important, .rst-content .wy-alert-danger.tip, .rst-content .wy-alert-danger.warning, .rst-content .wy-alert-danger.seealso, .rst-content .wy-alert-danger.admonition-todo { background: #fdf3f2 } .wy-alert.wy-alert-danger .wy-alert-title, .rst-content .wy-alert-danger.note .wy-alert-title, .rst-content .wy-alert-danger.attention .wy-alert-title, .rst-content .wy-alert-danger.caution .wy-alert-title, .rst-content .danger .wy-alert-title, .rst-content .error .wy-alert-title, .rst-content .wy-alert-danger.hint .wy-alert-title, .rst-content .wy-alert-danger.important .wy-alert-title, .rst-content .wy-alert-danger.tip .wy-alert-title, .rst-content .wy-alert-danger.warning .wy-alert-title, .rst-content .wy-alert-danger.seealso .wy-alert-title, .rst-content .wy-alert-danger.admonition-todo .wy-alert-title, .wy-alert.wy-alert-danger .rst-content .admonition-title, .rst-content .wy-alert.wy-alert-danger .admonition-title, .rst-content .wy-alert-danger.note .admonition-title, .rst-content .wy-alert-danger.attention .admonition-title, .rst-content .wy-alert-danger.caution .admonition-title, .rst-content .danger .admonition-title, .rst-content .error .admonition-title, .rst-content .wy-alert-danger.hint .admonition-title, .rst-content .wy-alert-danger.important .admonition-title, .rst-content .wy-alert-danger.tip .admonition-title, .rst-content .wy-alert-danger.warning .admonition-title, .rst-content .wy-alert-danger.seealso .admonition-title, .rst-content .wy-alert-danger.admonition-todo .admonition-title { background: #f29f97 } .wy-alert.wy-alert-warning, .rst-content .wy-alert-warning.note, .rst-content .attention, .rst-content .caution, .rst-content .wy-alert-warning.danger, .rst-content .wy-alert-warning.error, .rst-content .wy-alert-warning.hint, .rst-content .wy-alert-warning.important, .rst-content .wy-alert-warning.tip, .rst-content .warning, .rst-content .wy-alert-warning.seealso, .rst-content .admonition-todo { background: #ffedcc } .wy-alert.wy-alert-warning .wy-alert-title, .rst-content .wy-alert-warning.note .wy-alert-title, .rst-content .attention .wy-alert-title, .rst-content .caution .wy-alert-title, .rst-content .wy-alert-warning.danger .wy-alert-title, .rst-content .wy-alert-warning.error .wy-alert-title, .rst-content .wy-alert-warning.hint .wy-alert-title, .rst-content .wy-alert-warning.important .wy-alert-title, .rst-content .wy-alert-warning.tip .wy-alert-title, .rst-content .warning .wy-alert-title, .rst-content .wy-alert-warning.seealso .wy-alert-title, .rst-content .admonition-todo .wy-alert-title, .wy-alert.wy-alert-warning .rst-content .admonition-title, .rst-content .wy-alert.wy-alert-warning .admonition-title, .rst-content .wy-alert-warning.note .admonition-title, .rst-content .attention .admonition-title, .rst-content .caution .admonition-title, .rst-content .wy-alert-warning.danger .admonition-title, .rst-content .wy-alert-warning.error .admonition-title, .rst-content .wy-alert-warning.hint .admonition-title, .rst-content .wy-alert-warning.important .admonition-title, .rst-content .wy-alert-warning.tip .admonition-title, .rst-content .warning .admonition-title, .rst-content .wy-alert-warning.seealso .admonition-title, .rst-content .admonition-todo .admonition-title { background: #f0b37e } .wy-alert.wy-alert-info, .rst-content .note, .rst-content .wy-alert-info.attention, .rst-content .wy-alert-info.caution, .rst-content .wy-alert-info.danger, .rst-content .wy-alert-info.error, .rst-content .wy-alert-info.hint, .rst-content .wy-alert-info.important, .rst-content .wy-alert-info.tip, .rst-content .wy-alert-info.warning, .rst-content .seealso, .rst-content .wy-alert-info.admonition-todo { background: #e7f2fa } .wy-alert.wy-alert-info .wy-alert-title, .rst-content .note .wy-alert-title, .rst-content .wy-alert-info.attention .wy-alert-title, .rst-content .wy-alert-info.caution .wy-alert-title, .rst-content .wy-alert-info.danger .wy-alert-title, .rst-content .wy-alert-info.error .wy-alert-title, .rst-content .wy-alert-info.hint .wy-alert-title, .rst-content .wy-alert-info.important .wy-alert-title, .rst-content .wy-alert-info.tip .wy-alert-title, .rst-content .wy-alert-info.warning .wy-alert-title, .rst-content .seealso .wy-alert-title, .rst-content .wy-alert-info.admonition-todo .wy-alert-title, .wy-alert.wy-alert-info .rst-content .admonition-title, .rst-content .wy-alert.wy-alert-info .admonition-title, .rst-content .note .admonition-title, .rst-content .wy-alert-info.attention .admonition-title, .rst-content .wy-alert-info.caution .admonition-title, .rst-content .wy-alert-info.danger .admonition-title, .rst-content .wy-alert-info.error .admonition-title, .rst-content .wy-alert-info.hint .admonition-title, .rst-content .wy-alert-info.important .admonition-title, .rst-content .wy-alert-info.tip .admonition-title, .rst-content .wy-alert-info.warning .admonition-title, .rst-content .seealso .admonition-title, .rst-content .wy-alert-info.admonition-todo .admonition-title { background: #6ab0de } .wy-alert.wy-alert-success, .rst-content .wy-alert-success.note, .rst-content .wy-alert-success.attention, .rst-content .wy-alert-success.caution, .rst-content .wy-alert-success.danger, .rst-content .wy-alert-success.error, .rst-content .hint, .rst-content .important, .rst-content .tip, .rst-content .wy-alert-success.warning, .rst-content .wy-alert-success.seealso, .rst-content .wy-alert-success.admonition-todo { background: #dbfaf4 } .wy-alert.wy-alert-success .wy-alert-title, .rst-content .wy-alert-success.note .wy-alert-title, .rst-content .wy-alert-success.attention .wy-alert-title, .rst-content .wy-alert-success.caution .wy-alert-title, .rst-content .wy-alert-success.danger .wy-alert-title, .rst-content .wy-alert-success.error .wy-alert-title, .rst-content .hint .wy-alert-title, .rst-content .important .wy-alert-title, .rst-content .tip .wy-alert-title, .rst-content .wy-alert-success.warning .wy-alert-title, .rst-content .wy-alert-success.seealso .wy-alert-title, .rst-content .wy-alert-success.admonition-todo .wy-alert-title, .wy-alert.wy-alert-success .rst-content .admonition-title, .rst-content .wy-alert.wy-alert-success .admonition-title, .rst-content .wy-alert-success.note .admonition-title, .rst-content .wy-alert-success.attention .admonition-title, .rst-content .wy-alert-success.caution .admonition-title, .rst-content .wy-alert-success.danger .admonition-title, .rst-content .wy-alert-success.error .admonition-title, .rst-content .hint .admonition-title, .rst-content .important .admonition-title, .rst-content .tip .admonition-title, .rst-content .wy-alert-success.warning .admonition-title, .rst-content .wy-alert-success.seealso .admonition-title, .rst-content .wy-alert-success.admonition-todo .admonition-title { background: #1abc9c } .wy-alert.wy-alert-neutral, .rst-content .wy-alert-neutral.note, .rst-content .wy-alert-neutral.attention, .rst-content .wy-alert-neutral.caution, .rst-content .wy-alert-neutral.danger, .rst-content .wy-alert-neutral.error, .rst-content .wy-alert-neutral.hint, .rst-content .wy-alert-neutral.important, .rst-content .wy-alert-neutral.tip, .rst-content .wy-alert-neutral.warning, .rst-content .wy-alert-neutral.seealso, .rst-content .wy-alert-neutral.admonition-todo { background: #f3f6f6 } .wy-alert.wy-alert-neutral .wy-alert-title, .rst-content .wy-alert-neutral.note .wy-alert-title, .rst-content .wy-alert-neutral.attention .wy-alert-title, .rst-content .wy-alert-neutral.caution .wy-alert-title, .rst-content .wy-alert-neutral.danger .wy-alert-title, .rst-content .wy-alert-neutral.error .wy-alert-title, .rst-content .wy-alert-neutral.hint .wy-alert-title, .rst-content .wy-alert-neutral.important .wy-alert-title, .rst-content .wy-alert-neutral.tip .wy-alert-title, .rst-content .wy-alert-neutral.warning .wy-alert-title, .rst-content .wy-alert-neutral.seealso .wy-alert-title, .rst-content .wy-alert-neutral.admonition-todo .wy-alert-title, .wy-alert.wy-alert-neutral .rst-content .admonition-title, .rst-content .wy-alert.wy-alert-neutral .admonition-title, .rst-content .wy-alert-neutral.note .admonition-title, .rst-content .wy-alert-neutral.attention .admonition-title, .rst-content .wy-alert-neutral.caution .admonition-title, .rst-content .wy-alert-neutral.danger .admonition-title, .rst-content .wy-alert-neutral.error .admonition-title, .rst-content .wy-alert-neutral.hint .admonition-title, .rst-content .wy-alert-neutral.important .admonition-title, .rst-content .wy-alert-neutral.tip .admonition-title, .rst-content .wy-alert-neutral.warning .admonition-title, .rst-content .wy-alert-neutral.seealso .admonition-title, .rst-content .wy-alert-neutral.admonition-todo .admonition-title { color: #404040; background: #e1e4e5 } .wy-alert.wy-alert-neutral a, .rst-content .wy-alert-neutral.note a, .rst-content .wy-alert-neutral.attention a, .rst-content .wy-alert-neutral.caution a, .rst-content .wy-alert-neutral.danger a, .rst-content .wy-alert-neutral.error a, .rst-content .wy-alert-neutral.hint a, .rst-content .wy-alert-neutral.important a, .rst-content .wy-alert-neutral.tip a, .rst-content .wy-alert-neutral.warning a, .rst-content .wy-alert-neutral.seealso a, .rst-content .wy-alert-neutral.admonition-todo a { color: #008BF3 } .wy-alert p:last-child, .rst-content .note p:last-child, .rst-content .attention p:last-child, .rst-content .caution p:last-child, .rst-content .danger p:last-child, .rst-content .error p:last-child, .rst-content .hint p:last-child, .rst-content .important p:last-child, .rst-content .tip p:last-child, .rst-content .warning p:last-child, .rst-content .seealso p:last-child, .rst-content .admonition-todo p:last-child { margin-bottom: 0 } .wy-tray-container { position: fixed; bottom: 0; left: 0; z-index: 600 } .wy-tray-container li { display: block; width: 300px; background: transparent; color: #fff; text-align: center; box-shadow: 0 5px 5px 0 rgba(0,0,0,0.1); padding: 0 24px; min-width: 20%; opacity: 0; height: 0; line-height: 56px; overflow: hidden; -webkit-transition: all .3s ease-in; -moz-transition: all .3s ease-in; transition: all .3s ease-in } .wy-tray-container li.wy-tray-item-success { background: #27AE60 } .wy-tray-container li.wy-tray-item-info { background: #2980B9 } .wy-tray-container li.wy-tray-item-warning { background: #E67E22 } .wy-tray-container li.wy-tray-item-danger { background: #E74C3C } .wy-tray-container li.on { opacity: 1; height: 56px } @media screen and (max-width: 768px) { .wy-tray-container { bottom: auto; top: 0; width: 100% } .wy-tray-container li { width: 100% } } button { font-size: 100%; margin: 0; vertical-align: baseline; *vertical-align: middle; cursor: pointer; line-height: normal; -webkit-appearance: button; *overflow: visible } button::-moz-focus-inner, input::-moz-focus-inner { border: 0; padding: 0 } button[disabled] { cursor: default } .btn { font-family: "Source Sans Pro","proxima-nova","Helvetica Neue",Arial,sans-serif; display: inline-block; padding: 10px 30px; margin-bottom: 0; font-weight: 400; text-align: center; vertical-align: middle; -ms-touch-action: manipulation; touch-action: manipulation; cursor: pointer; background-image: none; background-color: #4a90e2; border: 1px solid transparent; white-space: nowrap; color: rgba(255,255,255,0.95); border-radius: 3px; -webkit-user-select: none; -moz-user-select: none; -ms-user-select: none; user-select: none; outline: none } .btn-hover { background: #2e8ece; color: #fff } .btn:disabled { background-image: none; filter: progid: DXImageTransform.Microsoft.gradient(enabled=false); filter: alpha(opacity=40); opacity: .4; cursor: not-allowed; box-shadow: none } .btn-disabled { background-image: none; filter: progid: DXImageTransform.Microsoft.gradient(enabled=false); filter: alpha(opacity=40); opacity: .4; cursor: not-allowed; box-shadow: none } .btn-disabled:hover, .btn-disabled:focus, .btn-disabled:active { background-image: none; filter: progid: DXImageTransform.Microsoft.gradient(enabled=false); filter: alpha(opacity=40); opacity: .4; cursor: not-allowed; box-shadow: none } .btn::-moz-focus-inner { padding: 0; border: 0 } .btn-small { font-size: 80% } .btn-info { background-color: #2980B9!important } .btn-info:hover { background-color: #2e8ece!important } .btn-neutral { font-size: 14px; color: #fff; font-weight: 300; /*background-color: #008bf3!important*/ } .btn-success { background-color: #27AE60!important } .btn-success:hover { background-color: #295!important } .btn-danger { background-color: #E74C3C!important } .btn-danger:hover { background-color: #ea6153!important } .btn-warning { background-color: #E67E22!important } .btn-warning:hover { background-color: #e98b39!important } .btn-invert { background-color: #222 } .btn-invert:hover { background-color: #2f2f2f!important } .btn-link { background-color: transparent!important; color: #2980B9; box-shadow: none; border-color: transparent!important } .btn-link:hover { background-color: transparent!important; color: #409ad5!important; box-shadow: none } .btn-link:active { background-color: transparent!important; color: #409ad5!important; box-shadow: none } .btn-link:visited { color: #9B59B6 } .wy-btn-group .btn, .wy-control .btn { vertical-align: middle } .wy-btn-group { margin-bottom: 24px; *zoom: 1 } .wy-btn-group:before, .wy-btn-group:after { display: table; content: "" } .wy-btn-group:after { clear: both } .wy-dropdown { position: relative; display: inline-block } .wy-dropdown-active .wy-dropdown-menu { display: block } .wy-dropdown-menu { position: absolute; left: 0; display: none; float: left; top: 100%; min-width: 100%; background: #fcfcfc; z-index: 100; border: solid 1px #cfd7dd; box-shadow: 0 2px 2px 0 rgba(0,0,0,0.1); padding: 12px } .wy-dropdown-menu>dd>a { display: block; clear: both; color: #404040; white-space: nowrap; font-size: 90%; padding: 0 12px; cursor: pointer } .wy-dropdown-menu>dd>a:hover { background: #2980B9; color: #fff } .wy-dropdown-menu>dd.divider { border-top: solid 1px #cfd7dd; margin: 6px 0 } .wy-dropdown-menu>dd.search { padding-bottom: 12px } .wy-dropdown-menu>dd.search input[type="search"] { width: 100% } .wy-dropdown-menu>dd.call-to-action { background: #e3e3e3; text-transform: uppercase; font-weight: 500; font-size: 80% } .wy-dropdown-menu>dd.call-to-action:hover { background: #e3e3e3 } .wy-dropdown-menu>dd.call-to-action .btn { color: #fff } .wy-dropdown.wy-dropdown-up .wy-dropdown-menu { bottom: 100%; top: auto; left: auto; right: 0 } .wy-dropdown.wy-dropdown-bubble .wy-dropdown-menu { background: #fcfcfc; margin-top: 2px } .wy-dropdown.wy-dropdown-bubble .wy-dropdown-menu a { padding: 6px 12px } .wy-dropdown.wy-dropdown-bubble .wy-dropdown-menu a:hover { background: #2980B9; color: #fff } .wy-dropdown.wy-dropdown-left .wy-dropdown-menu { right: 0; left: auto; text-align: right } .wy-dropdown-arrow:before { content: " "; border-bottom: 5px solid #f5f5f5; border-left: 5px solid transparent; border-right: 5px solid transparent; position: absolute; display: block; top: -4px; left: 50%; margin-left: -3px } .wy-dropdown-arrow.wy-dropdown-arrow-left:before { left: 11px } .wy-form-stacked select { display: block } .wy-form-aligned input, .wy-form-aligned textarea, .wy-form-aligned select, .wy-form-aligned .wy-help-inline, .wy-form-aligned label { display: inline-block; *display: inline; *zoom: 1; vertical-align: middle } .wy-form-aligned .wy-control-group>label { display: inline-block; vertical-align: middle; width: 10em; margin: 6px 12px 0 0; float: left } .wy-form-aligned .wy-control { float: left } .wy-form-aligned .wy-control label { display: block } .wy-form-aligned .wy-control select { margin-top: 6px } fieldset { border: 0; margin: 0; padding: 0 } legend { display: block; width: 100%; border: 0; padding: 0; white-space: normal; margin-bottom: 24px; font-size: 150%; *margin-left: -7px } label { display: block; margin: 0 0 .3125em; color: #333; font-size: 90% } input, select, textarea { font-size: 100%; margin: 0; vertical-align: baseline; *vertical-align: middle } .wy-control-group { margin-bottom: 24px; *zoom: 1; max-width: 68em; margin-left: auto; margin-right: auto; *zoom: 1 } .wy-control-group:before, .wy-control-group:after { display: table; content: "" } .wy-control-group:after { clear: both } .wy-control-group:before, .wy-control-group:after { display: table; content: "" } .wy-control-group:after { clear: both } .wy-control-group.wy-control-group-required>label:after { content: " *"; color: #E74C3C } .wy-control-group .wy-form-full, .wy-control-group .wy-form-halves, .wy-control-group .wy-form-thirds { padding-bottom: 12px } .wy-control-group .wy-form-full select, .wy-control-group .wy-form-halves select, .wy-control-group .wy-form-thirds select { width: 100% } .wy-control-group .wy-form-full input[type="text"], .wy-control-group .wy-form-full input[type="password"], .wy-control-group .wy-form-full input[type="email"], .wy-control-group .wy-form-full input[type="url"], .wy-control-group .wy-form-full input[type="date"], .wy-control-group .wy-form-full input[type="month"], .wy-control-group .wy-form-full input[type="time"], .wy-control-group .wy-form-full input[type="datetime"], .wy-control-group .wy-form-full input[type="datetime-local"], .wy-control-group .wy-form-full input[type="week"], .wy-control-group .wy-form-full input[type="number"], .wy-control-group .wy-form-full input[type="search"], .wy-control-group .wy-form-full input[type="tel"], .wy-control-group .wy-form-full input[type="color"], .wy-control-group .wy-form-halves input[type="text"], .wy-control-group .wy-form-halves input[type="password"], .wy-control-group .wy-form-halves input[type="email"], .wy-control-group .wy-form-halves input[type="url"], .wy-control-group .wy-form-halves input[type="date"], .wy-control-group .wy-form-halves input[type="month"], .wy-control-group .wy-form-halves input[type="time"], .wy-control-group .wy-form-halves input[type="datetime"], .wy-control-group .wy-form-halves input[type="datetime-local"], .wy-control-group .wy-form-halves input[type="week"], .wy-control-group .wy-form-halves input[type="number"], .wy-control-group .wy-form-halves input[type="search"], .wy-control-group .wy-form-halves input[type="tel"], .wy-control-group .wy-form-halves input[type="color"], .wy-control-group .wy-form-thirds input[type="text"], .wy-control-group .wy-form-thirds input[type="password"], .wy-control-group .wy-form-thirds input[type="email"], .wy-control-group .wy-form-thirds input[type="url"], .wy-control-group .wy-form-thirds input[type="date"], .wy-control-group .wy-form-thirds input[type="month"], .wy-control-group .wy-form-thirds input[type="time"], .wy-control-group .wy-form-thirds input[type="datetime"], .wy-control-group .wy-form-thirds input[type="datetime-local"], .wy-control-group .wy-form-thirds input[type="week"], .wy-control-group .wy-form-thirds input[type="number"], .wy-control-group .wy-form-thirds input[type="search"], .wy-control-group .wy-form-thirds input[type="tel"], .wy-control-group .wy-form-thirds input[type="color"] { width: 100% } .wy-control-group .wy-form-full { float: left; display: block; margin-right: 2.35765%; width: 100%; margin-right: 0 } .wy-control-group .wy-form-full:last-child { margin-right: 0 } .wy-control-group .wy-form-halves { float: left; display: block; margin-right: 2.35765%; width: 48.82117% } .wy-control-group .wy-form-halves:last-child { margin-right: 0 } .wy-control-group .wy-form-halves:nth-of-type(2n) { margin-right: 0 } .wy-control-group .wy-form-halves:nth-of-type(2n+1) { clear: left } .wy-control-group .wy-form-thirds { float: left; display: block; margin-right: 2.35765%; width: 31.76157% } .wy-control-group .wy-form-thirds:last-child { margin-right: 0 } .wy-control-group .wy-form-thirds:nth-of-type(3n) { margin-right: 0 } .wy-control-group .wy-form-thirds:nth-of-type(3n+1) { clear: left } .wy-control-group.wy-control-group-no-input .wy-control { margin: 6px 0 0; font-size: 90% } .wy-control-no-input { display: inline-block; margin: 6px 0 0; font-size: 90% } .wy-control-group.fluid-input input[type="text"], .wy-control-group.fluid-input input[type="password"], .wy-control-group.fluid-input input[type="email"], .wy-control-group.fluid-input input[type="url"], .wy-control-group.fluid-input input[type="date"], .wy-control-group.fluid-input input[type="month"], .wy-control-group.fluid-input input[type="time"], .wy-control-group.fluid-input input[type="datetime"], .wy-control-group.fluid-input input[type="datetime-local"], .wy-control-group.fluid-input input[type="week"], .wy-control-group.fluid-input input[type="number"], .wy-control-group.fluid-input input[type="search"], .wy-control-group.fluid-input input[type="tel"], .wy-control-group.fluid-input input[type="color"] { width: 100% } .wy-form-message-inline { display: inline-block; padding-left: .3em; color: #666; vertical-align: middle; font-size: 90% } .wy-form-message { display: block; color: #999; font-size: 70%; margin-top: .3125em; font-style: italic } .wy-form-message p { font-size: inherit; font-style: italic; margin-bottom: 6px } .wy-form-message p:last-child { margin-bottom: 0 } input { line-height: normal } input[type="button"], input[type="reset"], input[type="submit"] { -webkit-appearance: button; cursor: pointer; font-family: "Source Sans Pro","proxima-nova","Helvetica Neue",Arial,sans-serif; *overflow: visible } input[type="text"], input[type="password"], input[type="email"], input[type="url"], input[type="date"], input[type="month"], input[type="time"], input[type="datetime"], input[type="datetime-local"], input[type="week"], input[type="number"], input[type="search"], input[type="tel"], input[type="color"] { -webkit-appearance: none; padding: 6px; display: inline-block; border: 1px solid #ccc; font-size: 80%; font-family: "Source Sans Pro","proxima-nova","Helvetica Neue",Arial,sans-serif; box-shadow: inset 0 1px 3px #ddd; border-radius: 0; -webkit-transition: border .3s linear; -moz-transition: border .3s linear; transition: border .3s linear } input[type="datetime-local"] { padding: .34375em .625em } input[disabled] { cursor: default } input[type="checkbox"], input[type="radio"] { -webkit-box-sizing: border-box; -moz-box-sizing: border-box; box-sizing: border-box; padding: 0; margin-right: .3125em; *height: 13px; *width: 13px } input[type="search"] { -webkit-box-sizing: border-box; -moz-box-sizing: border-box; box-sizing: border-box } input[type="search"]::-webkit-search-cancel-button, input[type="search"]::-webkit-search-decoration { -webkit-appearance: none } input[type="text"]:focus, input[type="password"]:focus, input[type="email"]:focus, input[type="url"]:focus, input[type="date"]:focus, input[type="month"]:focus, input[type="time"]:focus, input[type="datetime"]:focus, input[type="datetime-local"]:focus, input[type="week"]:focus, input[type="number"]:focus, input[type="search"]:focus, input[type="tel"]:focus, input[type="color"]:focus { outline: 0; outline: thin dotted \9; border-color: #333 } input.no-focus:focus { border-color: #ccc!important } input[type="file"]:focus, input[type="radio"]:focus, input[type="checkbox"]:focus { outline: thin dotted #333; outline: 1px auto #129FEA } input[type="text"][disabled], input[type="password"][disabled], input[type="email"][disabled], input[type="url"][disabled], input[type="date"][disabled], input[type="month"][disabled], input[type="time"][disabled], input[type="datetime"][disabled], input[type="datetime-local"][disabled], input[type="week"][disabled], input[type="number"][disabled], input[type="search"][disabled], input[type="tel"][disabled], input[type="color"][disabled] { cursor: not-allowed; background-color: #fafafa } input:focus:invalid, textarea:focus:invalid, select:focus:invalid { color: #E74C3C; border: 1px solid #E74C3C } input:focus:invalid:focus, textarea:focus:invalid:focus, select:focus:invalid:focus { border-color: #E74C3C } input[type="file"]:focus:invalid:focus, input[type="radio"]:focus:invalid:focus, input[type="checkbox"]:focus:invalid:focus { outline-color: #E74C3C } input.wy-input-large { padding: 12px; font-size: 100% } textarea { overflow: auto; vertical-align: top; width: 100%; font-family: "Source Sans Pro","proxima-nova","Helvetica Neue",Arial,sans-serif } select, textarea { padding: .5em .625em; display: inline-block; border: 1px solid #ccc; font-size: 80%; box-shadow: inset 0 1px 3px #ddd; -webkit-transition: border .3s linear; -moz-transition: border .3s linear; transition: border .3s linear } select { border: 1px solid #ccc; background-color: #fff } select[multiple] { height: auto } select:focus, textarea:focus { outline: 0 } select[disabled], textarea[disabled], input[readonly], select[readonly], textarea[readonly] { cursor: not-allowed; background-color: #fafafa } input[type="radio"][disabled], input[type="checkbox"][disabled] { cursor: not-allowed } .wy-checkbox, .wy-radio { margin: 6px 0; color: #404040; display: block } .wy-checkbox input, .wy-radio input { vertical-align: baseline } .wy-form-message-inline { display: inline-block; *display: inline; *zoom: 1; vertical-align: middle } .wy-input-prefix, .wy-input-suffix { white-space: nowrap; padding: 6px } .wy-input-prefix .wy-input-context, .wy-input-suffix .wy-input-context { line-height: 27px; padding: 0 8px; display: inline-block; font-size: 80%; background-color: #f3f6f6; border: solid 1px #ccc; color: #999 } .wy-input-suffix .wy-input-context { border-left: 0 } .wy-input-prefix .wy-input-context { border-right: 0 } .wy-switch { width: 36px; height: 12px; margin: 12px 0; position: relative; border-radius: 4px; background: #ccc; cursor: pointer; -webkit-transition: all .2s ease-in-out; -moz-transition: all .2s ease-in-out; transition: all .2s ease-in-out } .wy-switch:before { position: absolute; content: ""; display: block; width: 18px; height: 18px; border-radius: 4px; background: #999; left: -3px; top: -3px; -webkit-transition: all .2s ease-in-out; -moz-transition: all .2s ease-in-out; transition: all .2s ease-in-out } .wy-switch:after { content: "false"; position: absolute; left: 48px; display: block; font-size: 12px; color: #ccc } .wy-switch.active { background: #1e8449 } .wy-switch.active:before { left: 24px; background: #27AE60 } .wy-switch.active:after { content: "true" } .wy-switch.disabled, .wy-switch.active.disabled { cursor: not-allowed } .wy-control-group.wy-control-group-error .wy-form-message, .wy-control-group.wy-control-group-error>label { color: #E74C3C } .wy-control-group.wy-control-group-error input[type="text"], .wy-control-group.wy-control-group-error input[type="password"], .wy-control-group.wy-control-group-error input[type="email"], .wy-control-group.wy-control-group-error input[type="url"], .wy-control-group.wy-control-group-error input[type="date"], .wy-control-group.wy-control-group-error input[type="month"], .wy-control-group.wy-control-group-error input[type="time"], .wy-control-group.wy-control-group-error input[type="datetime"], .wy-control-group.wy-control-group-error input[type="datetime-local"], .wy-control-group.wy-control-group-error input[type="week"], .wy-control-group.wy-control-group-error input[type="number"], .wy-control-group.wy-control-group-error input[type="search"], .wy-control-group.wy-control-group-error input[type="tel"], .wy-control-group.wy-control-group-error input[type="color"] { border: solid 1px #E74C3C } .wy-control-group.wy-control-group-error textarea { border: solid 1px #E74C3C } .wy-inline-validate { white-space: nowrap } .wy-inline-validate .wy-input-context { padding: .5em .625em; display: inline-block; font-size: 80% } .wy-inline-validate.wy-inline-validate-success .wy-input-context { color: #27AE60 } .wy-inline-validate.wy-inline-validate-danger .wy-input-context { color: #E74C3C } .wy-inline-validate.wy-inline-validate-warning .wy-input-context { color: #E67E22 } .wy-inline-validate.wy-inline-validate-info .wy-input-context { color: #2980B9 } .rotate-90 { -webkit-transform: rotate(90deg); -moz-transform: rotate(90deg); -ms-transform: rotate(90deg); -o-transform: rotate(90deg); transform: rotate(90deg) } .rotate-180 { -webkit-transform: rotate(180deg); -moz-transform: rotate(180deg); -ms-transform: rotate(180deg); -o-transform: rotate(180deg); transform: rotate(180deg) } .rotate-270 { -webkit-transform: rotate(270deg); -moz-transform: rotate(270deg); -ms-transform: rotate(270deg); -o-transform: rotate(270deg); transform: rotate(270deg) } .mirror { -webkit-transform: scaleX(-1); -moz-transform: scaleX(-1); -ms-transform: scaleX(-1); -o-transform: scaleX(-1); transform: scaleX(-1) } .mirror.rotate-90 { -webkit-transform: scaleX(-1) rotate(90deg); -moz-transform: scaleX(-1) rotate(90deg); -ms-transform: scaleX(-1) rotate(90deg); -o-transform: scaleX(-1) rotate(90deg); transform: scaleX(-1) rotate(90deg) } .mirror.rotate-180 { -webkit-transform: scaleX(-1) rotate(180deg); -moz-transform: scaleX(-1) rotate(180deg); -ms-transform: scaleX(-1) rotate(180deg); -o-transform: scaleX(-1) rotate(180deg); transform: scaleX(-1) rotate(180deg) } .mirror.rotate-270 { -webkit-transform: scaleX(-1) rotate(270deg); -moz-transform: scaleX(-1) rotate(270deg); -ms-transform: scaleX(-1) rotate(270deg); -o-transform: scaleX(-1) rotate(270deg); transform: scaleX(-1) rotate(270deg) } @media only screen and (max-width: 480px) { .wy-form button[type="submit"] { margin: .7em 0 0 } .wy-form input[type="text"], .wy-form input[type="password"], .wy-form input[type="email"], .wy-form input[type="url"], .wy-form input[type="date"], .wy-form input[type="month"], .wy-form input[type="time"], .wy-form input[type="datetime"], .wy-form input[type="datetime-local"], .wy-form input[type="week"], .wy-form input[type="number"], .wy-form input[type="search"], .wy-form input[type="tel"], .wy-form input[type="color"] { margin-bottom: .3em; display: block } .wy-form label { margin-bottom: .3em; display: block } .wy-form input[type="password"], .wy-form input[type="email"], .wy-form input[type="url"], .wy-form input[type="date"], .wy-form input[type="month"], .wy-form input[type="time"], .wy-form input[type="datetime"], .wy-form input[type="datetime-local"], .wy-form input[type="week"], .wy-form input[type="number"], .wy-form input[type="search"], .wy-form input[type="tel"], .wy-form input[type="color"] { margin-bottom: 0 } .wy-form-aligned .wy-control-group label { margin-bottom: .3em; text-align: left; display: block; width: 100% } .wy-form-aligned .wy-control { margin: 1.5em 0 0 } .wy-form .wy-help-inline, .wy-form-message-inline, .wy-form-message { display: block; font-size: 80%; padding: 6px 0 } } @media screen and (max-width: 768px) { .tablet-hide { display: none } } @media screen and (max-width: 480px) { .mobile-hide { display: none } } .float-left { float: left } .float-right { float: right } .full-width { width: 100% } .wy-table, .rst-content table.docutils, .rst-content table.field-list { border-collapse: collapse; border-spacing: 0; empty-cells: show; margin-bottom: 24px } .wy-table caption, .rst-content table.docutils caption, .rst-content table.field-list caption { color: #000; font: italic 85%/1 arial,sans-serif; padding: 1em 0; text-align: center } .wy-table td, .rst-content table.docutils td, .rst-content table.field-list td, .wy-table th, .rst-content table.docutils th, .rst-content table.field-list th { font-size: 16px; margin: 0; overflow: visible; padding: 6px 13px } .wy-table td:first-child, .rst-content table.docutils td:first-child, .rst-content table.field-list td:first-child, .wy-table th:first-child, .rst-content table.docutils th:first-child, .rst-content table.field-list th:first-child { border-left-width: 0 } .wy-table thead, .rst-content table.docutils thead, .rst-content table.field-list thead { color: #000; text-align: center; vertical-align: bottom; white-space: nowrap } .wy-table thead th, .rst-content table.docutils thead th, .rst-content table.field-list thead th { font-weight: 700; border-bottom: solid 1px #ddd; } .wy-table td, .rst-content table.docutils td, .rst-content table.field-list td { background-color: transparent; vertical-align: middle } .wy-table td p, .rst-content table.docutils td p, .rst-content table.field-list td p { line-height: 18px } .wy-table td p:last-child, .rst-content table.docutils td p:last-child, .rst-content table.field-list td p:last-child { margin-bottom: 0 } .wy-table .wy-table-cell-min, .rst-content table.docutils .wy-table-cell-min, .rst-content table.field-list .wy-table-cell-min { width: 1%; padding-right: 0 } .wy-table .wy-table-cell-min input[type=checkbox], .rst-content table.docutils .wy-table-cell-min input[type=checkbox], .rst-content table.field-list .wy-table-cell-min input[type=checkbox], .wy-table .wy-table-cell-min input[type=checkbox], .rst-content table.docutils .wy-table-cell-min input[type=checkbox], .rst-content table.field-list .wy-table-cell-min input[type=checkbox] { margin: 0 } .wy-table-secondary { color: gray; font-size: 90% } .wy-table-tertiary { color: gray; font-size: 80% } .wy-table-odd td, .wy-table-striped tr:nth-child(2n-1) td, .rst-content table.docutils:not(.field-list) tr:nth-child(2n-1) td { background-color: #fcfcfc; } .wy-table-odd td, .wy-table-striped tr:nth-child(2n) td, .rst-content table.docutils:not(.field-list) tr:nth-child(2n) td { background-color: #f8f8f8; } .wy-table-backed { background-color: #f8f8f8; } .wy-table-bordered-all, .rst-content table.docutils { border: 1px solid #ddd; line-height: 24px; } .wy-table-bordered-all td, .rst-content table.docutils td { border-bottom: 1px solid #ddd; border-left: 1px solid #ddd; } .wy-table-bordered-all tbody>tr:last-child td, .rst-content table.docutils tbody>tr:last-child td { border-bottom-width: 0 } .wy-table-bordered { border: 1px solid #ddd; } .wy-table-bordered-rows td { border-bottom: 1px solid #ddd; } .wy-table-bordered-rows tbody>tr:last-child td { border-bottom-width: 0 } .wy-table-horizontal tbody>tr:last-child td { border-bottom-width: 0 } .wy-table-horizontal td, .wy-table-horizontal th { border-width: 0 0 1px; border-bottom: 1px solid #ddd; } .wy-table-horizontal tbody>tr:last-child td { border-bottom-width: 0 } .wy-table-responsive { margin-bottom: 24px; max-width: 100%; overflow: auto } .wy-table-responsive table { margin-bottom: 0!important } .wy-table-responsive table td, .wy-table-responsive table th { white-space: nowrap } a { text-decoration: none; cursor: pointer } html { height: 100%; overflow-x: hidden } body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol"; font-weight: 400; color: #333; min-height: 100%; overflow-x: hidden; background: #edf0f2 } .wy-text-left { text-align: left } .wy-text-center { text-align: center } .wy-text-right { text-align: right } .wy-text-large { font-size: 120% } .wy-text-normal { font-size: 100% } .wy-text-small, small { font-size: 80% } .wy-text-strike { text-decoration: line-through } .wy-text-warning { color: #E67E22!important } a.wy-text-warning:hover { color: #eb9950!important } .wy-text-info { color: #2980B9!important } a.wy-text-info:hover { color: #409ad5!important } .wy-text-success { color: #27AE60!important } a.wy-text-success:hover { color: #36d278!important } .wy-text-danger { color: #E74C3C!important } a.wy-text-danger:hover { color: #ed7669!important } .wy-text-neutral { color: #404040!important } a.wy-text-neutral:hover { color: #595959!important } .h1, .h2, .h3, .h4, .h5, .h6, h1, h2, h3, h4, h5, h6, legend { margin-top: 0; font-weight: 700; font-family: "Source Sans Pro","ff-tisa-web-pro","Georgia",Arial,sans-serif; font-style: normal } p { line-height: 24px; margin: 0; font-size: 16px; margin-bottom: 24px } h1, .h1 { font-size: 175% } h2, .h2 { font-size: 150% } h3, .h3 { font-size: 125% } h4, .h4 { font-size: 115% } h5, .h5 { font-size: 110% } h6, .h6 { font-size: 100% } hr { display: block; height: 1px; border: 0; border-top: 1px solid #e1e4e5; margin: 24px 0; padding: 0 } code, .rst-content tt, .rst-content code { max-width: 100%; background: #fff; border: solid 1px #e1e4e5; font-size: 75%; padding: 0 5px; font-family: "Source Code Pro","Andale Mono WT","Andale Mono","Lucida Console","Lucida Sans Typewriter","DejaVu Sans Mono","Bitstream Vera Sans Mono","Liberation Mono","Nimbus Mono L",Monaco,"Courier New",Courier,monospace; color: #E74C3C; overflow-x: auto } code.code-large, .rst-content tt.code-large { font-size: 90% } .wy-plain-list-disc, .rst-content .section ul, .rst-content .toctree-wrapper ul, article ul { list-style: disc; line-height: 24px; margin-bottom: 24px } .wy-plain-list-disc li, .rst-content .section ul li, .rst-content .toctree-wrapper ul li, article ul li { list-style: disc; margin-left: 24px } .wy-plain-list-disc li p:last-child, .rst-content .section ul li p:last-child, .rst-content .toctree-wrapper ul li p:last-child, article ul li p:last-child { margin-bottom: 0 } .wy-plain-list-disc li ul, .rst-content .section ul li ul, .rst-content .toctree-wrapper ul li ul, article ul li ul { margin-bottom: 0 } .wy-plain-list-disc li li, .rst-content .section ul li li, .rst-content .toctree-wrapper ul li li, article ul li li { list-style: circle } .wy-plain-list-disc li li li, .rst-content .section ul li li li, .rst-content .toctree-wrapper ul li li li, article ul li li li { list-style: square } .wy-plain-list-disc li ol li, .rst-content .section ul li ol li, .rst-content .toctree-wrapper ul li ol li, article ul li ol li { list-style: decimal } .wy-plain-list-decimal, .rst-content .section ol, .rst-content ol.arabic, article ol { list-style: decimal; line-height: 24px; margin-bottom: 24px } .wy-plain-list-decimal li, .rst-content .section ol li, .rst-content ol.arabic li, article ol li { list-style: decimal; margin-left: 24px } .wy-plain-list-decimal li p:last-child, .rst-content .section ol li p:last-child, .rst-content ol.arabic li p:last-child, article ol li p:last-child { margin-bottom: 0 } .wy-plain-list-decimal li ul, .rst-content .section ol li ul, .rst-content ol.arabic li ul, article ol li ul { margin-bottom: 0 } .wy-plain-list-decimal li ul li, .rst-content .section ol li ul li, .rst-content ol.arabic li ul li, article ol li ul li { list-style: disc } footer { color: #999 } footer p { margin-bottom: 12px } .rst-footer-buttons { *zoom: 1 } .rst-footer-buttons:before, .rst-footer-buttons:after { display: table; content: "" } .rst-footer-buttons:after { clear: both } #search-results .search li { margin-bottom: 24px; border-bottom: solid 1px #e1e4e5; padding-bottom: 24px } #search-results .search li:first-child { border-top: solid 1px #e1e4e5; padding-top: 24px } #search-results .search li a { font-size: 120%; margin-bottom: 12px; display: inline-block } #search-results .context { color: gray; font-size: 90% } @media print { .rst-versions, footer, .wy-nav-side { display: none } .wy-nav-content-wrap { margin-left: 0 } } .rst-versions { position: fixed; bottom: 0; left: 0; width: 300px; color: #fcfcfc; background: #1f1d1d; border-top: solid 10px #343131; font-family: "Source Sans Pro","proxima-nova","Helvetica Neue",Arial,sans-serif; z-index: 400 } .rst-versions a { color: #2980B9; text-decoration: none } .rst-versions .rst-badge-small { display: none } .rst-versions .rst-current-version { padding: 12px; background-color: #272525; display: block; text-align: right; font-size: 90%; cursor: pointer; color: #27AE60; *zoom: 1 } .rst-versions .rst-current-version:before, .rst-versions .rst-current-version:after { display: table; content: "" } .rst-versions .rst-current-version:after { clear: both } .rst-versions .rst-current-version .fa, .rst-versions .rst-current-version .wy-menu-vertical li span.toctree-expand, .wy-menu-vertical li .rst-versions .rst-current-version span.toctree-expand, .rst-versions .rst-current-version .rst-content .admonition-title, .rst-content .rst-versions .rst-current-version .admonition-title, .rst-versions .rst-current-version .rst-content h1 .headerlink, .rst-content h1 .rst-versions .rst-current-version .headerlink, .rst-versions .rst-current-version .rst-content h2 .headerlink, .rst-content h2 .rst-versions .rst-current-version .headerlink, .rst-versions .rst-current-version .rst-content h3 .headerlink, .rst-content h3 .rst-versions .rst-current-version .headerlink, .rst-versions .rst-current-version .rst-content h4 .headerlink, .rst-content h4 .rst-versions .rst-current-version .headerlink, .rst-versions .rst-current-version .rst-content h5 .headerlink, .rst-content h5 .rst-versions .rst-current-version .headerlink, .rst-versions .rst-current-version .rst-content h6 .headerlink, .rst-content h6 .rst-versions .rst-current-version .headerlink, .rst-versions .rst-current-version .rst-content dl dt .headerlink, .rst-content dl dt .rst-versions .rst-current-version .headerlink, .rst-versions .rst-current-version .rst-content p.caption .headerlink, .rst-content p.caption .rst-versions .rst-current-version .headerlink, .rst-versions .rst-current-version .rst-content tt.download span:first-child, .rst-content tt.download .rst-versions .rst-current-version span:first-child, .rst-versions .rst-current-version .rst-content code.download span:first-child, .rst-content code.download .rst-versions .rst-current-version span:first-child, .rst-versions .rst-current-version .icon { color: #fcfcfc } .rst-versions .rst-current-version .fa-book, .rst-versions .rst-current-version .icon-book { float: left } .rst-versions .rst-current-version .icon-book { float: left } .rst-versions .rst-current-version.rst-out-of-date { background-color: #E74C3C; color: #fff } .rst-versions .rst-current-version.rst-active-old-version { background-color: #F1C40F; color: #000 } .rst-versions.shift-up .rst-other-versions { display: block } .rst-versions .rst-other-versions { font-size: 90%; padding: 12px; color: gray; display: none } .rst-versions .rst-other-versions hr { display: block; height: 1px; border: 0; margin: 20px 0; padding: 0; border-top: solid 1px #413d3d } .rst-versions .rst-other-versions dd { display: inline-block; margin: 0 } .rst-versions .rst-other-versions dd a { display: inline-block; padding: 6px; color: #fcfcfc } .rst-versions.rst-badge { width: auto; bottom: 20px; right: 20px; left: auto; border: none; max-width: 300px } .rst-versions.rst-badge .icon-book { float: none } .rst-versions.rst-badge .fa-book, .rst-versions.rst-badge .icon-book { float: none } .rst-versions.rst-badge.shift-up .rst-current-version { text-align: right } .rst-versions.rst-badge.shift-up .rst-current-version .fa-book, .rst-versions.rst-badge.shift-up .rst-current-version .icon-book { float: left } .rst-versions.rst-badge.shift-up .rst-current-version .icon-book { float: left } .rst-versions.rst-badge .rst-current-version { width: auto; height: 30px; line-height: 30px; padding: 0 6px; display: block; text-align: center } @media screen and (max-width: 768px) { .rst-versions { width: 85%; display: none } .rst-versions.shift { display: block } img { width: 100%; height: auto } } .rst-content img { max-width: 100%; height: auto!important } .rst-content div.figure { margin-bottom: 24px } .rst-content div.figure.align-center { text-align: center } .rst-content .section>img, .rst-content .section>a>img { margin-bottom: 24px } .rst-content blockquote { margin-left: 24px; line-height: 24px; margin-bottom: 24px } .rst-content .note .last, .rst-content .attention .last, .rst-content .caution .last, .rst-content .danger .last, .rst-content .error .last, .rst-content .hint .last, .rst-content .important .last, .rst-content .tip .last, .rst-content .warning .last, .rst-content .seealso .last, .rst-content .admonition-todo .last { margin-bottom: 0 } .rst-content .admonition-title:before { margin-right: 4px } .rst-content .admonition table { border-color: rgba(0,0,0,0.1) } .rst-content .admonition table td, .rst-content .admonition table th { background: transparent!important; border-color: rgba(0,0,0,0.1)!important } .rst-content .section ol.loweralpha, .rst-content .section ol.loweralpha li { list-style: lower-alpha } .rst-content .section ol.upperalpha, .rst-content .section ol.upperalpha li { list-style: upper-alpha } .rst-content .section ol p, .rst-content .section ul p { margin-bottom: 12px } .rst-content .line-block { margin-left: 24px } .rst-content .topic-title { font-weight: 700; margin-bottom: 12px } .rst-content .toc-backref { color: #404040 } .rst-content .align-right { float: right; margin: 0 0 24px 24px } .rst-content .align-left { float: left; margin: 0 24px 24px 0 } .rst-content .align-center { margin: auto; display: block } .rst-content h1 .headerlink, .rst-content h2 .headerlink, .rst-content h3 .headerlink, .rst-content h4 .headerlink, .rst-content h5 .headerlink, .rst-content h6 .headerlink, .rst-content dl dt .headerlink, .rst-content p.caption .headerlink { display: none; visibility: hidden; font-size: 14px } .rst-content h1 .headerlink:after, .rst-content h2 .headerlink:after, .rst-content h3 .headerlink:after, .rst-content h4 .headerlink:after, .rst-content h5 .headerlink:after, .rst-content h6 .headerlink:after, .rst-content dl dt .headerlink:after, .rst-content p.caption .headerlink:after { visibility: visible; content: ""; font-family: FontAwesome; display: inline-block } .rst-content h1:hover .headerlink, .rst-content h2:hover .headerlink, .rst-content h3:hover .headerlink, .rst-content h4:hover .headerlink, .rst-content h5:hover .headerlink, .rst-content h6:hover .headerlink, .rst-content dl dt:hover .headerlink, .rst-content p.caption:hover .headerlink { display: inline-block } .rst-content .sidebar { float: right; width: 40%; display: block; margin: 0 0 24px 24px; padding: 24px; background: #f3f6f6; border: solid 1px #e1e4e5 } .rst-content .sidebar p, .rst-content .sidebar ul, .rst-content .sidebar dl { font-size: 90% } .rst-content .sidebar .last { margin-bottom: 0 } .rst-content .sidebar .sidebar-title { display: block; font-family: "Source Sans Pro","ff-tisa-web-pro","Georgia",Arial,sans-serif; font-weight: 700; background: #e1e4e5; padding: 6px 12px; margin: -24px; margin-bottom: 24px; font-size: 100% } .rst-content .highlighted { background: #F1C40F; display: inline-block; font-weight: 700; padding: 0 6px } .rst-content .footnote-reference, .rst-content .citation-reference { vertical-align: super; font-size: 90% } .rst-content table.docutils.citation, .rst-content table.docutils.footnote { background: none; border: none; color: #999 } .rst-content table.docutils.citation td, .rst-content table.docutils.citation tr, .rst-content table.docutils.footnote td, .rst-content table.docutils.footnote tr { border: none; background-color: transparent!important; white-space: normal } .rst-content table.docutils.citation td.label, .rst-content table.docutils.footnote td.label { padding-left: 0; padding-right: 0; vertical-align: top } .rst-content table.field-list { border: none } .rst-content table.field-list td { border: none; padding-top: 5px } .rst-content table.field-list td>strong { display: inline-block; margin-top: 3px } .rst-content table.field-list .field-name { padding-right: 10px; text-align: left; white-space: nowrap } .rst-content table.field-list .field-body { text-align: left; padding-left: 0 } .rst-content tt, .rst-content tt, .rst-content code { color: #000 } .rst-content tt big, .rst-content tt em, .rst-content tt big, .rst-content code big, .rst-content tt em, .rst-content code em { font-size: 100%!important; line-height: normal } .rst-content tt .xref, a .rst-content tt, .rst-content tt .xref, .rst-content code .xref, a .rst-content tt, a .rst-content code { font-weight: 700 } .rst-content a tt, .rst-content a tt, .rst-content a code { color: #2980B9 } .rst-content dl { margin-bottom: 24px } .rst-content dl dt { font-weight: 700 } .rst-content dl p, .rst-content dl table, .rst-content dl ul, .rst-content dl ol { margin-bottom: 12px!important } .rst-content dl dd { margin: 0 0 12px 24px } .rst-content dl:not(.docutils) { margin-bottom: 24px } .rst-content dl:not(.docutils) dt { display: inline-block; margin: 6px 0; font-size: 90%; line-height: normal; background: #e7f2fa; color: #2980B9; border-top: solid 3px #6ab0de; padding: 6px; position: relative } .rst-content dl:not(.docutils) dt:before { color: #6ab0de } .rst-content dl:not(.docutils) dt .headerlink { color: #404040; font-size: 100%!important } .rst-content dl:not(.docutils) dl dt { margin-bottom: 6px; border: none; border-left: solid 3px #ccc; background: #f0f0f0; color: gray } .rst-content dl:not(.docutils) dl dt .headerlink { color: #404040; font-size: 100%!important } .rst-content dl:not(.docutils) dt:first-child { margin-top: 0 } .rst-content dl:not(.docutils) tt, .rst-content dl:not(.docutils) tt, .rst-content dl:not(.docutils) code { font-weight: 700 } .rst-content dl:not(.docutils) tt.descname, .rst-content dl:not(.docutils) tt.descclassname, .rst-content dl:not(.docutils) tt.descname, .rst-content dl:not(.docutils) code.descname, .rst-content dl:not(.docutils) tt.descclassname, .rst-content dl:not(.docutils) code.descclassname { background-color: transparent; border: none; padding: 0; font-size: 100%!important } .rst-content dl:not(.docutils) tt.descname, .rst-content dl:not(.docutils) tt.descname, .rst-content dl:not(.docutils) code.descname { font-weight: 700 } .rst-content dl:not(.docutils) .optional { display: inline-block; padding: 0 4px; color: #000; font-weight: 700 } .rst-content dl:not(.docutils) .property { display: inline-block; padding-right: 8px } .rst-content .viewcode-link, .rst-content .viewcode-back { display: inline-block; color: #27AE60; font-size: 80%; padding-left: 24px } .rst-content .viewcode-back { display: block; float: right } .rst-content p.rubric { margin-bottom: 12px; font-weight: 700 } .rst-content tt.download, .rst-content code.download { background: inherit; padding: inherit; font-family: inherit; font-size: inherit; color: inherit; border: inherit; white-space: inherit } .rst-content tt.download span:first-child:before, .rst-content code.download span:first-child:before { margin-right: 4px } @media screen and (max-width: 480px) { .rst-content .sidebar { width: 100% } } span[id*='MathJax-Span'] { color: #404040 } .math { text-align: center } @font-face { font-family: 'Source Sans Pro'; font-style: normal; font-weight: 100; src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_sans_pro/SourceSansPro-ExtraLight.ttf) format("truetype") } @font-face { font-family: 'Source Sans Pro'; font-style: italic; font-weight: 100; src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_sans_pro/SourceSansPro-ExtraLightItalic.ttf) format("truetype") } @font-face { font-family: 'Source Sans Pro'; font-style: normal; font-weight: 300; src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_sans_pro/SourceSansPro-Light.ttf) format("truetype") } @font-face { font-family: 'Source Sans Pro'; font-style: italic; font-weight: 300; src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_sans_pro/SourceSansPro-LightItalic.ttf) format("truetype") } @font-face { font-family: 'Source Sans Pro'; font-style: normal; font-weight: 400; src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_sans_pro/SourceSansPro-Regular.ttf) format("truetype") } @font-face { font-family: 'Source Sans Pro'; font-style: italic; font-weight: 400; src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_sans_pro/SourceSansPro-Italic.ttf) format("truetype") } @font-face { font-family: 'Source Sans Pro'; font-style: normal; font-weight: 500; src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_sans_pro/SourceSansPro-Semibold.ttf) format("truetype") } @font-face { font-family: 'Source Sans Pro'; font-style: italic; font-weight: 500; src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_sans_pro/SourceSansPro-SemiboldItalic.ttf) format("truetype") } @font-face { font-family: 'Source Sans Pro'; font-style: normal; font-weight: 700; src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_sans_pro/SourceSansPro-Bold.ttf) format("truetype") } @font-face { font-family: 'Source Sans Pro'; font-style: italic; font-weight: 700; src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_sans_pro/SourceSansPro-BoldItalic.ttf) format("truetype") } @font-face { font-family: 'Source Code Pro'; font-style: normal; font-weight: 300; src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_code_pro/SourceCodePro-Light.ttf) format("truetype") } @font-face { font-family: 'Source Code Pro'; font-style: normal; font-weight: 400; src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_code_pro/SourceCodePro-Regular.ttf) format("truetype") } @font-face { font-family: 'Source Code Pro'; font-style: normal; font-weight: 500; src: url(https://cdn.crashlytics.io/static_assets/web/typefaces/source_code_pro/SourceCodePro-Medium.ttf) format("truetype") } ================================================ FILE: docs/css/goreplay.css ================================================ code { font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace !important; border-bottom-left-radius: 3px; border-bottom-right-radius: 3px; border-top-left-radius: 3px; border-top-right-radius: 3px; background-color: #f7f7f7 !important; color: #333 !important; font-size: 14px !important; border: none !important; padding: 5px !important; } pre > code { line-height: 20px; padding: 16px !important; } blockquote { color: #777 !important; border-left: 4px solid rgb(221, 221, 221); padding-left: 16px; padding-right: 16px; margin-left: 0px !important; } /* Visited links purple is not a good idea */ a:visited { color: #008BF3; } a { color: #008BF3; } h3, h4, h5, h6 { color: #333 !important; } .rst-versions { display: none !important; } /* The next and previous button we don't need */ .rst-footer-buttons { display: none; } /* Foldable parts */ details > * { margin-left: 20px; } details > summary { margin-left: 0px; /* To overwrite the margin from above */ font-size: 120%; cursor: pointer } /* Fix the design not having the correct spacing */ .wy-menu-vertical .subnav li.current > a { padding: 0.4045em 2.427em; } .toctree-l3 { padding-left: 1.0em; } footer .fastlane { margin: 20px 0; } footer .fastlane iframe { vertical-align: middle; } /* Custom Syntax highlighting to look more like GitHub.com */ .hljs-symbol { color: #0086b3; } .hljs-keyword { font-weight: normal; color: #a71d5d; } .hljs-string { color: #183691; } /* Header Anchors */ /* From https://github.com/facelessuser/pymdown-extensions/blob/bf18d0635e9d91b0f98eacdcaa10f26e0ace0f20/doc_theme/css/theme_custom.css#L322-L395 */ .rst-content .headeranchor-link { position: absolute; top: 0; bottom: 0; left: 0; display: block; padding-right: 6px; padding-left: 30px; margin-left: -30px; } .rst-content .headeranchor-link:focus { outline: none; } .rst-content h1, .rst-content h2, .rst-content h3, .rst-content h4, .rst-content h5, .rst-content h6 { position: relative; } .rst-content h1 .headeranchor, .rst-content h2 .headeranchor, .rst-content h3 .headeranchor, .rst-content h4 .headeranchor, .rst-content h5 .headeranchor, .rst-content h6 .headeranchor { display: none; color: #000; vertical-align: middle; } .rst-content h1:hover .headeranchor-link, .rst-content h2:hover .headeranchor-link, .rst-content h3:hover .headeranchor-link, .rst-content h4:hover .headeranchor-link, .rst-content h5:hover .headeranchor-link, .rst-content h6:hover .headeranchor-link { height: 1em; padding-left: 8px; margin-left: -30px; text-decoration: none; } .rst-content h1:hover .headeranchor-link .headeranchor, .rst-content h2:hover .headeranchor-link .headeranchor, .rst-content h3:hover .headeranchor-link .headeranchor, .rst-content h4:hover .headeranchor-link .headeranchor, .rst-content h5:hover .headeranchor-link .headeranchor, .rst-content h6:hover .headeranchor-link .headeranchor { display: inline-block; } .rst-content .headeranchor { font: normal normal 16px FontAwesome; line-height: 1; display: inline-block; text-decoration: none; -webkit-font-smoothing: antialiased; -moz-osx-font-smoothing: grayscale; -webkit-user-select: none; -moz-user-select: none; -ms-user-select: none; user-select: none; } .rst-content .headeranchor:before { content: '\f0c1'; } /* index.md badges */ @media screen and (max-width: 768px) { .badge img { width: auto; } } ================================================ FILE: docs/css/sidenav.css ================================================ .wy-affix { position: fixed; top: 1.618em; } .wy-menu a:hover { text-decoration: none; } .wy-menu-vertical header, .wy-menu-vertical p.caption { height: 32px; display: inline-block; line-height: 32px; padding: 0 1.618em; margin-bottom: 0; margin-top: 14px; display: block; font-weight: 700; text-transform: uppercase; font-size: 85%; color: #ccc; white-space: nowrap; } .wy-menu-vertical span { color: #666; } .wy-menu-vertical ul { margin-bottom: 0; } .wy-menu-vertical li.divide-top { border-top: solid 1px #404040; } .wy-menu-vertical li.divide-bottom { border-bottom: solid 1px #404040; } .wy-menu-vertical li.current { background-color: #e5e5e5; } .wy-menu-vertical li.current a { color: rgba(0, 93, 255, 0.7); border-right: none; } .wy-menu-vertical li.current a:hover { color: rgba(0, 93, 255, 0.9); } .wy-menu-vertical li code, .wy-menu-vertical li .rst-content tt, .rst-content .wy-menu-vertical li tt { border: none; background: inherit; color: inherit; padding-left: 0; padding-right: 0 } .wy-menu-vertical li span.toctree-expand { display: block; float: left; margin-left: -1.2em; font-size: .8em; line-height: 1.6em; color: #999; } .wy-menu-vertical li.on a, .wy-menu-vertical li.current>a { color: rgba(0, 93, 255, 0.9); font-weight: 700; position: relative; background: #fafafa; border: none; } /*.wy-menu-vertical li.on a:hover span.toctree-expand, .wy-menu-vertical li.current>a:hover span.toctree-expand { color: gray; }*/ .wy-menu-vertical li.on a span.toctree-expand, .wy-menu-vertical li.current>a span.toctree-expand { display: block; font-size: .8em; line-height: 1.6em; color: #333; } .wy-menu-vertical li.toctree-l1.current li.toctree-l2>ul, .wy-menu-vertical li.toctree-l2.current li.toctree-l3>ul { display: none; } .wy-menu-vertical li.toctree-l1.current li.toctree-l2.current>ul, .wy-menu-vertical li.toctree-l2.current li.toctree-l3.current>ul { display: block; } .wy-menu-vertical li.toctree-l2.current li.toctree-l3>a { display: block; padding: .4045em 4.045em; } .wy-menu-vertical li.toctree-l2 a:hover span.toctree-expand { color: #999; } .wy-menu-vertical li.toctree-l2 span.toctree-expand { color: #999; } .wy-menu-vertical li.toctree-l3 { background-color: #eee; font-size: .9em; } .wy-menu-vertical li.toctree-l3.current>a { padding: .4045em 4.045em; } .wy-menu-vertical li.toctree-l3.current li.toctree-l4>a { display: block; padding: .4045em 5.663em; border-top: none; border-bottom: none; } .wy-menu-vertical li.toctree-l3 a:hover span.toctree-expand { color: rgba(0, 93, 255, 0.9); } .wy-menu-vertical li.toctree-l3 span.toctree-expand { color: #999; } .wy-menu-vertical li.toctree-l4 { font-size: .9em; } .wy-menu-vertical li.current ul { display: block; } .wy-menu-vertical .local-toc li ul { display: block; } .wy-menu-vertical li ul li a { margin-bottom: 0; color: rgba(0, 93, 255, 0.7); font-weight: 400; } .wy-menu-vertical a { display: inline-block; line-height: 18px; padding: .4045em 1.618em; display: block; position: relative; font-size: 90%; color: rgba(0, 93, 255, 0.7); } .wy-menu-vertical li.on a:hover, .wy-menu-vertical li.current>a:hover { background-color: #fafafa; } .wy-menu-vertical a:hover { color: rgba(0, 93, 255, 0.9); cursor: pointer; background-color: #fafafa; } .wy-menu-vertical a:hover span.toctree-expand { color: rgba(0, 93, 255, 0.5); } .wy-menu-vertical a:active span.toctree-expand { color: rgba(0, 93, 255, 0.7); } /* Search */ .wy-side-nav-search { z-index: 200; background-color: #fafafa; border-bottom: #333; text-align: center; padding: .809em; display: block; color: #333; margin-bottom: .809em } .wy-side-nav-search input[type=text] { width: 100%; color: #333; border-radius: 3px; outline: 0; padding: 10px; background-color: #fff; border: solid 1px #6d6d6d; box-shadow: none } .wy-side-nav-search img { display: block; margin: auto auto .809em; height: 45px; width: 45px; background-color: #2980B9; padding: 5px; border-radius: 100% } .wy-side-nav-search>a, .wy-side-nav-search .wy-dropdown>a { color: #333; font-size: 100%; font-weight: 700; display: inline-block; padding: 4px 6px; margin-bottom: .809em } .wy-side-nav-search>a:hover, .wy-side-nav-search .wy-dropdown>a:hover { background: rgba(255,255,255,0.1) } .wy-side-nav-search>a img.logo, .wy-side-nav-search .wy-dropdown>a img.logo { display: block; margin: 0 auto; height: auto; width: auto; border-radius: 0; max-width: 100%; background: transparent } .wy-side-nav-search>a.icon img.logo, .wy-side-nav-search .wy-dropdown>a.icon img.logo { margin-top: .85em } .wy-nav .wy-menu-vertical header { color: #2980B9 } .wy-nav .wy-menu-vertical a { color: #b3b3b3 } .wy-nav .wy-menu-vertical a:hover { background-color: #2980B9; color: #fff } [data-menu-wrap] { -webkit-transition: all .2s ease-in; -moz-transition: all .2s ease-in; transition: all .2s ease-in; position: absolute; opacity: 1; width: 100%; opacity: 0 } [data-menu-wrap].move-center { left: 0; right: auto; opacity: 1 } [data-menu-wrap].move-left { right: auto; left: -100%; opacity: 0 } [data-menu-wrap].move-right { right: -100%; left: auto; opacity: 0 } .wy-body-for-nav { background: left repeat-y #fcfcfc; background-image: url(data:image/png; base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyRpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMy1jMDExIDY2LjE0NTY2MSwgMjAxMi8wMi8wNi0xNDo1NjoyNyAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENTNiAoTWFjaW50b3NoKSIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDoxOERBMTRGRDBFMUUxMUUzODUwMkJCOThDMEVFNURFMCIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDoxOERBMTRGRTBFMUUxMUUzODUwMkJCOThDMEVFNURFMCI+IDx4bXBNTTpEZXJpdmVkRnJvbSBzdFJlZjppbnN0YW5jZUlEPSJ4bXAuaWlkOjE4REExNEZCMEUxRTExRTM4NTAyQkI5OEMwRUU1REUwIiBzdFJlZjpkb2N1bWVudElEPSJ4bXAuZGlkOjE4REExNEZDMEUxRTExRTM4NTAyQkI5OEMwRUU1REUwIi8+IDwvcmRmOkRlc2NyaXB0aW9uPiA8L3JkZjpSREY+IDwveDp4bXBtZXRhPiA8P3hwYWNrZXQgZW5kPSJyIj8+EwrlwAAAAA5JREFUeNpiMDU0BAgwAAE2AJgB9BnaAAAAAElFTkSuQmCC); background-size: 300px 1px } .wy-grid-for-nav { position: absolute; width: 100%; height: 100% } .wy-nav-side { position: fixed; top: 0; bottom: 0; left: 0; padding-bottom: 2em; width: 300px; overflow-x: hidden; overflow-y: scroll; min-height: 100%; background-color: #fafafa; z-index: 200; border-right: 2px solid #eee; } .wy-nav-top { display: none; background-color: #333; color: #fff; padding: .4045em .809em; position: relative; line-height: 50px; text-align: center; font-size: 100%; *zoom: 1 } .wy-nav-top:before, .wy-nav-top:after { display: table; content: "" } .wy-nav-top:after { clear: both } .wy-nav-top a { color: #fff; font-weight: 700 } .wy-nav-top img { margin-right: 12px; height: 45px; width: 45px; background-color: #2980B9; padding: 5px; border-radius: 100% } .wy-nav-top i { font-size: 30px; line-height: 50px; float: left; cursor: pointer } .wy-nav-content-wrap { margin-left: 300px; background: #fcfcfc; min-height: 100% } .wy-nav-content { padding: 1.618em 3.236em; height: 100%; max-width: 1100px; margin: auto } .wy-body-mask { position: fixed; width: 100%; height: 100%; background: rgba(0,0,0,0.2); display: none; z-index: 499 } .wy-body-mask.on { display: block } @media screen and (max-width: 768px) { .wy-body-for-nav { background: #fcfcfc } .wy-nav-top { display: block } .wy-nav-side { left: -300px } .wy-nav-side.shift { width: 85%; left: 0 } .wy-nav-content-wrap { margin-left: 0 } .wy-nav-content-wrap .wy-nav-content { padding: 1.618em } .wy-nav-content-wrap.shift { position: fixed; min-width: 100%; left: 85%; top: 0; height: 100%; overflow: hidden } } @media screen and (min-width: 1400px) { .wy-nav-content { margin: 0; background: #fcfcfc } } ================================================ FILE: docs/getting-started/basics.md ================================================ ### Overview Gor architecture tries to follow UNIX philosophy: everything made of pipes, various inputs multiplexing data to outputs. You can [rate limit](/rate-limiting.md), [filter](request-filtering.md), [rewrite](request-rewriting.md) requests or even use your own [middleware](middleware.md) to implement custom logic. Also, it is possible to replay requests at the higher rate for [load testing](saving-and-replaying-from-file.md). ### Available inputs * `--input-raw` - used to capture HTTP traffic, you should specify IP address or interface and application port. More about [[Capturing and replaying traffic]]. * `--input-file` - accepts file which previously was recorded using ` --output-file`. More about [[Saving and Replaying from file]] * `--input-tcp` - used by Gor aggregation instance if you decided forward traffic from multiple forwarder Gor instances to it. Read about using [[Aggregator-forwarder setup]]. ### Available outputs * `--output-http` - replay HTTP traffic to given endpoint, accepts base url. Read [more about it](Replaying HTTP traffic) * `--output-file` - records incoming traffic to the file. More about [[Saving and Replaying from file]] * `--output-tcp` - forward incoming data to another Gor instance, used in conjunction with `--input-tcp`. Read more about [[Aggregator-forwarder setup]]. * `--output-stdout` - used for debugging, outputs all data to stdout. ================================================ FILE: docs/getting-started/tutorial.md ================================================ ### Dependencies To start working with Gor, you need to have a web server running on your machine, and a terminal to run commands. If you are just poking around, you can quickly start the server by calling `gor file-server :8000`, this will start a simple file server of the current directory on port `8000`. ### Installing Gor Download the latest Gor binary from https://github.com/buger/gor/releases (we provide precompiled binaries for Windows, Linux x64 and Mac OS), or you can compile by yourself [[Compilation]]. Once the archive is downloaded and uncompressed, you can run Gor from the current directory, or you may want to copy binary to your PATH (for Linux and Mac OS it can be `/usr/local/bin`). ### Capturing web traffic Now run this command in terminal: `sudo ./gor --input-raw :8000 --output-stdout` This command says to listen for all network activity happening on port 8000 and log it to stdout. If you are familiar with `tcpdump`, we are going to implement similar functionality. > You may notice that it uses `sudo` and asks for the password: to analyze network, Gor needs permissions which are available only to super users. > However, it is possible to configure Gor [being run for non-root users](Running-as-non-root-user). Make a few requests by opening `http://localhost:8000` in your browser, or just by calling curl in terminal `curl http://localhost:8000`. You should see that `gor` outputs all the HTTP requests and responses right to the terminal window where it is running. **Gor is not a proxy:** you do not need to put 3-rd party tool to your critical path. Instead Gor just silently analyzes the traffic of your application and does not affect it anyhow. ### Replaying Now it's time to replay your original traffic to another environment. Let's start the same file web server but on a different port: `gor file-server :8001`. Instead of `--output-stdout` we will use `--output-http` and provide URL of second server: `sudo ./gor --input-raw :8000 --output-http="http://localhost:8001"` Make few requests to first server. You should see them replicated to the second one, voila! ### Saving requests to file and replaying them later Sometimes it's not possible to replay requests in real time; Gor allows you to save requests to the file and replay them later. First use `--output-file` to save them: `sudo ./gor --input-raw :8000 --output-file=requests.gor`. This will create new file and continuously write all captured requests to it. Let's re-run Gor, but now to replay requests from file: `./gor --input-file requests.gor --output-http="http://localhost:8001"`. You should see all the recorded requests coming to the second server, and they will be replayed in the same order and with exactly same timing as they were recorded. Next: [[The Basics]] ### Watch an overview: ![YOUTUBE](https://www.youtube.com/watch?v=CxuKZcMKaW4) ================================================ FILE: docs/index.md ================================================ Gor is an open-source tool for capturing and replaying live HTTP traffic into a test environment in order to continuously test your system with real data. It can be used to increase confidence in code deployments, configuration changes and infrastructure changes. Read for more info: * [[Getting Started]] * [[The Basics]] * [[Capturing and replaying traffic]] * [[Replaying HTTP traffic]] * [[[PRO] Replaying Binary protocols]] * [[[PRO] Recording and replaying keep alive TCP sessions]] * [[Saving and Replaying from file]] * [Performance testing](https://github.com/buger/gor/wiki/Saving-and-Replaying-from-file#performance-testing) * [[Rate limiting]] * [[Request filtering]] * [[Request rewriting]] * [[Middleware]] * [[Distributed configuration]] * [[Exporting to ElasticSearch]] * [[FAQ]] * [[Troubleshooting]] ## Commercial Aspects * [[Commercial Support]] * [[Commercial FAQ]] * [[Commercial collaboration]] Next: [Getting Started](Getting-Started) ================================================ FILE: docs/js/base.js ================================================ $(document).ready(function () { $('img[alt="YOUTUBE"]').each(function () { var id = $(this).attr('src').split('/')[$(this).attr('src').split('/').length - 1].split("=")[1]; var video = ''; $(this).replaceWith(video); }); }); ================================================ FILE: docs/js/turbolinks.js ================================================ /* Turbolinks 5.0.0 Copyright © 2016 Basecamp, LLC */ (function(){(function(){(function(){this.Turbolinks={supported:function(){return null!=window.history.pushState&&null!=window.requestAnimationFrame}(),visit:function(e,r){return t.controller.visit(e,r)},clearCache:function(){return t.controller.clearCache()}}}).call(this)}).call(this);var t=this.Turbolinks;(function(){(function(){var e,r;t.copyObject=function(t){var e,r,n;r={};for(e in t)n=t[e],r[e]=n;return r},t.closest=function(t,r){return e.call(t,r)},e=function(){var t,e;return t=document.documentElement,null!=(e=t.closest)?e:function(t){var e;for(e=this;e;){if(e.nodeType===Node.ELEMENT_NODE&&r.call(e,t))return e;e=e.parentNode}}}(),t.defer=function(t){return setTimeout(t,1)},t.dispatch=function(t,e){var r,n,o,i,s;return i=null!=e?e:{},s=i.target,r=i.cancelable,n=i.data,o=document.createEvent("Events"),o.initEvent(t,!0,r===!0),o.data=null!=n?n:{},(null!=s?s:document).dispatchEvent(o),o},t.match=function(t,e){return r.call(t,e)},r=function(){var t,e,r,n;return t=document.documentElement,null!=(e=null!=(r=null!=(n=t.matchesSelector)?n:t.webkitMatchesSelector)?r:t.msMatchesSelector)?e:t.mozMatchesSelector}(),t.uuid=function(){var t,e,r;for(r="",t=e=1;36>=e;t=++e)r+=9===t||14===t||19===t||24===t?"-":15===t?"4":20===t?(Math.floor(4*Math.random())+8).toString(16):Math.floor(15*Math.random()).toString(16);return r}}).call(this),function(){t.Location=function(){function t(t){var e,r;null==t&&(t=""),r=document.createElement("a"),r.href=t.toString(),this.absoluteURL=r.href,e=r.hash.length,2>e?this.requestURL=this.absoluteURL:(this.requestURL=this.absoluteURL.slice(0,-e),this.anchor=r.hash.slice(1))}var e,r,n,o;return t.wrap=function(t){return t instanceof this?t:new this(t)},t.prototype.getOrigin=function(){return this.absoluteURL.split("/",3).join("/")},t.prototype.getPath=function(){var t,e;return null!=(t=null!=(e=this.absoluteURL.match(/\/\/[^\/]*(\/[^?;]*)/))?e[1]:void 0)?t:"/"},t.prototype.getPathComponents=function(){return this.getPath().split("/").slice(1)},t.prototype.getLastPathComponent=function(){return this.getPathComponents().slice(-1)[0]},t.prototype.getExtension=function(){var t,e;return null!=(t=null!=(e=this.getLastPathComponent().match(/\.[^.]*$/))?e[0]:void 0)?t:""},t.prototype.isHTML=function(){return this.getExtension().match(/^(?:|\.(?:htm|html|xhtml))$/)},t.prototype.isPrefixedBy=function(t){var e;return e=r(t),this.isEqualTo(t)||o(this.absoluteURL,e)},t.prototype.isEqualTo=function(t){return this.absoluteURL===(null!=t?t.absoluteURL:void 0)},t.prototype.toCacheKey=function(){return this.requestURL},t.prototype.toJSON=function(){return this.absoluteURL},t.prototype.toString=function(){return this.absoluteURL},t.prototype.valueOf=function(){return this.absoluteURL},r=function(t){return e(t.getOrigin()+t.getPath())},e=function(t){return n(t,"/")?t:t+"/"},o=function(t,e){return t.slice(0,e.length)===e},n=function(t,e){return t.slice(-e.length)===e},t}()}.call(this),function(){var e=function(t,e){return function(){return t.apply(e,arguments)}};t.HttpRequest=function(){function r(r,n,o){this.delegate=r,this.requestCanceled=e(this.requestCanceled,this),this.requestTimedOut=e(this.requestTimedOut,this),this.requestFailed=e(this.requestFailed,this),this.requestLoaded=e(this.requestLoaded,this),this.requestProgressed=e(this.requestProgressed,this),this.url=t.Location.wrap(n).requestURL,this.referrer=t.Location.wrap(o).absoluteURL,this.createXHR()}return r.NETWORK_FAILURE=0,r.TIMEOUT_FAILURE=-1,r.timeout=60,r.prototype.send=function(){var t;return this.xhr&&!this.sent?(this.notifyApplicationBeforeRequestStart(),this.setProgress(0),this.xhr.send(),this.sent=!0,"function"==typeof(t=this.delegate).requestStarted?t.requestStarted():void 0):void 0},r.prototype.cancel=function(){return this.xhr&&this.sent?this.xhr.abort():void 0},r.prototype.requestProgressed=function(t){return t.lengthComputable?this.setProgress(t.loaded/t.total):void 0},r.prototype.requestLoaded=function(){return this.endRequest(function(t){return function(){var e;return 200<=(e=t.xhr.status)&&300>e?t.delegate.requestCompletedWithResponse(t.xhr.responseText,t.xhr.getResponseHeader("Turbolinks-Location")):(t.failed=!0,t.delegate.requestFailedWithStatusCode(t.xhr.status,t.xhr.responseText))}}(this))},r.prototype.requestFailed=function(){return this.endRequest(function(t){return function(){return t.failed=!0,t.delegate.requestFailedWithStatusCode(t.constructor.NETWORK_FAILURE)}}(this))},r.prototype.requestTimedOut=function(){return this.endRequest(function(t){return function(){return t.failed=!0,t.delegate.requestFailedWithStatusCode(t.constructor.TIMEOUT_FAILURE)}}(this))},r.prototype.requestCanceled=function(){return this.endRequest()},r.prototype.notifyApplicationBeforeRequestStart=function(){return t.dispatch("turbolinks:request-start",{data:{url:this.url,xhr:this.xhr}})},r.prototype.notifyApplicationAfterRequestEnd=function(){return t.dispatch("turbolinks:request-end",{data:{url:this.url,xhr:this.xhr}})},r.prototype.createXHR=function(){return this.xhr=new XMLHttpRequest,this.xhr.open("GET",this.url,!0),this.xhr.timeout=1e3*this.constructor.timeout,this.xhr.setRequestHeader("Accept","text/html, application/xhtml+xml"),this.xhr.setRequestHeader("Turbolinks-Referrer",this.referrer),this.xhr.onprogress=this.requestProgressed,this.xhr.onload=this.requestLoaded,this.xhr.onerror=this.requestFailed,this.xhr.ontimeout=this.requestTimedOut,this.xhr.onabort=this.requestCanceled},r.prototype.endRequest=function(t){return this.xhr?(this.notifyApplicationAfterRequestEnd(),null!=t&&t.call(this),this.destroy()):void 0},r.prototype.setProgress=function(t){var e;return this.progress=t,"function"==typeof(e=this.delegate).requestProgressed?e.requestProgressed(this.progress):void 0},r.prototype.destroy=function(){var t;return this.setProgress(1),"function"==typeof(t=this.delegate).requestFinished&&t.requestFinished(),this.delegate=null,this.xhr=null},r}()}.call(this),function(){var e=function(t,e){return function(){return t.apply(e,arguments)}};t.ProgressBar=function(){function t(){this.trickle=e(this.trickle,this),this.stylesheetElement=this.createStylesheetElement(),this.progressElement=this.createProgressElement()}var r;return r=300,t.defaultCSS=".turbolinks-progress-bar {\n position: fixed;\n display: block;\n top: 0;\n left: 0;\n height: 3px;\n background: #0076ff;\n z-index: 9999;\n transition: width "+r+"ms ease-out, opacity "+r/2+"ms "+r/2+"ms ease-in;\n transform: translate3d(0, 0, 0);\n}",t.prototype.show=function(){return this.visible?void 0:(this.visible=!0,this.installStylesheetElement(),this.installProgressElement(),this.startTrickling())},t.prototype.hide=function(){return this.visible&&!this.hiding?(this.hiding=!0,this.fadeProgressElement(function(t){return function(){return t.uninstallProgressElement(),t.stopTrickling(),t.visible=!1,t.hiding=!1}}(this))):void 0},t.prototype.setValue=function(t){return this.value=t,this.refresh()},t.prototype.installStylesheetElement=function(){return document.head.insertBefore(this.stylesheetElement,document.head.firstChild)},t.prototype.installProgressElement=function(){return this.progressElement.style.width=0,this.progressElement.style.opacity=1,document.documentElement.insertBefore(this.progressElement,document.body),this.refresh()},t.prototype.fadeProgressElement=function(t){return this.progressElement.style.opacity=0,setTimeout(t,1.5*r)},t.prototype.uninstallProgressElement=function(){return this.progressElement.parentNode?document.documentElement.removeChild(this.progressElement):void 0},t.prototype.startTrickling=function(){return null!=this.trickleInterval?this.trickleInterval:this.trickleInterval=setInterval(this.trickle,r)},t.prototype.stopTrickling=function(){return clearInterval(this.trickleInterval),this.trickleInterval=null},t.prototype.trickle=function(){return this.setValue(this.value+Math.random()/100)},t.prototype.refresh=function(){return requestAnimationFrame(function(t){return function(){return t.progressElement.style.width=10+90*t.value+"%"}}(this))},t.prototype.createStylesheetElement=function(){var t;return t=document.createElement("style"),t.type="text/css",t.textContent=this.constructor.defaultCSS,t},t.prototype.createProgressElement=function(){var t;return t=document.createElement("div"),t.className="turbolinks-progress-bar",t},t}()}.call(this),function(){var e=function(t,e){return function(){return t.apply(e,arguments)}};t.BrowserAdapter=function(){function r(r){this.controller=r,this.showProgressBar=e(this.showProgressBar,this),this.progressBar=new t.ProgressBar}var n,o,i,s;return s=t.HttpRequest,n=s.NETWORK_FAILURE,i=s.TIMEOUT_FAILURE,o=500,r.prototype.visitProposedToLocationWithAction=function(t,e){return this.controller.startVisitToLocationWithAction(t,e)},r.prototype.visitStarted=function(t){return t.issueRequest(),t.changeHistory(),t.loadCachedSnapshot()},r.prototype.visitRequestStarted=function(t){return this.progressBar.setValue(0),t.hasCachedSnapshot()||"restore"!==t.action?this.showProgressBarAfterDelay():this.showProgressBar()},r.prototype.visitRequestProgressed=function(t){return this.progressBar.setValue(t.progress)},r.prototype.visitRequestCompleted=function(t){return t.loadResponse()},r.prototype.visitRequestFailedWithStatusCode=function(t,e){switch(e){case n:case i:return this.reload();default:return t.loadResponse()}},r.prototype.visitRequestFinished=function(t){return this.hideProgressBar()},r.prototype.visitCompleted=function(t){return t.followRedirect()},r.prototype.pageInvalidated=function(){return this.reload()},r.prototype.showProgressBarAfterDelay=function(){return this.progressBarTimeout=setTimeout(this.showProgressBar,o)},r.prototype.showProgressBar=function(){return this.progressBar.show()},r.prototype.hideProgressBar=function(){return this.progressBar.hide(),clearTimeout(this.progressBarTimeout)},r.prototype.reload=function(){return window.location.reload()},r}()}.call(this),function(){var e,r=function(t,e){return function(){return t.apply(e,arguments)}};e=!1,addEventListener("load",function(){return t.defer(function(){return e=!0})},!1),t.History=function(){function n(t){this.delegate=t,this.onPopState=r(this.onPopState,this)}return n.prototype.start=function(){return this.started?void 0:(addEventListener("popstate",this.onPopState,!1),this.started=!0)},n.prototype.stop=function(){return this.started?(removeEventListener("popstate",this.onPopState,!1),this.started=!1):void 0},n.prototype.push=function(e,r){return e=t.Location.wrap(e),this.update("push",e,r)},n.prototype.replace=function(e,r){return e=t.Location.wrap(e),this.update("replace",e,r)},n.prototype.onPopState=function(e){var r,n,o,i;return this.shouldHandlePopState()&&(i=null!=(n=e.state)?n.turbolinks:void 0)?(r=t.Location.wrap(window.location),o=i.restorationIdentifier,this.delegate.historyPoppedToLocationWithRestorationIdentifier(r,o)):void 0},n.prototype.shouldHandlePopState=function(){return e===!0},n.prototype.update=function(t,e,r){var n;return n={turbolinks:{restorationIdentifier:r}},history[t+"State"](n,null,e)},n}()}.call(this),function(){t.Snapshot=function(){function e(t){var e,r;r=t.head,e=t.body,this.head=null!=r?r:document.createElement("head"),this.body=null!=e?e:document.createElement("body")}return e.wrap=function(t){return t instanceof this?t:this.fromHTML(t)},e.fromHTML=function(t){var e;return e=document.createElement("html"),e.innerHTML=t,this.fromElement(e)},e.fromElement=function(t){return new this({head:t.querySelector("head"),body:t.querySelector("body")})},e.prototype.clone=function(){return new e({head:this.head.cloneNode(!0),body:this.body.cloneNode(!0)})},e.prototype.getRootLocation=function(){var e,r;return r=null!=(e=this.getSetting("root"))?e:"/",new t.Location(r)},e.prototype.getCacheControlValue=function(){return this.getSetting("cache-control")},e.prototype.hasAnchor=function(t){try{return null!=this.body.querySelector("[id='"+t+"']")}catch(e){}},e.prototype.isPreviewable=function(){return"no-preview"!==this.getCacheControlValue()},e.prototype.isCacheable=function(){return"no-cache"!==this.getCacheControlValue()},e.prototype.getSetting=function(t){var e,r;return r=this.head.querySelectorAll("meta[name='turbolinks-"+t+"']"),e=r[r.length-1],null!=e?e.getAttribute("content"):void 0},e}()}.call(this),function(){var e=[].slice;t.Renderer=function(){function t(){}var r;return t.render=function(){var t,r,n,o;return n=arguments[0],r=arguments[1],t=3<=arguments.length?e.call(arguments,2):[],o=function(t,e,r){r.prototype=t.prototype;var n=new r,o=t.apply(n,e);return Object(o)===o?o:n}(this,t,function(){}),o.delegate=n,o.render(r),o},t.prototype.renderView=function(t){return this.delegate.viewWillRender(this.newBody),t(),this.delegate.viewRendered(this.newBody)},t.prototype.invalidateView=function(){return this.delegate.viewInvalidated()},t.prototype.createScriptElement=function(t){var e;return"false"===t.getAttribute("data-turbolinks-eval")?t:(e=document.createElement("script"),e.textContent=t.textContent,r(e,t),e)},r=function(t,e){var r,n,o,i,s,a,u;for(i=e.attributes,a=[],r=0,n=i.length;n>r;r++)s=i[r],o=s.name,u=s.value,a.push(t.setAttribute(o,u));return a},t}()}.call(this),function(){t.HeadDetails=function(){function t(t){var e,r,i,s,a,u,c;for(this.element=t,this.elements={},c=this.element.childNodes,s=0,u=c.length;u>s;s++)i=c[s],i.nodeType===Node.ELEMENT_NODE&&(a=i.outerHTML,r=null!=(e=this.elements)[a]?e[a]:e[a]={type:o(i),tracked:n(i),elements:[]},r.elements.push(i))}var e,r,n,o;return t.prototype.hasElementWithKey=function(t){return t in this.elements},t.prototype.getTrackedElementSignature=function(){var t,e;return function(){var r,n;r=this.elements,n=[];for(t in r)e=r[t].tracked,e&&n.push(t);return n}.call(this).join("")},t.prototype.getScriptElementsNotInDetails=function(t){return this.getElementsMatchingTypeNotInDetails("script",t)},t.prototype.getStylesheetElementsNotInDetails=function(t){return this.getElementsMatchingTypeNotInDetails("stylesheet",t)},t.prototype.getElementsMatchingTypeNotInDetails=function(t,e){var r,n,o,i,s,a;o=this.elements,s=[];for(n in o)i=o[n],a=i.type,r=i.elements,a!==t||e.hasElementWithKey(n)||s.push(r[0]);return s},t.prototype.getProvisionalElements=function(){var t,e,r,n,o,i,s;r=[],n=this.elements;for(e in n)o=n[e],s=o.type,i=o.tracked,t=o.elements,null!=s||i?t.length>1&&r.push.apply(r,t.slice(1)):r.push.apply(r,t);return r},o=function(t){return e(t)?"script":r(t)?"stylesheet":void 0},n=function(t){return"reload"===t.getAttribute("data-turbolinks-track")},e=function(t){var e;return e=t.tagName.toLowerCase(),"script"===e},r=function(t){var e;return e=t.tagName.toLowerCase(),"style"===e||"link"===e&&"stylesheet"===t.getAttribute("rel")},t}()}.call(this),function(){var e=function(t,e){function n(){this.constructor=t}for(var o in e)r.call(e,o)&&(t[o]=e[o]);return n.prototype=e.prototype,t.prototype=new n,t.__super__=e.prototype,t},r={}.hasOwnProperty;t.SnapshotRenderer=function(r){function n(e,r){this.currentSnapshot=e,this.newSnapshot=r,this.currentHeadDetails=new t.HeadDetails(this.currentSnapshot.head),this.newHeadDetails=new t.HeadDetails(this.newSnapshot.head),this.newBody=this.newSnapshot.body}return e(n,r),n.prototype.render=function(t){return this.trackedElementsAreIdentical()?(this.mergeHead(),this.renderView(function(e){return function(){return e.replaceBody(),e.focusFirstAutofocusableElement(),t()}}(this))):this.invalidateView()},n.prototype.mergeHead=function(){return this.copyNewHeadStylesheetElements(),this.copyNewHeadScriptElements(),this.removeCurrentHeadProvisionalElements(),this.copyNewHeadProvisionalElements()},n.prototype.replaceBody=function(){return this.activateBodyScriptElements(),this.importBodyPermanentElements(),this.assignNewBody()},n.prototype.trackedElementsAreIdentical=function(){return this.currentHeadDetails.getTrackedElementSignature()===this.newHeadDetails.getTrackedElementSignature()},n.prototype.copyNewHeadStylesheetElements=function(){var t,e,r,n,o;for(n=this.getNewHeadStylesheetElements(),o=[],e=0,r=n.length;r>e;e++)t=n[e],o.push(document.head.appendChild(t));return o},n.prototype.copyNewHeadScriptElements=function(){var t,e,r,n,o;for(n=this.getNewHeadScriptElements(),o=[],e=0,r=n.length;r>e;e++)t=n[e],o.push(document.head.appendChild(this.createScriptElement(t)));return o},n.prototype.removeCurrentHeadProvisionalElements=function(){var t,e,r,n,o;for(n=this.getCurrentHeadProvisionalElements(),o=[],e=0,r=n.length;r>e;e++)t=n[e],o.push(document.head.removeChild(t));return o},n.prototype.copyNewHeadProvisionalElements=function(){var t,e,r,n,o;for(n=this.getNewHeadProvisionalElements(),o=[],e=0,r=n.length;r>e;e++)t=n[e],o.push(document.head.appendChild(t));return o},n.prototype.importBodyPermanentElements=function(){var t,e,r,n,o,i;for(n=this.getNewBodyPermanentElements(),i=[],e=0,r=n.length;r>e;e++)o=n[e],(t=this.findCurrentBodyPermanentElement(o))?i.push(o.parentNode.replaceChild(t,o)):i.push(void 0);return i},n.prototype.activateBodyScriptElements=function(){var t,e,r,n,o,i;for(n=this.getNewBodyScriptElements(),i=[],e=0,r=n.length;r>e;e++)o=n[e],t=this.createScriptElement(o),i.push(o.parentNode.replaceChild(t,o));return i},n.prototype.assignNewBody=function(){return document.body=this.newBody},n.prototype.focusFirstAutofocusableElement=function(){var t;return null!=(t=this.findFirstAutofocusableElement())?t.focus():void 0},n.prototype.getNewHeadStylesheetElements=function(){return this.newHeadDetails.getStylesheetElementsNotInDetails(this.currentHeadDetails)},n.prototype.getNewHeadScriptElements=function(){return this.newHeadDetails.getScriptElementsNotInDetails(this.currentHeadDetails)},n.prototype.getCurrentHeadProvisionalElements=function(){return this.currentHeadDetails.getProvisionalElements()},n.prototype.getNewHeadProvisionalElements=function(){return this.newHeadDetails.getProvisionalElements()},n.prototype.getNewBodyPermanentElements=function(){return this.newBody.querySelectorAll("[id][data-turbolinks-permanent]")},n.prototype.findCurrentBodyPermanentElement=function(t){return document.body.querySelector("#"+t.id+"[data-turbolinks-permanent]")},n.prototype.getNewBodyScriptElements=function(){return this.newBody.querySelectorAll("script")},n.prototype.findFirstAutofocusableElement=function(){return document.body.querySelector("[autofocus]")},n}(t.Renderer)}.call(this),function(){var e=function(t,e){function n(){this.constructor=t}for(var o in e)r.call(e,o)&&(t[o]=e[o]);return n.prototype=e.prototype,t.prototype=new n,t.__super__=e.prototype,t},r={}.hasOwnProperty;t.ErrorRenderer=function(t){function r(t){this.html=t}return e(r,t),r.prototype.render=function(t){return this.renderView(function(e){return function(){return e.replaceDocumentHTML(),e.activateBodyScriptElements(),t()}}(this))},r.prototype.replaceDocumentHTML=function(){return document.documentElement.innerHTML=this.html},r.prototype.activateBodyScriptElements=function(){var t,e,r,n,o,i;for(n=this.getScriptElements(),i=[],e=0,r=n.length;r>e;e++)o=n[e],t=this.createScriptElement(o),i.push(o.parentNode.replaceChild(t,o));return i},r.prototype.getScriptElements=function(){return document.documentElement.querySelectorAll("script")},r}(t.Renderer)}.call(this),function(){t.View=function(){function e(t){this.delegate=t,this.element=document.documentElement}return e.prototype.getRootLocation=function(){return this.getSnapshot().getRootLocation()},e.prototype.getSnapshot=function(){return t.Snapshot.fromElement(this.element)},e.prototype.render=function(t,e){var r,n,o;return o=t.snapshot,r=t.error,n=t.isPreview,this.markAsPreview(n),null!=o?this.renderSnapshot(o,e):this.renderError(r,e)},e.prototype.markAsPreview=function(t){return t?this.element.setAttribute("data-turbolinks-preview",""):this.element.removeAttribute("data-turbolinks-preview")},e.prototype.renderSnapshot=function(e,r){return t.SnapshotRenderer.render(this.delegate,r,this.getSnapshot(),t.Snapshot.wrap(e))},e.prototype.renderError=function(e,r){return t.ErrorRenderer.render(this.delegate,r,e)},e}()}.call(this),function(){var e=function(t,e){return function(){return t.apply(e,arguments)}};t.ScrollManager=function(){function t(t){this.delegate=t,this.onScroll=e(this.onScroll,this)}return t.prototype.start=function(){return this.started?void 0:(addEventListener("scroll",this.onScroll,!1),this.onScroll(),this.started=!0)},t.prototype.stop=function(){return this.started?(removeEventListener("scroll",this.onScroll,!1),this.started=!1):void 0},t.prototype.scrollToElement=function(t){return t.scrollIntoView()},t.prototype.scrollToPosition=function(t){var e,r;return e=t.x,r=t.y,window.scrollTo(e,r)},t.prototype.onScroll=function(t){return this.updatePosition({x:window.pageXOffset,y:window.pageYOffset})},t.prototype.updatePosition=function(t){var e;return this.position=t,null!=(e=this.delegate)?e.scrollPositionChanged(this.position):void 0},t}()}.call(this),function(){t.SnapshotCache=function(){function e(t){this.size=t,this.keys=[],this.snapshots={}}var r;return e.prototype.has=function(t){var e;return e=r(t),e in this.snapshots},e.prototype.get=function(t){var e;if(this.has(t))return e=this.read(t),this.touch(t),e},e.prototype.put=function(t,e){return this.write(t,e),this.touch(t),e},e.prototype.read=function(t){var e;return e=r(t),this.snapshots[e]},e.prototype.write=function(t,e){var n;return n=r(t),this.snapshots[n]=e},e.prototype.touch=function(t){var e,n;return n=r(t),e=this.keys.indexOf(n),e>-1&&this.keys.splice(e,1),this.keys.unshift(n),this.trim()},e.prototype.trim=function(){var t,e,r,n,o;for(n=this.keys.splice(this.size),o=[],t=0,r=n.length;r>t;t++)e=n[t],o.push(delete this.snapshots[e]);return o},r=function(e){return t.Location.wrap(e).toCacheKey()},e}()}.call(this),function(){var e=function(t,e){return function(){return t.apply(e,arguments)}};t.Visit=function(){function r(r,n,o){this.controller=r,this.action=o,this.performScroll=e(this.performScroll,this),this.identifier=t.uuid(),this.location=t.Location.wrap(n),this.adapter=this.controller.adapter,this.state="initialized",this.timingMetrics={}}var n;return r.prototype.start=function(){return"initialized"===this.state?(this.recordTimingMetric("visitStart"),this.state="started",this.adapter.visitStarted(this)):void 0},r.prototype.cancel=function(){var t;return"started"===this.state?(null!=(t=this.request)&&t.cancel(),this.cancelRender(),this.state="canceled"):void 0},r.prototype.complete=function(){var t;return"started"===this.state?(this.recordTimingMetric("visitEnd"),this.state="completed","function"==typeof(t=this.adapter).visitCompleted&&t.visitCompleted(this),this.controller.visitCompleted(this)):void 0},r.prototype.fail=function(){var t;return"started"===this.state?(this.state="failed","function"==typeof(t=this.adapter).visitFailed?t.visitFailed(this):void 0):void 0},r.prototype.changeHistory=function(){var t,e;return this.historyChanged?void 0:(t=this.location.isEqualTo(this.referrer)?"replace":this.action,e=n(t),this.controller[e](this.location,this.restorationIdentifier),this.historyChanged=!0)},r.prototype.issueRequest=function(){return this.shouldIssueRequest()&&null==this.request?(this.progress=0,this.request=new t.HttpRequest(this,this.location,this.referrer),this.request.send()):void 0},r.prototype.getCachedSnapshot=function(){var t;return!(t=this.controller.getCachedSnapshotForLocation(this.location))||null!=this.location.anchor&&!t.hasAnchor(this.location.anchor)||"restore"!==this.action&&!t.isPreviewable()?void 0:t},r.prototype.hasCachedSnapshot=function(){return null!=this.getCachedSnapshot()},r.prototype.loadCachedSnapshot=function(){var t,e;return(e=this.getCachedSnapshot())?(t=this.shouldIssueRequest(),this.render(function(){var r;return this.cacheSnapshot(),this.controller.render({snapshot:e,isPreview:t},this.performScroll),"function"==typeof(r=this.adapter).visitRendered&&r.visitRendered(this),t?void 0:this.complete()})):void 0},r.prototype.loadResponse=function(){return null!=this.response?this.render(function(){var t,e;return this.cacheSnapshot(),this.request.failed?(this.controller.render({error:this.response},this.performScroll),"function"==typeof(t=this.adapter).visitRendered&&t.visitRendered(this),this.fail()):(this.controller.render({snapshot:this.response},this.performScroll),"function"==typeof(e=this.adapter).visitRendered&&e.visitRendered(this),this.complete())}):void 0},r.prototype.followRedirect=function(){return this.redirectedToLocation&&!this.followedRedirect?(this.location=this.redirectedToLocation,this.controller.replaceHistoryWithLocationAndRestorationIdentifier(this.redirectedToLocation,this.restorationIdentifier),this.followedRedirect=!0):void 0},r.prototype.requestStarted=function(){var t;return this.recordTimingMetric("requestStart"),"function"==typeof(t=this.adapter).visitRequestStarted?t.visitRequestStarted(this):void 0},r.prototype.requestProgressed=function(t){var e;return this.progress=t,"function"==typeof(e=this.adapter).visitRequestProgressed?e.visitRequestProgressed(this):void 0},r.prototype.requestCompletedWithResponse=function(e,r){return this.response=e,null!=r&&(this.redirectedToLocation=t.Location.wrap(r)),this.adapter.visitRequestCompleted(this)},r.prototype.requestFailedWithStatusCode=function(t,e){return this.response=e,this.adapter.visitRequestFailedWithStatusCode(this,t)},r.prototype.requestFinished=function(){var t;return this.recordTimingMetric("requestEnd"),"function"==typeof(t=this.adapter).visitRequestFinished?t.visitRequestFinished(this):void 0},r.prototype.performScroll=function(){return this.scrolled?void 0:("restore"===this.action?this.scrollToRestoredPosition()||this.scrollToTop():this.scrollToAnchor()||this.scrollToTop(),this.scrolled=!0)},r.prototype.scrollToRestoredPosition=function(){var t,e;return t=null!=(e=this.restorationData)?e.scrollPosition:void 0,null!=t?(this.controller.scrollToPosition(t),!0):void 0},r.prototype.scrollToAnchor=function(){return null!=this.location.anchor?(this.controller.scrollToAnchor(this.location.anchor),!0):void 0},r.prototype.scrollToTop=function(){return this.controller.scrollToPosition({x:0,y:0})},r.prototype.recordTimingMetric=function(t){var e;return null!=(e=this.timingMetrics)[t]?e[t]:e[t]=(new Date).getTime()},r.prototype.getTimingMetrics=function(){return t.copyObject(this.timingMetrics)},n=function(t){switch(t){case"replace":return"replaceHistoryWithLocationAndRestorationIdentifier";case"advance":case"restore":return"pushHistoryWithLocationAndRestorationIdentifier"}},r.prototype.shouldIssueRequest=function(){return"restore"===this.action?!this.hasCachedSnapshot():!0},r.prototype.cacheSnapshot=function(){return this.snapshotCached?void 0:(this.controller.cacheSnapshot(),this.snapshotCached=!0)},r.prototype.render=function(t){return this.cancelRender(),this.frame=requestAnimationFrame(function(e){return function(){return e.frame=null,t.call(e)}}(this))},r.prototype.cancelRender=function(){return this.frame?cancelAnimationFrame(this.frame):void 0},r}()}.call(this),function(){var e=function(t,e){return function(){return t.apply(e,arguments)}};t.Controller=function(){function r(){this.clickBubbled=e(this.clickBubbled,this),this.clickCaptured=e(this.clickCaptured,this),this.pageLoaded=e(this.pageLoaded,this),this.history=new t.History(this),this.view=new t.View(this),this.scrollManager=new t.ScrollManager(this),this.restorationData={},this.clearCache()}return r.prototype.start=function(){return t.supported&&!this.started?(addEventListener("click",this.clickCaptured,!0),addEventListener("DOMContentLoaded",this.pageLoaded,!1),this.scrollManager.start(),this.startHistory(),this.started=!0,this.enabled=!0):void 0},r.prototype.disable=function(){return this.enabled=!1},r.prototype.stop=function(){return this.started?(removeEventListener("click",this.clickCaptured,!0),removeEventListener("DOMContentLoaded",this.pageLoaded,!1),this.scrollManager.stop(),this.stopHistory(),this.started=!1):void 0},r.prototype.clearCache=function(){return this.cache=new t.SnapshotCache(10)},r.prototype.visit=function(e,r){var n,o;return null==r&&(r={}),e=t.Location.wrap(e),this.applicationAllowsVisitingLocation(e)?this.locationIsVisitable(e)?(n=null!=(o=r.action)?o:"advance",this.adapter.visitProposedToLocationWithAction(e,n)):window.location=e:void 0},r.prototype.startVisitToLocationWithAction=function(e,r,n){var o;return t.supported?(o=this.getRestorationDataForIdentifier(n),this.startVisit(e,r,{restorationData:o})):window.location=e},r.prototype.startHistory=function(){return this.location=t.Location.wrap(window.location),this.restorationIdentifier=t.uuid(),this.history.start(),this.history.replace(this.location,this.restorationIdentifier)},r.prototype.stopHistory=function(){return this.history.stop()},r.prototype.pushHistoryWithLocationAndRestorationIdentifier=function(e,r){return this.restorationIdentifier=r,this.location=t.Location.wrap(e),this.history.push(this.location,this.restorationIdentifier)},r.prototype.replaceHistoryWithLocationAndRestorationIdentifier=function(e,r){return this.restorationIdentifier=r,this.location=t.Location.wrap(e),this.history.replace(this.location,this.restorationIdentifier)},r.prototype.historyPoppedToLocationWithRestorationIdentifier=function(e,r){var n;return this.restorationIdentifier=r,this.enabled?(n=this.getRestorationDataForIdentifier(this.restorationIdentifier),this.startVisit(e,"restore",{restorationIdentifier:this.restorationIdentifier,restorationData:n,historyChanged:!0}),this.location=t.Location.wrap(e)):this.adapter.pageInvalidated()},r.prototype.getCachedSnapshotForLocation=function(t){var e;return e=this.cache.get(t),e?e.clone():void 0},r.prototype.shouldCacheSnapshot=function(){return this.view.getSnapshot().isCacheable()},r.prototype.cacheSnapshot=function(){var t;return this.shouldCacheSnapshot()?(this.notifyApplicationBeforeCachingSnapshot(),t=this.view.getSnapshot(),this.cache.put(this.lastRenderedLocation,t.clone())):void 0},r.prototype.scrollToAnchor=function(t){var e;return(e=document.getElementById(t))?this.scrollToElement(e):this.scrollToPosition({x:0,y:0})},r.prototype.scrollToElement=function(t){return this.scrollManager.scrollToElement(t)},r.prototype.scrollToPosition=function(t){return this.scrollManager.scrollToPosition(t)},r.prototype.scrollPositionChanged=function(t){var e;return e=this.getCurrentRestorationData(),e.scrollPosition=t},r.prototype.render=function(t,e){return this.view.render(t,e)},r.prototype.viewInvalidated=function(){return this.adapter.pageInvalidated()},r.prototype.viewWillRender=function(t){return this.notifyApplicationBeforeRender(t)},r.prototype.viewRendered=function(){return this.lastRenderedLocation=this.currentVisit.location,this.notifyApplicationAfterRender()},r.prototype.pageLoaded=function(){return this.lastRenderedLocation=this.location,this.notifyApplicationAfterPageLoad()},r.prototype.clickCaptured=function(){return removeEventListener("click",this.clickBubbled,!1),addEventListener("click",this.clickBubbled,!1)},r.prototype.clickBubbled=function(t){var e,r,n;return this.enabled&&this.clickEventIsSignificant(t)&&(r=this.getVisitableLinkForNode(t.target))&&(n=this.getVisitableLocationForLink(r))&&this.applicationAllowsFollowingLinkToLocation(r,n)?(t.preventDefault(),e=this.getActionForLink(r),this.visit(n,{action:e})):void 0},r.prototype.applicationAllowsFollowingLinkToLocation=function(t,e){var r;return r=this.notifyApplicationAfterClickingLinkToLocation(t,e),!r.defaultPrevented},r.prototype.applicationAllowsVisitingLocation=function(t){var e;return e=this.notifyApplicationBeforeVisitingLocation(t),!e.defaultPrevented},r.prototype.notifyApplicationAfterClickingLinkToLocation=function(e,r){return t.dispatch("turbolinks:click",{target:e,data:{url:r.absoluteURL},cancelable:!0})},r.prototype.notifyApplicationBeforeVisitingLocation=function(e){return t.dispatch("turbolinks:before-visit",{data:{url:e.absoluteURL},cancelable:!0})},r.prototype.notifyApplicationAfterVisitingLocation=function(e){return t.dispatch("turbolinks:visit",{data:{url:e.absoluteURL}})},r.prototype.notifyApplicationBeforeCachingSnapshot=function(){return t.dispatch("turbolinks:before-cache")},r.prototype.notifyApplicationBeforeRender=function(e){return t.dispatch("turbolinks:before-render",{data:{newBody:e}})},r.prototype.notifyApplicationAfterRender=function(){return t.dispatch("turbolinks:render")},r.prototype.notifyApplicationAfterPageLoad=function(e){return null==e&&(e={}),t.dispatch("turbolinks:load",{data:{url:this.location.absoluteURL,timing:e}})},r.prototype.startVisit=function(t,e,r){var n;return null!=(n=this.currentVisit)&&n.cancel(),this.currentVisit=this.createVisit(t,e,r),this.currentVisit.start(),this.notifyApplicationAfterVisitingLocation(t)},r.prototype.createVisit=function(e,r,n){ var o,i,s,a,u;return i=null!=n?n:{},a=i.restorationIdentifier,s=i.restorationData,o=i.historyChanged,u=new t.Visit(this,e,r),u.restorationIdentifier=null!=a?a:t.uuid(),u.restorationData=t.copyObject(s),u.historyChanged=o,u.referrer=this.location,u},r.prototype.visitCompleted=function(t){return this.notifyApplicationAfterPageLoad(t.getTimingMetrics())},r.prototype.clickEventIsSignificant=function(t){return!(t.defaultPrevented||t.target.isContentEditable||t.which>1||t.altKey||t.ctrlKey||t.metaKey||t.shiftKey)},r.prototype.getVisitableLinkForNode=function(e){return this.nodeIsVisitable(e)?t.closest(e,"a[href]:not([target])"):void 0},r.prototype.getVisitableLocationForLink=function(e){var r;return r=new t.Location(e.getAttribute("href")),this.locationIsVisitable(r)?r:void 0},r.prototype.getActionForLink=function(t){var e;return null!=(e=t.getAttribute("data-turbolinks-action"))?e:"advance"},r.prototype.nodeIsVisitable=function(e){var r;return(r=t.closest(e,"[data-turbolinks]"))?"false"!==r.getAttribute("data-turbolinks"):!0},r.prototype.locationIsVisitable=function(t){return t.isPrefixedBy(this.view.getRootLocation())&&t.isHTML()},r.prototype.getCurrentRestorationData=function(){return this.getRestorationDataForIdentifier(this.restorationIdentifier)},r.prototype.getRestorationDataForIdentifier=function(t){var e;return null!=(e=this.restorationData)[t]?e[t]:e[t]={}},r}()}.call(this),function(){var e,r,n;t.start=function(){return r()?(null==t.controller&&(t.controller=e()),t.controller.start()):void 0},r=function(){return null==window.Turbolinks&&(window.Turbolinks=t),n()},e=function(){var e;return e=new t.Controller,e.adapter=new t.BrowserAdapter(e),e},n=function(){return window.Turbolinks===t},n()&&t.start()}.call(this)}).call(this),"object"==typeof module&&module.exports?module.exports=t:false}).call(this); ================================================ FILE: docs/pro/recording-and-replaying-keep-alive-tcp-sessions.md ================================================ > **This feature available only in PRO version. See https://goreplay.org/pro.html for details.** By default, GoReplay does not guarantee that when you record keep-alive TCP session, it will be replayed in the same TCP connection as well. This is ok for most of the cases, but it does not give an accurate number of TCP sessions while replaying, also may cause issues if your application state depends on TCP session (do not mess with HTTP session). [GoReplay PRO](https://goreplay.org/pro.html) extension adds support for accurate recording and replaying of keep-alive TCP sessions. Separate connection to your server is created per original session and it makes benchmarks and tests incredibly accurate. To enable session recognition you just need to pass `--recognize-tcp-sessions` option. ``` gor --input-raw :80 --recognize-tcp-sessions --output-http http://test.target ``` Note that enabling this option also change algorithm of distributing traffic when using `--split-output`, see [Distributed configuration]. ================================================ FILE: docs/pro/replaying-binary-protocols.md ================================================ > **This feature available only in PRO version. See https://goreplay.org/pro.html for details.** Gor includes basic support for working with binary formats like `thrift` or `protocol-buffers`. To start set `--input-raw-protocol` to 'binary' (by default 'http'). For replaying, you should use `--output-binary`, example: ``` gor --input-raw :80 --input-raw-protocol binary --output-binary staging:8081 ``` While working with `--input-raw` you may notice a 2-second delay before messages are emitted to the outputs. This behaviour is expected and happening because for general binary protocol it is impossible to know when TCP message ends, so Gor has to set inactivity timeout. Each protocol has own rules (for example write message length as first bytes), and require individual handling to know message length. We consider improving detailed protocol support for `thrift`, `protocol-buffer` and etc. Note that you can use all load testing features for binary protocols. For example, the following command will loop and replay recorded payload on 10x speed for 30 seconds: ``` gor --input-file './binary*.gor|1000%' --output-binary staging:9091 --input-file-loop --exit-after 30s ``` ================================================ FILE: elasticsearch.go ================================================ package goreplay import ( "encoding/json" "github.com/buger/goreplay/proto" "log" "net/url" "strings" "time" elastigo "github.com/mattbaird/elastigo/lib" ) type ESUriErorr struct{} func (e *ESUriErorr) Error() string { return "Wrong ElasticSearch URL format. Expected to be: scheme://host/index_name" } type ESPlugin struct { Active bool ApiPort string eConn *elastigo.Conn Host string Index string indexor *elastigo.BulkIndexer done chan bool } type ESRequestResponse struct { ReqURL string `json:"Req_URL"` ReqMethod string `json:"Req_Method"` ReqUserAgent string `json:"Req_User-Agent"` ReqAcceptLanguage string `json:"Req_Accept-Language,omitempty"` ReqAccept string `json:"Req_Accept,omitempty"` ReqAcceptEncoding string `json:"Req_Accept-Encoding,omitempty"` ReqIfModifiedSince string `json:"Req_If-Modified-Since,omitempty"` ReqConnection string `json:"Req_Connection,omitempty"` ReqCookies string `json:"Req_Cookies,omitempty"` RespStatus string `json:"Resp_Status"` RespStatusCode string `json:"Resp_Status-Code"` RespProto string `json:"Resp_Proto,omitempty"` RespContentLength string `json:"Resp_Content-Length,omitempty"` RespContentType string `json:"Resp_Content-Type,omitempty"` RespTransferEncoding string `json:"Resp_Transfer-Encoding,omitempty"` RespContentEncoding string `json:"Resp_Content-Encoding,omitempty"` RespExpires string `json:"Resp_Expires,omitempty"` RespCacheControl string `json:"Resp_Cache-Control,omitempty"` RespVary string `json:"Resp_Vary,omitempty"` RespSetCookie string `json:"Resp_Set-Cookie,omitempty"` Rtt int64 `json:"RTT"` Timestamp time.Time } // Parse ElasticSearch URI // // Proper format is: scheme://[userinfo@]host/index_name // userinfo is: user[:password] // net/url.Parse() does not fail if scheme is not provided but actually does not // handle URI properly. // So we must 'validate' URI format to match requirements to use net/url.Parse() func parseURI(URI string) (err error, index string) { parsedUrl, parseErr := url.Parse(URI) if parseErr != nil { err = new(ESUriErorr) return } // check URL validity by extracting host and index values. host := parsedUrl.Host urlPathParts := strings.Split(parsedUrl.Path, "/") index = urlPathParts[len(urlPathParts)-1] // force index specification in uri : ie no implicit index if host == "" || index == "" { err = new(ESUriErorr) } return } func (p *ESPlugin) Init(URI string) { var err error err, p.Index = parseURI(URI) if err != nil { log.Fatal("Can't initialize ElasticSearch plugin.", err) } p.eConn = elastigo.NewConn() p.eConn.SetFromUrl(URI) p.indexor = p.eConn.NewBulkIndexerErrors(50, 60) p.done = make(chan bool) p.indexor.Start() go p.ErrorHandler() Debug(1, "Initialized Elasticsearch Plugin") return } func (p *ESPlugin) IndexerShutdown() { p.indexor.Stop() return } func (p *ESPlugin) ErrorHandler() { for { errBuf := <-p.indexor.ErrorChannel Debug(1, "[ELASTICSEARCH]", errBuf.Err) } } func (p *ESPlugin) RttDurationToMs(d time.Duration) int64 { sec := d / time.Second nsec := d % time.Second fl := float64(sec) + float64(nsec)*1e-6 return int64(fl) } // ResponseAnalyze send req and resp to ES func (p *ESPlugin) ResponseAnalyze(req, resp []byte, start, stop time.Time) { if len(resp) == 0 { // nil http response - skipped elasticsearch export for this request return } t := time.Now() rtt := p.RttDurationToMs(stop.Sub(start)) esResp := ESRequestResponse{ ReqURL: string(proto.Path(req)), ReqMethod: string(proto.Method(req)), ReqUserAgent: string(proto.Header(req, []byte("User-Agent"))), ReqAcceptLanguage: string(proto.Header(req, []byte("Accept-Language"))), ReqAccept: string(proto.Header(req, []byte("Accept"))), ReqAcceptEncoding: string(proto.Header(req, []byte("Accept-Encoding"))), ReqIfModifiedSince: string(proto.Header(req, []byte("If-Modified-Since"))), ReqConnection: string(proto.Header(req, []byte("Connection"))), ReqCookies: string(proto.Header(req, []byte("Cookie"))), RespStatus: string(proto.Status(resp)), RespStatusCode: string(proto.Status(resp)), RespProto: string(proto.Method(resp)), RespContentLength: string(proto.Header(resp, []byte("Content-Length"))), RespContentType: string(proto.Header(resp, []byte("Content-Type"))), RespTransferEncoding: string(proto.Header(resp, []byte("Transfer-Encoding"))), RespContentEncoding: string(proto.Header(resp, []byte("Content-Encoding"))), RespExpires: string(proto.Header(resp, []byte("Expires"))), RespCacheControl: string(proto.Header(resp, []byte("Cache-Control"))), RespVary: string(proto.Header(resp, []byte("Vary"))), RespSetCookie: string(proto.Header(resp, []byte("Set-Cookie"))), Rtt: rtt, Timestamp: t, } j, err := json.Marshal(&esResp) if err != nil { Debug(0, "[ELASTIC-RESPONSE]", err) } else { p.indexor.Index(p.Index, "RequestResponse", "", "", "", &t, j) } return } ================================================ FILE: elasticsearch_test.go ================================================ package goreplay import ( "testing" ) const expectedIndex = "gor" func assertExpectedGorIndex(index string, t *testing.T) { if expectedIndex != index { t.Fatalf("Expected index %s but got %s", expectedIndex, index) } } func assertExpectedIndex(expectedIndex string, index string, t *testing.T) { if expectedIndex != index { t.Fatalf("Expected index %s but got %s", expectedIndex, index) } } func assertExpectedError(returnedError error, t *testing.T) { expectedError := new(ESUriErorr) if expectedError != returnedError { t.Errorf("Expected err %s but got %s", expectedError, returnedError) } } func assertNoError(returnedError error, t *testing.T) { if nil != returnedError { t.Errorf("Expected no err but got %s", returnedError) } } // Argument host:port/index_name // i.e : localhost:9200/gor // Fail because scheme is mandatory func TestElasticConnectionBuildFailWithoutScheme(t *testing.T) { uri := "localhost:9200/" + expectedIndex err, _ := parseURI(uri) assertExpectedError(err, t) } // Argument scheme://Host:port // i.e : http://localhost:9200 // Fail : explicit index is required func TestElasticConnectionBuildFailWithoutIndex(t *testing.T) { uri := "http://localhost:9200" err, index := parseURI(uri) assertExpectedIndex("", index, t) assertExpectedError(err, t) } // Argument scheme://Host/index_name // i.e : http://localhost/gor func TestElasticConnectionBuildFailWithoutPort(t *testing.T) { uri := "http://localhost/" + expectedIndex err, index := parseURI(uri) assertNoError(err, t) assertExpectedGorIndex(index, t) } // Argument scheme://Host:port/index_name // i.e : http://localhost:9200/gor func TestElasticLocalConnectionBuild(t *testing.T) { uri := "http://localhost:9200/" + expectedIndex err, index := parseURI(uri) assertNoError(err, t) assertExpectedGorIndex(index, t) } // Argument scheme://Host:port/index_name // i.e : http://localhost.local:9200/gor or https://localhost.local:9200/gor func TestElasticSimpleLocalWithSchemeConnectionBuild(t *testing.T) { uri := "http://localhost.local:9200/" + expectedIndex err, index := parseURI(uri) assertNoError(err, t) assertExpectedGorIndex(index, t) } // Argument scheme://Host:port/index_name // i.e : http://localhost.local:9200/gor or https://localhost.local:9200/gor func TestElasticSimpleLocalWithHTTPSConnectionBuild(t *testing.T) { uri := "https://localhost.local:9200/" + expectedIndex err, index := parseURI(uri) assertNoError(err, t) assertExpectedGorIndex(index, t) } // Argument scheme://Host:port/index_name // i.e : localhost.local:9200/pathtoElastic/gor func TestElasticLongPathConnectionBuild(t *testing.T) { uri := "http://localhost.local:9200/pathtoElastic/" + expectedIndex err, index := parseURI(uri) assertNoError(err, t) assertExpectedGorIndex(index, t) } // Argument scheme://Host:userinfo@port/index_name // i.e : http://user:password@localhost.local:9200/gor func TestElasticBasicAuthConnectionBuild(t *testing.T) { uri := "http://user:password@localhost.local:9200/" + expectedIndex err, index := parseURI(uri) assertNoError(err, t) assertExpectedGorIndex(index, t) } // Argument scheme://Host:port/path/index_name // i.e : http://localhost.local:9200/path/gor or https://localhost.local:9200/path/gor func TestElasticComplexPathConnectionBuild(t *testing.T) { uri := "http://localhost.local:9200/path/" + expectedIndex err, index := parseURI(uri) assertNoError(err, t) assertExpectedGorIndex(index, t) } ================================================ FILE: emitter.go ================================================ package goreplay import ( "fmt" "github.com/buger/goreplay/internal/byteutils" "hash/fnv" "io" "log" "sync" "github.com/coocood/freecache" ) // Emitter represents an abject to manage plugins communication type Emitter struct { sync.WaitGroup plugins *InOutPlugins } // NewEmitter creates and initializes new Emitter object. func NewEmitter() *Emitter { return &Emitter{} } // Start initialize loop for sending data from inputs to outputs func (e *Emitter) Start(plugins *InOutPlugins, middlewareCmd string) { if Settings.CopyBufferSize < 1 { Settings.CopyBufferSize = 5 << 20 } e.plugins = plugins if middlewareCmd != "" { middleware := NewMiddleware(middlewareCmd) for _, in := range plugins.Inputs { middleware.ReadFrom(in) } e.plugins.Inputs = append(e.plugins.Inputs, middleware) e.plugins.All = append(e.plugins.All, middleware) e.Add(1) go func() { defer e.Done() if err := CopyMulty(middleware, plugins.Outputs...); err != nil { Debug(2, fmt.Sprintf("[EMITTER] error during copy: %q", err)) } }() } else { for _, in := range plugins.Inputs { e.Add(1) go func(in PluginReader) { defer e.Done() if err := CopyMulty(in, plugins.Outputs...); err != nil { Debug(2, fmt.Sprintf("[EMITTER] error during copy: %q", err)) } }(in) } } } // Close closes all the goroutine and waits for it to finish. func (e *Emitter) Close() { for _, p := range e.plugins.All { if cp, ok := p.(io.Closer); ok { cp.Close() } } if len(e.plugins.All) > 0 { // wait for everything to stop e.Wait() } e.plugins.All = nil // avoid Close to make changes again } // CopyMulty copies from 1 reader to multiple writers func CopyMulty(src PluginReader, writers ...PluginWriter) error { wIndex := 0 modifier := NewHTTPModifier(&Settings.ModifierConfig) filteredRequests := freecache.NewCache(200 * 1024 * 1024) // 200M for { msg, err := src.PluginRead() if err != nil { if err == ErrorStopped || err == io.EOF { return nil } return err } if msg != nil && len(msg.Data) > 0 { if len(msg.Data) > int(Settings.CopyBufferSize) { msg.Data = msg.Data[:Settings.CopyBufferSize] } meta := payloadMeta(msg.Meta) if len(meta) < 3 { Debug(2, fmt.Sprintf("[EMITTER] Found malformed record %q from %q", msg.Meta, src)) continue } requestID := meta[1] // start a subroutine only when necessary if Settings.Verbose >= 3 { Debug(3, "[EMITTER] input: ", byteutils.SliceToString(msg.Meta[:len(msg.Meta)-1]), " from: ", src) } if modifier != nil { Debug(3, "[EMITTER] modifier:", requestID, "from:", src) if isRequestPayload(msg.Meta) { msg.Data = modifier.Rewrite(msg.Data) // If modifier tells to skip request if len(msg.Data) == 0 { filteredRequests.Set(requestID, []byte{}, 60) // continue } Debug(3, "[EMITTER] Rewritten input:", requestID, "from:", src) } else { _, err := filteredRequests.Get(requestID) if err == nil { filteredRequests.Del(requestID) continue } } } if Settings.PrettifyHTTP { msg.Data = prettifyHTTP(msg.Data) if len(msg.Data) == 0 { continue } } if Settings.SplitOutput { if Settings.RecognizeTCPSessions { if !PRO { log.Fatal("Detailed TCP sessions work only with PRO license") } hasher := fnv.New32a() hasher.Write(meta[1]) wIndex = int(hasher.Sum32()) % len(writers) if _, err := writers[wIndex].PluginWrite(msg); err != nil { return err } } else { // Simple round robin if _, err := writers[wIndex].PluginWrite(msg); err != nil { return err } wIndex = (wIndex + 1) % len(writers) } } else { for _, dst := range writers { if _, err := dst.PluginWrite(msg); err != nil && err != io.ErrClosedPipe { return err } } } } } } ================================================ FILE: emitter_test.go ================================================ package goreplay import ( "fmt" "os" "sync" "sync/atomic" "testing" "time" ) func TestMain(m *testing.M) { PRO = true code := m.Run() os.Exit(code) } func TestEmitter(t *testing.T) { wg := new(sync.WaitGroup) input := NewTestInput() output := NewTestOutput(func(*Message) { wg.Done() }) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output}, } plugins.All = append(plugins.All, input, output) emitter := NewEmitter() go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 1000; i++ { wg.Add(1) input.EmitGET() } wg.Wait() emitter.Close() } func TestEmitterFiltered(t *testing.T) { wg := new(sync.WaitGroup) input := NewTestInput() input.skipHeader = true output := NewTestOutput(func(*Message) { wg.Done() }) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output}, } plugins.All = append(plugins.All, input, output) methods := HTTPMethods{[]byte("GET")} Settings.ModifierConfig = HTTPModifierConfig{Methods: methods} emitter := &Emitter{} go emitter.Start(plugins, "") wg.Add(2) id := uuid() reqh := payloadHeader(RequestPayload, id, time.Now().UnixNano(), -1) reqb := append(reqh, []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...) resh := payloadHeader(ResponsePayload, id, time.Now().UnixNano()+1, 1) respb := append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...) input.EmitBytes(reqb) input.EmitBytes(respb) id = uuid() reqh = payloadHeader(RequestPayload, id, time.Now().UnixNano(), -1) reqb = append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...) resh = payloadHeader(ResponsePayload, id, time.Now().UnixNano()+1, 1) respb = append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...) input.EmitBytes(reqb) input.EmitBytes(respb) wg.Wait() emitter.Close() Settings.ModifierConfig = HTTPModifierConfig{} } func TestEmitterSplitRoundRobin(t *testing.T) { wg := new(sync.WaitGroup) input := NewTestInput() var counter1, counter2 int32 output1 := NewTestOutput(func(*Message) { atomic.AddInt32(&counter1, 1) wg.Done() }) output2 := NewTestOutput(func(*Message) { atomic.AddInt32(&counter2, 1) wg.Done() }) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output1, output2}, } Settings.SplitOutput = true emitter := NewEmitter() go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 1000; i++ { wg.Add(1) input.EmitGET() } wg.Wait() emitter.Close() if counter1 == 0 || counter2 == 0 || counter1 != counter2 { t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2) } Settings.SplitOutput = false } func TestEmitterRoundRobin(t *testing.T) { wg := new(sync.WaitGroup) input := NewTestInput() var counter1, counter2 int32 output1 := NewTestOutput(func(*Message) { counter1++ wg.Done() }) output2 := NewTestOutput(func(*Message) { counter2++ wg.Done() }) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output1, output2}, } plugins.All = append(plugins.All, input, output1, output2) Settings.SplitOutput = true emitter := NewEmitter() go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 1000; i++ { wg.Add(1) input.EmitGET() } wg.Wait() emitter.Close() if counter1 == 0 || counter2 == 0 { t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2) } Settings.SplitOutput = false } func TestEmitterSplitSession(t *testing.T) { wg := new(sync.WaitGroup) wg.Add(200) input := NewTestInput() input.skipHeader = true var counter1, counter2 int32 output1 := NewTestOutput(func(msg *Message) { if payloadID(msg.Meta)[0] == 'a' { counter1++ } wg.Done() }) output2 := NewTestOutput(func(msg *Message) { if payloadID(msg.Meta)[0] == 'b' { counter2++ } wg.Done() }) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output1, output2}, } Settings.SplitOutput = true Settings.RecognizeTCPSessions = true emitter := NewEmitter() go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 200; i++ { // Keep session but randomize id := make([]byte, 20) if i&1 == 0 { // for recognizeTCPSessions one should be odd and other will be even number id[0] = 'a' } else { id[0] = 'b' } input.EmitBytes([]byte(fmt.Sprintf("1 %s 1 1\nGET / HTTP/1.1\r\n\r\n", id[:20]))) } wg.Wait() if counter1 != counter2 { t.Errorf("Round robin should split traffic equally: %d vs %d", counter1, counter2) } Settings.SplitOutput = false Settings.RecognizeTCPSessions = false emitter.Close() } func BenchmarkEmitter(b *testing.B) { wg := new(sync.WaitGroup) input := NewTestInput() output := NewTestOutput(func(*Message) { wg.Done() }) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output}, } plugins.All = append(plugins.All, input, output) emitter := NewEmitter() go emitter.Start(plugins, Settings.Middleware) b.ResetTimer() for i := 0; i < b.N; i++ { wg.Add(1) input.EmitGET() } wg.Wait() emitter.Close() } ================================================ FILE: examples/middleware/echo.clj ================================================ (ns echo.core (:gen-class) (:require [clojure.string :as cs] [clojure.java.io :as io]) (:import org.apache.commons.codec.binary.Hex java.io.BufferedReader java.io.IOException java.io.InputStreamReader)) (defn transform-http-msg "Function that transforms/filters the incoming HTTP messages." [headers body] ;; do actual transformations here [headers body]) (defn decode-hex-string "Decode an Hex-encoded string." [s] (String. (Hex/decodeHex (.toCharArray s)))) (defn encode-hex-string "Encode a string to a hex-encoded string." [^String s] (String. (Hex/encodeHex (.getBytes s)))) (defn -main [& args] (let [br (BufferedReader. (InputStreamReader. System/in))] (try (loop [hex-line (.readLine br)] (let [decoded-req (decode-hex-string hex-line) ;; empty line separates headers from body http-request (partition-by empty? (cs/split-lines decoded-req)) headers (first http-request) ;; HTTP messages can contain no body: body (when (= 3 (count http-request)) (last http-request)) [new-headers new-body] (transform-http-msg headers body)] (println (encode-hex-string (str (cs/join "\n" headers) (when body (str "\n\n" (cs/join "\n" body))))))) (when-let [line (.readLine br)] (recur line))) (catch IOException e nil)))) ================================================ FILE: examples/middleware/echo.java ================================================ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Hex; class Echo { public static String decodeHexString(String s) throws DecoderException { return new String(Hex.decodeHex(s.toCharArray())); } public static String encodeHexString(String s) { return new String(Hex.encodeHex(s.getBytes())); } public static String transformHTTPMessage(String req) { // do actual transformations here return req; } public static void main(String[] args) throws DecoderException { if(args != null){ for(String arg : args){ System.out.println(arg); } } BufferedReader stdin = new BufferedReader(new InputStreamReader( System.in)); String line = null; try { while ((line = stdin.readLine()) != null) { String decodedLine = decodeHexString(line); String transformedLine = transformHTTPMessage(decodedLine); String encodedLine = encodeHexString(transformedLine); System.out.println(encodedLine); } } catch (IOException e) { } } } ================================================ FILE: examples/middleware/echo.js ================================================ #!/usr/bin/env node const readline = require("readline"); const StringDecoder = require("string_decoder").StringDecoder const rl = readline.createInterface({ input: process.stdin }); var ignoreIds = new Set(); var ignoreAddresses = "/api"; const decoder = new StringDecoder("utf8"); function convertHexString(hex) { var bytes = []; for (var i = 0; i < hex.length - 1; i += 2) { bytes.push(parseInt(hex.substr(i, 2), 16)); } return decoder.write(Buffer.from(bytes)); } function log(output) { console.error("==================="); console.error(output); } function shouldOutputLine(request) { const components = request.split("\n"); const header = components[0].split(" "); const type = parseInt(header[0]); const tag = header[1]; if (type === 3) { return true; } if (type === 1) { // Check if it's oauth const endpoint = components[1].split(" ")[1]; if (!endpoint.startsWith(ignoreAddresses)) { ignoreIds.add(tag); return false; } } else if (type === 2) { if (ignoreIds.has(tag)) { ignoreIds.delete(tag); return false; } } return true; } rl.on("line", (input) => { const str = convertHexString(input); console.log(input); if (shouldOutputLine(str)) { log(str); } }); ================================================ FILE: examples/middleware/echo.py ================================================ #! /usr/bin/env python3 # -*- coding: utf-8 -*- import sys import fileinput import binascii # Used to find end of the Headers section EMPTY_LINE = b'\r\n\r\n' def log(msg): """ Logging to STDERR as STDOUT and STDIN used for data transfer @type msg: str or byte string @param msg: Message to log to STDERR """ try: msg = str(msg) + '\n' except: pass sys.stderr.write(msg) sys.stderr.flush() def find_end_of_headers(byte_data): """ Finds where the header portion ends and the content portion begins. @type byte_data: str or byte string @param byte_data: Hex decoded req or resp string """ return byte_data.index(EMPTY_LINE) + 4 def process_stdin(): """ Process STDIN and output to STDOUT """ for raw_line in fileinput.input(): line = raw_line.rstrip() # Decode base64 encoded line decoded = bytes.fromhex(line) # Split into metadata and payload, the payload is headers + body (raw_metadata, payload) = decoded.split(b'\n', 1) # Split into headers and payload headers_pos = find_end_of_headers(payload) raw_headers = payload[:headers_pos] raw_content = payload[headers_pos:] log('===================================') request_type_id = int(raw_metadata.split(b' ')[0]) log('Request type: {}'.format({ 1: 'Request', 2: 'Original Response', 3: 'Replayed Response' }[request_type_id])) log('===================================') log('Original data:') log(line) log('Decoded request:') log(decoded) encoded = binascii.hexlify(raw_metadata + b'\n' + raw_headers + raw_content).decode('ascii') log('Encoded data:') log(encoded) sys.stdout.write(encoded + '\n') if __name__ == '__main__': process_stdin() ================================================ FILE: examples/middleware/echo.rb ================================================ #!/usr/bin/env ruby # encoding: utf-8 while data = STDIN.gets # continuously read line from STDIN next unless data data = data.chomp # remove end of line symbol decoded = [data].pack("H*") # decode base64 encoded request # decoded value is raw HTTP payload, example: # # POST /post HTTP/1.1 # Content-Length: 7 # Host: www.w3.org # # a=1&b=2" encoded = decoded.unpack("H*").first # encoding back to base64 # Emit request back # You can skip this if want to filter out request STDOUT.puts encoded end ================================================ FILE: examples/middleware/echo.sh ================================================ #!/usr/bin/env bash # # `xxd` utility included into vim-common package # It allow hex decoding/encoding # # This example may broke if you request contains `null` string, you may consider using pipes instead. # See: https://github.com/buger/gor/issues/309 # function log { if [[ -n "$GOR_TEST" ]]; then # if we are not testing # Logging to stderr, because stdout/stdin used for data transfer >&2 echo "[DEBUG][ECHO] $1" fi } while read line; do decoded=$(echo -e "$line" | xxd -r -p) header=$(echo -e "$decoded" | head -n +1) payload=$(echo -e "$decoded" | tail -n +2) encoded=$(echo -e "$header\n$payload" | xxd -p | tr -d "\\n") log "" log "===================================" case ${header:0:1} in "1") log "Request type: Request" ;; "2") log "Request type: Original Response" ;; "3") log "Request type: Replayed Response" ;; *) log "Unknown request type $header" esac echo "$encoded" log "===================================" log "Original data: $line" log "Decoded request: $decoded" log "Encoded data: $encoded" done; ================================================ FILE: examples/middleware/token_modifier.go ================================================ /* This middleware made for auth system that randomly generate access tokens, which used later for accessing secure content. Since there is no pre-defined token value, naive approach without middleware (or if middleware use only request payloads) will fail, because replayed server have own tokens, not synced with origin. To fix this, our middleware should take in account responses of replayed and origin server, store `originalToken -> replayedToken` aliases and rewrite all requests using this token to use replayed alias. See `middleware_test.go#TestTokenMiddleware` test for examples of using this middleware. How middleware works: Original request +--------------+ +-------------+----------STDIN---------->+ | | Gor input | | Middleware | +-------------+----------STDIN---------->+ | Original response +------+---+---+ | ^ +-------------+ Modified request v | | Gor output +<---------STDOUT-----------------+ | +-----+-------+ | | | | Replayed response | +------------------STDIN----------------->----+ */ package main import ( "bufio" "bytes" "encoding/hex" "fmt" "github.com/buger/goreplay/proto" "os" ) // requestID -> originalToken var originalTokens map[string][]byte // originalToken -> replayedToken var tokenAliases map[string][]byte func main() { originalTokens = make(map[string][]byte) tokenAliases = make(map[string][]byte) scanner := bufio.NewScanner(os.Stdin) for scanner.Scan() { encoded := scanner.Bytes() buf := make([]byte, len(encoded)/2) hex.Decode(buf, encoded) process(buf) } } func process(buf []byte) { // First byte indicate payload type, possible values: // 1 - Request // 2 - Response // 3 - ReplayedResponse payloadType := buf[0] headerSize := bytes.IndexByte(buf, '\n') + 1 header := buf[:headerSize-1] // Header contains space separated values of: request type, request id, and request start time (or round-trip time for responses) meta := bytes.Split(header, []byte(" ")) // For each request you should receive 3 payloads (request, response, replayed response) with same request id reqID := string(meta[1]) payload := buf[headerSize:] Debug("Received payload:", string(buf)) switch payloadType { case '1': // Request if bytes.Equal(proto.Path(payload), []byte("/token")) { originalTokens[reqID] = []byte{} Debug("Found token request:", reqID) } else { token, vs, _ := proto.PathParam(payload, []byte("token")) if vs != -1 { // If there is GET token param if alias, ok := tokenAliases[string(token)]; ok { // Rewrite original token to alias payload = proto.SetPathParam(payload, []byte("token"), alias) // Copy modified payload to our buffer buf = append(buf[:headerSize], payload...) } } } // Emitting data back os.Stdout.Write(encode(buf)) case '2': // Original response if _, ok := originalTokens[reqID]; ok { // Token is inside response body secureToken := proto.Body(payload) originalTokens[reqID] = secureToken Debug("Remember origial token:", string(secureToken)) } case '3': // Replayed response if originalToken, ok := originalTokens[reqID]; ok { delete(originalTokens, reqID) secureToken := proto.Body(payload) tokenAliases[string(originalToken)] = secureToken Debug("Create alias for new token token, was:", string(originalToken), "now:", string(secureToken)) } } } func encode(buf []byte) []byte { dst := make([]byte, len(buf)*2+1) hex.Encode(dst, buf) dst[len(dst)-1] = '\n' return dst } func Debug(args ...interface{}) { if os.Getenv("GOR_TEST") == "" { // if we are not testing fmt.Fprint(os.Stderr, "[DEBUG][TOKEN-MOD] ") fmt.Fprintln(os.Stderr, args...) } } ================================================ FILE: go.mod ================================================ module github.com/buger/goreplay go 1.21 require ( github.com/Shopify/sarama v1.38.1 github.com/aws/aws-sdk-go v1.44.262 github.com/coocood/freecache v1.2.3 github.com/google/gopacket v1.1.20-0.20210429153827-3eaba0894325 github.com/gorilla/websocket v1.5.0 github.com/klauspost/compress v1.16.5 // indirect github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b github.com/stretchr/testify v1.8.2 github.com/xdg-go/scram v1.1.2 golang.org/x/net v0.34.0 golang.org/x/sys v0.29.0 k8s.io/apimachinery v0.27.1 k8s.io/client-go v0.27.1 ) require ( github.com/araddon/gou v0.0.0-20211019181548-e7d08105776c // indirect github.com/bitly/go-hostpool v0.1.0 // indirect github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/eapache/go-resiliency v1.3.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect github.com/eapache/queue v1.1.0 // indirect github.com/emicklei/go-restful/v3 v3.10.2 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/gnostic v0.6.9 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pierrec/lz4/v4 v4.1.17 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/smartystreets/goconvey v1.7.2 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect golang.org/x/crypto v0.32.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect golang.org/x/term v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/api v0.27.1 // indirect k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect k8s.io/utils v0.0.0-20230505201702-9f6742963106 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) ================================================ FILE: go.sum ================================================ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/sarama v1.38.1 h1:lqqPUPQZ7zPqYlWpTh+LQ9bhYNu2xJL6k1SJN4WVe2A= github.com/Shopify/sarama v1.38.1/go.mod h1:iwv9a67Ha8VNa+TifujYoWGxWnu2kNVAQdSdZ4X2o5g= github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc= github.com/Shopify/toxiproxy/v2 v2.5.0/go.mod h1:yhM2epWtAmel9CB8r2+L+PCmhH6yH2pITaPAo7jxJl0= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/araddon/gou v0.0.0-20211019181548-e7d08105776c h1:XUqw//RExYoxW4Eie8MuKp8sEDAZI1gMHX/daUFgZww= github.com/araddon/gou v0.0.0-20211019181548-e7d08105776c/go.mod h1:ikc1XA58M+Rx7SEbf0bLJCfBkwayZ8T5jBo5FXK8Uz8= github.com/aws/aws-sdk-go v1.44.262 h1:gyXpcJptWoNkK+DiAiaBltlreoWKQXjAIh6FRh60F+I= github.com/aws/aws-sdk-go v1.44.262/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2ioR0= github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coocood/freecache v1.2.3 h1:lcBwpZrwBZRZyLk/8EMyQVXRiFl663cCuMOrjCALeto= github.com/coocood/freecache v1.2.3/go.mod h1:RBUWa/Cy+OHdfTGFEhEuE1pMCMX51Ncizj7rthiQ3vk= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0= github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 h1:8yY/I9ndfrgrXUbOGObLHKBR4Fl3nZXwM2c7OYTT8hM= github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/emicklei/go-restful/v3 v3.10.2 h1:hIovbnmBTLjHXkqEBUz3HGpXZdM7ZrE9fJIZIqlJLqE= github.com/emicklei/go-restful/v3 v3.10.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.20-0.20210429153827-3eaba0894325 h1:YmIcZ5Var3BAQ64AW98Iiys5Ih4fiU0xK41+8isC5Ec= github.com/google/gopacket v1.1.20-0.20210429153827-3eaba0894325/go.mod h1:riddUzxTSBpJXk3qBHtYr4qOhFhT6k/1c0E3qkQjQpA= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg= github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8= github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b h1:v29yPGHhOqw7VHEnTeQFAth3SsBrmwc8JfuhNY0G34k= github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b/go.mod h1:5MWrJXKRQyhQdUCF+vu6U5c4nQpg70vW3eHaU0/AYbU= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/onsi/ginkgo/v2 v2.9.1 h1:zie5Ly042PD3bsCvsSOPvRnFwyo3rKe64TJlD6nu0mk= github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo= github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E= github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ= github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.27.1 h1:Z6zUGQ1Vd10tJ+gHcNNNgkV5emCyW+v2XTmn+CLjSd0= k8s.io/api v0.27.1/go.mod h1:z5g/BpAiD+f6AArpqNjkY+cji8ueZDU/WV1jcj5Jk4E= k8s.io/apimachinery v0.27.1 h1:EGuZiLI95UQQcClhanryclaQE6xjg1Bts6/L3cD7zyc= k8s.io/apimachinery v0.27.1/go.mod h1:5ikh59fK3AJ287GUvpUsryoMFtH9zj/ARfWCo3AyXTM= k8s.io/client-go v0.27.1 h1:oXsfhW/qncM1wDmWBIuDzRHNS2tLhK3BZv512Nc59W8= k8s.io/client-go v0.27.1/go.mod h1:f8LHMUkVb3b9N8bWturc+EDtVVVwZ7ueTVquFAJb2vA= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg= k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= k8s.io/utils v0.0.0-20230505201702-9f6742963106 h1:EObNQ3TW2D+WptiYXlApGNLVy0zm/JIBVY9i+M4wpAU= k8s.io/utils v0.0.0-20230505201702-9f6742963106/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= ================================================ FILE: gor_stat.go ================================================ package goreplay import ( "runtime" "strconv" "time" ) type GorStat struct { statName string rateMs int latest int mean int max int count int } func NewGorStat(statName string, rateMs int) (s *GorStat) { s = new(GorStat) s.statName = statName s.rateMs = rateMs s.latest = 0 s.mean = 0 s.max = 0 s.count = 0 if Settings.Stats { go s.reportStats() } return } func (s *GorStat) Write(latest int) { if Settings.Stats { if latest > s.max { s.max = latest } if latest != 0 { s.mean = ((s.mean * s.count) + latest) / (s.count + 1) } s.latest = latest s.count = s.count + 1 } } func (s *GorStat) Reset() { s.latest = 0 s.max = 0 s.mean = 0 s.count = 0 } func (s *GorStat) String() string { return s.statName + ":" + strconv.Itoa(s.latest) + "," + strconv.Itoa(s.mean) + "," + strconv.Itoa(s.max) + "," + strconv.Itoa(s.count) + "," + strconv.Itoa(s.count/(s.rateMs/1000.0)) + "," + strconv.Itoa(runtime.NumGoroutine()) } func (s *GorStat) reportStats() { Debug(0, "\n", s.statName+":latest,mean,max,count,count/second,gcount") for { Debug(0, "\n", s) s.Reset() time.Sleep(time.Duration(s.rateMs) * time.Millisecond) } } ================================================ FILE: homebrew/gor.rb ================================================ require "language/go" class Gor < Formula desc "Real-time HTTP traffic replay tool written in Go" homepage "https://gortool.com" url "https://github.com/buger/gor/archive/v0.14.0.tar.gz" sha256 "62260a6f5cabde571b91d5762fba9c47691643df0a58565cbe808854cd064dc8" head "https://github.com/buger/gor.git" bottle do cellar :any_skip_relocation sha256 "c382403de70a41b7445920a02051f5e82030704aaaae70cfcd4e8f401cc87f6a" => :el_capitan sha256 "4b76b3785584897800e87967f1af9510208faefe46f57d7bd6f8b40a7133c19b" => :yosemite sha256 "d186cb1566d33ab8f78215e69934f49dd96becb1c236905b4502d94399ae1974" => :mavericks end depends_on "go" => :build go_resource "github.com/bitly/go-hostpool" do url "https://github.com/bitly/go-hostpool.git", :revision => "d0e59c22a56e8dadfed24f74f452cea5a52722d2" end go_resource "github.com/buger/elastigo" do url "https://github.com/buger/elastigo.git", :revision => "23fcfd9db0d8be2189a98fdab77a4c90fcc3a1e9" end go_resource "github.com/google/gopacket" do url "https://github.com/google/gopacket.git", :revision => "aa09ced736460d76535444c825932a0742975f7d" end def install ENV["GOPATH"] = buildpath mkdir_p buildpath/"src/github.com/buger/" ln_sf buildpath, buildpath/"src/github.com/buger/gor" Language::Go.stage_deps resources, buildpath/"src" system "go", "build", "-o", "#{bin}/gor", "-ldflags", "-X main.VERSION \"#{version}\"" end test do assert_match version.to_s, shell_output("#{bin}/gor", 1) end end ================================================ FILE: http_modifier.go ================================================ package goreplay import ( "bytes" "encoding/base64" "github.com/buger/goreplay/proto" "hash/fnv" "strings" ) type HTTPModifier struct { config *HTTPModifierConfig } func NewHTTPModifier(config *HTTPModifierConfig) *HTTPModifier { // Optimization to skip modifier completely if we do not need it if len(config.URLRegexp) == 0 && len(config.URLNegativeRegexp) == 0 && len(config.URLRewrite) == 0 && len(config.HeaderRewrite) == 0 && len(config.HeaderFilters) == 0 && len(config.HeaderNegativeFilters) == 0 && len(config.HeaderBasicAuthFilters) == 0 && len(config.HeaderHashFilters) == 0 && len(config.ParamHashFilters) == 0 && len(config.Params) == 0 && len(config.Headers) == 0 && len(config.Methods) == 0 { return nil } return &HTTPModifier{config: config} } func (m *HTTPModifier) Rewrite(payload []byte) (response []byte) { if !proto.HasRequestTitle(payload) { return payload } if len(m.config.Methods) > 0 { method := proto.Method(payload) matched := false for _, m := range m.config.Methods { if bytes.Equal(method, m) { matched = true break } } if !matched { return } } if len(m.config.Headers) > 0 { for _, header := range m.config.Headers { payload = proto.SetHeader(payload, []byte(header.Name), []byte(header.Value)) } } if len(m.config.Params) > 0 { for _, param := range m.config.Params { payload = proto.SetPathParam(payload, param.Name, param.Value) } } if len(m.config.URLRegexp) > 0 { path := proto.Path(payload) matched := false for _, f := range m.config.URLRegexp { if f.regexp.Match(path) { matched = true break } } if !matched { return } } if len(m.config.URLNegativeRegexp) > 0 { path := proto.Path(payload) for _, f := range m.config.URLNegativeRegexp { if f.regexp.Match(path) { return } } } if len(m.config.HeaderFilters) > 0 { for _, f := range m.config.HeaderFilters { value := proto.Header(payload, f.name) if len(value) == 0 { return } if !f.regexp.Match(value) { return } } } if len(m.config.HeaderNegativeFilters) > 0 { for _, f := range m.config.HeaderNegativeFilters { value := proto.Header(payload, f.name) if len(value) > 0 && f.regexp.Match(value) { return } } } if len(m.config.HeaderBasicAuthFilters) > 0 { for _, f := range m.config.HeaderBasicAuthFilters { value := proto.Header(payload, []byte("Authorization")) if len(value) > 0 { valueString := string(value) trimmedBasicAuthEncoded := strings.TrimPrefix(valueString, "Basic ") if strings.Compare(valueString, trimmedBasicAuthEncoded) != 0 { decodedAuth, _ := base64.StdEncoding.DecodeString(trimmedBasicAuthEncoded) if !f.regexp.Match(decodedAuth) { return } } } } } if len(m.config.HeaderHashFilters) > 0 { for _, f := range m.config.HeaderHashFilters { value := proto.Header(payload, f.name) if len(value) > 0 { hasher := fnv.New32a() hasher.Write(value) if (hasher.Sum32() % 100) >= f.percent { return } } } } if len(m.config.ParamHashFilters) > 0 { for _, f := range m.config.ParamHashFilters { value, s, _ := proto.PathParam(payload, f.name) if s != -1 { hasher := fnv.New32a() hasher.Write(value) if (hasher.Sum32() % 100) >= f.percent { return } } } } if len(m.config.URLRewrite) > 0 { path := proto.Path(payload) for _, f := range m.config.URLRewrite { if f.src.Match(path) { path = f.src.ReplaceAll(path, f.target) payload = proto.SetPath(payload, path) break } } } if len(m.config.HeaderRewrite) > 0 { for _, f := range m.config.HeaderRewrite { value := proto.Header(payload, f.header) if len(value) == 0 { break } if f.src.Match(value) { newValue := f.src.ReplaceAll(value, f.target) payload = proto.SetHeader(payload, f.header, newValue) } } } return payload } ================================================ FILE: http_modifier_settings.go ================================================ package goreplay import ( "errors" "fmt" "regexp" "strconv" "strings" ) // HTTPModifierConfig holds configuration options for built-in traffic modifier type HTTPModifierConfig struct { URLNegativeRegexp HTTPURLRegexp `json:"http-disallow-url"` URLRegexp HTTPURLRegexp `json:"http-allow-url"` URLRewrite URLRewriteMap `json:"http-rewrite-url"` HeaderRewrite HeaderRewriteMap `json:"http-rewrite-header"` HeaderFilters HTTPHeaderFilters `json:"http-allow-header"` HeaderNegativeFilters HTTPHeaderFilters `json:"http-disallow-header"` HeaderBasicAuthFilters HTTPHeaderBasicAuthFilters `json:"http-basic-auth-filter"` HeaderHashFilters HTTPHashFilters `json:"http-header-limiter"` ParamHashFilters HTTPHashFilters `json:"http-param-limiter"` Params HTTPParams `json:"http-set-param"` Headers HTTPHeaders `json:"http-set-header"` Methods HTTPMethods `json:"http-allow-method"` } // Handling of --http-allow-header, --http-disallow-header options type headerFilter struct { name []byte regexp *regexp.Regexp } // HTTPHeaderFilters holds list of headers and their regexps type HTTPHeaderFilters []headerFilter func (h *HTTPHeaderFilters) String() string { return fmt.Sprint(*h) } // Set method to implement flags.Value func (h *HTTPHeaderFilters) Set(value string) error { valArr := strings.SplitN(value, ":", 2) if len(valArr) < 2 { return errors.New("need both header and value, colon-delimited (ex. user_id:^169$)") } val := strings.TrimSpace(valArr[1]) r, err := regexp.Compile(val) if err != nil { return err } *h = append(*h, headerFilter{name: []byte(valArr[0]), regexp: r}) return nil } // Handling of --http-basic-auth-filter option type basicAuthFilter struct { regexp *regexp.Regexp } // HTTPHeaderBasicAuthFilters holds list of regxp to match basic Auth header values type HTTPHeaderBasicAuthFilters []basicAuthFilter func (h *HTTPHeaderBasicAuthFilters) String() string { return fmt.Sprint(*h) } // Set method to implement flags.Value func (h *HTTPHeaderBasicAuthFilters) Set(value string) error { r, err := regexp.Compile(value) if err != nil { return err } *h = append(*h, basicAuthFilter{regexp: r}) return nil } // Handling of --http-allow-header-hash and --http-allow-param-hash options type hashFilter struct { name []byte percent uint32 } // HTTPHashFilters represents a slice of header hash filters type HTTPHashFilters []hashFilter func (h *HTTPHashFilters) String() string { return fmt.Sprint(*h) } // Set method to implement flags.Value func (h *HTTPHashFilters) Set(value string) error { valArr := strings.SplitN(value, ":", 2) if len(valArr) < 2 { return errors.New("need both header and value, colon-delimited (ex. user_id:50%)") } f := hashFilter{name: []byte(valArr[0])} val := strings.TrimSpace(valArr[1]) if strings.Contains(val, "%") { p, _ := strconv.ParseInt(val[:len(val)-1], 0, 0) f.percent = uint32(p) } else if strings.Contains(val, "/") { // DEPRECATED format var num, den uint64 fracArr := strings.Split(val, "/") num, _ = strconv.ParseUint(fracArr[0], 10, 64) den, _ = strconv.ParseUint(fracArr[1], 10, 64) f.percent = uint32((float64(num) / float64(den)) * 100) } else { return errors.New("Value should be percent and contain '%'") } *h = append(*h, f) return nil } // Handling of --http-set-header option type httpHeader struct { Name string Value string } // HTTPHeaders is a slice of headers that must appended type HTTPHeaders []httpHeader func (h *HTTPHeaders) String() string { return fmt.Sprint(*h) } // Set method to implement flags.Value func (h *HTTPHeaders) Set(value string) error { v := strings.SplitN(value, ":", 2) if len(v) != 2 { return errors.New("Expected `Key: Value`") } header := httpHeader{ strings.TrimSpace(v[0]), strings.TrimSpace(v[1]), } *h = append(*h, header) return nil } // Handling of --http-set-param option type httpParam struct { Name []byte Value []byte } // HTTPParams filters for --http-set-param type HTTPParams []httpParam func (h *HTTPParams) String() string { return fmt.Sprint(*h) } // Set method to implement flags.Value func (h *HTTPParams) Set(value string) error { v := strings.SplitN(value, "=", 2) if len(v) != 2 { return errors.New("Expected `Key=Value`") } param := httpParam{ []byte(strings.TrimSpace(v[0])), []byte(strings.TrimSpace(v[1])), } *h = append(*h, param) return nil } // // Handling of --http-allow-method option // // HTTPMethods holds values for method allowed type HTTPMethods [][]byte func (h *HTTPMethods) String() string { return fmt.Sprint(*h) } // Set method to implement flags.Value func (h *HTTPMethods) Set(value string) error { *h = append(*h, []byte(value)) return nil } // Handling of --http-rewrite-url option type urlRewrite struct { src *regexp.Regexp target []byte } // URLRewriteMap holds regexp and data to modify URL type URLRewriteMap []urlRewrite func (r *URLRewriteMap) String() string { return fmt.Sprint(*r) } // Set method to implement flags.Value func (r *URLRewriteMap) Set(value string) error { valArr := strings.SplitN(value, ":", 2) if len(valArr) < 2 { return errors.New("need both src and target, colon-delimited (ex. /a:/b)") } regexp, err := regexp.Compile(valArr[0]) if err != nil { return err } *r = append(*r, urlRewrite{src: regexp, target: []byte(valArr[1])}) return nil } // Handling of --http-rewrite-header option type headerRewrite struct { header []byte src *regexp.Regexp target []byte } // HeaderRewriteMap holds regexp and data to rewrite headers type HeaderRewriteMap []headerRewrite func (r *HeaderRewriteMap) String() string { return fmt.Sprint(*r) } // Set method to implement flags.Value func (r *HeaderRewriteMap) Set(value string) error { headerArr := strings.SplitN(value, ":", 2) if len(headerArr) < 2 { return errors.New("need both header, regexp and rewrite target, colon-delimited (ex. Header: regexp,target)") } header := headerArr[0] valArr := strings.SplitN(strings.TrimSpace(headerArr[1]), ",", 2) if len(valArr) < 2 { return errors.New("need both header, regexp and rewrite target, colon-delimited (ex. Header: regexp,target)") } regexp, err := regexp.Compile(valArr[0]) if err != nil { return err } *r = append(*r, headerRewrite{header: []byte(header), src: regexp, target: []byte(valArr[1])}) return nil } // Handling of --http-allow-url option type urlRegexp struct { regexp *regexp.Regexp } // HTTPURLRegexp a slice of regexp to match URLs type HTTPURLRegexp []urlRegexp func (r *HTTPURLRegexp) String() string { return fmt.Sprint(*r) } // Set method to implement flags.Value func (r *HTTPURLRegexp) Set(value string) error { regexp, err := regexp.Compile(value) *r = append(*r, urlRegexp{regexp: regexp}) return err } ================================================ FILE: http_modifier_settings_test.go ================================================ package goreplay import ( "testing" ) func TestHTTPHeaderFilters(t *testing.T) { filters := HTTPHeaderFilters{} err := filters.Set("Header1:^$") if err != nil { t.Error("Should not error on Header1:^$") } err = filters.Set("Header2:^:$") if err != nil { t.Error("Should not error on Header2:^:$") } // Missing colon err = filters.Set("Header3-^$") if err == nil { t.Error("Should error on Header2:^:$") } } func TestHTTPHashFilters(t *testing.T) { filters := HTTPHashFilters{} err := filters.Set("Header1:1/2") if err != nil { t.Error("Should support old syntax") } if filters[0].percent != 50 { t.Error("Wrong percentage", filters[0].percent) } err = filters.Set("Header2:1") if err == nil { t.Error("Should error on Header2 because no % symbol") } err = filters.Set("Header2:10%") if err != nil { t.Error("Should pass") } if filters[1].percent != 10 { t.Error("Wrong percentage", filters[1].percent) } } func TestUrlRewriteMap(t *testing.T) { var err error rewrites := URLRewriteMap{} if err = rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping"); err != nil { t.Error("Should set mapping", err) } if err = rewrites.Set("/v1/user/([^\\/]+)/ping"); err == nil { t.Error("Should not set mapping without :") } } ================================================ FILE: http_modifier_test.go ================================================ package goreplay import ( "bytes" "github.com/buger/goreplay/proto" "testing" ) func TestHTTPModifierWithoutConfig(t *testing.T) { if NewHTTPModifier(&HTTPModifierConfig{}) != nil { t.Error("If no config specified should not be initialized") } } func TestHTTPModifierHeaderFilters(t *testing.T) { filters := HTTPHeaderFilters{} filters.Set("Host:^www.w3.org$") modifier := NewHTTPModifier(&HTTPModifierConfig{ HeaderFilters: filters, }) payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") if len(modifier.Rewrite(payload)) == 0 { t.Error("Request should pass filters") } filters = HTTPHeaderFilters{} // Setting filter that not match our header filters.Set("Host:^www.w4.org$") modifier = NewHTTPModifier(&HTTPModifierConfig{ HeaderFilters: filters, }) if len(modifier.Rewrite(payload)) != 0 { t.Error("Request should not pass filters") } } func TestHTTPModifierHeaderNegativeFilters(t *testing.T) { filters := HTTPHeaderFilters{} filters.Set("Host:^www.w3.org$") modifier := NewHTTPModifier(&HTTPModifierConfig{ HeaderNegativeFilters: filters, }) payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w4.org\r\n\r\na=1&b=2") if len(modifier.Rewrite(payload)) == 0 { t.Error("Request should pass filters") } filters = HTTPHeaderFilters{} // Setting filter that not match our header filters.Set("Host:^www.w4.org$") modifier = NewHTTPModifier(&HTTPModifierConfig{ HeaderNegativeFilters: filters, }) if len(modifier.Rewrite(payload)) != 0 { t.Error("Request should not pass filters") } filters = HTTPHeaderFilters{} // Setting filter that not match our header filters.Set("Host: www*") modifier = NewHTTPModifier(&HTTPModifierConfig{ HeaderNegativeFilters: filters, }) if len(modifier.Rewrite(payload)) != 0 { t.Error("Request should not pass filters") } } func TestHTTPHeaderBasicAuthFilters(t *testing.T) { filters := HTTPHeaderBasicAuthFilters{} filters.Set("^customer[0-9].*") modifier := NewHTTPModifier(&HTTPModifierConfig{ HeaderBasicAuthFilters: filters, }) //Encoded UserId:Password = customer3:welcome payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nAuthorization: Basic Y3VzdG9tZXIzOndlbGNvbWU=\r\n\r\na=1&b=2") if len(modifier.Rewrite(payload)) == 0 { t.Error("Request should pass filters") } //customer6:rest@123^TEST payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 88\r\nAuthorization: Basic Y3VzdG9tZXI2OnJlc3RAMTIzXlRFU1Q==\r\n\r\na=1&b=2") if len(modifier.Rewrite(payload)) == 0 { t.Error("Request should pass filters") } filters = HTTPHeaderBasicAuthFilters{} // Setting filter that not match our header filters.Set("^(homer simpson|mickey mouse).*") modifier = NewHTTPModifier(&HTTPModifierConfig{ HeaderBasicAuthFilters: filters, }) if len(modifier.Rewrite(payload)) != 0 { t.Error("Request should not pass filters") } //mickey mouse:happy123 payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 88\r\nAuthorization: Basic bWlja2V5IG1vdXNlOmhhcHB5MTIz\r\n\r\na=1&b=2") if len(modifier.Rewrite(payload)) == 0 { t.Error("Request should pass filters") } } func TestHTTPModifierURLRewrite(t *testing.T) { var url, newURL []byte rewrites := URLRewriteMap{} payload := func(url []byte) []byte { return []byte("POST " + string(url) + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") } err := rewrites.Set("/v1/user/([^\\/]+)/ping:/v2/user/$1/ping") if err != nil { t.Error("Should not error on /v1/user/([^\\/]+)/ping:/v2/user/$1/ping") } modifier := NewHTTPModifier(&HTTPModifierConfig{ URLRewrite: rewrites, }) url = []byte("/v1/user/joe/ping") if newURL = proto.Path(modifier.Rewrite(payload(url))); bytes.Equal(newURL, url) { t.Error("Request url should have been rewritten, wasn't", string(newURL)) } url = []byte("/v1/user/ping") if newURL = proto.Path(modifier.Rewrite(payload(url))); !bytes.Equal(newURL, url) { t.Error("Request url should have been rewritten, wasn't", string(newURL)) } } func TestHTTPModifierHeaderRewrite(t *testing.T) { var header, newHeader []byte rewrites := HeaderRewriteMap{} payload := []byte("GET / HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") err := rewrites.Set("Host: (.*).w3.org,$1.beta.w3.org") if err != nil { t.Error("Should not error", err) } modifier := NewHTTPModifier(&HTTPModifierConfig{ HeaderRewrite: rewrites, }) header = []byte("www.beta.w3.org") if newHeader = proto.Header(modifier.Rewrite(payload), []byte("Host")); !bytes.Equal(newHeader, header) { t.Error("Request header should have been rewritten, wasn't", string(newHeader), string(header)) } } func TestHTTPModifierHeaderHashFilters(t *testing.T) { filters := HTTPHashFilters{} filters.Set("Header2:1/2") modifier := NewHTTPModifier(&HTTPModifierConfig{ HeaderHashFilters: filters, }) payload := func(header []byte) []byte { return []byte("POST / HTTP/1.1\r\n" + string(header) + "Content-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") } if p := modifier.Rewrite(payload([]byte(""))); len(p) == 0 { t.Error("Request should pass filters if Header does not exist") } if p := modifier.Rewrite(payload([]byte("Header2: 3\r\n"))); len(p) > 0 { t.Error("Request should not pass filters, Header2 hash too high") } if p := modifier.Rewrite(payload([]byte("Header2: 1\r\n"))); len(p) == 0 { t.Error("Request should pass filters") } } func TestHTTPModifierParamHashFilters(t *testing.T) { filters := HTTPHashFilters{} filters.Set("user_id:1/2") modifier := NewHTTPModifier(&HTTPModifierConfig{ ParamHashFilters: filters, }) payload := func(value []byte) []byte { return []byte("POST /" + string(value) + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") } if p := modifier.Rewrite(payload([]byte(""))); len(p) == 0 { t.Error("Request should pass filters if param does not exist") } if p := modifier.Rewrite(payload([]byte("?user_id=3"))); len(p) > 0 { t.Error("Request should not pass filters", string(p)) } if p := modifier.Rewrite(payload([]byte("?user_id=1"))); len(p) == 0 { t.Error("Request should pass filters") } } func TestHTTPModifierHeaders(t *testing.T) { headers := HTTPHeaders{} headers.Set("Header1:1") headers.Set("Host:localhost") modifier := NewHTTPModifier(&HTTPModifierConfig{ Headers: headers, }) payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") newPayload := []byte("POST /post HTTP/1.1\r\nHeader1: 1\r\nContent-Length: 7\r\nHost: localhost\r\n\r\na=1&b=2") if payload = modifier.Rewrite(payload); !bytes.Equal(payload, newPayload) { t.Error("Should update request headers", string(payload)) } } func TestHTTPModifierURLRegexp(t *testing.T) { filters := HTTPURLRegexp{} filters.Set("/v1/app") filters.Set("/v1/api") modifier := NewHTTPModifier(&HTTPModifierConfig{ URLRegexp: filters, }) payload := func(url string) []byte { return []byte("POST " + url + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") } if len(modifier.Rewrite(payload("/v1/app/test"))) == 0 { t.Error("Should pass url") } if len(modifier.Rewrite(payload("/v1/api/test"))) == 0 { t.Error("Should pass url") } if len(modifier.Rewrite(payload("/other"))) > 0 { t.Error("Should not pass url") } } func TestHTTPModifierURLNegativeRegexp(t *testing.T) { filters := HTTPURLRegexp{} filters.Set("/restricted1") filters.Set("/some/restricted2") modifier := NewHTTPModifier(&HTTPModifierConfig{ URLNegativeRegexp: filters, }) payload := func(url string) []byte { return []byte("POST " + url + " HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") } if len(modifier.Rewrite(payload("/v1/app/test"))) == 0 { t.Error("Should pass url") } if len(modifier.Rewrite(payload("/restricted1"))) > 0 { t.Error("Should not pass url") } if len(modifier.Rewrite(payload("/some/restricted2"))) > 0 { t.Error("Should not pass url") } } func TestHTTPModifierSetHeader(t *testing.T) { filters := HTTPHeaders{} filters.Set("User-Agent:Gor") modifier := NewHTTPModifier(&HTTPModifierConfig{ Headers: filters, }) payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") payloadAfter := []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") if payload = modifier.Rewrite(payload); !bytes.Equal(payloadAfter, payload) { t.Error("Should add new header", string(payload)) } } func TestHTTPModifierSetParam(t *testing.T) { filters := HTTPParams{} filters.Set("api_key=1") modifier := NewHTTPModifier(&HTTPModifierConfig{ Params: filters, }) payload := []byte("POST /post?api_key=1234 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") payloadAfter := []byte("POST /post?api_key=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") if payload = modifier.Rewrite(payload); !bytes.Equal(payloadAfter, payload) { t.Error("Should override param", string(payload)) } } ================================================ FILE: http_prettifier.go ================================================ package goreplay import ( "bytes" "compress/gzip" "fmt" "github.com/buger/goreplay/proto" "io/ioutil" "net/http/httputil" "strconv" ) func prettifyHTTP(p []byte) []byte { tEnc := bytes.Equal(proto.Header(p, []byte("Transfer-Encoding")), []byte("chunked")) cEnc := bytes.Equal(proto.Header(p, []byte("Content-Encoding")), []byte("gzip")) if !(tEnc || cEnc) { return p } headersPos := proto.MIMEHeadersEndPos(p) if headersPos < 5 || headersPos > len(p) { return p } headers := p[:headersPos] content := p[headersPos:] if tEnc { buf := bytes.NewReader(content) r := httputil.NewChunkedReader(buf) content, _ = ioutil.ReadAll(r) headers = proto.DeleteHeader(headers, []byte("Transfer-Encoding")) newLen := strconv.Itoa(len(content)) headers = proto.SetHeader(headers, []byte("Content-Length"), []byte(newLen)) } if cEnc { buf := bytes.NewReader(content) g, err := gzip.NewReader(buf) if err != nil { Debug(1, "[Prettifier] GZIP encoding error:", err) return []byte{} } content, err = ioutil.ReadAll(g) if err != nil { Debug(1, fmt.Sprintf("[HTTP-PRETTIFIER] %q", err)) return p } headers = proto.DeleteHeader(headers, []byte("Content-Encoding")) newLen := strconv.Itoa(len(content)) headers = proto.SetHeader(headers, []byte("Content-Length"), []byte(newLen)) } newPayload := append(headers, content...) return newPayload } ================================================ FILE: http_prettifier_test.go ================================================ package goreplay import ( "bytes" "compress/gzip" "github.com/buger/goreplay/proto" "strconv" "testing" ) func TestHTTPPrettifierGzip(t *testing.T) { b := bytes.NewBufferString("") w := gzip.NewWriter(b) w.Write([]byte("test")) w.Close() size := strconv.Itoa(len(b.Bytes())) payload := []byte("HTTP/1.1 200 OK\r\nContent-Length: " + size + "\r\nContent-Encoding: gzip\r\n\r\n") payload = append(payload, b.Bytes()...) newPayload := prettifyHTTP(payload) if string(newPayload) != "HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\ntest" { t.Errorf("Payload not match %q", string(newPayload)) } } func TestHTTPPrettifierChunked(t *testing.T) { payload := []byte("POST / HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n") payload = prettifyHTTP(payload) if string(proto.Header(payload, []byte("Content-Length"))) != "23" { t.Errorf("payload should have content length of 23") } } ================================================ FILE: input_dummy.go ================================================ package goreplay import ( "time" ) // DummyInput used for debugging. It generate 1 "GET /"" request per second. type DummyInput struct { data chan []byte quit chan struct{} } // NewDummyInput constructor for DummyInput func NewDummyInput(options string) (di *DummyInput) { di = new(DummyInput) di.data = make(chan []byte) di.quit = make(chan struct{}) go di.emit() return } // PluginRead reads message from this plugin func (i *DummyInput) PluginRead() (*Message, error) { var msg Message select { case <-i.quit: return nil, ErrorStopped case buf := <-i.data: msg.Meta, msg.Data = payloadMetaWithBody(buf) return &msg, nil } } func (i *DummyInput) emit() { ticker := time.NewTicker(time.Second) for range ticker.C { uuid := uuid() reqh := payloadHeader(RequestPayload, uuid, time.Now().UnixNano(), -1) i.data <- append(reqh, []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n")...) resh := payloadHeader(ResponsePayload, uuid, time.Now().UnixNano()+1, 1) i.data <- append(resh, []byte("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")...) } } func (i *DummyInput) String() string { return "Dummy Input" } // Close closes this plugins func (i *DummyInput) Close() error { close(i.quit) return nil } ================================================ FILE: input_file.go ================================================ package goreplay import ( "bufio" "bytes" "compress/gzip" "container/heap" "errors" "expvar" "fmt" "io" "math" "os" "path/filepath" "strconv" "strings" "sync" "sync/atomic" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" ) type filePayload struct { data []byte timestamp int64 } // An IntHeap is a min-heap of ints. type payloadQueue struct { sync.RWMutex s []*filePayload } func (h payloadQueue) Len() int { return len(h.s) } func (h payloadQueue) Less(i, j int) bool { return h.s[i].timestamp < h.s[j].timestamp } func (h payloadQueue) Swap(i, j int) { h.s[i], h.s[j] = h.s[j], h.s[i] } func (h *payloadQueue) Push(x interface{}) { // Push and Pop use pointer receivers because they modify the slice's length, // not just its contents. h.s = append(h.s, x.(*filePayload)) } func (h *payloadQueue) Pop() interface{} { old := h.s n := len(old) x := old[n-1] h.s = old[0 : n-1] return x } func (h payloadQueue) Idx(i int) *filePayload { return h.s[i] } type fileInputReader struct { reader *bufio.Reader file io.ReadCloser closed int32 // Value of 0 indicates that the file is still open. s3 bool queue payloadQueue readDepth int dryRun bool path string } func (f *fileInputReader) parse(init chan struct{}) error { payloadSeparatorAsBytes := []byte(payloadSeparator) var buffer bytes.Buffer var initialized bool lineNum := 0 for { line, err := f.reader.ReadBytes('\n') lineNum++ if err != nil { if err != io.EOF { Debug(1, err) } f.Close() if !initialized { close(init) initialized = true } return err } if bytes.Equal(payloadSeparatorAsBytes[1:], line) { asBytes := buffer.Bytes() meta := payloadMeta(asBytes) if len(meta) < 3 { Debug(1, fmt.Sprintf("Found malformed record, file: %s, line %d", f.path, lineNum)) buffer = bytes.Buffer{} continue } timestamp, _ := strconv.ParseInt(string(meta[2]), 10, 64) data := asBytes[:len(asBytes)-1] f.queue.Lock() heap.Push(&f.queue, &filePayload{ timestamp: timestamp, data: data, }) f.queue.Unlock() for { if f.queue.Len() < f.readDepth { break } if !initialized { close(init) initialized = true } if !f.dryRun { time.Sleep(100 * time.Millisecond) } } buffer = bytes.Buffer{} continue } buffer.Write(line) } } func (f *fileInputReader) wait() { for { if atomic.LoadInt32(&f.closed) == 1 { return } if f.queue.Len() > 0 { return } if !f.dryRun { time.Sleep(100 * time.Millisecond) } } return } // Close closes this plugin func (f *fileInputReader) Close() error { if atomic.LoadInt32(&f.closed) == 0 { atomic.StoreInt32(&f.closed, 1) f.file.Close() } return nil } func newFileInputReader(path string, readDepth int, dryRun bool) *fileInputReader { var file io.ReadCloser var err error if strings.HasPrefix(path, "s3://") { file = NewS3ReadCloser(path) } else { file, err = os.Open(path) } if err != nil { Debug(0, fmt.Sprintf("[INPUT-FILE] err: %q", err)) return nil } r := &fileInputReader{path: path, file: file, closed: 0, readDepth: readDepth, dryRun: dryRun} if strings.HasSuffix(path, ".gz") { gzReader, err := gzip.NewReader(file) if err != nil { Debug(0, fmt.Sprintf("[INPUT-FILE] err: %q", err)) return nil } r.reader = bufio.NewReader(gzReader) } else { r.reader = bufio.NewReader(file) } heap.Init(&r.queue) init := make(chan struct{}) go r.parse(init) <-init return r } // FileInput can read requests generated by FileOutput type FileInput struct { mu sync.Mutex data chan []byte exit chan bool path string readers []*fileInputReader speedFactor float64 loop bool readDepth int dryRun bool maxWait time.Duration stats *expvar.Map } // NewFileInput constructor for FileInput. Accepts file path as argument. func NewFileInput(path string, loop bool, readDepth int, maxWait time.Duration, dryRun bool) (i *FileInput) { i = new(FileInput) i.data = make(chan []byte, 1000) i.exit = make(chan bool) i.path = path i.speedFactor = 1 i.loop = loop i.readDepth = readDepth i.stats = expvar.NewMap("file-" + path) i.dryRun = dryRun i.maxWait = maxWait if err := i.init(); err != nil { return } go i.emit() return } func parseS3Url(path string) (bucket, key string) { path = path[5:] // stripping `s3://` sep := strings.IndexByte(path, '/') bucket = path[:sep] key = path[sep+1:] return bucket, key } func (i *FileInput) init() (err error) { defer i.mu.Unlock() i.mu.Lock() var matches []string if strings.HasPrefix(i.path, "s3://") { sess := session.Must(session.NewSession(awsConfig())) svc := s3.New(sess) bucket, key := parseS3Url(i.path) params := &s3.ListObjectsInput{ Bucket: aws.String(bucket), Prefix: aws.String(key), } resp, err := svc.ListObjects(params) if err != nil { Debug(2, "[INPUT-FILE] Error while retrieving list of files from S3", i.path, err) return err } for _, c := range resp.Contents { matches = append(matches, "s3://"+bucket+"/"+(*c.Key)) } } else if matches, err = filepath.Glob(i.path); err != nil { Debug(2, "[INPUT-FILE] Wrong file pattern", i.path, err) return } if len(matches) == 0 { Debug(2, "[INPUT-FILE] No files match pattern: ", i.path) return errors.New("no matching files") } i.readers = make([]*fileInputReader, len(matches)) for idx, p := range matches { i.readers[idx] = newFileInputReader(p, i.readDepth, i.dryRun) } i.stats.Add("reader_count", int64(len(matches))) return nil } // PluginRead reads message from this plugin func (i *FileInput) PluginRead() (*Message, error) { var msg Message select { case <-i.exit: return nil, ErrorStopped case buf := <-i.data: i.stats.Add("read_from", 1) msg.Meta, msg.Data = payloadMetaWithBody(buf) return &msg, nil } } func (i *FileInput) String() string { return "File input: " + i.path } // Find reader with smallest timestamp e.g next payload in row func (i *FileInput) nextReader() (next *fileInputReader) { for _, r := range i.readers { if r == nil { continue } r.wait() if r.queue.Len() == 0 { continue } if next == nil || r.queue.Idx(0).timestamp < next.queue.Idx(0).timestamp { next = r continue } } return } func (i *FileInput) emit() { var lastTime int64 = -1 var maxWait, firstWait, minWait int64 minWait = math.MaxInt64 i.stats.Add("negative_wait", 0) for { select { case <-i.exit: return default: } reader := i.nextReader() if reader == nil { if i.loop { i.init() lastTime = -1 continue } else { break } } reader.queue.RLock() payload := heap.Pop(&reader.queue).(*filePayload) i.stats.Add("total_counter", 1) i.stats.Add("total_bytes", int64(len(payload.data))) reader.queue.RUnlock() if lastTime != -1 { diff := payload.timestamp - lastTime if firstWait == 0 { firstWait = diff } if i.speedFactor != 1 { diff = int64(float64(diff) / i.speedFactor) } if i.maxWait > 0 && diff > int64(i.maxWait) { diff = int64(i.maxWait) } if diff >= 0 { lastTime = payload.timestamp if !i.dryRun { time.Sleep(time.Duration(diff)) } i.stats.Add("total_wait", diff) if diff > maxWait { maxWait = diff } if diff < minWait { minWait = diff } } else { i.stats.Add("negative_wait", 1) } } else { lastTime = payload.timestamp } // Recheck if we have exited since last check. select { case <-i.exit: return default: if !i.dryRun { i.data <- payload.data } } } i.stats.Set("first_wait", time.Duration(firstWait)) i.stats.Set("max_wait", time.Duration(maxWait)) i.stats.Set("min_wait", time.Duration(minWait)) Debug(2, fmt.Sprintf("[INPUT-FILE] FileInput: end of file '%s'\n", i.path)) if i.dryRun { fmt.Printf("Records found: %v\nFiles processed: %v\nBytes processed: %v\nMax wait: %v\nMin wait: %v\nFirst wait: %v\nIt will take `%v` to replay at current speed.\nFound %v records with out of order timestamp\n", i.stats.Get("total_counter"), i.stats.Get("reader_count"), i.stats.Get("total_bytes"), i.stats.Get("max_wait"), i.stats.Get("min_wait"), i.stats.Get("first_wait"), time.Duration(i.stats.Get("total_wait").(*expvar.Int).Value()), i.stats.Get("negative_wait"), ) } } // Close closes this plugin func (i *FileInput) Close() error { defer i.mu.Unlock() i.mu.Lock() close(i.exit) for _, r := range i.readers { r.Close() } return nil } ================================================ FILE: input_file_test.go ================================================ package goreplay import ( "bytes" "errors" "fmt" "io/ioutil" "math/rand" "os" "sync" "testing" "time" ) func TestInputFileWithGET(t *testing.T) { input := NewTestInput() rg := NewRequestGenerator([]PluginReader{input}, func() { input.EmitGET() }, 1) readPayloads := []*Message{} // Given a capture file with a GET request expectedCaptureFile := CreateCaptureFile(rg) defer expectedCaptureFile.TearDown() // When the request is read from the capture file err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(msg *Message) { readPayloads = append(readPayloads, msg) }) // The read request should match the original request if err != nil { t.Error(err) } else if !expectedCaptureFile.PayloadsEqual(readPayloads) { t.Error("Request read back from file should match") } } func TestInputFileWithPayloadLargerThan64Kb(t *testing.T) { input := NewTestInput() rg := NewRequestGenerator([]PluginReader{input}, func() { input.EmitSizedPOST(64 * 1024) }, 1) readPayloads := []*Message{} // Given a capture file with a request over 64Kb expectedCaptureFile := CreateCaptureFile(rg) defer expectedCaptureFile.TearDown() // When the request is read from the capture file err := ReadFromCaptureFile(expectedCaptureFile.file, 1, func(msg *Message) { readPayloads = append(readPayloads, msg) }) // The read request should match the original request if err != nil { t.Error(err) } else if !expectedCaptureFile.PayloadsEqual(readPayloads) { t.Error("Request read back from file should match") } } func TestInputFileWithGETAndPOST(t *testing.T) { input := NewTestInput() rg := NewRequestGenerator([]PluginReader{input}, func() { input.EmitGET() input.EmitPOST() }, 2) readPayloads := []*Message{} // Given a capture file with a GET request expectedCaptureFile := CreateCaptureFile(rg) defer expectedCaptureFile.TearDown() // When the requests are read from the capture file err := ReadFromCaptureFile(expectedCaptureFile.file, 2, func(msg *Message) { readPayloads = append(readPayloads, msg) }) // The read requests should match the original request if err != nil { t.Error(err) } else if !expectedCaptureFile.PayloadsEqual(readPayloads) { t.Error("Request read back from file should match") } } func TestInputFileMultipleFilesWithRequestsOnly(t *testing.T) { rnd := rand.Int63() file1, _ := os.OpenFile(fmt.Sprintf("/tmp/%d_0", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) file1.Write([]byte("1 1 1\ntest1")) file1.Write([]byte(payloadSeparator)) file1.Write([]byte("1 1 3\ntest2")) file1.Write([]byte(payloadSeparator)) file1.Close() file2, _ := os.OpenFile(fmt.Sprintf("/tmp/%d_1", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) file2.Write([]byte("1 1 2\ntest3")) file2.Write([]byte(payloadSeparator)) file2.Write([]byte("1 1 4\ntest4")) file2.Write([]byte(payloadSeparator)) file2.Close() input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false, 100, 0, false) for i := '1'; i <= '4'; i++ { msg, _ := input.PluginRead() if msg.Meta[4] != byte(i) { t.Error("Should emit requests in right order", string(msg.Meta)) } } os.Remove(file1.Name()) os.Remove(file2.Name()) } func TestInputFileRequestsWithLatency(t *testing.T) { rnd := rand.Int63() file, _ := os.OpenFile(fmt.Sprintf("/tmp/%d", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) defer file.Close() file.Write([]byte("1 1 100000000\nrequest1")) file.Write([]byte(payloadSeparator)) file.Write([]byte("1 2 150000000\nrequest2")) file.Write([]byte(payloadSeparator)) file.Write([]byte("1 3 250000000\nrequest3")) file.Write([]byte(payloadSeparator)) input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), false, 100, 0, false) start := time.Now().UnixNano() for i := 0; i < 3; i++ { input.PluginRead() } end := time.Now().UnixNano() var expectedLatency int64 = 300000000 - 100000000 realLatency := end - start if realLatency > expectedLatency { t.Errorf("Should emit requests respecting latency. Expected: %v, real: %v", expectedLatency, realLatency) } } func TestInputFileMultipleFilesWithRequestsAndResponses(t *testing.T) { rnd := rand.Int63() file1, _ := os.OpenFile(fmt.Sprintf("/tmp/%d_0", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) file1.Write([]byte("1 1 1\nrequest1")) file1.Write([]byte(payloadSeparator)) file1.Write([]byte("2 1 1\nresponse1")) file1.Write([]byte(payloadSeparator)) file1.Write([]byte("1 2 3\nrequest2")) file1.Write([]byte(payloadSeparator)) file1.Write([]byte("2 2 3\nresponse2")) file1.Write([]byte(payloadSeparator)) file1.Close() file2, _ := os.OpenFile(fmt.Sprintf("/tmp/%d_1", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) file2.Write([]byte("1 3 2\nrequest3")) file2.Write([]byte(payloadSeparator)) file2.Write([]byte("2 3 2\nresponse3")) file2.Write([]byte(payloadSeparator)) file2.Write([]byte("1 4 4\nrequest4")) file2.Write([]byte(payloadSeparator)) file2.Write([]byte("2 4 4\nresponse4")) file2.Write([]byte(payloadSeparator)) file2.Close() input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false, 100, 0, false) for i := '1'; i <= '4'; i++ { msg, _ := input.PluginRead() if msg.Meta[0] != '1' && msg.Meta[4] != byte(i) { t.Error("Shound emit requests in right order", string(msg.Meta)) } msg, _ = input.PluginRead() if msg.Meta[0] != '2' && msg.Meta[4] != byte(i) { t.Error("Shound emit responses in right order", string(msg.Meta)) } } os.Remove(file1.Name()) os.Remove(file2.Name()) } func TestInputFileLoop(t *testing.T) { rnd := rand.Int63() file, _ := os.OpenFile(fmt.Sprintf("/tmp/%d", rnd), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) file.Write([]byte("1 1 1\ntest1")) file.Write([]byte(payloadSeparator)) file.Write([]byte("1 1 2\ntest2")) file.Write([]byte(payloadSeparator)) file.Close() input := NewFileInput(fmt.Sprintf("/tmp/%d", rnd), true, 100, 0, false) // Even if we have just 2 requests in file, it should indifinitly loop for i := 0; i < 1000; i++ { input.PluginRead() } input.Close() os.Remove(file.Name()) } func TestInputFileCompressed(t *testing.T) { rnd := rand.Int63() output := NewFileOutput(fmt.Sprintf("/tmp/%d_0.gz", rnd), &FileOutputConfig{FlushInterval: time.Minute, Append: true}) for i := 0; i < 1000; i++ { output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")}) } name1 := output.file.Name() output.Close() output2 := NewFileOutput(fmt.Sprintf("/tmp/%d_1.gz", rnd), &FileOutputConfig{FlushInterval: time.Minute, Append: true}) for i := 0; i < 1000; i++ { output2.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")}) } name2 := output2.file.Name() output2.Close() input := NewFileInput(fmt.Sprintf("/tmp/%d*", rnd), false, 100, 0, false) for i := 0; i < 2000; i++ { input.PluginRead() } os.Remove(name1) os.Remove(name2) } type CaptureFile struct { msgs []*Message file *os.File } func NewExpectedCaptureFile(msgs []*Message, file *os.File) *CaptureFile { ecf := new(CaptureFile) ecf.file = file ecf.msgs = msgs return ecf } func (expectedCaptureFile *CaptureFile) TearDown() { if expectedCaptureFile.file != nil { os.Remove(expectedCaptureFile.file.Name()) } } type RequestGenerator struct { inputs []PluginReader emit func() wg *sync.WaitGroup } func NewRequestGenerator(inputs []PluginReader, emit func(), count int) (rg *RequestGenerator) { rg = new(RequestGenerator) rg.inputs = inputs rg.emit = emit rg.wg = new(sync.WaitGroup) rg.wg.Add(count) return } func (expectedCaptureFile *CaptureFile) PayloadsEqual(other []*Message) bool { if len(expectedCaptureFile.msgs) != len(other) { return false } for i, payload := range other { if !bytes.Equal(expectedCaptureFile.msgs[i].Meta, payload.Meta) { return false } if !bytes.Equal(expectedCaptureFile.msgs[i].Data, payload.Data) { return false } } return true } func CreateCaptureFile(requestGenerator *RequestGenerator) *CaptureFile { f, err := ioutil.TempFile("", "testmainconf") if err != nil { panic(err) } readPayloads := []*Message{} output := NewTestOutput(func(msg *Message) { readPayloads = append(readPayloads, msg) requestGenerator.wg.Done() }) outputFile := NewFileOutput(f.Name(), &FileOutputConfig{FlushInterval: time.Second, Append: true}) plugins := &InOutPlugins{ Inputs: requestGenerator.inputs, Outputs: []PluginWriter{output, outputFile}, } for _, input := range requestGenerator.inputs { plugins.All = append(plugins.All, input) } plugins.All = append(plugins.All, output, outputFile) emitter := NewEmitter() go emitter.Start(plugins, Settings.Middleware) requestGenerator.emit() requestGenerator.wg.Wait() time.Sleep(100 * time.Millisecond) emitter.Close() return NewExpectedCaptureFile(readPayloads, f) } func ReadFromCaptureFile(captureFile *os.File, count int, callback writeCallback) (err error) { wg := new(sync.WaitGroup) input := NewFileInput(captureFile.Name(), false, 100, 0, false) output := NewTestOutput(func(msg *Message) { callback(msg) wg.Done() }) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output}, } plugins.All = append(plugins.All, input, output) wg.Add(count) emitter := NewEmitter() go emitter.Start(plugins, Settings.Middleware) done := make(chan int, 1) go func() { wg.Wait() done <- 1 }() select { case <-done: break case <-time.After(2 * time.Second): err = errors.New("Timed out") } emitter.Close() return } ================================================ FILE: input_http.go ================================================ package goreplay import ( "log" "net" "net/http" "net/http/httputil" "time" ) // HTTPInput used for sending requests to Gor via http type HTTPInput struct { data chan []byte address string listener net.Listener stop chan bool // Channel used only to indicate goroutine should shutdown } // NewHTTPInput constructor for HTTPInput. Accepts address with port which it will listen on. func NewHTTPInput(address string) (i *HTTPInput) { i = new(HTTPInput) i.data = make(chan []byte, 1000) i.stop = make(chan bool) i.listen(address) return } // PluginRead reads message from this plugin func (i *HTTPInput) PluginRead() (*Message, error) { var msg Message select { case <-i.stop: return nil, ErrorStopped case buf := <-i.data: msg.Data = buf msg.Meta = payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1) return &msg, nil } } // Close closes this plugin func (i *HTTPInput) Close() error { close(i.stop) return nil } func (i *HTTPInput) handler(w http.ResponseWriter, r *http.Request) { r.URL.Scheme = "http" r.URL.Host = i.address buf, _ := httputil.DumpRequestOut(r, true) http.Error(w, http.StatusText(200), 200) i.data <- buf } func (i *HTTPInput) listen(address string) { var err error mux := http.NewServeMux() mux.HandleFunc("/", i.handler) i.listener, err = net.Listen("tcp", address) if err != nil { log.Fatal("HTTP input listener failure:", err) } i.address = i.listener.Addr().String() go func() { err = http.Serve(i.listener, mux) if err != nil && err != http.ErrServerClosed { log.Fatal("HTTP input serve failure ", err) } }() } func (i *HTTPInput) String() string { return "HTTP input: " + i.address } ================================================ FILE: input_http_test.go ================================================ package goreplay import ( "bytes" "net/http" "strings" "sync" "testing" "time" ) func TestHTTPInput(t *testing.T) { wg := new(sync.WaitGroup) input := NewHTTPInput("127.0.0.1:0") time.Sleep(time.Millisecond) output := NewTestOutput(func(*Message) { wg.Done() }) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output}, } plugins.All = append(plugins.All, input, output) emitter := NewEmitter() go emitter.Start(plugins, Settings.Middleware) address := strings.Replace(input.address, "[::]", "127.0.0.1", -1) for i := 0; i < 100; i++ { wg.Add(1) http.Get("http://" + address + "/") } wg.Wait() emitter.Close() } func TestInputHTTPLargePayload(t *testing.T) { wg := new(sync.WaitGroup) const n = 10 << 20 // 10MB var large [n]byte large[n-1] = '0' input := NewHTTPInput("127.0.0.1:0") output := NewTestOutput(func(msg *Message) { _len := len(msg.Data) if _len >= n { // considering http body CRLF t.Errorf("expected body to be >= %d", n) } wg.Done() }) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output}, } plugins.All = append(plugins.All, input, output) emitter := NewEmitter() defer emitter.Close() go emitter.Start(plugins, Settings.Middleware) address := strings.Replace(input.address, "[::]", "127.0.0.1", -1) var req *http.Request var err error req, err = http.NewRequest("POST", "http://"+address, bytes.NewBuffer(large[:])) if err != nil { t.Error(err) return } wg.Add(1) _, err = http.DefaultClient.Do(req) if err != nil { t.Error(err) return } wg.Wait() } ================================================ FILE: input_kafka.go ================================================ package goreplay import ( "encoding/json" "log" "strconv" "strings" "time" "github.com/Shopify/sarama" "github.com/Shopify/sarama/mocks" ) // KafkaInput is used for receiving Kafka messages and // transforming them into HTTP payloads. type KafkaInput struct { config *InputKafkaConfig consumers []sarama.PartitionConsumer messages chan *sarama.ConsumerMessage speedFactor float64 quit chan struct{} kafkaTimer *kafkaTimer } func getOffsetOfPartitions(offsetCfg string) int64 { offset, err := strconv.ParseInt(offsetCfg, 10, 64) if err != nil || offset < -2 { log.Fatalln("Failed to parse offset: "+offsetCfg, err) } return offset } // NewKafkaInput creates instance of kafka consumer client with TLS config func NewKafkaInput(offsetCfg string, config *InputKafkaConfig, tlsConfig *KafkaTLSConfig) *KafkaInput { c := NewKafkaConfig(&config.SASLConfig, tlsConfig) var con sarama.Consumer if mock, ok := config.consumer.(*mocks.Consumer); ok && mock != nil { con = config.consumer } else { var err error con, err = sarama.NewConsumer(strings.Split(config.Host, ","), c) if err != nil { log.Fatalln("Failed to start Sarama(Kafka) consumer:", err) } } partitions, err := con.Partitions(config.Topic) if err != nil { log.Fatalln("Failed to collect Sarama(Kafka) partitions:", err) } i := &KafkaInput{ config: config, consumers: make([]sarama.PartitionConsumer, len(partitions)), messages: make(chan *sarama.ConsumerMessage, 256), speedFactor: 1, quit: make(chan struct{}), kafkaTimer: new(kafkaTimer), } i.config.Offset = offsetCfg for index, partition := range partitions { consumer, err := con.ConsumePartition(config.Topic, partition, getOffsetOfPartitions(offsetCfg)) if err != nil { log.Fatalln("Failed to start Sarama(Kafka) partition consumer:", err) } go func(consumer sarama.PartitionConsumer) { defer consumer.Close() for message := range consumer.Messages() { i.messages <- message } }(consumer) go i.ErrorHandler(consumer) i.consumers[index] = consumer } return i } // ErrorHandler should receive errors func (i *KafkaInput) ErrorHandler(consumer sarama.PartitionConsumer) { for err := range consumer.Errors() { Debug(1, "Failed to read access log entry:", err) } } // PluginRead a reads message from this plugin func (i *KafkaInput) PluginRead() (*Message, error) { var message *sarama.ConsumerMessage var msg Message select { case <-i.quit: return nil, ErrorStopped case message = <-i.messages: } inputTs := "" msg.Data = message.Value if i.config.UseJSON { var kafkaMessage KafkaMessage json.Unmarshal(message.Value, &kafkaMessage) inputTs = kafkaMessage.ReqTs var err error msg.Data, err = kafkaMessage.Dump() if err != nil { Debug(1, "[INPUT-KAFKA] failed to decode access log entry:", err) return nil, err } } // does it have meta if isOriginPayload(msg.Data) { msg.Meta, msg.Data = payloadMetaWithBody(msg.Data) inputTs = string(payloadMeta(msg.Meta)[2]) } i.timeWait(inputTs) return &msg, nil } func (i *KafkaInput) String() string { return "Kafka Input: " + i.config.Host + "/" + i.config.Topic } // Close closes this plugin func (i *KafkaInput) Close() error { close(i.quit) return nil } func (i *KafkaInput) timeWait(curInputTs string) { if i.config.Offset == "-1" || curInputTs == "" { return } // implement for Kafka input showdown or speedup emitting timer := i.kafkaTimer curTs := time.Now().UnixNano() curInput, err := strconv.ParseInt(curInputTs, 10, 64) if timer.latestInputTs == 0 || timer.latestOutputTs == 0 { timer.latestInputTs = curInput timer.latestOutputTs = curTs return } if err != nil { log.Fatalln("Fatal to parse timestamp err: ", err) } diffTs := curInput - timer.latestInputTs pastTs := curTs - timer.latestOutputTs diff := diffTs - pastTs if i.speedFactor != 1 { diff = int64(float64(diff) / i.speedFactor) } if diff > 0 { time.Sleep(time.Duration(diff)) } timer.latestInputTs = curInput timer.latestOutputTs = curTs } type kafkaTimer struct { latestInputTs int64 latestOutputTs int64 } ================================================ FILE: input_kafka_test.go ================================================ package goreplay import ( "testing" "github.com/Shopify/sarama" "github.com/Shopify/sarama/mocks" ) func TestInputKafkaRAW(t *testing.T) { consumer := mocks.NewConsumer(t, nil) defer consumer.Close() consumer.ExpectConsumePartition("test", 0, mocks.AnyOffset).YieldMessage(&sarama.ConsumerMessage{Value: []byte("1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n")}) consumer.SetTopicMetadata( map[string][]int32{"test": {0}}, ) input := NewKafkaInput("-1", &InputKafkaConfig{ consumer: consumer, Topic: "test", UseJSON: false, }, nil) msg, err := input.PluginRead() if err != nil { t.Fatal(err) } if string(append(msg.Meta, msg.Data...)) != "1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n" { t.Error("Message not properly decoded") } } func TestInputKafkaJSON(t *testing.T) { consumer := mocks.NewConsumer(t, nil) defer consumer.Close() consumer.ExpectConsumePartition("test", 0, mocks.AnyOffset).YieldMessage(&sarama.ConsumerMessage{Value: []byte(`{"Req_URL":"/","Req_Type":"1","Req_ID":"2","Req_Ts":"3","Req_Method":"GET","Req_Headers":{"Header":"1"}}`)}) consumer.SetTopicMetadata( map[string][]int32{"test": {0}}, ) input := NewKafkaInput("-1", &InputKafkaConfig{ consumer: consumer, Topic: "test", UseJSON: true, }, nil) msg, err := input.PluginRead() if err != nil { t.Fatal(err) } if string(append(msg.Meta, msg.Data...)) != "1 2 3\nGET / HTTP/1.1\r\nHeader: 1\r\n\r\n" { t.Error("Message not properly decoded") } } ================================================ FILE: input_raw.go ================================================ package goreplay import ( "context" "fmt" "github.com/buger/goreplay/internal/capture" "github.com/buger/goreplay/internal/tcp" "github.com/buger/goreplay/proto" "log" "net" "strconv" "strings" "sync" ) // RAWInputConfig represents configuration that can be applied on raw input type RAWInputConfig = capture.PcapOptions // RAWInput used for intercepting traffic for given address type RAWInput struct { sync.Mutex config RAWInputConfig messageStats []tcp.Stats listener *capture.Listener messageParser *tcp.MessageParser cancelListener context.CancelFunc closed bool quit chan bool // Channel used only to indicate goroutine should shutdown host string ports []uint16 } // NewRAWInput constructor for RAWInput. Accepts raw input config as arguments. func NewRAWInput(address string, config RAWInputConfig) (i *RAWInput) { i = new(RAWInput) i.config = config i.quit = make(chan bool) host, _ports, err := net.SplitHostPort(address) if err != nil { // If we are reading pcap file, no port needed if strings.HasSuffix(address, "pcap") { host = address _ports = "0" err = nil } else if strings.HasPrefix(address, "k8s://") { portIndex := strings.LastIndex(address, ":") host = address[:portIndex] _ports = address[portIndex+1:] } else { log.Fatalf("input-raw: error while parsing address: %s", err) } } if strings.HasSuffix(host, "pcap") { i.config.Engine = capture.EnginePcapFile } var ports []uint16 if _ports != "" { portsStr := strings.Split(_ports, ",") for _, portStr := range portsStr { port, err := strconv.Atoi(strings.TrimSpace(portStr)) if err != nil { log.Fatalf("parsing port error: %v", err) } ports = append(ports, uint16(port)) } } i.host = host i.ports = ports i.listen(address) return } // PluginRead reads meassage from this plugin func (i *RAWInput) PluginRead() (*Message, error) { var msgTCP *tcp.Message var msg Message select { case <-i.quit: return nil, ErrorStopped case msgTCP = <-i.listener.Messages(): msg.Data = msgTCP.Data() } var msgType byte = ResponsePayload if msgTCP.Direction == tcp.DirIncoming { msgType = RequestPayload if i.config.RealIPHeader != "" { msg.Data = proto.SetHeader(msg.Data, []byte(i.config.RealIPHeader), []byte(msgTCP.SrcAddr)) } } msg.Meta = payloadHeader(msgType, msgTCP.UUID(), msgTCP.Start.UnixNano(), msgTCP.End.UnixNano()-msgTCP.Start.UnixNano()) // to be removed.... if msgTCP.Truncated { Debug(2, "[INPUT-RAW] message truncated, increase copy-buffer-size") } // to be removed... if msgTCP.TimedOut { Debug(2, "[INPUT-RAW] message timeout reached, increase input-raw-expire") } if i.config.Stats { stat := msgTCP.Stats go i.addStats(stat) } msgTCP = nil return &msg, nil } func (i *RAWInput) listen(address string) { var err error i.listener, err = capture.NewListener(i.host, i.ports, i.config) if err != nil { log.Fatal(err) } err = i.listener.Activate() if err != nil { log.Fatal(err) } var ctx context.Context ctx, i.cancelListener = context.WithCancel(context.Background()) errCh := i.listener.ListenBackground(ctx) <-i.listener.Reading Debug(1, i) go func() { <-errCh // the listener closed voluntarily i.Close() }() } func (i *RAWInput) String() string { return fmt.Sprintf("Intercepting traffic from: %s:%s", i.host, strings.Join(strings.Fields(fmt.Sprint(i.ports)), ",")) } // GetStats returns the stats so far and reset the stats func (i *RAWInput) GetStats() []tcp.Stats { i.Lock() defer func() { i.messageStats = []tcp.Stats{} i.Unlock() }() return i.messageStats } // Close closes the input raw listener func (i *RAWInput) Close() error { i.Lock() defer i.Unlock() if i.closed { return nil } i.cancelListener() close(i.quit) i.closed = true return nil } func (i *RAWInput) addStats(mStats tcp.Stats) { i.Lock() if len(i.messageStats) >= 10000 { i.messageStats = []tcp.Stats{} } i.messageStats = append(i.messageStats, mStats) i.Unlock() } ================================================ FILE: input_raw_test.go ================================================ package goreplay import ( "bytes" "github.com/buger/goreplay/internal/capture" "github.com/buger/goreplay/internal/tcp" "github.com/buger/goreplay/proto" "io/ioutil" "net" "net/http" "net/http/httptest" "net/http/httputil" "os/exec" "strings" "sync" "sync/atomic" "testing" "time" ) const testRawExpire = time.Millisecond * 200 func TestRAWInputIPv4(t *testing.T) { wg := new(sync.WaitGroup) listener, err := net.Listen("tcp4", "127.0.0.1:0") if err != nil { t.Error(err) return } origin := &http.Server{ Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("ab")) }), ReadTimeout: 10 * time.Second, WriteTimeout: 10 * time.Second, } go origin.Serve(listener) defer listener.Close() _, port, _ := net.SplitHostPort(listener.Addr().String()) var respCounter, reqCounter int64 conf := RAWInputConfig{ Engine: capture.EnginePcap, Expire: 0, Protocol: tcp.ProtocolHTTP, TrackResponse: true, RealIPHeader: "X-Real-IP", } input := NewRAWInput(listener.Addr().String(), conf) output := NewTestOutput(func(msg *Message) { if msg.Meta[0] == '1' { if len(proto.Header(msg.Data, []byte("X-Real-IP"))) == 0 { t.Error("Should have X-Real-IP header") } reqCounter++ } else { respCounter++ } wg.Done() }) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output}, } plugins.All = append(plugins.All, input, output) addr := "http://127.0.0.1:" + port emitter := NewEmitter() defer emitter.Close() go emitter.Start(plugins, Settings.Middleware) // time.Sleep(time.Second) for i := 0; i < 1; i++ { wg.Add(2) _, err = http.Get(addr) if err != nil { t.Error(err) return } } wg.Wait() const want = 10 if reqCounter != respCounter && reqCounter != want { t.Errorf("want %d requests and %d responses, got %d requests and %d responses", want, want, reqCounter, respCounter) } } func TestRAWInputNoKeepAlive(t *testing.T) { wg := new(sync.WaitGroup) listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } origin := &http.Server{ Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("ab")) }), ReadTimeout: 10 * time.Second, WriteTimeout: 10 * time.Second, } origin.SetKeepAlivesEnabled(false) go origin.Serve(listener) defer listener.Close() _, port, _ := net.SplitHostPort(listener.Addr().String()) conf := RAWInputConfig{ Engine: capture.EnginePcap, Expire: testRawExpire, Protocol: tcp.ProtocolHTTP, TrackResponse: true, } input := NewRAWInput(":"+port, conf) var respCounter, reqCounter int64 output := NewTestOutput(func(msg *Message) { if msg.Meta[0] == '1' { atomic.AddInt64(&reqCounter, 1) wg.Done() } else { atomic.AddInt64(&respCounter, 1) wg.Done() } }) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output}, } plugins.All = append(plugins.All, input, output) addr := "http://127.0.0.1:" + port emitter := NewEmitter() go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 10; i++ { // request + response wg.Add(2) _, err = http.Get(addr) if err != nil { t.Error(err) return } } wg.Wait() const want = 10 if reqCounter != respCounter && reqCounter != want { t.Errorf("want %d requests and %d responses, got %d requests and %d responses", want, want, reqCounter, respCounter) } emitter.Close() } func TestRAWInputIPv6(t *testing.T) { wg := new(sync.WaitGroup) listener, err := net.Listen("tcp", "[::1]:0") if err != nil { return } origin := &http.Server{ Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("ab")) }), ReadTimeout: 10 * time.Second, WriteTimeout: 10 * time.Second, } go origin.Serve(listener) defer listener.Close() _, port, _ := net.SplitHostPort(listener.Addr().String()) originAddr := "[::1]:" + port var respCounter, reqCounter int64 conf := RAWInputConfig{ Engine: capture.EnginePcap, Protocol: tcp.ProtocolHTTP, TrackResponse: true, } input := NewRAWInput(originAddr, conf) output := NewTestOutput(func(msg *Message) { if msg.Meta[0] == '1' { atomic.AddInt64(&reqCounter, 1) } else { atomic.AddInt64(&respCounter, 1) } wg.Done() }) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output}, } emitter := NewEmitter() addr := "http://" + originAddr go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 10; i++ { // request + response wg.Add(2) _, err = http.Get(addr) if err != nil { t.Error(err) return } } wg.Wait() const want = 10 if reqCounter != respCounter && reqCounter != want { t.Errorf("want %d requests and %d responses, got %d requests and %d responses", want, want, reqCounter, respCounter) } emitter.Close() } func TestInputRAWChunkedEncoding(t *testing.T) { wg := new(sync.WaitGroup) fileContent, _ := ioutil.ReadFile("README.md") // Origing and Replay server initialization origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() ioutil.ReadAll(r.Body) wg.Done() })) originAddr := strings.Replace(origin.Listener.Addr().String(), "[::]", "127.0.0.1", -1) conf := RAWInputConfig{ Engine: capture.EnginePcap, Expire: time.Second, Protocol: tcp.ProtocolHTTP, TrackResponse: true, AllowIncomplete: true, } input := NewRAWInput(originAddr, conf) replay := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() body, _ := ioutil.ReadAll(r.Body) if !bytes.Equal(body, fileContent) { buf, _ := httputil.DumpRequest(r, true) t.Error("Wrong POST body:", string(buf)) } wg.Done() })) defer replay.Close() httpOutput := NewHTTPOutput(replay.URL, &HTTPOutputConfig{}) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{httpOutput}, } plugins.All = append(plugins.All, input, httpOutput) emitter := NewEmitter() defer emitter.Close() go emitter.Start(plugins, Settings.Middleware) wg.Add(2) curl := exec.Command("curl", "http://"+originAddr, "--header", "Transfer-Encoding: chunked", "--header", "Expect:", "--data-binary", "@README.md") err := curl.Run() if err != nil { t.Error(err) return } wg.Wait() } func BenchmarkRAWInputWithReplay(b *testing.B) { var respCounter, reqCounter, replayCounter uint32 wg := &sync.WaitGroup{} listener, err := net.Listen("tcp4", "127.0.0.1:0") if err != nil { b.Error(err) return } listener0, err := net.Listen("tcp4", "127.0.0.1:0") if err != nil { b.Error(err) return } origin := http.Server{ Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("ab")) }), } go origin.Serve(listener) defer origin.Close() originAddr := listener.Addr().String() replay := http.Server{ Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { atomic.AddUint32(&replayCounter, 1) w.Write(nil) wg.Done() }), } go replay.Serve(listener0) defer replay.Close() replayAddr := listener0.Addr().String() conf := RAWInputConfig{ Engine: capture.EnginePcap, Expire: testRawExpire, Protocol: tcp.ProtocolHTTP, TrackResponse: true, } input := NewRAWInput(originAddr, conf) testOutput := NewTestOutput(func(msg *Message) { if msg.Meta[0] == '1' { reqCounter++ } else { respCounter++ } wg.Done() }) httpOutput := NewHTTPOutput("http://"+replayAddr, &HTTPOutputConfig{}) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{testOutput, httpOutput}, } emitter := NewEmitter() go emitter.Start(plugins, Settings.Middleware) addr := "http://" + originAddr b.ResetTimer() for i := 0; i < b.N; i++ { wg.Add(3) // reqCounter + replayCounter + respCounter resp, err := http.Get(addr) if err != nil { wg.Add(-3) } resp.Body.Close() } wg.Wait() b.ReportMetric(float64(reqCounter), "requests") b.ReportMetric(float64(respCounter), "responses") b.ReportMetric(float64(replayCounter), "replayed") emitter.Close() } ================================================ FILE: input_tcp.go ================================================ package goreplay import ( "bufio" "bytes" "crypto/tls" "fmt" "io" "log" "net" ) // TCPInput used for internal communication type TCPInput struct { data chan *Message listener net.Listener address string config *TCPInputConfig stop chan bool // Channel used only to indicate goroutine should shutdown } // TCPInputConfig represents configuration of a TCP input plugin type TCPInputConfig struct { Secure bool `json:"input-tcp-secure"` CertificatePath string `json:"input-tcp-certificate"` KeyPath string `json:"input-tcp-certificate-key"` } // NewTCPInput constructor for TCPInput, accepts address with port func NewTCPInput(address string, config *TCPInputConfig) (i *TCPInput) { i = new(TCPInput) i.data = make(chan *Message, 1000) i.address = address i.config = config i.stop = make(chan bool) i.listen(address) return } // PluginRead returns data and details read from plugin func (i *TCPInput) PluginRead() (msg *Message, err error) { select { case <-i.stop: return nil, ErrorStopped case msg = <-i.data: return msg, nil } } // Close closes the plugin func (i *TCPInput) Close() error { close(i.stop) i.listener.Close() return nil } func (i *TCPInput) listen(address string) { if i.config.Secure { cer, err := tls.LoadX509KeyPair(i.config.CertificatePath, i.config.KeyPath) if err != nil { log.Fatalln("error while loading --input-tcp TLS certificate:", err) } config := &tls.Config{Certificates: []tls.Certificate{cer}} listener, err := tls.Listen("tcp", address, config) if err != nil { log.Fatalln("[INPUT-TCP] failed to start INPUT-TCP listener:", err) } i.listener = listener } else { listener, err := net.Listen("tcp", address) if err != nil { log.Fatalln("failed to start INPUT-TCP listener:", err) } i.listener = listener } go func() { for { conn, err := i.listener.Accept() if err == nil { go i.handleConnection(conn) continue } if isTemporaryNetworkError(err) { continue } if operr, ok := err.(*net.OpError); ok && operr.Err.Error() != "use of closed network connection" { Debug(0, fmt.Sprintf("[INPUT-TCP] listener closed, err: %q", err)) } break } }() } var payloadSeparatorAsBytes = []byte(payloadSeparator) func (i *TCPInput) handleConnection(conn net.Conn) { defer conn.Close() reader := bufio.NewReader(conn) var buffer bytes.Buffer for { line, err := reader.ReadBytes('\n') if err != nil { if isTemporaryNetworkError(err) { continue } if err != io.EOF { Debug(0, fmt.Sprintf("[INPUT-TCP] connection error: %q", err)) } break } if bytes.Equal(payloadSeparatorAsBytes[1:], line) { // unread the '\n' before monkeys buffer.UnreadByte() var msg Message msg.Meta, msg.Data = payloadMetaWithBody(buffer.Bytes()) i.data <- &msg buffer.Reset() } else { buffer.Write(line) } } } func (i *TCPInput) String() string { return "TCP input: " + i.address } func isTemporaryNetworkError(err error) bool { if nerr, ok := err.(net.Error); ok && nerr.Temporary() { return true } if operr, ok := err.(*net.OpError); ok && operr.Temporary() { return true } return false } ================================================ FILE: input_tcp_test.go ================================================ package goreplay import ( "bytes" "crypto/rand" "crypto/rsa" "crypto/tls" "crypto/x509" "encoding/pem" "io/ioutil" "log" "math/big" "net" "os" "sync" "testing" "time" ) func TestTCPInput(t *testing.T) { wg := new(sync.WaitGroup) input := NewTCPInput("127.0.0.1:0", &TCPInputConfig{}) output := NewTestOutput(func(*Message) { wg.Done() }) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output}, } plugins.All = append(plugins.All, input, output) emitter := NewEmitter() go emitter.Start(plugins, Settings.Middleware) tcpAddr, err := net.ResolveTCPAddr("tcp", input.listener.Addr().String()) if err != nil { log.Fatal(err) } conn, err := net.DialTCP("tcp", nil, tcpAddr) if err != nil { log.Fatal(err) } msg := []byte("1 1 1\nGET / HTTP/1.1\r\n\r\n") for i := 0; i < 100; i++ { wg.Add(1) if _, err = conn.Write(msg); err == nil { _, err = conn.Write(payloadSeparatorAsBytes) } if err != nil { t.Error(err) return } } wg.Wait() emitter.Close() } func genCertificate(template *x509.Certificate) ([]byte, []byte) { priv, _ := rsa.GenerateKey(rand.Reader, 2048) serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) serialNumber, _ := rand.Int(rand.Reader, serialNumberLimit) template.SerialNumber = serialNumber template.BasicConstraintsValid = true template.NotBefore = time.Now() template.NotAfter = time.Now().Add(time.Hour) derBytes, _ := x509.CreateCertificate(rand.Reader, template, template, &priv.PublicKey, priv) var certPem, keyPem bytes.Buffer pem.Encode(&certPem, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) pem.Encode(&keyPem, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}) return certPem.Bytes(), keyPem.Bytes() } func TestTCPInputSecure(t *testing.T) { serverCertPem, serverPrivPem := genCertificate(&x509.Certificate{ DNSNames: []string{"localhost"}, IPAddresses: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::")}, }) serverCertPemFile, _ := ioutil.TempFile("", "server.crt") serverCertPemFile.Write(serverCertPem) serverCertPemFile.Close() serverPrivPemFile, _ := ioutil.TempFile("", "server.key") serverPrivPemFile.Write(serverPrivPem) serverPrivPemFile.Close() defer func() { os.Remove(serverPrivPemFile.Name()) os.Remove(serverCertPemFile.Name()) }() wg := new(sync.WaitGroup) input := NewTCPInput("127.0.0.1:0", &TCPInputConfig{ Secure: true, CertificatePath: serverCertPemFile.Name(), KeyPath: serverPrivPemFile.Name(), }) output := NewTestOutput(func(*Message) { wg.Done() }) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output}, } plugins.All = append(plugins.All, input, output) emitter := NewEmitter() go emitter.Start(plugins, Settings.Middleware) conf := &tls.Config{ InsecureSkipVerify: true, } conn, err := tls.Dial("tcp", input.listener.Addr().String(), conf) if err != nil { t.Fatal(err) } defer conn.Close() msg := []byte("1 1 1\nGET / HTTP/1.1\r\n\r\n") for i := 0; i < 100; i++ { wg.Add(1) conn.Write(msg) conn.Write([]byte(payloadSeparator)) } wg.Wait() emitter.Close() } ================================================ FILE: internal/byteutils/byteutils.go ================================================ // Package byteutils provides helpers for working with byte slices package byteutils import ( "unsafe" ) // Cut elements from slice for a given range func Cut(a []byte, from, to int) []byte { copy(a[from:], a[to:]) a = a[:len(a)-to+from] return a } // Insert new slice at specified position func Insert(a []byte, i int, b []byte) []byte { a = append(a, make([]byte, len(b))...) copy(a[i+len(b):], a[i:]) copy(a[i:i+len(b)], b) return a } // Replace function unlike bytes.Replace allows you to specify range func Replace(a []byte, from, to int, new []byte) []byte { lenDiff := len(new) - (to - from) if lenDiff > 0 { // Extend if new segment bigger a = append(a, make([]byte, lenDiff)...) copy(a[to+lenDiff:], a[to:]) copy(a[from:from+len(new)], new) return a } if lenDiff < 0 { copy(a[from:], new) copy(a[from+len(new):], a[to:]) return a[:len(a)+lenDiff] } // same size copy(a[from:], new) return a } // SliceToString preferred for large body payload (zero allocation and faster) func SliceToString(buf []byte) string { return *(*string)(unsafe.Pointer(&buf)) } ================================================ FILE: internal/byteutils/byteutils_test.go ================================================ package byteutils import ( "bytes" "testing" ) func TestCut(t *testing.T) { if !bytes.Equal(Cut([]byte("123456"), 2, 4), []byte("1256")) { t.Error("Should properly cut") } } func TestInsert(t *testing.T) { if !bytes.Equal(Insert([]byte("123456"), 2, []byte("abcd")), []byte("12abcd3456")) { t.Error("Should insert into middle of slice") } } func TestReplace(t *testing.T) { if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("ab")), []byte("12ab56")) { t.Error("Should replace when same length") } if !bytes.Equal(Replace([]byte("123456"), 2, 4, []byte("abcd")), []byte("12abcd56")) { t.Error("Should replace when replacement length bigger") } if !bytes.Equal(Replace([]byte("123456"), 2, 5, []byte("ab")), []byte("12ab6")) { t.Error("Should replace when replacement length bigger") } } func BenchmarkStringtoSlice(b *testing.B) { var s string var buf [1 << 20]byte for i := 0; i < b.N; i++ { s = SliceToString(buf[:]) } _ = s // avoid gc to optimize away the loop body } ================================================ FILE: internal/capture/af_packet.go ================================================ //go:build !linux package capture import ( "fmt" "time" "github.com/google/gopacket" ) func newAfpacketHandle(device string, snaplen int, block_size int, num_blocks int, useVLAN bool, timeout time.Duration) (*afpacketHandle, error) { return nil, fmt.Errorf("Not implemented") } func afpacketComputeSize(targetSizeMb int, snaplen int, pageSize int) ( frameSize int, blockSize int, numBlocks int, err error) { return 0, 0, 0, fmt.Errorf("Not implemented") } type afpacketHandle struct{} // ReadPacketData satisfies PacketDataSource interface func (h *afpacketHandle) ReadPacketData() (data []byte, ci gopacket.CaptureInfo, err error) { return nil, gopacket.CaptureInfo{}, fmt.Errorf("Not implemented") } // SetBPFFilter translates a BPF filter string into BPF RawInstruction and applies them. func (h *afpacketHandle) SetBPFFilter(filter string, snaplen int) (err error) { return fmt.Errorf("Not implemented") } ================================================ FILE: internal/capture/af_packet_linux.go ================================================ //go:build linux package capture import ( "fmt" "time" "github.com/google/gopacket" "github.com/google/gopacket/afpacket" "github.com/google/gopacket/layers" "github.com/google/gopacket/pcap" "golang.org/x/net/bpf" _ "github.com/google/gopacket/layers" ) type afpacketHandle struct { TPacket *afpacket.TPacket } func newAfpacketHandle(device string, snaplen int, block_size int, num_blocks int, useVLAN bool, timeout time.Duration) (*afpacketHandle, error) { h := &afpacketHandle{} var err error if device == "any" { h.TPacket, err = afpacket.NewTPacket( afpacket.OptFrameSize(snaplen), afpacket.OptBlockSize(block_size), afpacket.OptNumBlocks(num_blocks), afpacket.OptAddVLANHeader(false), afpacket.OptPollTimeout(timeout), afpacket.SocketRaw, afpacket.TPacketVersion3) } else { h.TPacket, err = afpacket.NewTPacket( afpacket.OptInterface(device), afpacket.OptFrameSize(snaplen), afpacket.OptBlockSize(block_size), afpacket.OptNumBlocks(num_blocks), afpacket.OptAddVLANHeader(false), afpacket.OptPollTimeout(timeout), afpacket.SocketRaw, afpacket.TPacketVersion3) } return h, err } // ReadPacketData satisfies PacketDataSource interface func (h *afpacketHandle) ReadPacketData() (data []byte, ci gopacket.CaptureInfo, err error) { return h.TPacket.ReadPacketData() } // SetBPFFilter translates a BPF filter string into BPF RawInstruction and applies them. func (h *afpacketHandle) SetBPFFilter(filter string, snaplen int) (err error) { pcapBPF, err := pcap.CompileBPFFilter(layers.LinkTypeEthernet, snaplen, filter) if err != nil { return err } bpfIns := []bpf.RawInstruction{} for _, ins := range pcapBPF { bpfIns2 := bpf.RawInstruction{ Op: ins.Code, Jt: ins.Jt, Jf: ins.Jf, K: ins.K, } bpfIns = append(bpfIns, bpfIns2) } if h.TPacket.SetBPF(bpfIns); err != nil { return err } return h.TPacket.SetBPF(bpfIns) } // LinkType returns ethernet link type. func (h *afpacketHandle) LinkType() layers.LinkType { return layers.LinkTypeEthernet } // Close will close afpacket source. func (h *afpacketHandle) Close() { h.TPacket.Close() } // SocketStats prints received, dropped, queue-freeze packet stats. func (h *afpacketHandle) SocketStats() (as afpacket.SocketStats, asv afpacket.SocketStatsV3, err error) { return h.TPacket.SocketStats() } // afpacketComputeSize computes the block_size and the num_blocks in such a way that the // allocated mmap buffer is close to but smaller than target_size_mb. // The restriction is that the block_size must be divisible by both the // frame size and page size. func afpacketComputeSize(targetSizeMb int, snaplen int, pageSize int) ( frameSize int, blockSize int, numBlocks int, err error) { if snaplen < pageSize { frameSize = pageSize / (pageSize / snaplen) } else { frameSize = (snaplen/pageSize + 1) * pageSize } // 128 is the default from the gopacket library so just use that blockSize = frameSize * 128 numBlocks = (targetSizeMb * 1024 * 1024) / blockSize if numBlocks == 0 { return 0, 0, 0, fmt.Errorf("Interface buffersize is too small") } return frameSize, blockSize, numBlocks, nil } ================================================ FILE: internal/capture/capture.go ================================================ package capture import ( "context" "errors" "expvar" "fmt" "github.com/buger/goreplay/internal/size" "github.com/buger/goreplay/internal/tcp" "github.com/buger/goreplay/proto" "io" "log" "net" "os" "runtime" "strings" "sync" "syscall" "time" "github.com/google/gopacket" "github.com/google/gopacket/layers" "github.com/google/gopacket/pcap" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ) var stats *expvar.Map func init() { stats = expvar.NewMap("raw") stats.Init() } // PacketHandler is a function that is used to handle packets type PacketHandler func(*tcp.Packet) type PcapStatProvider interface { Stats() (*pcap.Stats, error) } type PcapSetFilter interface { SetBPFFilter(string) error } // PcapOptions options that can be set on a pcap capture handle, // these options take effect on inactive pcap handles type PcapOptions struct { BufferTimeout time.Duration `json:"input-raw-buffer-timeout"` TimestampType string `json:"input-raw-timestamp-type"` BPFFilter string `json:"input-raw-bpf-filter"` BufferSize size.Size `json:"input-raw-buffer-size"` Promiscuous bool `json:"input-raw-promisc"` Monitor bool `json:"input-raw-monitor"` Snaplen bool `json:"input-raw-override-snaplen"` Engine EngineType `json:"input-raw-engine"` VXLANPort int `json:"input-raw-vxlan-port"` VXLANVNIs []int `json:"input-raw-vxlan-vni"` VLAN bool `json:"input-raw-vlan"` VLANVIDs []int `json:"input-raw-vlan-vid"` Expire time.Duration `json:"input-raw-expire"` TrackResponse bool `json:"input-raw-track-response"` Protocol tcp.TCPProtocol `json:"input-raw-protocol"` RealIPHeader string `json:"input-raw-realip-header"` Stats bool `json:"input-raw-stats"` AllowIncomplete bool `json:"input-raw-allow-incomplete"` IgnoreInterface []string `json:"input-raw-ignore-interface"` Transport string } // Listener handle traffic capture, this is its representation. type Listener struct { sync.Mutex config PcapOptions Activate func() error // function is used to activate the engine. it must be called before reading packets Handles map[string]packetHandle Interfaces []pcap.Interface loopIndex int Reading chan bool // this channel is closed when the listener has started reading packets messages chan *tcp.Message ports []uint16 host string // pcap file name or interface (name, hardware addr, index or ip address) closeDone chan struct{} quit chan struct{} closed bool } type packetHandle struct { handler gopacket.PacketDataSource ips []net.IP } // EngineType ... type EngineType uint8 // Available engines for intercepting traffic const ( EnginePcap EngineType = 1 << iota EnginePcapFile EngineRawSocket EngineAFPacket EngineVXLAN ) // Set is here so that EngineType can implement flag.Var func (eng *EngineType) Set(v string) error { switch v { case "", "libpcap": *eng = EnginePcap case "pcap_file": *eng = EnginePcapFile case "raw_socket": *eng = EngineRawSocket case "af_packet": *eng = EngineAFPacket case "vxlan": *eng = EngineVXLAN default: return fmt.Errorf("invalid engine %s", v) } return nil } func (eng *EngineType) String() (e string) { switch *eng { case EnginePcapFile: e = "pcap_file" case EnginePcap: e = "libpcap" case EngineRawSocket: e = "raw_socket" case EngineAFPacket: e = "af_packet" case EngineVXLAN: e = "vxlan" default: e = "" } return e } // NewListener creates and initialize a new Listener. if transport or/and engine are invalid/unsupported // is "tcp" and "pcap", are assumed. l.Engine and l.Transport can help to get the values used. // if there is an error it will be associated with getting network interfaces func NewListener(host string, ports []uint16, config PcapOptions) (l *Listener, err error) { l = &Listener{} l.host = host if l.host == "localhost" { l.host = "127.0.0.1" } l.ports = ports l.config = config l.config.Transport = "tcp" l.Handles = make(map[string]packetHandle) l.closeDone = make(chan struct{}) l.quit = make(chan struct{}) l.Reading = make(chan bool) l.messages = make(chan *tcp.Message, 10000) if strings.HasPrefix(l.host, "k8s://") { l.config.BPFFilter = l.Filter(pcap.Interface{}, k8sIPs(l.host[6:])...) } switch config.Engine { default: l.Activate = l.activatePcap case EngineRawSocket: l.Activate = l.activateRawSocket case EngineAFPacket: l.Activate = l.activateAFPacket case EnginePcapFile: l.Activate = l.activatePcapFile return case EngineVXLAN: l.Activate = l.activateVxLanSocket return } err = l.setInterfaces() if err != nil { return nil, err } return } // Listen listens for packets from the handles, and call handler on every packet received // until the context done signal is sent or there is unrecoverable error on all handles. // this function must be called after activating pcap handles func (l *Listener) Listen(ctx context.Context) (err error) { l.Lock() for key, handle := range l.Handles { go l.readHandle(key, handle) } l.Unlock() go func() { for { time.Sleep(time.Second) if l.closed { return } // Check for Pod IP changes if strings.HasPrefix(l.host, "k8s://") { newFilter := l.Filter(pcap.Interface{}, k8sIPs(l.host[6:])...) if newFilter != l.config.BPFFilter { fmt.Println("k8s pods configuration changed, new filter: ", newFilter) for _, h := range l.Handles { if _, ok := h.handler.(PcapSetFilter); ok { h.handler.(PcapSetFilter).SetBPFFilter(newFilter) } } l.config.BPFFilter = newFilter } } var prevInterfaces []string for _, in := range l.Interfaces { prevInterfaces = append(prevInterfaces, in.Name) } l.setInterfaces() for _, in := range l.Interfaces { var found bool for _, prev := range prevInterfaces { if in.Name == prev { found = true } } if !found { fmt.Println("Found new interface:", in.Name) l.Lock() l.Activate() for key, handle := range l.Handles { if key == in.Name { fmt.Println("Activating capture on:", in.Name) go l.readHandle(key, handle) break } } l.Unlock() } } } }() close(l.Reading) done := ctx.Done() select { case <-done: close(l.quit) // signal close on all handles <-l.closeDone // wait all handles to be closed err = ctx.Err() case <-l.closeDone: // all handles closed voluntarily } l.closed = true return } // ListenBackground is like listen but can run concurrently and signal error through channel func (l *Listener) ListenBackground(ctx context.Context) chan error { err := make(chan error, 1) go func() { defer close(err) if e := l.Listen(ctx); err != nil { err <- e } }() return err } // Allowed format: // // [namespace/]pod/[pod_name] // [namespace/]deployment/[deployment_name] // [namespace/]daemonset/[daemonset_name] // [namespace/]labelSelector/[selector] // [namespace/]fieldSelector/[selector] func k8sIPs(addr string) []string { config, err := rest.InClusterConfig() if err != nil { panic(err.Error()) } // creates the clientset clientset, err := kubernetes.NewForConfig(config) if err != nil { panic(err.Error()) } sections := strings.Split(addr, "/") if len(sections) < 2 { panic("Not supported k8s scheme. Allowed values: [namespace/]pod/[pod_name], [namespace/]deployment/[deployment_name], [namespace/]daemonset/[daemonset_name], [namespace/]label/[label-name]/[label-value]") } // If no namespace passed, assume it is ALL switch sections[0] { case "pod", "deployment", "daemonset", "labelSelector", "fieldSelector": sections = append([]string{""}, sections...) } namespace, selectorType, selectorValue := sections[0], sections[1], sections[2] labelSelector := "" fieldSelector := "" switch selectorType { case "pod": fieldSelector = "metadata.name=" + selectorValue case "deployment": labelSelector = "app=" + selectorValue case "daemonset": labelSelector = "pod-template-generation=1,name=" + selectorValue case "labelSelector": labelSelector = selectorValue case "fieldSelector": fieldSelector = selectorValue } pods, err := clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: labelSelector, FieldSelector: fieldSelector}) if err != nil { panic(err.Error()) } var podIPs []string for _, pod := range pods.Items { for _, podIP := range pod.Status.PodIPs { podIPs = append(podIPs, podIP.IP) } } return podIPs } // Filter returns automatic filter applied by goreplay // to a pcap handle of a specific interface func (l *Listener) Filter(ifi pcap.Interface, hosts ...string) (filter string) { // https://www.tcpdump.org/manpages/pcap-filter.7.html if len(hosts) == 0 { // If k8s have not found any IPs if strings.HasPrefix(l.host, "k8s://") { hosts = []string{} } else { hosts = []string{l.host} if listenAll(l.host) || isDevice(l.host, ifi) { hosts = interfaceAddresses(ifi) } } } filter = portsFilter(l.config.Transport, "dst", l.ports) if len(hosts) != 0 && !l.config.Promiscuous { filter = fmt.Sprintf("((%s) and (%s))", filter, hostsFilter("dst", hosts)) } else { filter = fmt.Sprintf("(%s)", filter) } if l.config.TrackResponse { responseFilter := portsFilter(l.config.Transport, "src", l.ports) if len(hosts) != 0 && !l.config.Promiscuous { responseFilter = fmt.Sprintf("((%s) and (%s))", responseFilter, hostsFilter("src", hosts)) } else { responseFilter = fmt.Sprintf("(%s)", responseFilter) } filter = fmt.Sprintf("%s or %s", filter, responseFilter) } if l.config.VLAN { if len(l.config.VLANVIDs) > 0 { for _, vi := range l.config.VLANVIDs { filter = fmt.Sprintf("vlan %d and ", vi) + filter } } else { filter = "vlan and " + filter } } return } // PcapHandle returns new pcap Handle from dev on success. // this function should be called after setting all necessary options for this listener func (l *Listener) PcapHandle(ifi pcap.Interface) (handle *pcap.Handle, err error) { var inactive *pcap.InactiveHandle inactive, err = pcap.NewInactiveHandle(ifi.Name) if err != nil { return nil, fmt.Errorf("inactive handle error: %q, interface: %q", err, ifi.Name) } defer inactive.CleanUp() if l.config.TimestampType != "" && l.config.TimestampType != "go" { var ts pcap.TimestampSource ts, err = pcap.TimestampSourceFromString(l.config.TimestampType) fmt.Println("Setting custom Timestamp Source. Supported values: `go`, ", inactive.SupportedTimestamps()) err = inactive.SetTimestampSource(ts) if err != nil { return nil, fmt.Errorf("%q: supported timestamps: %q, interface: %q", err, inactive.SupportedTimestamps(), ifi.Name) } } if l.config.Promiscuous { if err = inactive.SetPromisc(l.config.Promiscuous); err != nil { return nil, fmt.Errorf("promiscuous mode error: %q, interface: %q", err, ifi.Name) } } if l.config.Monitor { if err = inactive.SetRFMon(l.config.Monitor); err != nil && !errors.Is(err, pcap.CannotSetRFMon) { return nil, fmt.Errorf("monitor mode error: %q, interface: %q", err, ifi.Name) } } var snap int if !l.config.Snaplen { infs, _ := net.Interfaces() for _, i := range infs { if i.Name == ifi.Name { snap = i.MTU + 200 } } } if snap == 0 { snap = 64<<10 + 200 } err = inactive.SetSnapLen(snap) if err != nil { return nil, fmt.Errorf("snapshot length error: %q, interface: %q", err, ifi.Name) } if l.config.BufferSize > 0 { err = inactive.SetBufferSize(int(l.config.BufferSize)) if err != nil { return nil, fmt.Errorf("handle buffer size error: %q, interface: %q", err, ifi.Name) } } if l.config.BufferTimeout == 0 { l.config.BufferTimeout = 2000 * time.Millisecond } err = inactive.SetTimeout(l.config.BufferTimeout) if err != nil { return nil, fmt.Errorf("handle buffer timeout error: %q, interface: %q", err, ifi.Name) } handle, err = inactive.Activate() if err != nil { return nil, fmt.Errorf("PCAP Activate device error: %q, interface: %q", err, ifi.Name) } bpfFilter := l.config.BPFFilter if bpfFilter == "" { bpfFilter = l.Filter(ifi) } fmt.Println("Interface:", ifi.Name, ". BPF Filter:", bpfFilter) err = handle.SetBPFFilter(bpfFilter) if err != nil { handle.Close() return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, bpfFilter, ifi.Name) } return } // SocketHandle returns new unix ethernet handle associated with this listener settings func (l *Listener) SocketHandle(ifi pcap.Interface) (handle Socket, err error) { handle, err = NewSocket(ifi) if err != nil { return nil, fmt.Errorf("sock raw error: %q, interface: %q", err, ifi.Name) } if err = handle.SetPromiscuous(l.config.Promiscuous || l.config.Monitor); err != nil { return nil, fmt.Errorf("promiscuous mode error: %q, interface: %q", err, ifi.Name) } if l.config.BPFFilter == "" { l.config.BPFFilter = l.Filter(ifi) } fmt.Println("BPF Filter: ", l.config.BPFFilter) if err = handle.SetBPFFilter(l.config.BPFFilter); err != nil { handle.Close() return nil, fmt.Errorf("BPF filter error: %q%s, interface: %q", err, l.config.BPFFilter, ifi.Name) } handle.SetLoopbackIndex(int32(l.loopIndex)) return } func http1StartHint(pckt *tcp.Packet) (isRequest, isResponse bool) { if proto.HasRequestTitle(pckt.Payload) { return true, false } if proto.HasResponseTitle(pckt.Payload) { return false, true } // No request or response detected return false, false } func http1EndHint(m *tcp.Message) bool { if m.MissingChunk() { return false } req, res := http1StartHint(m.Packets()[0]) return proto.HasFullPayload(m, m.PacketData()...) && (req || res) } func (l *Listener) readHandle(key string, hndl packetHandle) { runtime.LockOSThread() defer l.closeHandles(key) linkSize := 14 linkType := int(layers.LinkTypeEthernet) if _, ok := hndl.handler.(*pcap.Handle); ok { linkType = int(hndl.handler.(*pcap.Handle).LinkType()) linkSize, ok = pcapLinkTypeLength(linkType, l.config.VLAN) if !ok { if os.Getenv("GORDEBUG") != "0" { log.Printf("can not identify link type of an interface '%s'\n", key) } return // can't find the linktype size } } messageParser := tcp.NewMessageParser(l.messages, l.ports, hndl.ips, l.config.Expire, l.config.AllowIncomplete) if l.config.Protocol == tcp.ProtocolHTTP { messageParser.Start = http1StartHint messageParser.End = http1EndHint } timer := time.NewTicker(1 * time.Second) for { select { case <-l.quit: return case <-timer.C: if h, ok := hndl.handler.(PcapStatProvider); ok { s, err := h.Stats() if err == nil { stats.Add("packets_received", int64(s.PacketsReceived)) stats.Add("packets_dropped", int64(s.PacketsDropped)) stats.Add("packets_if_dropped", int64(s.PacketsIfDropped)) } } default: data, ci, err := hndl.handler.ReadPacketData() if err == nil { if l.config.TimestampType == "go" { ci.Timestamp = time.Now() } messageParser.PacketHandler(&tcp.PcapPacket{ Data: data, LType: linkType, LTypeLen: linkSize, Ci: &ci, }) continue } if enext, ok := err.(pcap.NextError); ok && enext == pcap.NextErrorTimeoutExpired { continue } if eno, ok := err.(syscall.Errno); ok && eno.Temporary() { continue } if enet, ok := err.(*net.OpError); ok && (enet.Temporary() || enet.Timeout()) { continue } if err == io.EOF || err == io.ErrClosedPipe { log.Printf("stopped reading from %s interface with error %s\n", key, err) return } log.Printf("stopped reading from %s interface with error %s\n", key, err) return } } } func (l *Listener) Messages() chan *tcp.Message { return l.messages } func (l *Listener) closeHandles(key string) { l.Lock() defer l.Unlock() if handle, ok := l.Handles[key]; ok { if c, ok := handle.handler.(io.Closer); ok { c.Close() } delete(l.Handles, key) if len(l.Handles) == 0 { close(l.closeDone) } } } func (l *Listener) activatePcap() error { var e error var msg string for _, ifi := range l.Interfaces { if _, found := l.Handles[ifi.Name]; found { continue } var handle *pcap.Handle handle, e = l.PcapHandle(ifi) if e != nil { msg += ("\n" + e.Error()) continue } l.Handles[ifi.Name] = packetHandle{ handler: handle, ips: interfaceIPs(ifi), } } if len(l.Handles) == 0 { return fmt.Errorf("pcap handles error:%s", msg) } return nil } func (l *Listener) activateVxLanSocket() error { handler, err := newVXLANHandler(l.config.VXLANPort, l.config.VXLANVNIs) if err != nil { return err } l.Handles["vxlan"] = packetHandle{ handler: handler, } return nil } func (l *Listener) activateRawSocket() error { if runtime.GOOS != "linux" { return fmt.Errorf("sock_raw is not stabilized on OS other than linux") } var msg string var e error for _, ifi := range l.Interfaces { if _, found := l.Handles[ifi.Name]; found { continue } var handle Socket handle, e = l.SocketHandle(ifi) if e != nil { msg += ("\n" + e.Error()) continue } l.Handles[ifi.Name] = packetHandle{ handler: handle, ips: interfaceIPs(ifi), } } if len(l.Handles) == 0 { return fmt.Errorf("raw socket handles error:%s", msg) } return nil } func (l *Listener) activatePcapFile() (err error) { var handle *pcap.Handle var e error if handle, e = pcap.OpenOffline(l.host); e != nil { return fmt.Errorf("open pcap file error: %q", e) } tmp := l.host l.host = "" l.config.BPFFilter = l.Filter(pcap.Interface{}) l.host = tmp if e = handle.SetBPFFilter(l.config.BPFFilter); e != nil { handle.Close() return fmt.Errorf("BPF filter error: %q, filter: %s", e, l.config.BPFFilter) } fmt.Println("BPF Filter:", l.config.BPFFilter) l.Handles["pcap_file"] = packetHandle{ handler: handle, } return } func (l *Listener) activateAFPacket() error { szFrame, szBlock, numBlocks, err := afpacketComputeSize(32, 32<<10, os.Getpagesize()) if err != nil { return err } var msg string for _, ifi := range l.Interfaces { if _, found := l.Handles[ifi.Name]; found { continue } handle, err := newAfpacketHandle(ifi.Name, szFrame, szBlock, numBlocks, false, pcap.BlockForever) if err != nil { msg += ("\n" + err.Error()) continue } if l.config.BPFFilter == "" { l.config.BPFFilter = l.Filter(ifi) } fmt.Println("Interface:", ifi.Name, ". BPF Filter:", l.config.BPFFilter) handle.SetBPFFilter(l.config.BPFFilter, 64<<10) l.Handles[ifi.Name] = packetHandle{ handler: handle, ips: interfaceIPs(ifi), } } if len(l.Handles) == 0 { return fmt.Errorf("pcap handles error:%s", msg) } return nil } func (l *Listener) setInterfaces() (err error) { var pifis []pcap.Interface pifis, err = pcap.FindAllDevs() ifis, _ := net.Interfaces() l.Interfaces = []pcap.Interface{} if err != nil { return } for _, pi := range pifis { ignore := false for _, ig := range l.config.IgnoreInterface { if pi.Name == ig { ignore = true break } } if ignore { continue } if strings.HasPrefix(l.host, "k8s://") { if !strings.HasPrefix(pi.Name, "veth") { continue } } if isDevice(l.host, pi) { l.Interfaces = []pcap.Interface{pi} return } var ni net.Interface for _, i := range ifis { if i.Name == pi.Name { ni = i break } addrs, _ := i.Addrs() for _, a := range addrs { for _, pa := range pi.Addresses { if a.String() == pa.IP.String() { ni = i break } } } } if ni.Flags&net.FlagLoopback != 0 { l.loopIndex = ni.Index } if runtime.GOOS != "windows" { if len(pi.Addresses) == 0 { continue } if ni.Flags&net.FlagUp == 0 { continue } } l.Interfaces = append(l.Interfaces, pi) } return } func isDevice(addr string, ifi pcap.Interface) bool { // Windows npcap loopback have no IPs if addr == "127.0.0.1" && ifi.Name == `\Device\NPF_Loopback` { return true } if addr == ifi.Name { return true } if strings.HasSuffix(addr, "*") { if strings.HasPrefix(ifi.Name, addr[:len(addr)-1]) { return true } } for _, _addr := range ifi.Addresses { if _addr.IP.String() == addr { return true } } return false } func interfaceAddresses(ifi pcap.Interface) []string { var hosts []string for _, addr := range ifi.Addresses { hosts = append(hosts, addr.IP.String()) } return hosts } func interfaceIPs(ifi pcap.Interface) []net.IP { var ips []net.IP for _, addr := range ifi.Addresses { ips = append(ips, addr.IP) } return ips } func listenAll(addr string) bool { switch addr { case "", "0.0.0.0", "[::]", "::": return true } return false } func portsFilter(transport string, direction string, ports []uint16) string { if len(ports) == 0 || ports[0] == 0 { return fmt.Sprintf("%s %s portrange 0-%d", transport, direction, 1<<16-1) } var filters []string for _, port := range ports { filters = append(filters, fmt.Sprintf("%s %s port %d", transport, direction, port)) } return strings.Join(filters, " or ") } func hostsFilter(direction string, hosts []string) string { var hostsFilters []string for _, host := range hosts { hostsFilters = append(hostsFilters, fmt.Sprintf("%s host %s", direction, host)) } return strings.Join(hostsFilters, " or ") } func pcapLinkTypeLength(lType int, vlan bool) (int, bool) { switch layers.LinkType(lType) { case layers.LinkTypeEthernet: if vlan { return 18, true } else { return 14, true } case layers.LinkTypeNull, layers.LinkTypeLoop: return 4, true case layers.LinkTypeRaw, 12, 14: return 0, true case layers.LinkTypeIPv4, layers.LinkTypeIPv6: // (TODO:) look out for IP encapsulation? return 0, true case layers.LinkTypeLinuxSLL: return 16, true case layers.LinkTypeFDDI: return 13, true case 226 /*DLT_IPNET*/ : // https://www.tcpdump.org/linktypes/LINKTYPE_IPNET.html return 24, true default: return 0, false } } ================================================ FILE: internal/capture/capture_test.go ================================================ package capture import ( "testing" ) func TestSetInterfaces(t *testing.T) { listener := &Listener{ loopIndex: 99999, } listener.setInterfaces() for _, nic := range listener.Interfaces { if (len(nic.Addresses)) == 0 { t.Errorf("nic %s was captured with 0 addresses", nic.Name) } } if listener.loopIndex == 99999 { t.Errorf("loopback nic index was not found") } } ================================================ FILE: internal/capture/doc.go ================================================ /* Package capture provides traffic sniffier using AF_PACKET, pcap or pcap file. it allows you to listen for traffic from any port (e.g. sniffing) because they operate on IP level. Ports is TCP/IP feature, same as flow control, reliable transmission and etc. Currently this package implements TCP layer: flow control is managed under tcp package. BPF filters can also be applied. example: // for the transport should be "tcp" listener, err := capture.NewListener(host, port, transport, engine, trackResponse) if err != nil { // handle error } listener.SetPcapOptions(opts) err = listner.Activate() if err != nil { // handle it } if err := listener.Listen(context.Background(), handler); err != nil { // handle error } // or errCh := listener.ListenBackground(context.Background(), handler) // runs in the background select { case err := <- errCh: // handle error case <-quit: // case <- l.Reading: // if we have started reading } */ package capture // import github.com/buger/goreplay/capture ================================================ FILE: internal/capture/dump.go ================================================ // https://github.com/google/gopacket/blob/403ca653c4/pcapgo/read.go package capture import ( "encoding/binary" "fmt" "io" "time" "github.com/google/gopacket" "github.com/google/gopacket/layers" ) // Writer wraps an underlying io.Writer to write packet data in PCAP // format. See http://wiki.wireshark.org/Development/LibpcapFileFormat // for information on the file format. // // For those that care, we currently write v2.4 files with nanosecond // or microsecond timestamp resolution and little-endian encoding. type Writer struct { w io.Writer tsScaler int // Moving this into the struct seems to save an allocation for each call to writePacketHeader buf [16]byte } const magicNanoseconds = 0xA1B23C4D const magicMicroseconds = 0xA1B2C3D4 const versionMajor = 2 const versionMinor = 4 // NewWriterNanos returns a new writer object, for writing packet data out // to the given writer. If this is a new empty writer (as opposed to // an append), you must call WriteFileHeader before WritePacket. Packet // timestamps are written with nanosecond precision. // // // Write a new file: // f, _ := os.Create("/tmp/file.pcap") // w := pcapgo.NewWriterNanos(f) // w.WriteFileHeader(65536, layers.LinkTypeEthernet) // new file, must do this. // w.WritePacket(gopacket.CaptureInfo{...}, data1) // f.Close() // // Append to existing file (must have same snaplen and linktype) // f2, _ := os.OpenFile("/tmp/fileNano.pcap", os.O_APPEND, 0700) // w2 := pcapgo.NewWriter(f2) // // no need for file header, it's already written. // w2.WritePacket(gopacket.CaptureInfo{...}, data2) // f2.Close() func NewWriterNanos(w io.Writer) *Writer { return &Writer{w: w, tsScaler: nanosPerNano} } // NewWriter returns a new writer object, for writing packet data out // to the given writer. If this is a new empty writer (as opposed to // an append), you must call WriteFileHeader before WritePacket. // Packet timestamps are written with microsecond precision. // // // Write a new file: // f, _ := os.Create("/tmp/file.pcap") // w := pcapgo.NewWriter(f) // w.WriteFileHeader(65536, layers.LinkTypeEthernet) // new file, must do this. // w.WritePacket(gopacket.CaptureInfo{...}, data1) // f.Close() // // Append to existing file (must have same snaplen and linktype) // f2, _ := os.OpenFile("/tmp/file.pcap", os.O_APPEND, 0700) // w2 := pcapgo.NewWriter(f2) // // no need for file header, it's already written. // w2.WritePacket(gopacket.CaptureInfo{...}, data2) // f2.Close() func NewWriter(w io.Writer) *Writer { return &Writer{w: w, tsScaler: nanosPerMicro} } // WriteFileHeader writes a file header out to the writer. // This must be called exactly once per output. func (w *Writer) WriteFileHeader(snaplen uint32, linktype layers.LinkType) error { var buf [24]byte if w.tsScaler == nanosPerMicro { binary.LittleEndian.PutUint32(buf[0:4], magicMicroseconds) } else { binary.LittleEndian.PutUint32(buf[0:4], magicNanoseconds) } binary.LittleEndian.PutUint16(buf[4:6], versionMajor) binary.LittleEndian.PutUint16(buf[6:8], versionMinor) // bytes 8:12 stay 0 (timezone = UTC) // bytes 12:16 stay 0 (sigfigs is always set to zero, according to // http://wiki.wireshark.org/Development/LibpcapFileFormat binary.LittleEndian.PutUint32(buf[16:20], snaplen) binary.LittleEndian.PutUint32(buf[20:24], uint32(linktype)) _, err := w.w.Write(buf[:]) return err } const nanosPerMicro = 1000 const nanosPerNano = 1 func (w *Writer) writePacketHeader(ci gopacket.CaptureInfo) error { t := ci.Timestamp if t.IsZero() { t = time.Now() } secs := t.Unix() usecs := t.Nanosecond() / w.tsScaler binary.LittleEndian.PutUint32(w.buf[0:4], uint32(secs)) binary.LittleEndian.PutUint32(w.buf[4:8], uint32(usecs)) binary.LittleEndian.PutUint32(w.buf[8:12], uint32(ci.CaptureLength)) binary.LittleEndian.PutUint32(w.buf[12:16], uint32(ci.Length)) _, err := w.w.Write(w.buf[:]) return err } // WritePacket writes the given packet data out to the file. func (w *Writer) WritePacket(ci gopacket.CaptureInfo, data []byte) error { if ci.CaptureLength != len(data) { return fmt.Errorf("capture length %d does not match data length %d", ci.CaptureLength, len(data)) } if ci.CaptureLength > ci.Length { return fmt.Errorf("invalid capture info %+v: capture length > length", ci) } if err := w.writePacketHeader(ci); err != nil { return fmt.Errorf("error writing packet header: %v", err) } _, err := w.w.Write(data) return err } ================================================ FILE: internal/capture/sock_linux.go ================================================ //go:build linux && !arm64 package capture import ( "fmt" "net" "sync" "time" "unsafe" "golang.org/x/sys/unix" "github.com/google/gopacket" "github.com/google/gopacket/layers" "github.com/google/gopacket/pcap" ) const ( // ETHALL htons(ETH_P_ALL) ETHALL uint16 = unix.ETH_P_ALL<<8 | unix.ETH_P_ALL>>8 // BLOCKSIZE ring buffer block_size BLOCKSIZE = 64 << 10 // BLOCKNR ring buffer block_nr BLOCKNR = (2 << 20) / BLOCKSIZE // 2mb / 64kb // FRAMESIZE ring buffer frame_size FRAMESIZE = BLOCKSIZE // FRAMENR ring buffer frame_nr FRAMENR = BLOCKNR * BLOCKSIZE / FRAMESIZE // MAPHUGE2MB 2mb huge map MAPHUGE2MB = 21 << unix.MAP_HUGE_SHIFT ) var tpacket2hdrlen = tpAlign(int(unsafe.Sizeof(unix.Tpacket2Hdr{}))) // SockRaw is a linux M'maped af_packet socket type SockRaw struct { mu sync.Mutex fd int ifindex int snaplen int pollTimeout uintptr frame uint32 // current frame buf []byte // points to the memory space of the ring buffer shared with the kernel. loopIndex int32 // this field must filled to avoid reading packet twice on a loopback device } // NewSocket returns new M'maped sock_raw on packet version 2. func NewSocket(pifi pcap.Interface) (*SockRaw, error) { var ifi net.Interface infs, _ := net.Interfaces() found := false for _, i := range infs { if i.Name == pifi.Name { ifi = i found = true break } } if !found { return nil, fmt.Errorf("can't find matching interface") } // sock create fd, err := unix.Socket(unix.AF_PACKET, unix.SOCK_RAW, int(ETHALL)) if err != nil { return nil, err } sock := &SockRaw{ fd: fd, ifindex: ifi.Index, snaplen: FRAMESIZE, pollTimeout: ^uintptr(0), } // set packet version err = unix.SetsockoptInt(fd, unix.SOL_PACKET, unix.PACKET_VERSION, unix.TPACKET_V2) if err != nil { unix.Close(fd) return nil, fmt.Errorf("setsockopt packet_version: %v", err) } // bind to interface addr := unix.RawSockaddrLinklayer{ Family: unix.AF_PACKET, Protocol: ETHALL, Ifindex: int32(ifi.Index), } _, _, e := unix.Syscall( unix.SYS_BIND, uintptr(fd), uintptr(unsafe.Pointer(&addr)), uintptr(unix.SizeofSockaddrLinklayer), ) if e != 0 { unix.Close(fd) return nil, e } // create shared-memory ring buffer tp := &unix.TpacketReq{ Block_size: BLOCKSIZE, Block_nr: BLOCKNR, Frame_size: FRAMESIZE, Frame_nr: FRAMENR, } err = unix.SetsockoptTpacketReq(sock.fd, unix.SOL_PACKET, unix.PACKET_RX_RING, tp) if err != nil { unix.Close(fd) return nil, fmt.Errorf("setsockopt packet_rx_ring: %v", err) } sock.buf, err = unix.Mmap( sock.fd, 0, BLOCKSIZE*BLOCKNR, unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED|MAPHUGE2MB, ) if err != nil { unix.Close(fd) return nil, fmt.Errorf("socket mmap error: %v", err) } return sock, nil } // ReadPacketData implements gopacket.PacketDataSource. func (sock *SockRaw) ReadPacketData() (buf []byte, ci gopacket.CaptureInfo, err error) { sock.mu.Lock() defer sock.mu.Unlock() var tpHdr *unix.Tpacket2Hdr poll := &unix.PollFd{ Fd: int32(sock.fd), Events: unix.POLLIN, } var i int read: i = int(sock.frame * FRAMESIZE) tpHdr = (*unix.Tpacket2Hdr)(unsafe.Pointer(&sock.buf[i])) sock.frame = (sock.frame + 1) % FRAMENR if tpHdr.Status&unix.TP_STATUS_USER == 0 { _, _, e := unix.Syscall(unix.SYS_POLL, uintptr(unsafe.Pointer(poll)), 1, sock.pollTimeout) if e != 0 && e != unix.EINTR { return buf, ci, e } // it might be some other frame with data! if tpHdr.Status&unix.TP_STATUS_USER == 0 { goto read } } tpHdr.Status = unix.TP_STATUS_KERNEL sockAddr := (*unix.RawSockaddrLinklayer)(unsafe.Pointer(&sock.buf[i+tpacket2hdrlen])) // parse out repeating packets on loopback if sockAddr.Ifindex == sock.loopIndex && sock.frame%2 != 0 { goto read } ci.Length = int(tpHdr.Len) ci.Timestamp = time.Unix(int64(tpHdr.Sec), int64(tpHdr.Nsec)) ci.InterfaceIndex = int(sockAddr.Ifindex) buf = make([]byte, tpHdr.Snaplen) ci.CaptureLength = copy(buf, sock.buf[i+int(tpHdr.Mac):]) return } // Close closes the underlying socket func (sock *SockRaw) Close() (err error) { sock.mu.Lock() defer sock.mu.Unlock() if sock.fd != -1 { unix.Munmap(sock.buf) sock.buf = nil err = unix.Close(sock.fd) sock.fd = -1 } return } // SetSnapLen sets the maximum capture length to the given value. // for this to take effects on the kernel level SetBPFilter should be called too. func (sock *SockRaw) SetSnapLen(snap int) error { sock.mu.Lock() defer sock.mu.Unlock() if snap < 0 { return fmt.Errorf("expected %d snap length to be at least 0", snap) } if snap > FRAMESIZE { snap = FRAMESIZE } sock.snaplen = snap return nil } // SetTimeout sets poll wait timeout for the socket. // negative value will block forever func (sock *SockRaw) SetTimeout(t time.Duration) error { sock.mu.Lock() defer sock.mu.Unlock() sock.pollTimeout = uintptr(t) return nil } // GetSnapLen returns the maximum capture length func (sock *SockRaw) GetSnapLen() int { sock.mu.Lock() defer sock.mu.Unlock() return sock.snaplen } // SetBPFFilter compiles and sets a BPF filter for the socket handle. func (sock *SockRaw) SetBPFFilter(expr string) error { sock.mu.Lock() defer sock.mu.Unlock() if expr == "" { return unix.SetsockoptInt(sock.fd, unix.SOL_SOCKET, unix.SO_DETACH_FILTER, 0) } filter, err := pcap.CompileBPFFilter(layers.LinkTypeEthernet, sock.snaplen, expr) if err != nil { return err } if len(filter) > int(^uint16(0)) { return fmt.Errorf("filters out of range 0-%d", ^uint16(0)) } if len(filter) == 0 { return unix.SetsockoptInt(sock.fd, unix.SOL_SOCKET, unix.SO_DETACH_FILTER, 0) } fprog := &unix.SockFprog{ Len: uint16(len(filter)), Filter: &(*(*[]unix.SockFilter)(unsafe.Pointer(&filter)))[0], } return unix.SetsockoptSockFprog(sock.fd, unix.SOL_SOCKET, unix.SO_ATTACH_FILTER, fprog) } // SetPromiscuous sets promiscuous mode to the required value. for better result capture on all interfaces instead. // If it is enabled, traffic not destined for the interface will also be captured. func (sock *SockRaw) SetPromiscuous(b bool) error { sock.mu.Lock() defer sock.mu.Unlock() mreq := unix.PacketMreq{ Ifindex: int32(sock.ifindex), Type: unix.PACKET_MR_PROMISC, } opt := unix.PACKET_ADD_MEMBERSHIP if !b { opt = unix.PACKET_DROP_MEMBERSHIP } return unix.SetsockoptPacketMreq(sock.fd, unix.SOL_PACKET, opt, &mreq) } // Stats returns number of packets and dropped packets. This will be the number of packets/dropped packets since the last call to stats (not the cummulative sum!). func (sock *SockRaw) Stats() (*unix.TpacketStats, error) { sock.mu.Lock() defer sock.mu.Unlock() return unix.GetsockoptTpacketStats(sock.fd, unix.SOL_PACKET, unix.PACKET_STATISTICS) } // SetLoopbackIndex necessary to avoid reading packet twice on a loopback device func (sock *SockRaw) SetLoopbackIndex(i int32) { sock.mu.Lock() defer sock.mu.Unlock() sock.loopIndex = i } // WritePacketData transmits a raw packet. func (sock *SockRaw) WritePacketData(pkt []byte) error { _, err := unix.Write(sock.fd, pkt) return err } func tpAlign(x int) int { return int((uint(x) + unix.TPACKET_ALIGNMENT - 1) &^ (unix.TPACKET_ALIGNMENT - 1)) } ================================================ FILE: internal/capture/sock_others.go ================================================ //go:build !linux || arm64 || darwin package capture import ( "errors" "github.com/google/gopacket/pcap" ) // NewSocket returns new M'maped sock_raw on packet version 2. func NewSocket(_ pcap.Interface) (Socket, error) { return nil, errors.New("afpacket socket is only available on linux") } ================================================ FILE: internal/capture/socket.go ================================================ package capture import ( "time" "github.com/google/gopacket" ) // Socket is any interface that defines the behaviors of Socket type Socket interface { ReadPacketData() ([]byte, gopacket.CaptureInfo, error) WritePacketData([]byte) error SetBPFFilter(string) error SetPromiscuous(bool) error SetSnapLen(int) error GetSnapLen() int SetTimeout(time.Duration) error SetLoopbackIndex(i int32) Close() error } ================================================ FILE: internal/capture/vxlan.go ================================================ package capture import ( "errors" "fmt" "github.com/google/gopacket" "github.com/google/gopacket/layers" "net" "time" ) const VxLanPacketSize = 1526 //vxlan 8 B + ethernet II 1518 B type vxlanHandle struct { connection *net.UDPConn packetChannel chan gopacket.Packet vnis []int } func newVXLANHandler(port int, vnis []int) (*vxlanHandle, error) { if port == 0 { port = 4789 } addr := net.UDPAddr{ Port: port, IP: net.ParseIP("0.0.0.0"), } vxlanHandle := &vxlanHandle{} con, err := net.ListenUDP("udp", &addr) if err != nil { return nil, fmt.Errorf(err.Error()) } vxlanHandle.connection = con vxlanHandle.packetChannel = make(chan gopacket.Packet, 1000) vxlanHandle.vnis = vnis go vxlanHandle.reader() return vxlanHandle, nil } func (v *vxlanHandle) reader() { for { inputBytes := make([]byte, VxLanPacketSize) length, _, err := v.connection.ReadFromUDP(inputBytes) if err != nil { if errors.Is(err, net.ErrClosed) { return } continue } packet := gopacket.NewPacket(inputBytes[:length], layers.LayerTypeVXLAN, gopacket.NoCopy) ci := packet.Metadata() ci.Timestamp = time.Now() ci.CaptureLength = length ci.Length = length if len(v.vnis) > 0 && !v.vniIsAllowed(packet) { continue } v.packetChannel <- packet } } func (v *vxlanHandle) vniIsAllowed(packet gopacket.Packet) bool { defaultState := false if layer := packet.Layer(layers.LayerTypeVXLAN); layer != nil { vxlan, _ := layer.(*layers.VXLAN) for _, vn := range v.vnis { if vn > 0 && int(vxlan.VNI) == vn { return true } if vn < 0 { if int(vxlan.VNI) == -vn { return false } defaultState = true } } } return defaultState } func (v *vxlanHandle) ReadPacketData() ([]byte, gopacket.CaptureInfo, error) { packet := <-v.packetChannel layer := packet.Layer(layers.LayerTypeVXLAN) bytes := layer.LayerPayload() return bytes, packet.Metadata().CaptureInfo, nil } func (v *vxlanHandle) Close() error { if v.connection != nil { return v.connection.Close() } return nil } ================================================ FILE: internal/ring/ring.go ================================================ package ring import ( "errors" "runtime" "sync/atomic" "time" ) var ( // ErrDisposed is returned when an operation is performed on a disposed // queue. ErrDisposed = errors.New(`queue: disposed`) // ErrTimeout is returned when an applicable queue operation times out. ErrTimeout = errors.New(`queue: poll timed out`) // ErrEmptyQueue is returned when an non-applicable queue operation was called // due to the queue's empty item state ErrEmptyQueue = errors.New(`queue: empty queue`) ) // roundUp takes a uint64 greater than 0 and rounds it up to the next // power of 2. func roundUp(v uint64) uint64 { v-- v |= v >> 1 v |= v >> 2 v |= v >> 4 v |= v >> 8 v |= v >> 16 v |= v >> 32 v++ return v } type node struct { position uint64 data interface{} } type nodes []node // RingBuffer is a MPMC buffer that achieves threadsafety with CAS operations // only. A put on full or get on empty call will block until an item // is put or retrieved. Calling Dispose on the RingBuffer will unblock // any blocked threads with an error. This buffer is similar to the buffer // described here: http://www.1024cores.net/home/lock-free-algorithms/queues/bounded-mpmc-queue // with some minor additions. type RingBuffer struct { _padding0 [8]uint64 queue uint64 _padding1 [8]uint64 dequeue uint64 _padding2 [8]uint64 mask, disposed uint64 _padding3 [8]uint64 nodes nodes } func (rb *RingBuffer) init(size uint64) { size = roundUp(size) rb.nodes = make(nodes, size) for i := uint64(0); i < size; i++ { rb.nodes[i] = node{position: i} } rb.mask = size - 1 // so we don't have to do this with every put/get operation } // Put adds the provided item to the queue. If the queue is full, this // call will block until an item is added to the queue or Dispose is called // on the queue. An error will be returned if the queue is disposed. func (rb *RingBuffer) Put(item interface{}) error { _, err := rb.put(item, false) return err } // Offer adds the provided item to the queue if there is space. If the queue // is full, this call will return false. An error will be returned if the // queue is disposed. func (rb *RingBuffer) Offer(item interface{}) (bool, error) { return rb.put(item, true) } func (rb *RingBuffer) put(item interface{}, offer bool) (bool, error) { var n *node pos := atomic.LoadUint64(&rb.queue) L: for { if atomic.LoadUint64(&rb.disposed) == 1 { return false, ErrDisposed } n = &rb.nodes[pos&rb.mask] seq := atomic.LoadUint64(&n.position) switch dif := seq - pos; { case dif == 0: if atomic.CompareAndSwapUint64(&rb.queue, pos, pos+1) { break L } case dif < 0: panic(`Ring buffer in a compromised state during a put operation.`) default: pos = atomic.LoadUint64(&rb.queue) } if offer { return false, nil } runtime.Gosched() // free up the cpu before the next iteration } n.data = item atomic.StoreUint64(&n.position, pos+1) return true, nil } // Get will return the next item in the queue. This call will block // if the queue is empty. This call will unblock when an item is added // to the queue or Dispose is called on the queue. An error will be returned // if the queue is disposed. func (rb *RingBuffer) Get() (interface{}, error) { return rb.Poll(0) } // Poll will return the next item in the queue. This call will block // if the queue is empty. This call will unblock when an item is added // to the queue, Dispose is called on the queue, or the timeout is reached. An // error will be returned if the queue is disposed or a timeout occurs. A // non-positive timeout will block indefinitely. func (rb *RingBuffer) Poll(timeout time.Duration) (interface{}, error) { var ( n *node pos = atomic.LoadUint64(&rb.dequeue) start time.Time ) if timeout > 0 { start = time.Now() } L: for { if atomic.LoadUint64(&rb.disposed) == 1 { return nil, ErrDisposed } n = &rb.nodes[pos&rb.mask] seq := atomic.LoadUint64(&n.position) switch dif := seq - (pos + 1); { case dif == 0: if atomic.CompareAndSwapUint64(&rb.dequeue, pos, pos+1) { break L } case dif < 0: panic(`Ring buffer in compromised state during a get operation.`) default: pos = atomic.LoadUint64(&rb.dequeue) } if timeout > 0 && time.Since(start) >= timeout { return nil, ErrTimeout } if timeout < 0 { return nil, ErrTimeout } runtime.Gosched() // free up the cpu before the next iteration } data := n.data n.data = nil atomic.StoreUint64(&n.position, pos+rb.mask+1) return data, nil } // Len returns the number of items in the queue. func (rb *RingBuffer) Len() uint64 { return atomic.LoadUint64(&rb.queue) - atomic.LoadUint64(&rb.dequeue) } // Cap returns the capacity of this ring buffer. func (rb *RingBuffer) Cap() uint64 { return uint64(len(rb.nodes)) } // Dispose will dispose of this queue and free any blocked threads // in the Put and/or Get methods. Calling those methods on a disposed // queue will return an error. func (rb *RingBuffer) Dispose() { atomic.CompareAndSwapUint64(&rb.disposed, 0, 1) } // IsDisposed will return a bool indicating if this queue has been // disposed. func (rb *RingBuffer) IsDisposed() bool { return atomic.LoadUint64(&rb.disposed) == 1 } // NewRingBuffer will allocate, initialize, and return a ring buffer // with the specified size. func NewRingBuffer(size uint64) *RingBuffer { rb := &RingBuffer{} rb.init(size) return rb } ================================================ FILE: internal/simpletime/time.go ================================================ package simpletime import ( "time" ) var Now time.Time func init() { go func() { for { // Accurate enough Now = time.Now() time.Sleep(100 * time.Millisecond) } }() } ================================================ FILE: internal/size/size.go ================================================ package size import ( "fmt" "regexp" "strconv" ) // Size represents size that implements flag.Var type Size int64 // the following regexes follow Go semantics https://golang.org/ref/spec#Letters_and_digits var ( rB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+$`) rKB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+kb$`) rMB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+mb$`) rGB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+gb$`) rTB = regexp.MustCompile(`(?i)^(?:0b|0x|0o)?[\da-f_]+tb$`) ) // Set parses size to integer from different bases and data units func (siz *Size) Set(size string) (err error) { if size == "" { return } const ( _ = 1 << (iota * 10) KB MB GB TB ) var ( lmt = len(size) - 2 s = []byte(size) ) var _len int64 switch { case rB.Match(s): _len, err = strconv.ParseInt(size, 0, 64) case rKB.Match(s): _len, err = strconv.ParseInt(size[:lmt], 0, 64) _len *= KB case rMB.Match(s): _len, err = strconv.ParseInt(size[:lmt], 0, 64) _len *= MB case rGB.Match(s): _len, err = strconv.ParseInt(size[:lmt], 0, 64) _len *= GB case rTB.Match(s): _len, err = strconv.ParseInt(size[:lmt], 0, 64) _len *= TB default: return fmt.Errorf("invalid _len %q", size) } *siz = Size(_len) return } func (siz *Size) String() string { return fmt.Sprintf("%d", *siz) } ================================================ FILE: internal/size/size_test.go ================================================ package size import "testing" func TestParseDataUnit(t *testing.T) { var d = map[string]int{ "42mb": 42 << 20, "4_2": 42, "00": 0, "0": 0, "0_600tb": 384 << 40, "0600Tb": 384 << 40, "0o12Mb": 10 << 20, "0b_10010001111_1kb": 2335 << 10, "1024": 1 << 10, "0b111": 7, "0x12gB": 18 << 30, "0x_67_7a_2f_cc_40_c6": 113774485586118, "121562380192901": 121562380192901, } var buf Size var err error for k, v := range d { err = buf.Set(k) if err != nil || buf != Size(v) { t.Errorf("Error parsing %s: %v", k, err) } } } ================================================ FILE: internal/tcp/doc.go ================================================ /* Package tcp implements TCP transport layer protocol, it is responsible for parsing, reassembling tcp packets, handling communication with engine listeners(github.com/buger/goreplay/capture), and reporting errors and statistics of packets. the packets are parsed by following TCP way(https://en.wikipedia.org/wiki/Transmission_Control_Protocol#TCP_segment_structure). example: import "github.com/buger/goreplay/tcp" messageExpire := time.Second*5 maxSize := 5 << 20 debugger := func(debugLevel int, data ...interface{}){} // debugger can also be nil messageHandler := func(mssg *tcp.Message){} mssgPool := tcp.NewMessageParser(maxMessageSize, messageExpire, debugger, messageHandler) listener.Listen(ctx, mssgPool.Handler) you can use pool.End or/and pool.Start to set custom session behaviors debugLevel in debugger function indicates the priority of the logs, the bigger the number the lower the priority. errors are signified by debug level 4 for errors, 5 for discarded packets, and 6 for received packets. */ package tcp // import github.com/buger/goreplay/tcp ================================================ FILE: internal/tcp/tcp_message.go ================================================ package tcp import ( "encoding/binary" "encoding/hex" "fmt" "github.com/buger/goreplay/proto" "net" "reflect" "sort" "time" "unsafe" ) // TCPProtocol is a number to indicate type of protocol type TCPProtocol uint8 const ( // ProtocolHTTP ... ProtocolHTTP TCPProtocol = iota // ProtocolBinary ... ProtocolBinary ) // Set is here so that TCPProtocol can implement flag.Var func (protocol *TCPProtocol) Set(v string) error { switch v { case "", "http": *protocol = ProtocolHTTP case "binary": *protocol = ProtocolBinary default: return fmt.Errorf("unsupported protocol %s", v) } return nil } func (protocol *TCPProtocol) String() string { switch *protocol { case ProtocolBinary: return "binary" case ProtocolHTTP: return "http" default: return "" } } // Stats every message carry its own stats object type Stats struct { LostData int Length int // length of the data Start time.Time // first packet's timestamp End time.Time // last packet's timestamp SrcAddr string DstAddr string Direction Dir TimedOut bool // timeout before getting the whole message Truncated bool // last packet truncated due to max message size IPversion byte } // Message is the representation of a tcp message type Message struct { packets []*Packet parser *MessageParser feedback interface{} continueAdjusted bool Stats } // UUID returns the UUID of a TCP request and its response. func (m *Message) UUID() []byte { var streamID uint64 pckt := m.packets[0] // check if response or request have generated the ID before. if m.Direction == DirIncoming { streamID = uint64(pckt.SrcPort)<<48 | uint64(pckt.DstPort)<<32 | uint64(ip2int(pckt.SrcIP)) } else { streamID = uint64(pckt.DstPort)<<48 | uint64(pckt.SrcPort)<<32 | uint64(ip2int(pckt.DstIP)) } id := make([]byte, 12) binary.BigEndian.PutUint64(id, streamID) if m.Direction == DirIncoming { binary.BigEndian.PutUint32(id[8:], pckt.Ack) } else { binary.BigEndian.PutUint32(id[8:], pckt.Seq) } uuidHex := make([]byte, 24) hex.Encode(uuidHex[:], id[:]) return uuidHex } func (m *Message) add(packet *Packet) bool { // Skip duplicates for _, p := range m.packets { if p.Seq == packet.Seq { return false } } // Packets not always captured in same Seq order, and sometimes we need to prepend if len(m.packets) == 0 || packet.Seq > m.packets[len(m.packets)-1].Seq { m.packets = append(m.packets, packet) } else if packet.Seq < m.packets[0].Seq { m.packets = append([]*Packet{packet}, m.packets...) } else { // insert somewhere in the middle... for i, p := range m.packets { if packet.Seq < p.Seq { m.packets = append(m.packets[:i], append([]*Packet{packet}, m.packets[i:]...)...) break } } } m.Length += len(packet.Payload) m.LostData += int(packet.Lost) if packet.Timestamp.After(m.End) || m.End.IsZero() { m.End = packet.Timestamp } return true } // Packets returns packets of the message func (m *Message) Packets() []*Packet { return m.packets } func (m *Message) MissingChunk() bool { nextSeq := m.packets[0].Seq for _, p := range m.packets { if p.Seq != nextSeq { return true } nextSeq += uint32(len(p.Payload)) } return false } func (m *Message) PacketData() [][]byte { tmp := make([][]byte, len(m.packets)) for i, p := range m.packets { tmp[i] = p.Payload } return tmp } // Data returns data in this message func (m *Message) Data() []byte { packetData := m.PacketData() tmp := packetData[0] if len(packetData) > 0 { tmp, _ = copySlice(tmp, len(packetData[0]), packetData[1:]...) } // Remove Expect header, since its replay not fully supported if state, ok := m.feedback.(*proto.HTTPState); ok { if state.Continue100 { tmp = proto.DeleteHeader(tmp, []byte("Expect")) } } return tmp } // SetProtocolState set feedback/data that can be used later, e.g with End or Start hint func (m *Message) SetProtocolState(feedback interface{}) { m.feedback = feedback } // ProtocolState returns feedback associated to this message func (m *Message) ProtocolState() interface{} { return m.feedback } // Sort a helper to sort packets func (m *Message) Sort() { sort.SliceStable(m.packets, func(i, j int) bool { return m.packets[i].Seq < m.packets[j].Seq }) } // Emitter message handler type Emitter func(*Message) // HintEnd hints the parser to stop the session, see MessageParser.End // when set, it will be executed before checking FIN or RST flag type HintEnd func(*Message) bool // HintStart hints the parser to start the reassembling the message, see MessageParser.Start // when set, it will be called after checking SYN flag type HintStart func(*Packet) (IsRequest, IsOutgoing bool) // MessageParser holds data of all tcp messages in progress(still receiving/sending packets). // message is identified by its source port and dst port, and last 4bytes of src IP. type MessageParser struct { m map[uint64]*Message messageExpire time.Duration // the maximum time to wait for the final packet, minimum is 100ms allowIncompete bool End HintEnd Start HintStart ticker *time.Ticker messages chan *Message packets chan *PcapPacket close chan struct{} // to signal that we are able to close ports []uint16 ips []net.IP } // NewMessageParser returns a new instance of message parser func NewMessageParser(messages chan *Message, ports []uint16, ips []net.IP, messageExpire time.Duration, allowIncompete bool) (parser *MessageParser) { parser = new(MessageParser) parser.messageExpire = messageExpire if parser.messageExpire == 0 { parser.messageExpire = time.Millisecond * 1000 } parser.allowIncompete = allowIncompete parser.packets = make(chan *PcapPacket, 10000) if messages == nil { messages = make(chan *Message, 1000) } parser.messages = messages parser.m = make(map[uint64]*Message) parser.ticker = time.NewTicker(time.Millisecond * 100) parser.close = make(chan struct{}, 1) parser.ports = ports parser.ips = ips go parser.wait() return parser } var packetLen int // Packet returns packet handler func (parser *MessageParser) PacketHandler(packet *PcapPacket) { packetLen++ parser.packets <- packet } func (parser *MessageParser) wait() { var ( now time.Time ) for { select { case pckt := <-parser.packets: parser.processPacket(parser.parsePacket(pckt)) case now = <-parser.ticker.C: parser.timer(now) case <-parser.close: parser.ticker.Stop() // parser.Close should wait for this function to return parser.close <- struct{}{} return // default: } } } func (parser *MessageParser) parsePacket(pcapPkt *PcapPacket) *Packet { pckt, err := ParsePacket(pcapPkt.Data, pcapPkt.LType, pcapPkt.LTypeLen, pcapPkt.Ci, false) if err != nil { if _, empty := err.(EmptyPacket); !empty { stats.Add("packet_error", 1) } return nil } for _, p := range parser.ports { if pckt.DstPort == p && containsOrEmpty(pckt.DstIP, parser.ips) { pckt.Direction = DirIncoming break } else if pckt.SrcPort == p && containsOrEmpty(pckt.SrcIP, parser.ips) { pckt.Direction = DirOutcoming break } } return pckt } func containsOrEmpty(element net.IP, ipList []net.IP) bool { if len(ipList) == 0 { return true } for _, ip := range ipList { if ip.Equal(element) { return true } } return false } func (parser *MessageParser) processPacket(pckt *Packet) { if pckt == nil { return } // Trying to build unique hash, but there is small chance of collision // No matter if it is request or response, all packets in the same message have same m, ok := parser.m[pckt.MessageID()] switch { case ok: if m.Direction == DirUnknown { if in, out := parser.Start(pckt); in || out { if in { m.Direction = DirIncoming } else { m.Direction = DirOutcoming } } } parser.addPacket(m, pckt) return case pckt.Direction == DirUnknown && parser.Start != nil: if in, out := parser.Start(pckt); in || out { if in { pckt.Direction = DirIncoming } else { pckt.Direction = DirOutcoming } } } m = new(Message) m.Direction = pckt.Direction m.SrcAddr = pckt.SrcIP.String() m.DstAddr = pckt.DstIP.String() parser.m[pckt.MessageID()] = m m.Start = pckt.Timestamp m.parser = parser parser.addPacket(m, pckt) } func (parser *MessageParser) addPacket(m *Message, pckt *Packet) bool { if !m.add(pckt) { return false } // If we are using protocol parsing, like HTTP, depend on its parsing func. // For the binary procols wait for message to expire if parser.End != nil { if parser.End(m) { parser.Emit(m) return true } parser.Fix100Continue(m) } return true } func (parser *MessageParser) Fix100Continue(m *Message) { // Only adjust a message once if state, ok := m.feedback.(*proto.HTTPState); ok && state.Continue100 && !m.continueAdjusted { // Shift Ack by given offset // Size of "HTTP/1.1 100 Continue\r\n\r\n" message for _, p := range m.packets { p.messageID = 0 p.Ack += 25 } // If next section was aready approved and received, merge messages if next, found := parser.m[m.packets[0].MessageID()]; found { for _, p := range next.packets { parser.addPacket(m, p) } } // Re-add (or override) again with new message and ID parser.m[m.packets[0].MessageID()] = m m.continueAdjusted = true } } func (parser *MessageParser) Read() *Message { m := <-parser.messages return m } func (parser *MessageParser) Emit(m *Message) { stats.Add("message_count", 1) delete(parser.m, m.packets[0].MessageID()) parser.messages <- m } func GetUnexportedField(field reflect.Value) interface{} { return reflect.NewAt(field.Type(), unsafe.Pointer(field.UnsafeAddr())).Elem().Interface() } var failMsg int func (parser *MessageParser) timer(now time.Time) { packetLen = 0 packetQueueLen.Set(int64(len(parser.packets))) messageQueueLen.Set(int64(len(parser.m))) for _, m := range parser.m { if now.Sub(m.End) > parser.messageExpire { m.TimedOut = true stats.Add("message_timeout_count", 1) failMsg++ if parser.End == nil || parser.allowIncompete { parser.Emit(m) } delete(parser.m, m.packets[0].MessageID()) } } } func (parser *MessageParser) Close() error { parser.close <- struct{}{} <-parser.close // wait for timer to be closed! return nil } ================================================ FILE: internal/tcp/tcp_packet.go ================================================ package tcp import ( "encoding/binary" "expvar" "fmt" "net" "time" "github.com/google/gopacket" ) func copySlice(to []byte, skip int, from ...[]byte) ([]byte, int) { var totalLen int for _, s := range from { totalLen += len(s) } totalLen += skip if len(to) < totalLen { diff := totalLen - len(to) to = append(to, make([]byte, diff)...) } for _, s := range from { skip += copy(to[skip:], s) } return to, skip } var stats *expvar.Map var packetQueueLen, messageQueueLen *expvar.Int func init() { packetQueueLen = new(expvar.Int) messageQueueLen = new(expvar.Int) stats = expvar.NewMap("tcp") stats.Init() stats.Set("packet_queue", packetQueueLen) stats.Set("message_queue", messageQueueLen) } type Dir int const ( DirUnknown = iota DirIncoming DirOutcoming ) /* Packet represent data and layers of packet. parser extracts information from pcap Packet. functions of *Packet doesn't validate if packet is nil, calllers must make sure that ParsePacket has'nt returned any error before calling any other function. */ type Packet struct { Direction Dir messageID uint64 SrcIP, DstIP net.IP Version uint8 SrcPort, DstPort uint16 Ack, Seq uint32 ACK, SYN, FIN, RST bool Lost uint32 Retry int CaptureLength int Timestamp time.Time Payload []byte buf []byte created time.Time gc bool } type PcapPacket struct { Data []byte LType int LTypeLen int Ci *gopacket.CaptureInfo } // ParsePacket parse raw packets func ParsePacket(data []byte, lType, lTypeLen int, ci *gopacket.CaptureInfo, allowEmpty bool) (pckt *Packet, err error) { pckt = new(Packet) if err := pckt.parse(data, lType, lTypeLen, ci, allowEmpty); err != nil { return nil, err } return pckt, nil } func (pckt *Packet) parse(data []byte, lType, lTypeLen int, cp *gopacket.CaptureInfo, allowEmpty bool) error { pckt.Retry = 0 pckt.messageID = 0 pckt.buf = pckt.buf[:] // TODO: check resolution pckt.Timestamp = cp.Timestamp if len(data) < lTypeLen { return ErrHdrLength("Link") } if len(data) <= lTypeLen { return ErrHdrMissing("IPv4 or IPv6") } ldata := data[lTypeLen:] var proto byte var netLayer, transLayer []byte if ldata[0]>>4 == 4 { // IPv4 header if len(ldata) < 20 { return ErrHdrLength("IPv4") } proto = ldata[9] ihl := int(ldata[0]&0x0F) * 4 if ihl < 20 { return ErrHdrInvalid("IPv4's IHL") } if len(ldata) < ihl { return ErrHdrLength("IPv4 opts") } netLayer = ldata[:ihl] } else if ldata[0]>>4 == 6 { if len(ldata) < 40 { return ErrHdrLength("IPv6") } proto = ldata[6] totalLen := 40 for ipv6ExtensionHdr(proto) { hdr := len(ldata) - totalLen if hdr < 8 { return ErrHdrExpected("IPv6 opts") } extLen := 8 if proto != 44 { extLen = int(ldata[totalLen+1]+1) * 8 } if hdr < extLen { return ErrHdrLength("IPv6 opts") } proto = ldata[totalLen] totalLen += extLen } netLayer = ldata[:totalLen] } else { return ErrHdrExpected("IPv4 or IPv6") } if proto != 6 { return ErrHdrExpected("TCP") } if len(data) <= len(netLayer) { return ErrHdrMissing("TCP") } ndata := ldata[len(netLayer):] // TCP header if len(ndata) < 20 { return ErrHdrLength("TCP") } dOf := int(ndata[12]>>4) * 4 if dOf < 20 { return ErrHdrInvalid("TCP's ndata offset") } if len(ndata) < dOf { return ErrHdrLength("TCP opts") } // There are case when packet have padding but dOf shows its not empty := true for i := 0; i < len(ndata[dOf:]); i++ { if ndata[dOf:][i] != 0 { empty = false break } } if !allowEmpty && empty { return EmptyPacket("") } if (netLayer[0] >> 4) == 4 { // IPv4 header pckt.Version = 4 pckt.SrcIP = netLayer[12:16] pckt.DstIP = netLayer[16:20] } else { // IPv6 header pckt.Version = 6 pckt.SrcIP = netLayer[8:24] pckt.DstIP = netLayer[24:40] } transLayer = ndata[:dOf] pckt.CaptureLength = cp.CaptureLength pckt.SrcPort = binary.BigEndian.Uint16(transLayer[0:2]) pckt.DstPort = binary.BigEndian.Uint16(transLayer[2:4]) pckt.Seq = binary.BigEndian.Uint32(transLayer[4:8]) pckt.Ack = binary.BigEndian.Uint32(transLayer[8:12]) pckt.FIN = transLayer[13]&0x01 != 0 pckt.SYN = transLayer[13]&0x02 != 0 pckt.RST = transLayer[13]&0x04 != 0 pckt.ACK = transLayer[13]&0x10 != 0 pckt.Lost = uint32(cp.Length - cp.CaptureLength) pckt.Payload = ndata[dOf:] return nil } func (pckt *Packet) MessageID() uint64 { if pckt.messageID == 0 { // All packets in the same message will share the same ID pckt.messageID = uint64(pckt.SrcPort)<<48 | uint64(pckt.DstPort)<<32 | (uint64(ip2int(pckt.SrcIP)) + uint64(ip2int(pckt.DstIP)) + uint64(pckt.Ack)) } return pckt.messageID } // Src returns the source socket of a packet func (pckt *Packet) Src() string { return fmt.Sprintf("%s:%d", pckt.SrcIP, pckt.SrcPort) } // Dst returns destination socket func (pckt *Packet) Dst() string { return fmt.Sprintf("%s:%d", pckt.DstIP, pckt.DstPort) } type EmptyPacket string func (err EmptyPacket) Error() string { return "Empty packet" } // ErrHdrLength returned on short header length type ErrHdrLength string func (err ErrHdrLength) Error() string { return "short " + string(err) + " length" } // ErrHdrMissing returned on missing header(s) type ErrHdrMissing string func (err ErrHdrMissing) Error() string { return "missing " + string(err) + " header(s)" } // ErrHdrExpected returned when header(s) are different from the one expected type ErrHdrExpected string func (err ErrHdrExpected) Error() string { return "expected " + string(err) + " header(s)" } // ErrHdrInvalid returned when header(s) are different from the one expected type ErrHdrInvalid string func (err ErrHdrInvalid) Error() string { return "invalid " + string(err) + " value" } // https://en.wikipedia.org/wiki/IPv6_packet#Extension_headers func ipv6ExtensionHdr(b byte) bool { // TODO: support all extension headers return b == 0 || b == 43 || b == 44 } func ip2int(ip net.IP) uint32 { if len(ip) == 0 { return 0 } if len(ip) == 16 { return binary.BigEndian.Uint32(ip[12:16]) } return binary.BigEndian.Uint32(ip) } ================================================ FILE: internal/tcp/tcp_test.go ================================================ package tcp import ( "bytes" "encoding/binary" "github.com/buger/goreplay/proto" // "runtime" "testing" "time" "github.com/stretchr/testify/assert" "github.com/google/gopacket" "github.com/google/gopacket/layers" ) func generateHeader(request bool, seq uint32, length uint16) []byte { hdr := make([]byte, 4+24+24) binary.BigEndian.PutUint32(hdr, uint32(layers.ProtocolFamilyIPv4)) ip := hdr[4:] ip[0] = 4<<4 | 6 binary.BigEndian.PutUint16(ip[2:4], length+24+24) ip[9] = uint8(layers.IPProtocolTCP) copy(ip[12:16], []byte{127, 0, 0, 1}) copy(ip[16:], []byte{127, 0, 0, 1}) // set tcp header tcp := ip[24:] tcp[12] = 6 << 4 if request { binary.BigEndian.PutUint16(tcp, 5535) binary.BigEndian.PutUint16(tcp[2:], 8000) } else { binary.BigEndian.PutUint16(tcp, 8000) binary.BigEndian.PutUint16(tcp[2:], 5535) } binary.BigEndian.PutUint32(tcp[4:], seq) return hdr } func GetPackets(request bool, start uint32, _len int, payload []byte) []*Packet { var packets = make([]*Packet, _len) var err error for i := start; i < start+uint32(_len); i++ { d := append(generateHeader(request, i, uint16(len(payload))), payload...) ci := &gopacket.CaptureInfo{Length: len(d), CaptureLength: len(d), Timestamp: time.Now()} packets[i-start], err = ParsePacket(d, int(layers.LinkTypeLoop), 4, ci, true) if request { packets[i-start].Direction = DirIncoming } else { packets[i-start].Direction = DirOutcoming } if err != nil { panic(err) } } return packets } func TestRequestResponseMapping(t *testing.T) { packets := []*Packet{ {SrcPort: 60000, DstPort: 80, Ack: 1, Seq: 1, Direction: DirIncoming, Timestamp: time.Unix(1, 0), Payload: []byte("GET / HTTP/1.1\r\n")}, {SrcPort: 60000, DstPort: 80, Ack: 1, Seq: 17, Direction: DirIncoming, Timestamp: time.Unix(2, 0), Payload: []byte("Host: localhost\r\n\r\n")}, // Seq of first response packet match Ack of first request packet {SrcPort: 80, DstPort: 60000, Ack: 36, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(3, 0), Payload: []byte("HTTP/1.1 200 OK\r\n")}, {SrcPort: 80, DstPort: 60000, Ack: 36, Seq: 18, Direction: DirOutcoming, Timestamp: time.Unix(4, 0), Payload: []byte("Content-Length: 0\r\n\r\n")}, // Same TCP stream {SrcPort: 60000, DstPort: 80, Ack: 39, Seq: 36, Direction: DirIncoming, Timestamp: time.Unix(5, 0), Payload: []byte("GET / HTTP/1.1\r\n")}, {SrcPort: 60000, DstPort: 80, Ack: 39, Seq: 52, Direction: DirIncoming, Timestamp: time.Unix(6, 0), Payload: []byte("Host: localhost\r\n\r\n")}, // Seq of first response packet match Ack of first request packet {SrcPort: 80, DstPort: 60000, Ack: 71, Seq: 39, Direction: DirOutcoming, Timestamp: time.Unix(7, 0), Payload: []byte("HTTP/1.1 200 OK\r\n")}, {SrcPort: 80, DstPort: 60000, Ack: 71, Seq: 56, Direction: DirOutcoming, Timestamp: time.Unix(8, 0), Payload: []byte("Content-Length: 0\r\n\r\n")}, } parser := NewMessageParser(nil, nil, nil, time.Second, false) parser.Start = func(pckt *Packet) (bool, bool) { return proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload) } parser.End = func(m *Message) bool { return proto.HasFullPayload(m, m.PacketData()...) } for _, packet := range packets { parser.processPacket(packet) } messages := []*Message{} for i := 0; i < 4; i++ { m := parser.Read() messages = append(messages, m) } assert.Equal(t, int(messages[0].Direction), int(DirIncoming)) assert.Equal(t, int(messages[1].Direction), int(DirOutcoming)) assert.Equal(t, int(messages[2].Direction), int(DirIncoming)) assert.Equal(t, int(messages[3].Direction), int(DirOutcoming)) assert.Equal(t, messages[0].UUID(), messages[1].UUID()) assert.Equal(t, messages[2].UUID(), messages[3].UUID()) assert.NotEqual(t, messages[0].UUID(), messages[2].UUID()) } func TestMessageParserWithHint(t *testing.T) { parser := NewMessageParser(nil, nil, nil, time.Second, false) parser.Start = func(pckt *Packet) (bool, bool) { return proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload) } parser.End = func(m *Message) bool { return proto.HasFullPayload(m, m.PacketData()...) } packets := []*Packet{ // Seq of first response packet match Ack of first request packet {SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(1, 0), Payload: []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\n")}, {SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 18, Direction: DirOutcoming, Timestamp: time.Unix(2, 0), Payload: []byte("\r\nMozilla\r\n9\r\nDeveloper\r")}, {SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 42, Direction: DirOutcoming, Timestamp: time.Unix(3, 0), Payload: []byte("\n7\r\nNetwork\r\n0\r\n\r\n")}, {SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 1, Direction: DirIncoming, Timestamp: time.Unix(4, 0), Payload: []byte("POST / HTTP/1.1\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\n")}, {SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 66, Direction: DirIncoming, Timestamp: time.Unix(5, 0), Payload: []byte("MozillaDeveloper")}, {SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 82, Direction: DirIncoming, Timestamp: time.Unix(6, 0), Payload: []byte("Network")}, {SrcPort: 80, DstPort: 60000, Ack: 89, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(7, 0), Payload: []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\n\r\n")}, } for _, p := range packets { parser.processPacket(p) } messages := []*Message{} for i := 0; i < 3; i++ { m := parser.Read() messages = append(messages, m) } if !bytes.HasSuffix(messages[0].Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n")) { t.Errorf("expected to %q to have suffix %q", messages[0].Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n")) } if !bytes.HasSuffix(messages[1].Data(), []byte("Network")) { t.Errorf("expected to %q to have suffix %q", messages[1].Data(), []byte("Network")) } if !bytes.HasSuffix(messages[2].Data(), []byte("Content-Length: 0\r\n\r\n")) { t.Errorf("expected to %q to have suffix %q", messages[2].Data(), []byte("Content-Length: 0\r\n\r\n")) } } func TestMessageParserWrongOrder(t *testing.T) { parser := NewMessageParser(nil, nil, nil, time.Second, false) parser.Start = func(pckt *Packet) (bool, bool) { return proto.HasRequestTitle(pckt.Payload), proto.HasResponseTitle(pckt.Payload) } parser.End = func(m *Message) bool { return proto.HasFullPayload(m, m.PacketData()...) } packets := []*Packet{ // Seq of first response packet match Ack of first request packet {SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 66, Direction: DirIncoming, Timestamp: time.Unix(5, 0), Payload: []byte("MozillaDeveloper")}, {SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(1, 0), Payload: []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n7\r\n")}, {SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 42, Direction: DirOutcoming, Timestamp: time.Unix(3, 0), Payload: []byte("\n7\r\nNetwork\r\n0\r\n\r\n")}, {SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 1, Direction: DirIncoming, Timestamp: time.Unix(4, 0), Payload: []byte("POST / HTTP/1.1\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\n")}, {SrcPort: 80, DstPort: 60000, Ack: 1, Seq: 18, Direction: DirOutcoming, Timestamp: time.Unix(2, 0), Payload: []byte("\r\nMozilla\r\n9\r\nDeveloper\r")}, {SrcPort: 80, DstPort: 60000, Ack: 89, Seq: 1, Direction: DirOutcoming, Timestamp: time.Unix(7, 0), Payload: []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 0\r\n\r\n")}, {SrcPort: 60000, DstPort: 80, Ack: 60, Seq: 82, Direction: DirIncoming, Timestamp: time.Unix(6, 0), Payload: []byte("Network")}, } for _, p := range packets { parser.processPacket(p) } m := parser.Read() if !bytes.HasSuffix(m.Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n")) { t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("\n7\r\nNetwork\r\n0\r\n\r\n")) } m = parser.Read() if !bytes.HasSuffix(m.Data(), []byte("Content-Length: 0\r\n\r\n")) { t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("Content-Length: 0\r\n\r\n")) } m = parser.Read() if !bytes.HasSuffix(m.Data(), []byte("Network")) { t.Errorf("expected to %q to have suffix %q", m.Data(), []byte("Network")) } } func TestMessageParserWithoutHint(t *testing.T) { var data [63 << 10]byte packets := GetPackets(true, 1, 10, data[:]) p := NewMessageParser(nil, nil, nil, time.Second, false) for _, v := range packets { p.processPacket(v) } m := p.Read() if m.Length != 63<<10*10 { t.Errorf("expected %d to equal %d", m.Length, 63<<10*10) } } func TestMessageTimeoutReached(t *testing.T) { const size = 63 << 11 var data [size >> 1]byte packets := GetPackets(true, 1, 2, data[:]) p := NewMessageParser(nil, nil, nil, 100*time.Millisecond, true) p.processPacket(packets[0]) time.Sleep(time.Millisecond * 20) p.processPacket(packets[1]) m := p.Read() if m.Length != size { t.Errorf("expected %d to equal %d", m.Length, size) } if !m.TimedOut { t.Error("expected message to be timeout") } } func BenchmarkMessageUUID(b *testing.B) { packets := GetPackets(true, 1, 5, nil) var uuid []byte parser := NewMessageParser(nil, nil, nil, 10*time.Millisecond, true) for _, p := range packets { parser.processPacket(p) } msg := parser.Read() b.ResetTimer() for i := 0; i < b.N; i++ { uuid = msg.UUID() } _ = uuid } func BenchmarkPacketParseAndSort(b *testing.B) { m := new(Message) m.packets = make([]*Packet, 100) for i, v := range GetPackets(true, 1, 100, nil) { m.packets[i] = v } b.ResetTimer() for i := 0; i < b.N; i++ { m.Sort() } } func BenchmarkMessageParserWithoutHint(b *testing.B) { var chunk = []byte("111111111111111111111111111111") packets := GetPackets(true, 1, 1000, chunk) p := NewMessageParser(nil, nil, nil, 2*time.Second, false) b.ResetTimer() b.ReportMetric(float64(1000), "packets/op") for i := 0; i < b.N; i++ { for _, v := range packets { p.processPacket(v) } p.Read() } } func BenchmarkMessageParserWithHint(b *testing.B) { var buf [1002][]byte var chunk = []byte("1e\r\n111111111111111111111111111111\r\n") buf[0] = []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n") for i := 1; i < 1000; i++ { buf[i] = chunk } buf[1001] = []byte("0\r\n\r\n") packets := make([]*Packet, len(buf)) for i := 0; i < len(buf); i++ { packets[i] = GetPackets(false, 1, 1, buf[i])[0] } parser := NewMessageParser(nil, nil, nil, 2*time.Second, false) parser.Start = func(pckt *Packet) (bool, bool) { return false, proto.HasResponseTitle(pckt.Payload) } parser.End = func(m *Message) bool { return proto.HasFullPayload(m, m.PacketData()...) } b.ResetTimer() b.ReportMetric(float64(len(packets)), "packets/op") b.ReportMetric(float64(1000), "chunks/op") for i := 0; i < b.N; i++ { for j := range packets { parser.processPacket(packets[j]) } parser.Read() } } func BenchmarkNewAndParsePacket(b *testing.B) { data := append(generateHeader(true, 1024, 10), make([]byte, 10)...) b.ResetTimer() for i := 0; i < b.N; i++ { ParsePacket(data, int(layers.LinkTypeLoop), 4, &gopacket.CaptureInfo{}, true) } } ================================================ FILE: k8s/README.md ================================================ # Native k8s integration After following steps below, you will be able to capture traffic inside k8s like this: ``` gor --input-raw k8s://namespace/deployment/app:80 --output-http http://replay.com ``` GoReplay will running as a daemonset (e.g. on each phisical k8s node. It will also require giving required permission to have read access to K8s APIs, so it can dynamically filter traffic for a specific pods. Supported format for filtering required pods: ``` k8s://[namespace/]pod/[pod_name] - k8s://default/pod/nginx-7848d4b86f-5nxz8 k8s://[namespace/]deployment/[deployment_name] - k8s://default/deployment/nginx k8s://[namespace/]daemonset/[daemonset_name] - k8s://default/daemonset/nginx k8s://[namespace/]labelSelector/[selector] - k8s://default/labelSelector/app=nginx k8s://[namespace/]fieldSelector/[selector] - k8s://default/fieldSelector/metadata.name=nginx-7848d4b86f-5nxz8 ``` `namespace` is optional, omit to use all namespaces: `k8s://labelSelector/app=replay` ## 1. Create a namespace `kubectl create namespace goreplay` ## 2. Create the Kubernetes service account in the namespace: `kubectl create serviceaccount goreplay --namespace goreplay` ## 3. Create Cluster Role which gives read-only access to the pods: `kubectl -n goreplay -f clusterrole.yaml apply` ```yaml kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: pod-reader rules: - apiGroups: [""] resources: ["pods"] verbs: ["get", "watch", "list"] - apiGroups: [""] resources: ["deployments"] verbs: ["get", "watch", "list"] - apiGroups: [""] resources: ["daemonset"] verbs: ["get", "watch", "list"] ``` ## 4. Attach role to goreplay service account `kubectl -n goreplay -f rolebinding.yaml apply` ```yaml kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: goreplay-reader-binding subjects: - kind: ServiceAccount name: goreplay namespace: goreplay roleRef: kind: ClusterRole name: pod-reader apiGroup: "" ``` ## 5. Start goreplay daemonset `kubectl -n goreplay -f goreplay.yaml apply` In arguments, specify which service you want to capture. Following format supported: ```yaml apiVersion: apps/v1 kind: DaemonSet metadata: name: goreplay-daemon spec: selector: matchLabels: app: goreplay template: metadata: labels: app: goreplay spec: hostNetwork: true serviceAccountName: goreplay containers: - name: goreplay image: buger/goreplay:2.0.0-rc4 args: - "--input-raw" - "k8s://deployments/nginx:80" - "--output-stdout" ``` ## 6. Create a simple http service (Optionally) `kubectl -n default -f nginx.yaml apply` ```yaml apiVersion: apps/v1 kind: Deployment metadata: name: nginx labels: app: nginx spec: replicas: 1 selector: matchLabels: app: nginx template: metadata: labels: app: nginx spec: containers: - name: nginx image: nginx ports: - containerPort: 80 --- apiVersion: v1 kind: Service metadata: name: ngnix-service spec: selector: app: nginx type: NodePort ports: - protocol: TCP port: 80 targetPort: 80 ``` ## 7. Verify installation, and debugging tips Find url for your service using `kubectl get svc` or `minikube service --url ngnix-service -n http`, and make a call to it. Get GoReplay logs, and check if it capture traffic of your service. `kubectl logs -n goreplay -l app=goreplay --all-containers` Describe daemonset: `kubectl describe daemonset goreplay-daemon -n goreplay` Get GoReplay pod list: `kubectl get pods -n goreplay -l app=goreplay` Get logs for specific pod (take data from previous step): `kubectl logs goreplay-daemon- -n goreplay` Get related k8s events: `kubectl get events -n goreplay --field-selector involvedObject.name=goreplay-daemon-` ## 8. Debuggin telemetry We provde a special script which will get all require logs, and help you find installation errors. `curl -s https://raw.githubusercontent.com/buger/goreplay/refs/heads/master/k8s/collect_goreplay_telemetry.sh | bash` If you are using microk8s or similar, you can also specific prefix for your kubectl command like this: `curl -s https://raw.githubusercontent.com/buger/goreplay/refs/heads/master/k8s/collect_goreplay_telemetry.sh | bash -s -- "microk8s kubectl"` ================================================ FILE: k8s/clusterrole.yaml ================================================ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: pod-reader rules: - apiGroups: [""] resources: ["pods"] verbs: ["get", "watch", "list"] - apiGroups: ["apps"] resources: ["deployments"] verbs: ["get", "watch", "list"] - apiGroups: ["apps"] resources: ["daemonsets"] verbs: ["get", "watch", "list"] ================================================ FILE: k8s/collect_goreplay_telemetry.sh ================================================ #!/usr/bin/env bash # # collect_goreplay_telemetry.sh # # Gathers telemetry from a GoReplay DaemonSet in the 'goreplay' namespace. # Works on macOS and Linux, assuming 'kubectl' (or compatible) is installed. # # Usage examples: # ./collect_goreplay_telemetry.sh # ./collect_goreplay_telemetry.sh "microk8s kubectl" set -euo pipefail ######################################## # Determine kubectl command ######################################## if [[ $# -gt 0 ]]; then # If an argument was provided, use that as the kubectl command KUBECTL="$*" else # Default to 'kubectl' KUBECTL="kubectl" fi ######################################## # Check that the base command exists ######################################## # For "microk8s kubectl", we only check "microk8s" in PATH. For "oc" we check "oc". BASE_CMD="${KUBECTL%% *}" # everything before the first space if ! command -v "${BASE_CMD}" >/dev/null 2>&1; then echo "ERROR: '${BASE_CMD}' not found in PATH. Please install or configure it first." exit 1 fi echo "Using kubectl command: $KUBECTL" echo ######################################## # Helper function to print and run commands ######################################## run_cmd() { echo "Command: $*" eval "$*" } ######################################## # 1. Print logs from ALL GoReplay pods ######################################## echo "==================================================" echo "1. Gathering logs from all goreplay pods (all containers)..." echo "==================================================" run_cmd "$KUBECTL logs -n goreplay -l app=goreplay --all-containers" || { echo "WARNING: Failed to get logs from pods with label app=goreplay" } ######################################## # 2. Describe the GoReplay DaemonSet ######################################## echo echo "==================================================" echo "2. Describing DaemonSet goreplay-daemon..." echo "==================================================" run_cmd "$KUBECTL describe daemonset goreplay-daemon -n goreplay" || { echo "WARNING: Failed to describe daemonset goreplay-daemon" } ######################################## # 3. Get list of GoReplay pods (full output) ######################################## echo echo "==================================================" echo "3. Listing goreplay pods (full output)..." echo "==================================================" # Print full output (no -o name here): run_cmd "$KUBECTL get pods -n goreplay -l app=goreplay" # Then retrieve just the names for further processing: echo echo "Getting goreplay pod names for telemetry collection..." pods=$($KUBECTL get pods -n goreplay -l app=goreplay -o name 2>/dev/null) || { echo "ERROR: Failed to list pods with label app=goreplay" exit 1 } echo "Found pods:" echo "$pods" echo ######################################## # 4. For each pod, gather logs, describe, and get events ######################################## for pod in $pods; do # pod looks like "pod/goreplay-daemon-xyz" pod_name="${pod##*/}" # remove "pod/" prefix echo "==================================================" echo "LOGS for pod: ${pod_name}" echo "==================================================" run_cmd "$KUBECTL logs ${pod_name} -n goreplay" || { echo "WARNING: Failed to get logs for pod ${pod_name}" } echo echo "--------------------------------------------------" echo "DESCRIBE for pod: ${pod_name}" echo "--------------------------------------------------" run_cmd "$KUBECTL describe pod -n goreplay ${pod_name}" || { echo "WARNING: Failed to describe pod ${pod_name}" } echo echo "--------------------------------------------------" echo "EVENTS for pod: ${pod_name}" echo "--------------------------------------------------" run_cmd "$KUBECTL get events -n goreplay --field-selector involvedObject.name=${pod_name}" || { echo "WARNING: Failed to get events for pod ${pod_name}" } echo done echo "==================================================" echo "Telemetry collection complete." echo "==================================================" ================================================ FILE: k8s/goreplay.yaml ================================================ apiVersion: apps/v1 kind: DaemonSet metadata: name: goreplay-daemon spec: selector: matchLabels: app: goreplay template: metadata: labels: app: goreplay spec: hostNetwork: true serviceAccountName: goreplay containers: - name: goreplay image: buger/gor:v2.0.0-rc4 args: - "--input-raw k8s://deployments/nginx:80" - "--output-stdout" - "--verbose" ================================================ FILE: k8s/nginx.yaml ================================================ apiVersion: apps/v1 kind: Deployment metadata: name: nginx labels: app: nginx spec: replicas: 1 selector: matchLabels: app: nginx template: metadata: labels: app: nginx spec: containers: - name: nginx image: nginx ports: - containerPort: 80 --- apiVersion: v1 kind: Service metadata: name: ngnix-service spec: selector: app: nginx type: NodePort ports: - protocol: TCP port: 80 targetPort: 80 ================================================ FILE: k8s/rolebinding.yaml ================================================ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: goreplay-reader-binding subjects: - kind: ServiceAccount name: goreplay namespace: goreplay roleRef: kind: ClusterRole name: pod-reader apiGroup: "" ================================================ FILE: kafka.go ================================================ package goreplay import ( "bytes" "crypto/sha256" "crypto/sha512" "crypto/tls" "crypto/x509" "errors" "fmt" "github.com/buger/goreplay/proto" "io/ioutil" "log" "github.com/Shopify/sarama" "github.com/xdg-go/scram" ) // SASLKafkaConfig SASL configuration type SASLKafkaConfig struct { UseSASL bool `json:"input-kafka-use-sasl"` Mechanism string `json:"input-kafka-mechanism"` Username string `json:"input-kafka-username"` Password string `json:"input-kafka-password"` } // InputKafkaConfig should contains required information to // build producers. type InputKafkaConfig struct { consumer sarama.Consumer Host string `json:"input-kafka-host"` Topic string `json:"input-kafka-topic"` UseJSON bool `json:"input-kafka-json-format"` Offset string `json:"input-kafka-offset"` SASLConfig SASLKafkaConfig } // OutputKafkaConfig is the representation of kfka output configuration type OutputKafkaConfig struct { producer sarama.AsyncProducer Host string `json:"output-kafka-host"` Topic string `json:"output-kafka-topic"` UseJSON bool `json:"output-kafka-json-format"` SASLConfig SASLKafkaConfig } // KafkaTLSConfig should contains TLS certificates for connecting to secured Kafka clusters type KafkaTLSConfig struct { CACert string `json:"kafka-tls-ca-cert"` ClientCert string `json:"kafka-tls-client-cert"` ClientKey string `json:"kafka-tls-client-key"` } // KafkaMessage should contains catched request information that should be // passed as Json to Apache Kafka. type KafkaMessage struct { ReqURL string `json:"Req_URL"` ReqType string `json:"Req_Type"` ReqID string `json:"Req_ID"` ReqTs string `json:"Req_Ts"` ReqMethod string `json:"Req_Method"` ReqBody string `json:"Req_Body,omitempty"` ReqHeaders map[string]string `json:"Req_Headers,omitempty"` } // NewTLSConfig loads TLS certificates func NewTLSConfig(clientCertFile, clientKeyFile, caCertFile string) (*tls.Config, error) { tlsConfig := tls.Config{} if clientCertFile != "" && clientKeyFile == "" { return &tlsConfig, errors.New("Missing key of client certificate in kafka") } if clientCertFile == "" && clientKeyFile != "" { return &tlsConfig, errors.New("missing TLS client certificate in kafka") } // Load client cert if (clientCertFile != "") && (clientKeyFile != "") { cert, err := tls.LoadX509KeyPair(clientCertFile, clientKeyFile) if err != nil { return &tlsConfig, err } tlsConfig.Certificates = []tls.Certificate{cert} } // Load CA cert if caCertFile != "" { caCert, err := ioutil.ReadFile(caCertFile) if err != nil { return &tlsConfig, err } caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) tlsConfig.RootCAs = caCertPool } return &tlsConfig, nil } // NewKafkaConfig returns Kafka config with or without TLS func NewKafkaConfig(saslConfig *SASLKafkaConfig, tlsConfig *KafkaTLSConfig) *sarama.Config { config := sarama.NewConfig() // Configuration options go here if tlsConfig != nil && (tlsConfig.ClientCert != "" || tlsConfig.CACert != "") { config.Net.TLS.Enable = true tlsConfig, err := NewTLSConfig(tlsConfig.ClientCert, tlsConfig.ClientKey, tlsConfig.CACert) if err != nil { log.Fatal(err) } config.Net.TLS.Config = tlsConfig } if saslConfig.UseSASL { mechanism := sarama.SASLMechanism(saslConfig.Mechanism) config.Net.SASL.Enable = saslConfig.UseSASL config.Net.SASL.Mechanism = mechanism config.Net.SASL.User = saslConfig.Username config.Net.SASL.Password = saslConfig.Password if mechanism == sarama.SASLTypeSCRAMSHA256 { config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &XDGSCRAMClient{HashGeneratorFcn: SHA256} } } else if mechanism == sarama.SASLTypeSCRAMSHA512 { config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &XDGSCRAMClient{HashGeneratorFcn: SHA512} } } } return config } // Dump returns the given request in its HTTP/1.x wire // representation. func (m KafkaMessage) Dump() ([]byte, error) { var b bytes.Buffer b.WriteString(fmt.Sprintf("%s %s %s\n", m.ReqType, m.ReqID, m.ReqTs)) b.WriteString(fmt.Sprintf("%s %s HTTP/1.1", m.ReqMethod, m.ReqURL)) b.Write(proto.CRLF) for key, value := range m.ReqHeaders { b.WriteString(fmt.Sprintf("%s: %s", key, value)) b.Write(proto.CRLF) } b.Write(proto.CRLF) b.WriteString(m.ReqBody) return b.Bytes(), nil } var ( // SHA256 SASLMechanism SHA256 scram.HashGeneratorFcn = sha256.New // SHA512 SASLMechanism SHA512 scram.HashGeneratorFcn = sha512.New ) // XDGSCRAMClient for SASL-Protocol type XDGSCRAMClient struct { *scram.Client *scram.ClientConversation scram.HashGeneratorFcn } // Begin of XDGSCRAMClient func (x *XDGSCRAMClient) Begin(userName, password, authzID string) (err error) { x.Client, err = x.HashGeneratorFcn.NewClient(userName, password, authzID) if err != nil { return err } x.ClientConversation = x.Client.NewConversation() return nil } // Step of XDGSCRAMClient func (x *XDGSCRAMClient) Step(challenge string) (response string, err error) { response, err = x.ClientConversation.Step(challenge) return } // Done of XDGSCRAMClient func (x *XDGSCRAMClient) Done() bool { return x.ClientConversation.Done() } ================================================ FILE: limiter.go ================================================ package goreplay import ( "fmt" "io" "math/rand" "strconv" "strings" "time" ) // Limiter is a wrapper for input or output plugin which adds rate limiting type Limiter struct { plugin interface{} limit int isPercent bool currentRPS int currentTime int64 } func parseLimitOptions(options string) (limit int, isPercent bool) { if n := strings.Index(options, "%"); n > 0 { limit, _ = strconv.Atoi(options[:n]) isPercent = true } else { limit, _ = strconv.Atoi(options) isPercent = false } return } func newLimiterExceptions(l *Limiter) { if !l.isPercent { return } speedFactor := float64(l.limit) / float64(100) // FileInput、KafkaInput have its own rate limiting. Unlike other inputs we not just dropping requests, we can slow down or speed up request emittion. switch input := l.plugin.(type) { case *FileInput: input.speedFactor = speedFactor case *KafkaInput: input.speedFactor = speedFactor } } // NewLimiter constructor for Limiter, accepts plugin and options // `options` allow to sprcify relatve or absolute limiting func NewLimiter(plugin interface{}, options string) PluginReadWriter { l := new(Limiter) l.limit, l.isPercent = parseLimitOptions(options) l.plugin = plugin l.currentTime = time.Now().UnixNano() newLimiterExceptions(l) return l } func (l *Limiter) isLimitedExceptions() bool { if !l.isPercent { return false } // Fileinput、Kafkainput have its own limiting algorithm switch l.plugin.(type) { case *FileInput: return true case *KafkaInput: return true default: return false } } func (l *Limiter) isLimited() bool { if l.isLimitedExceptions() { return false } if l.isPercent { return l.limit <= rand.Intn(100) } if (time.Now().UnixNano() - l.currentTime) > time.Second.Nanoseconds() { l.currentTime = time.Now().UnixNano() l.currentRPS = 0 } if l.currentRPS >= l.limit { return true } l.currentRPS++ return false } // PluginWrite writes message to this plugin func (l *Limiter) PluginWrite(msg *Message) (n int, err error) { if l.isLimited() { return 0, nil } if w, ok := l.plugin.(PluginWriter); ok { return w.PluginWrite(msg) } // avoid further writing return 0, io.ErrClosedPipe } // PluginRead reads message from this plugin func (l *Limiter) PluginRead() (msg *Message, err error) { if r, ok := l.plugin.(PluginReader); ok { msg, err = r.PluginRead() } else { // avoid further reading return nil, io.ErrClosedPipe } if l.isLimited() { return nil, nil } return } func (l *Limiter) String() string { return fmt.Sprintf("Limiting %s to: %d (isPercent: %v)", l.plugin, l.limit, l.isPercent) } // Close closes the resources. func (l *Limiter) Close() error { if fi, ok := l.plugin.(io.Closer); ok { fi.Close() } return nil } ================================================ FILE: limiter_test.go ================================================ //go:build !race package goreplay import ( "sync" "testing" ) func TestOutputLimiter(t *testing.T) { wg := new(sync.WaitGroup) input := NewTestInput() output := NewLimiter(NewTestOutput(func(*Message) { wg.Done() }), "10") wg.Add(10) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output}, } plugins.All = append(plugins.All, input, output) emitter := NewEmitter() go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 100; i++ { input.EmitGET() } wg.Wait() emitter.Close() } func TestInputLimiter(t *testing.T) { wg := new(sync.WaitGroup) input := NewLimiter(NewTestInput(), "10") output := NewTestOutput(func(*Message) { wg.Done() }) wg.Add(10) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output}, } plugins.All = append(plugins.All, input, output) emitter := NewEmitter() go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 100; i++ { input.(*Limiter).plugin.(*TestInput).EmitGET() } wg.Wait() emitter.Close() } // Should limit all requests func TestPercentLimiter1(t *testing.T) { wg := new(sync.WaitGroup) input := NewTestInput() output := NewLimiter(NewTestOutput(func(*Message) { wg.Done() }), "0%") plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output}, } plugins.All = append(plugins.All, input, output) emitter := NewEmitter() go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 100; i++ { input.EmitGET() } wg.Wait() } // Should not limit at all func TestPercentLimiter2(t *testing.T) { wg := new(sync.WaitGroup) input := NewTestInput() output := NewLimiter(NewTestOutput(func(*Message) { wg.Done() }), "100%") wg.Add(100) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output}, } plugins.All = append(plugins.All, input, output) emitter := NewEmitter() go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 100; i++ { input.EmitGET() } wg.Wait() } ================================================ FILE: middleware/README.md ================================================ # GoReplay middleware GoReplay support protocol for writing middleware in any language, which allows you to implement custom logic like authentification or complex rewriting and filterting. See protocol description here: https://github.com/buger/goreplay/wiki/Middleware, but the basic idea that middleware process receive hex encoded data via STDIN and emits it back via STDOUT. STDERR for loggin inside middleware. Yes, that's simple. To simplify middleware creation we provide packages for NodeJS and Go (upcoming). If you want to get access to original and replayed responses, do not forget adding `--output-http-track-response` and `--input-raw-track-response` options. ## NodeJS Before starting, you should install the package via npm: `npm install goreplay_middleware`. And initialize middleware the following way: ```javascript var gor = require("goreplay_middleware"); // `init` will initialize STDIN listener gor.init(); ``` Basic idea is that you write callbacks which respond to `request`, `response`, `replay`, or `message` events, which contain request meta information and actuall http paylod. Depending on your needs you may compare, override or filter incoming requests and responses. You can respond to the incoming events using `on` function, by providing callbacks: ```javascript // valid events are `request`, `response` (original response), `replay` (replayed response), and `message` (all events) gor.on('request', function(data) { // `data` contains incoming message its meta information. data // Raw HTTP payload of `Buffer` type // Example (hidden character for line endings shown on purpose): // GET / HTTP/1.1\r\n // User-Agent: Golang\r\n // \r\n data.http // Meta is an array size of 4, containing: // 1. request type - 1, 2 or 3 (which maps to `request`, `respose` and `replay`) // 2. uuid - request unique identifier. Request responses have the same ID as their request. // 3. timestamp of when request was made (for responses it is time of request start too) // 4. latency - time difference between request start and finish. For `request` is zero. data.meta // Unique request ID. It should be same for `request`, `response` and `replay` events of the same request. data.ID // You should return data at the end of function, even if you not changed request, if you do not want to filter it out. // If you just `return` nothing, request will be filtered return data }) ``` ### Mapping requests and responses You can provide request ID as additional argument to `on` function, which allow you to map related requests and responses. Below is example of middleware which checks that original and replayed response have same HTTP status code. ```javascript // Example of very basic way to compare if replayed traffic have no errors gor.on("request", function(req) { gor.on("response", req.ID, function(resp) { gor.on("replay", req.ID, function(repl) { if (gor.httpStatus(resp.http) != gor.httpStatus(repl.http)) { // Note that STDERR is used for logging, and it actually will be send to `Gor` STDOUT. // This trick is used because SDTIN and STDOUT already used for process communication. // You can write logger that writes to files insead. console.error(`${gor.httpPath(req.http)} STATUS NOT MATCH: 'Expected ${gor.httpStatus(resp.http)}' got '${gor.httpStatus(repl.http)}'`) } return repl; }) return resp; }) return req }) ``` This middleware includes `searchResponses` helper which is used to compare value of original response with the replayed response. If authentication system or xsrf protection returns unique tokens in headers or the response, it will be helpful to rewrite your requests based on them. Because tokens are unique, and the value contained in original and replayed responses will be different. So, you need to extract value from both responses, and rewrite requests based on those mappings. `searchResponses` accepts request id, regexp pattern for searching the compared value (should include capture group), and callback which returns both original and replayed matched value. Example: ```javascript // Compare HTTP headers for response and replayed response, and map values let tokMap = {}; gor.on("request", function(req) { let tok = gor.httpHeader(req.http, "Auth-Token"); if (tok && tokMap[tok]) { req.http = gor.setHttpHeader(req.http, "Auth-Token", tokMap[tok]) } gor.searchResponses(req.ID, "X-Set-Token: (\w+)$", function(respTok, replTok) { if (respTok && replTok) tokMap[respTok] = replTok; }) return req; }) ``` ### API documentation Package expose following functions to process raw HTTP payloads: * `init` - initialize middleware object, start reading from STDIN. * `httpPath` - URL path of the request: `gor.httpPath(req.http)` * `httpMethod` - Http method: 'GET', 'POST', etc. `gor.httpMethod(req.http)`. * `setHttpPath` - update URL path: `req.http = gor.setHttpPath(req.http, newPath)` * `httpPathParam` - get param from URL path: `gor.httpPathParam(req.http, queryParam)` * `setHttpPathParam` - set URL param: `req.http = gor.setHttpPathParam(req.http, queryParam, value)` * `httpStatus` - response status code * `httpHeaders` - get all headers: `gor.httpHeaders(req.http)` * `httpHeader` - get HTTP header: `gor.httpHeader(req.http, "Content-Length")` * `setHttpHeader` - Set HTTP header, returns modified payload: `req.http = gor.setHttpHeader(req.http, "X-Replayed", "1")` * `httpBody` - get HTTP Body: `gor.httpBody(req.http)` * `setHttpBody` - Set HTTP Body and ensures that `Content-Length` header have proper value. Returns modified payload: `req.http = gor.setHttpBody(req.http, Buffer.from('hello!'))`. * `httpBodyParam` - get POST body param: `gor.httpBodyParam(req.http, param)` * `setHttpBodyParam` - set POST body param: `req.http = gor.setHttpBodyParam(req.http, param, value)` * `httpCookie` - get HTTP cookie: `gor.httpCookie(req.http, "SESSSION_ID")` * `setHttpCookie` - set HTTP cookie, returns modified payload: `req.http = gor.setHttpCookie(req.http, "iam", "cuckoo")` * `deleteHttpCookie` - delete HTTP cookie, returns modified payload: `req.http = gor.deleteHttpCookie(req.http, "iam")` Also it is totally legit to use standard `Buffer` functions like `indexOf` for processing the HTTP payload. Just do not forget that if you modify the body, update the `Content-Length` header with a new value. And if you modify any of the headers, line endings should be `\r\n`. Rest is up to your imagination. ## Masking PII Data This middleware provides functionality to mask Personally Identifiable Information (PII) data in HTTP requests based on specified headers and JSON paths. It allows you to define a configuration object that specifies which headers and JSON fields should be masked and the type of data they represent. ```javascript const gor = require("goreplay_middleware"); const faker = require("faker"); // Initialize the middleware gor.init(); // Configuration for masking PII data const maskConfig = { headers: [ { name: "Authorization", type: "token" }, { name: "X-API-Key", type: "token" }, { name: "X-User-Email", type: "email" }, { name: "X-User-Name", type: "name" }, ], jsonPaths: [ { path: "$.user.email", type: "email" }, { path: "$.user.name", type: "name" }, { path: "$.user.phone", type: "phone" }, { path: "$.user.address", type: "address" }, ], }; // Function to mask a value based on its type function maskValue(type) { switch (type) { case "email": return faker.internet.email(); case "name": return faker.name.findName(); case "phone": return faker.phone.phoneNumber(); case "address": return faker.address.streetAddress(); case "token": return faker.random.alphaNumeric(32); default: return "***"; } } // Middleware function to mask PII data gor.on("message", (data) => { // Mask headers maskConfig.headers.forEach((header) => { const value = gor.httpHeader(data.http, header.name); if (value) { data.http = gor.setHttpHeader(data.http, header.name, maskValue(header.type)); } }); // Mask JSON fields const body = gor.httpBody(data.http); if (body) { try { const jsonBody = JSON.parse(body.toString()); maskConfig.jsonPaths.forEach((field) => { const value = eval(`jsonBody${field.path.slice(1)}`); if (value) { eval(`jsonBody${field.path.slice(1)} = maskValue(field.type)`); } }); data.http = gor.setHttpBody(data.http, Buffer.from(JSON.stringify(jsonBody))); } catch (error) { console.error("Error parsing JSON body:", error); } } return data; }); ``` ### Configuration The `maskConfig` object is used to configure the masking behavior. It consists of two properties: - `headers`: An array of objects representing the headers to be masked. Each object should have the following properties: - `name`: The name of the header. - `type`: The type of data the header represents (e.g., "email", "name", "token"). - `jsonPaths`: An array of objects representing the JSON paths to be masked. Each object should have the following properties: - `path`: The JSON path to the field to be masked (e.g., "$.user.email"). - `type`: The type of data the field represents (e.g., "email", "name", "phone", "address"). ### Masking Function The `maskValue` function is responsible for generating masked values based on the data type. It uses the Faker library to generate realistic-looking masked data for different types such as email, name, phone, address, and token. You can extend this function to support additional data types or customize the masking behavior. ### Middleware Function The middleware function is triggered for each HTTP message (request or response) processed by GoReplay. It performs the following steps: 1. Iterate over the specified headers in the `maskConfig` and mask their values using the `maskValue` function based on the associated data type. 2. Parse the JSON body of the request (if present) and iterate over the specified JSON paths in the `maskConfig`. If a value exists at a given path, replace it with a masked value generated by the `maskValue` function based on the associated data type. 3. Update the request body with the masked JSON data. 4. Return the modified HTTP message. Note: The middleware uses the `eval` function to dynamically access and modify the JSON object based on the provided paths. Exercise caution when using `eval` and ensure that the paths are properly validated to prevent potential security risks. To use this middleware, make sure to install the required dependencies (`goreplay_middleware` and `faker`), configure the `maskConfig` object according to your needs, and run the middleware with GoReplay. ## Support Feel free to ask questions here and by sending email to [support@goreplay.org](mailto:support@goreplay.org). Commercial support is available and welcomed 🙈. ================================================ FILE: middleware/middleware.js ================================================ // ======= GoReplay Middleware helper ============= // Created by Leonid Bugaev in 2017 // // For questions use GitHub or support@goreplay.org // // GoReplay: https://github.com/buger/goreplay // Middleware package: https://github.com/buger/goreplay/middleware var middleware; function init() { var proxy = { ch: {}, on: function(chan, id, cb) { if (!cb && id) { cb = id; } else if (cb && id) { chan = chan + "#" + id; } if (!proxy.ch[chan]) { proxy.ch[chan] = []; } proxy.ch[chan].push({ created: new Date(), cb: cb }); return proxy; }, emit: function(msg, raw) { var chanPrefix; switch(msg.type) { case "1": chanPrefix = "request"; break; case "2": chanPrefix = "response"; break; case "3": chanPrefix = "replay"; break; } let resp = msg; ["message", chanPrefix, chanPrefix + "#" + msg.ID].forEach(function(chanID, idx){ if (proxy.ch[chanID]) { proxy.ch[chanID].forEach(function(ch){ let r = ch.cb(msg); if (resp) resp = r; // If one of callback decided not to send response back, do not override it in global callbacks }) // Cleanup Individual message channels to avoid memory leaks if (idx == 2) { delete proxy.ch[chanID] } } }) if (resp) { process.stdout.write(`${resp.rawMeta.toString('hex')}${Buffer.from("\n").toString("hex")}${resp.http.toString('hex')}\n`) } return resp } } // Clean up old messaged ID specific channels if they are older then 60s let gc = function(gcTime){ let now = new Date(); for (k in proxy.ch) { if (k.indexOf("#") == -1) continue; proxy.ch[k] = proxy.ch[k].filter(function(ch){ return (now - ch.created) < gcTime }) if (proxy.ch[k].length == 0) { delete proxy.ch[k] } } } proxy.gc = gc setInterval(function(){ gc(10 * 1000) }, 1000); const readline = require('readline'); const rl = readline.createInterface({ input: process.stdin }); rl.on('line', function(line) { let msg = parseMessage(line) if (msg) { proxy.emit(msg, line) } }); middleware = proxy; return proxy; } function parseMessage(msg) { try { let payload = Buffer.from(msg, "hex"); let metaPos = payload.indexOf("\n"); let meta = payload.slice(0, metaPos); let metaArr = meta.toString("ascii").split(" "); let pType = metaArr[0]; let pID = metaArr[1]; let raw = payload.slice(metaPos + 1, payload.length); return { type: pType, ID: pID, rawMeta: meta, meta: metaArr, http: raw } } catch(e) { fail(`Error while parsing incoming request: ${msg}`) } } // Used to compare values from original and replayed responses // Accepts request id, regexp pattern for searching the compared value (should include capture group), and callback which returns both original and replayed matched value. // Example: // // // Compare HTTP headers for response and replayed response, and map values // let tokMap = {}; // // gor.on("request", function(req) { // let tok = gor.httpHeader(req.http, "Auth-Token"); // if (tok && tokMap[tok]) { // req.http = gor.setHttpHeader(req.http, "Auth-Token", tokMap[tok]) // } // // gor.searchResponses(req.ID, "X-Set-Token: (\w+)$", function(respTok, replTok) { // tokMap[respTok] = replTok; // }) // // return req; // }) // function searchResponses(id, searchPattern, callback) { let re = new RegExp(searchPattern); // Using regexp require converting buffer to string // Before converting to string we can use initial `Buffer.indexOf` check let indexPattern = searchPattern.split("(")[0]; if (!indexPattern) { console.error("Search regexp should include capture group, pointing to the value: `prefix-(.*)`") return } middleware.on("response", id, function(resp){ if (resp.http.indexOf(indexPattern) == -1) { callback() return resp } let respMatch = resp.http.toString('utf-8').match(re); if (!respMatch) { callback() return resp } middleware.on("replay", id, function(repl) { if (repl.http.indexOf(indexPattern) == -1) { callback(respMatch[1]); return repl; } let replMatch = repl.http.toString('utf-8').match(re); if (!replMatch) { callback(respMatch[1]); return repl; } callback(respMatch[1], replMatch[1]); return repl; }) return resp; }) } // =========== HTTP parsing ================= // Example HTTP payload record (including hidden characters): // // POST / HTTP/1.1\r\n // User-Agent: Node\r\n // Content-Length: 5\r\n // \r\n // hello function httpMethod(payload) { var pEnd = payload.indexOf(' '); return payload.slice(0, pEnd).toString("ascii"); } function httpPath(payload) { var pStart = payload.indexOf(' ') + 1; var pEnd = payload.indexOf(' ', pStart); return payload.slice(pStart, pEnd).toString("ascii"); } function setHttpPath(payload, newPath) { var pStart = payload.indexOf(' ') + 1; var pEnd = payload.indexOf(' ', pStart); return Buffer.concat([payload.slice(0, pStart), Buffer.from(newPath), payload.slice(pEnd, payload.length)]) } function httpPathParam(payload, name) { let path = httpPath(payload); let re = new RegExp(name + "=([^&$]+)"); let match = path.match(re); if (match) return decodeURI(match[1]); } function setHttpPathParam(payload, name, value) { let path = httpPath(payload); let re = new RegExp(name + "=([^&$]+)"); let newPath = path.replace(re, name + "=" + encodeURI(value)); // If we should add new param instead if (newPath == path) { if (newPath.indexOf("?") == -1) { newPath += "?" } else { newPath += "&" } newPath += name + "=" + encodeURI(value); } return setHttpPath(payload, newPath) } // HTTP response have status code in same position as `path` for requests function httpStatus(payload) { return httpPath(payload); } function setHttpStatus(payload, newStatus) { return setHttpPath(payload, newStatus); } function httpHeaders(payload) { var httpHeaderString = payload.slice(0,payload.indexOf("\r\n\r\n") + 4).toString().split("\n").slice(1); var headers = {}; for (var item in httpHeaderString) { var parts = httpHeaderString[item].split(":"); if (parts.length > 1) { headers[parts[0]] = parts.slice(1).join(":").trim(); } } return headers; } function httpHeader(payload, name) { var currentLine = 0; var i = 0; var header = { start: -1, end: -1, valueStart: -1 } var nameBuf = Buffer.from(name); var nameBufLower = Buffer.from(name.toLowerCase()); while(c = payload[i]) { if (c == 13) { // new line "\n" currentLine++; i++ header.end = i if (currentLine > 0 && header.start > 0 && header.valueStart > 0) { if (nameBuf.compare(payload, header.start, header.valueStart - 1) == 0 || nameBufLower.compare(payload, header.start, header.valueStart - 1) == 0) { // ensure that headers are not case sensitive header.value = payload.slice(header.valueStart, header.end - 1).toString("utf-8").trim(); header.name = payload.slice(header.start, header.valueStart - 1).toString("utf-8"); return header } } header.start = -1 header.valueStart = -1 continue; } else if (c == 10) { // "\r" i++ continue; } else if (c == 58) { // ":" Header/value separator symbol if (header.valueStart == -1) { header.valueStart = i + 1; i++ continue; } } if (header.start == -1) header.start = i; i++ } return } function setHttpHeader(payload, name, value) { let header = httpHeader(payload, name); if (!header) { let headerStart = payload.indexOf(13) + 1; return Buffer.concat([payload.slice(0, headerStart + 1), Buffer.from(name + ": " + value + "\r\n"), payload.slice(headerStart + 1, payload.length)]) } else { return Buffer.concat([payload.slice(0, header.valueStart), Buffer.from(" " + value + "\r\n"), payload.slice(header.end + 1, payload.length)]) } } function deleteHttpHeader(payload, name) { let header = httpHeader(payload, name); if (header) { return Buffer.concat([payload.slice(0, header.start), payload.slice(header.end+1, payload.length)]) } return payload } function httpBody(payload) { let bodyIndex = payload.indexOf("\r\n\r\n"); if (-1 != bodyIndex){ return payload.slice(bodyIndex + 4, payload.length); } else { return null; } } function setHttpBody(payload, newBody) { let p = setHttpHeader(payload, "Content-Length", newBody.length) let headerEnd = p.indexOf("\r\n\r\n") + 4; return Buffer.concat([p.slice(0, headerEnd), newBody]) } function httpBodyParam(payload, name) { let body = httpBody(payload); let re = new RegExp(name + "=([^&$]+)"); if (body.indexOf(name + "=") != -1) { let param = body.toString('utf-8').match(re); if (param) { return decodeURI(param[1]); } } } function setHttpBodyParam(payload, name, value) { let body = httpBody(payload); let re = new RegExp(name + "=([^&$]+)"); let newBody = body.toString('utf-8'); if (newBody.indexOf(name + "=") != -1 ) { newBody = newBody.replace(re, name + "=" + encodeURI(value)); } else { if (newBody.indexOf("=") != -1) { newBody += "&"; } newBody += name + "=" + value; } return setHttpBody(payload, Buffer.from(newBody)); } function setHttpCookie(payload, name, value) { let h = httpHeader(payload, "Cookie"); let cookie = h ? h.value : ""; let cookies = cookie.split("; ").filter(function(v){ return v.indexOf(name + "=") != 0 }) cookies.push(name + "=" + value) return setHttpHeader(payload, "Cookie", cookies.join("; ")) } function deleteHttpCookie(payload, name) { let h = httpHeader(payload, "Cookie"); let cookie = h ? h.value : ""; let cookies = cookie.split("; ").filter(function(v){ return v.indexOf(name + "=") != 0 }) return setHttpHeader(payload, "Cookie", cookies.join("; ")) } function httpCookie(payload, name) { let h = httpHeader(payload, "Cookie"); let cookie = h ? h.value : ""; let value; let cookies = cookie.split("; ").forEach(function(v){ if (v.indexOf(name + "=") == 0) { value = v.substr(name.length + 1); } }) return value; } module.exports = { init: init, on: function(){ return middleware.on.apply(this, arguments) }, parseMessage: parseMessage, searchResponses: searchResponses, httpPath: httpPath, httpMethod: httpMethod, setHttpPath: setHttpPath, httpPathParam: httpPathParam, setHttpPathParam: setHttpPathParam, httpStatus: httpStatus, setHttpStatus: setHttpStatus, httpHeader: httpHeader, setHttpHeader: setHttpHeader, deleteHttpHeader: deleteHttpHeader, httpBody: httpBody, setHttpBody: setHttpBody, httpBodyParam: httpBodyParam, setHttpBodyParam: setHttpBodyParam, httpCookie: httpCookie, setHttpCookie: setHttpCookie, deleteHttpCookie: deleteHttpCookie, test: testRunner, benchmark: testBenchmark, httpHeaders: httpHeaders } // =========== Tests ============== function testRunner(){ ["init", "filter", "parseMessage", "httpMethod", "httpPath", "setHttpHeader", "deleteHttpHeader", "httpPathParam", "httpHeader", "httpBody", "setHttpBody", "httpBodyParam", "httpCookie", "setHttpCookie", "deleteHttpCookie", "httpHeaders"].forEach(function(t){ console.log(`====== Start ${t} =======`) eval(`TEST_${t}()`) console.log(`====== End ${t} =======`) }) } function testBenchmark(){ const child_process = require('child_process'); let gor = init(); gor.on("message", function(){ }); gor.on("request", function(){ }); for (var i = 0; i<256; i++) { let req = parseMessage(Buffer.from("1 2 3\nGET / HTTP/1.1\r\n\r\n").toString('hex')); req.ID = +Date.now() gor.emit(req); gor.on("request", req.ID+"", function(){ gor.on("response", req.ID+"", function(){ }) }) if ( i % 3 == 0 ) { let resp = parseMessage(Buffer.from("2 2 3\nHTTP/1.1 200 OK\r\n\r\n").toString('hex')); resp.ID = req.ID gor.emit(resp); } } child_process.execSync("sleep 0.01"); gor.gc(1) fail(JSON.stringify(gor.ch)) } // Just print in red color function fail(message) { console.error("\x1b[31m[MIDDLEWARE] %s\x1b[0m", message) } function log(message) { console.error(message) } function TEST_init() { const child_process = require('child_process'); let received = 0; let gor = init(); gor.on("message", function(){ received++; // should be called 3 times for for every request }); gor.on("request", function(){ received++; // should be called 1 time only for request }); gor.on("response", "2", function(){ received++; // should be called 1 time only for specific response }) if (Object.keys(gor.ch).length != 3) { return fail("Should create 3 channels"); } let req = parseMessage(Buffer.from("1 2 3\nGET / HTTP/1.1\r\n\r\n").toString('hex')); let resp = parseMessage(Buffer.from("2 2 3\nHTTP/1.1 200 OK\r\n\r\n").toString('hex')); let resp2 = parseMessage(Buffer.from("2 3 3\nHTTP/1.1 200 OK\r\n\r\n").toString('hex')); gor.emit(req); gor.emit(resp); gor.emit(resp2); child_process.execSync("sleep 0.01"); if (received != 5) { fail(`Should receive 5 messages: ${received}`); } } function TEST_filter() { const child_process = require('child_process'); let gor = init(); gor.on("request", function(req){ if (httpPath(req.http) != "/filter") { return req } }); gor.on("request", function(req){ return req }); let reqPass = parseMessage(Buffer.from("1 2 3\nGET / HTTP/1.1\r\n\r\n").toString('hex')); let reqFilter = parseMessage(Buffer.from("1 2 3\nGET /filter HTTP/1.1\r\n\r\n").toString('hex')); if (!gor.emit(reqPass)) { return fail("Should not filter request") } if (gor.emit(reqFilter)) { return fail("Should filter request even if one middleware rejected it") } } function TEST_parseMessage() { const exampleMessage = Buffer.from("1 2 3\nGET / HTTP/1.1\r\n\r\n").toString('hex') let msg = parseMessage(exampleMessage) let expected = { type: '1', ID: '2', meta: ["1", "2", "3"], http: Buffer.from("GET / HTTP/1.1\r\n\r\n") } Object.keys(expected).forEach(function(k){ if (msg[k].toString() != expected[k].toString()) { fail(`${k}: '${expected[k]}' != '${msg[k]}'`) } }) } function TEST_httpPath() { const examplePayload = "GET /test HTTP/1.1\r\n\r\n"; let payload = Buffer.from(examplePayload); let path = httpPath(payload); if (path != "/test") { return fail(`Path '${patj}' != '/test'`) } let newPayload = setHttpPath(payload, '/') if (newPayload.toString() != "GET / HTTP/1.1\r\n\r\n") { return fail(`Malformed payload '${newPayload}'`) } newPayload = setHttpPath(payload, '/bigger') if (newPayload.toString() != "GET /bigger HTTP/1.1\r\n\r\n") { return fail(`Malformed payload '${newPayload}'`) } } function TEST_httpMethod() { const examplePayload = "GET /test HTTP/1.1\r\n\r\n"; let payload = Buffer.from(examplePayload); let method = httpMethod(payload); if (method != "GET") { return fail(`Path '${method}' != 'GET'`) } } function TEST_httpPathParam() { let p = Buffer.from("GET / HTTP/1.1\r\n\r\n"); if (httpPathParam(p, "test")) { return fail("Should not found param") } p = setHttpPathParam(p, "test", "123"); if (httpPath(p) != "/?test=123") { return fail("Should set first param: " + httpPath(p)); } if (httpPathParam(p, "test") != "123") { return fail("Should get first param: " + httpPathParam(p, "test")); } p = setHttpPathParam(p, "qwer", "ty"); if (httpPath(p) != "/?test=123&qwer=ty") { return fail("Should set second param: " + httpPath(p)); } p = setHttpPathParam(p, "test", "4321"); if (httpPath(p) != "/?test=4321&qwer=ty") { return fail("Should update first param: " + httpPath(p)); } if (httpPathParam(p, "test") != "4321") { return fail("Should update first param: " + httpPath(p)); } } function TEST_httpBodyParam() { let p = Buffer.from("POST / HTTP/1.1\r\n\r\n"); if (httpBodyParam(p, "test")) { return fail("Should not found param") } p = setHttpBodyParam(p, "test", "123"); if (httpBody(p).toString() != "test=123") { return fail("Should set first param: " + httpBody(p).toString()); } if (httpBodyParam(p, "test") != "123") { return fail("Should get first param: " + httpBodyParam(p, "test")); } p = setHttpBodyParam(p, "qwer", "ty"); if (httpBody(p).toString() != "test=123&qwer=ty") { return fail("Should set second param: " + httpBody(p).toString()); } p = setHttpBodyParam(p, "test", "4321"); if (httpBody(p).toString() != "test=4321&qwer=ty") { return fail("Should update first param: " + httpBody(p).toString()); } if (httpBodyParam(p, "test") != "4321") { return fail("Should update first param: " + httpBody(p).toString()); } } function TEST_httpHeader() { const examplePayload = "GET / HTTP/1.1\r\nHost: localhost:3000\r\nUser-Agent: Node\r\nContent-Length:5\r\n\r\nhello"; let expected = {"Host": "localhost:3000", "User-Agent": "Node", "Content-Length": "5"} Object.keys(expected).forEach(function(name){ let payload = Buffer.from(examplePayload); let header = httpHeader(payload, name); if (!header) { fail(`Header not found. Was looking for: ${name}`) } if (header && header.value != expected[name]) { fail(`${name}: '${expected[name]}' != '${header.value}'`) } }) } function TEST_setHttpHeader() { const examplePayload = "GET / HTTP/1.1\r\nUser-Agent: Node\r\nContent-Length: 5\r\n\r\nhello"; // Modify existing header ["", "1", "Long test header"].forEach(function(ua){ let expected = `GET / HTTP/1.1\r\nUser-Agent: ${ua}\r\nContent-Length: 5\r\n\r\nhello`; let p = Buffer.from(examplePayload); p = setHttpHeader(p, "User-Agent", ua); if (p != expected) { console.error(`setHeader failed, expected User-Agent value: ${ua}.\n${p}`) } }) // Adding new header let expected = `GET / HTTP/1.1\r\nX-Test: test\r\nUser-Agent: Node\r\nContent-Length: 5\r\n\r\nhello`; let p = Buffer.from(examplePayload); p = setHttpHeader(p, "X-Test", "test"); if (p != expected) { console.error(`setHeader failed, expected new header 'X-Test' header: ${p}`) } } function TEST_deleteHttpHeader() { const examplePayload = "GET / HTTP/1.1\r\nUser-Agent: Node\r\nContent-Length: 5\r\n\r\nhello"; // Adding new header let expected = `GET / HTTP/1.1\r\nContent-Length: 5\r\n\r\nhello`; let p = Buffer.from(examplePayload); p = deleteHttpHeader(p, "User-Agent", "test"); if (p != expected) { console.error(`setHeader failed, expected delete header 'User-Agent' header: ${p}`) } } function TEST_httpBody() { const examplePayload = "GET / HTTP/1.1\r\nUser-Agent: Node\r\nContent-Length: 5\r\n\r\nhello"; let body = httpBody(Buffer.from(examplePayload)); if (body != "hello") { fail(`'${body}' != 'hello'`) } const exampleInvalidPayload = "Invalid HTTP Response by Network issue"; let invalidBody = httpBody(Buffer.from(exampleInvalidPayload)); if (invalidBody != null) { fail(`'${invalidBody}' != 'null'`) } } function TEST_setHttpBody() { const examplePayload = "GET / HTTP/1.1\r\nUser-Agent: Node\r\nContent-Length: 5\r\n\r\nhello"; let p = setHttpBody(Buffer.from(examplePayload), Buffer.from("hello, world!")); if (p != "GET / HTTP/1.1\r\nUser-Agent: Node\r\nContent-Length: 13\r\n\r\nhello, world!") { fail(`Wrong body: '${p}'`) } } function TEST_httpCookie() { const examplePayload = "GET / HTTP/1.1\r\nCookie: a=b; test=zxc\r\n\r\n"; let c = httpCookie(Buffer.from(examplePayload), "test"); if (c != "zxc") { return fail(`Should get cookie: ${c}`); } c = httpCookie(Buffer.from(examplePayload), "nope"); if (c != null) { return fail(`Should not find cookie: ${c}`); } } function TEST_setHttpCookie() { const examplePayload = "GET / HTTP/1.1\r\nCookie: a=b; test=zxc\r\n\r\n"; let p = setHttpCookie(Buffer.from(examplePayload), "test", "1"); if (p != "GET / HTTP/1.1\r\nCookie: a=b; test=1\r\n\r\n") { return fail(`Should update cookie: ${p}`) } p = setHttpCookie(Buffer.from(examplePayload), "new", "one"); if (p != "GET / HTTP/1.1\r\nCookie: a=b; test=zxc; new=one\r\n\r\n") { return fail(`Should add new cookie: ${p}`) } } function TEST_deleteHttpCookie() { const examplePayload = "GET / HTTP/1.1\r\nCookie: a=b; test=zxc\r\n\r\n"; let p = deleteHttpCookie(Buffer.from(examplePayload), "a"); if (p != "GET / HTTP/1.1\r\nCookie: test=zxc\r\n\r\n") { return fail(`Should delete cookie: ${p}`) } } function TEST_httpHeaders() { const examplePayload = "GET / HTTP/1.1\r\nHost: localhost:3000\r\nUser-Agent: Node\r\nContent-Length:5\r\n\r\nhello"; let expectedHeaders = {"Host": "localhost:3000", "User-Agent": "Node", "Content-Length": "5"} let payload = Buffer.from(examplePayload); let headers = httpHeaders(payload); ["Host", "User-Agent", "Content-Length"].forEach(function(header){ let actual = headers[header]; let expected = expectedHeaders[header]; if (!actual) { fail(`${header} Header was not found`); } if (actual != expected) { fail(`${header} Header not Equal to Expected: ${expected} was ${actual}`); } }) } ================================================ FILE: middleware/package.json ================================================ { "name": "goreplay_middleware", "version": "1.0.0", "description": "Package for writing middleware for GoReplay https://goreplay.org", "main": "middleware.js", "scripts": { "test": "node -e \"var gor = require('./middleware.js'); gor.test(); process.exit()\"", "benchmark": "node -e \"var gor = require('./middleware.js'); gor.benchmark(); process.exit()\"" }, "keywords": [ "middleware", "goreplay" ], "author": "Leonid Bugaev", "license": "LGPL-3.0" } ================================================ FILE: middleware.go ================================================ package goreplay import ( "bufio" "context" "encoding/hex" "fmt" "io" "os" "os/exec" "strings" "sync" "syscall" ) // Middleware represents a middleware object type Middleware struct { command string data chan *Message Stdin io.Writer Stdout io.Reader commandCancel context.CancelFunc stop chan bool // Channel used only to indicate goroutine should shutdown closed bool mu sync.RWMutex } // NewMiddleware returns new middleware func NewMiddleware(command string) *Middleware { m := new(Middleware) m.command = command m.data = make(chan *Message, 1000) m.stop = make(chan bool) commands := strings.Split(command, " ") ctx, cancl := context.WithCancel(context.Background()) m.commandCancel = cancl cmd := exec.CommandContext(ctx, commands[0], commands[1:]...) m.Stdout, _ = cmd.StdoutPipe() m.Stdin, _ = cmd.StdinPipe() cmd.Stderr = os.Stderr go m.read(m.Stdout) go func() { defer m.Close() var err error if err = cmd.Start(); err == nil { err = cmd.Wait() } if err != nil { if e, ok := err.(*exec.ExitError); ok { status := e.Sys().(syscall.WaitStatus) if status.Signal() == syscall.SIGKILL /*killed or context canceld */ { return } } Debug(0, fmt.Sprintf("[MIDDLEWARE] command[%q] error: %q", command, err.Error())) } }() return m } // ReadFrom start a worker to read from this plugin func (m *Middleware) ReadFrom(plugin PluginReader) { Debug(2, fmt.Sprintf("[MIDDLEWARE] command[%q] Starting reading from %q", m.command, plugin)) go m.copy(m.Stdin, plugin) } func (m *Middleware) copy(to io.Writer, from PluginReader) { var buf, dst []byte for { msg, err := from.PluginRead() if err != nil { return } if msg == nil || len(msg.Data) == 0 { continue } buf = msg.Data if Settings.PrettifyHTTP { buf = prettifyHTTP(msg.Data) } dstLen := (len(buf)+len(msg.Meta))*2 + 1 // if enough space was previously allocated use it instead if dstLen > len(dst) { dst = make([]byte, dstLen) } n := hex.Encode(dst, msg.Meta) n += hex.Encode(dst[n:], buf) dst[n] = '\n' n, err = to.Write(dst[:n+1]) if err == nil { continue } if m.isClosed() { return } } } func (m *Middleware) read(from io.Reader) { reader := bufio.NewReader(from) var line []byte var e error for { if line, e = reader.ReadBytes('\n'); e != nil { if m.isClosed() { return } continue } buf := make([]byte, (len(line)-1)/2) if _, err := hex.Decode(buf, line[:len(line)-1]); err != nil { Debug(0, fmt.Sprintf("[MIDDLEWARE] command[%q] failed to decode err: %q", m.command, err)) continue } var msg Message msg.Meta, msg.Data = payloadMetaWithBody(buf) select { case <-m.stop: return case m.data <- &msg: } } } // PluginRead reads message from this plugin func (m *Middleware) PluginRead() (msg *Message, err error) { select { case <-m.stop: return nil, ErrorStopped case msg = <-m.data: } return } func (m *Middleware) String() string { return fmt.Sprintf("Modifying traffic using %q command", m.command) } func (m *Middleware) isClosed() bool { m.mu.RLock() defer m.mu.RUnlock() return m.closed } // Close closes this plugin func (m *Middleware) Close() error { if m.isClosed() { return nil } m.mu.Lock() defer m.mu.Unlock() m.commandCancel() close(m.stop) m.closed = true return nil } ================================================ FILE: middleware_test.go ================================================ package goreplay import ( "bytes" "context" "os" "os/exec" "strings" "sync/atomic" "syscall" "testing" ) const echoSh = "./examples/middleware/echo.sh" const tokenModifier = "go run ./examples/middleware/token_modifier.go" var withDebug = append(syscall.Environ(), "GOR_TEST=1") func initMiddleware(cmd *exec.Cmd, cancl context.CancelFunc, l PluginReader, c func(error)) *Middleware { var m Middleware m.data = make(chan *Message, 1000) m.stop = make(chan bool) m.commandCancel = cancl m.Stdout, _ = cmd.StdoutPipe() m.Stdin, _ = cmd.StdinPipe() cmd.Stderr = os.Stderr go m.read(m.Stdout) go func() { defer m.Close() var err error if err = cmd.Start(); err == nil { err = cmd.Wait() } if err != nil { c(err) } }() m.ReadFrom(l) return &m } func initCmd(command string, env []string) (*exec.Cmd, context.CancelFunc) { commands := strings.Split(command, " ") ctx, cancl := context.WithCancel(context.Background()) cmd := exec.CommandContext(ctx, commands[0], commands[1:]...) cmd.Env = env return cmd, cancl } func TestMiddlewareEarlyClose(t *testing.T) { quit := make(chan struct{}) in := NewTestInput() cmd, cancl := initCmd(echoSh, withDebug) midd := initMiddleware(cmd, cancl, in, func(err error) { if err != nil { if e, ok := err.(*exec.ExitError); ok { status := e.Sys().(syscall.WaitStatus) if status.Signal() != syscall.SIGKILL { t.Errorf("expected error to be signal killed. got %s", status.Signal().String()) } } } quit <- struct{}{} }) var body = []byte("OPTIONS / HTTP/1.1\r\nHost: example.org\r\n\r\n") count := uint32(0) out := NewTestOutput(func(msg *Message) { if !bytes.Equal(body, msg.Data) { t.Errorf("expected %q to equal %q", body, msg.Data) } atomic.AddUint32(&count, 1) if atomic.LoadUint32(&count) == 5 { quit <- struct{}{} } }) pl := &InOutPlugins{} pl.Inputs = []PluginReader{midd, in} pl.Outputs = []PluginWriter{out} pl.All = []interface{}{midd, out, in} e := NewEmitter() go e.Start(pl, "") for i := 0; i < 5; i++ { in.EmitBytes(body) } <-quit midd.Close() <-quit } //func TestTokenMiddleware(t *testing.T) { // quit := make(chan struct{}) // in := NewTestInput() // in.skipHeader = true // cmd, cancl := initCmd(tokenModifier, withDebug) // midd := initMiddleware(cmd, cancl, in, func(err error) {}) // req := []byte("1 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nGET /token HTTP/1.1\r\nHost: example.org\r\n\r\n") // res := []byte("2 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nHTTP/1.1 200 OK\r\nContent-Length: 10\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n17d823647c") // rep := []byte("3 932079936fa4306fc308d67588178d17d823647c 1439818823587396305 200\nHTTP/1.1 200 OK\r\nContent-Length: 15\r\nContent-Type: text/plain; charset=utf-8\r\n\r\n932079936fa4306") // count := uint32(0) // out := NewTestOutput(func(msg *Message) { // if msg.Meta[0] == '1' && !bytes.Equal(payloadID(msg.Meta), payloadID(req)) { // token, _, _ := proto.PathParam(msg.Data, []byte("token")) // if !bytes.Equal(token, proto.Body(rep)) { // t.Errorf("expected the token %s to be equal to the replayed response's token %s", token, proto.Body(rep)) // } // } // atomic.AddUint32(&count, 1) // if atomic.LoadUint32(&count) == 2 { // quit <- struct{}{} // } // }) // pl := &InOutPlugins{} // pl.Inputs = []PluginReader{midd, in} // pl.Outputs = []PluginWriter{out} // pl.All = []interface{}{midd, out, in} // e := NewEmitter() // go e.Start(pl, "") // in.EmitBytes(req) // emit original request // in.EmitBytes(res) // emit its response // in.EmitBytes(rep) // emit replayed response // // emit the request which should have modified token // token := []byte("1 8e091765ae902fef8a2b7d9dd96 14398188235873 100\nGET /?token=17d823647c HTTP/1.1\r\nHost: example.org\r\n\r\n") // in.EmitBytes(token) // <-quit // midd.Close() //} //func TestMiddlewareWithPrettify(t *testing.T) { // Settings.PrettifyHTTP = true // quit := make(chan struct{}) // in := NewTestInput() // cmd, cancl := initCmd(echoSh, withDebug) // midd := initMiddleware(cmd, cancl, in, func(err error) {}) // var b1 = []byte("POST / HTTP/1.1\r\nHost: example.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n") // var b2 = []byte("POST / HTTP/1.1\r\nHost: example.org\r\nContent-Length: 25\r\n\r\nWikipedia in\r\n\r\nchunks.") // out := NewTestOutput(func(msg *Message) { // if !bytes.Equal(proto.Body(b2), proto.Body(msg.Data)) { // t.Errorf("expected %q body to equal %q body", b2, msg.Data) // } // quit <- struct{}{} // }) // pl := &InOutPlugins{} // pl.Inputs = []PluginReader{midd, in} // pl.Outputs = []PluginWriter{out} // pl.All = []interface{}{midd, out, in} // e := NewEmitter() // go e.Start(pl, "") // in.EmitBytes(b1) // <-quit // midd.Close() // Settings.PrettifyHTTP = false //} ================================================ FILE: mkdocs.yml ================================================ site_name: My Docs theme: readthedocs ================================================ FILE: nfpm.yaml ================================================ # nfpm example config file # # check https://nfpm.goreleaser.com/configuration for detailed usage # name: "GoReplay" arch: ${PLATFORM} platform: "linux" version: ${VERSION} section: "default" priority: "extra" provides: - goreplay maintainer: "Leonid Bugaev " description: | GoReplay is the simplest and safest way to test your app using real traffic before you put it into production. vendor: "GoReplay" homepage: "https://goreplay.org" license: "AGPL" contents: - src: ./${BIN_NAME} dst: /usr/local/bin ================================================ FILE: output_binary.go ================================================ //go:build !pro package goreplay import ( "errors" "time" "github.com/buger/goreplay/internal/size" ) var _ PluginWriter = (*BinaryOutput)(nil) // BinaryOutputConfig struct for holding binary output configuration type BinaryOutputConfig struct { Workers int `json:"output-binary-workers"` Timeout time.Duration `json:"output-binary-timeout"` BufferSize size.Size `json:"output-tcp-response-buffer"` Debug bool `json:"output-binary-debug"` TrackResponses bool `json:"output-binary-track-response"` } // BinaryOutput plugin manage pool of workers which send request to replayed server // By default workers pool is dynamic and starts with 10 workers // You can specify fixed number of workers using `--output-tcp-workers` type BinaryOutput struct { address string } // NewBinaryOutput constructor for BinaryOutput // Initialize workers func NewBinaryOutput(address string, config *BinaryOutputConfig) PluginReadWriter { return &BinaryOutput{address: address} } // PluginWrite writes a message to this plugin func (o *BinaryOutput) PluginWrite(msg *Message) (n int, err error) { return 0, errors.New("binary output is only available in PRO version") } // PluginRead reads a message from this plugin func (o *BinaryOutput) PluginRead() (*Message, error) { return nil, errors.New("binary output is only available in PRO version") } func (o *BinaryOutput) String() string { return "Binary output: " + o.address + " (PRO version required)" } // Close closes this plugin for reading func (o *BinaryOutput) Close() error { return nil } ================================================ FILE: output_binary_pro.go ================================================ //go:build pro package goreplay import ( "sync/atomic" "time" "github.com/buger/goreplay/internal/size" ) var _ PluginWriter = (*BinaryOutput)(nil) // BinaryOutputConfig struct for holding binary output configuration type BinaryOutputConfig struct { Workers int `json:"output-binary-workers"` Timeout time.Duration `json:"output-binary-timeout"` BufferSize size.Size `json:"output-tcp-response-buffer"` Debug bool `json:"output-binary-debug"` TrackResponses bool `json:"output-binary-track-response"` } // BinaryOutput plugin manage pool of workers which send request to replayed server // By default workers pool is dynamic and starts with 10 workers // You can specify fixed number of workers using `--output-tcp-workers` type BinaryOutput struct { // Keep this as first element of struct because it guarantees 64bit // alignment. atomic.* functions crash on 32bit machines if operand is not // aligned at 64bit. See https://github.com/golang/go/issues/599 activeWorkers int64 address string queue chan *Message responses chan response needWorker chan int quit chan struct{} config *BinaryOutputConfig queueStats *GorStat } // NewBinaryOutput constructor for BinaryOutput // Initialize workers func NewBinaryOutput(address string, config *BinaryOutputConfig) PluginReadWriter { o := new(BinaryOutput) o.address = address o.config = config o.queue = make(chan *Message, 1000) o.responses = make(chan response, 1000) o.needWorker = make(chan int, 1) o.quit = make(chan struct{}) // Initial workers count if o.config.Workers == 0 { o.needWorker <- initialDynamicWorkers } else { o.needWorker <- o.config.Workers } go o.workerMaster() return o } func (o *BinaryOutput) workerMaster() { for { newWorkers := <-o.needWorker for i := 0; i < newWorkers; i++ { go o.startWorker() } // Disable dynamic scaling if workers poll fixed size if o.config.Workers != 0 { return } } } func (o *BinaryOutput) startWorker() { client := NewTCPClient(o.address, &TCPClientConfig{ Debug: o.config.Debug, Timeout: o.config.Timeout, ResponseBufferSize: int(o.config.BufferSize), }) deathCount := 0 atomic.AddInt64(&o.activeWorkers, 1) for { select { case msg := <-o.queue: o.sendRequest(client, msg) deathCount = 0 case <-time.After(time.Millisecond * 100): // When dynamic scaling enabled workers die after 2s of inactivity if o.config.Workers == 0 { deathCount++ } else { continue } if deathCount > 20 { workersCount := atomic.LoadInt64(&o.activeWorkers) // At least 1 startWorker should be alive if workersCount != 1 { atomic.AddInt64(&o.activeWorkers, -1) return } } } } } // PluginWrite writes a message tothis plugin func (o *BinaryOutput) PluginWrite(msg *Message) (n int, err error) { if !isRequestPayload(msg.Meta) { return len(msg.Data), nil } o.queue <- msg if o.config.Workers == 0 { workersCount := atomic.LoadInt64(&o.activeWorkers) if len(o.queue) > int(workersCount) { o.needWorker <- len(o.queue) } } return len(msg.Data) + len(msg.Meta), nil } // PluginRead reads a message from this plugin func (o *BinaryOutput) PluginRead() (*Message, error) { var resp response var msg Message select { case <-o.quit: return nil, ErrorStopped case resp = <-o.responses: } msg.Data = resp.payload msg.Meta = payloadHeader(ReplayedResponsePayload, resp.uuid, resp.startedAt, resp.roundTripTime) return &msg, nil } func (o *BinaryOutput) sendRequest(client *TCPClient, msg *Message) { if !isRequestPayload(msg.Meta) { return } uuid := payloadID(msg.Meta) start := time.Now() resp, err := client.Send(msg.Data) stop := time.Now() if err != nil { Debug(1, "Request error:", err) } if o.config.TrackResponses { o.responses <- response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()} } } func (o *BinaryOutput) String() string { return "Binary output: " + o.address } // Close closes this plugin for reading func (o *BinaryOutput) Close() error { close(o.quit) return nil } ================================================ FILE: output_dummy.go ================================================ package goreplay import ( "os" ) // DummyOutput used for debugging, prints all incoming requests type DummyOutput struct { } // NewDummyOutput constructor for DummyOutput func NewDummyOutput() (di *DummyOutput) { di = new(DummyOutput) return } // PluginWrite writes message to this plugin func (i *DummyOutput) PluginWrite(msg *Message) (int, error) { var n, nn int var err error n, err = os.Stdout.Write(msg.Meta) nn, err = os.Stdout.Write(msg.Data) n += nn nn, err = os.Stdout.Write(payloadSeparatorAsBytes) n += nn return n, err } func (i *DummyOutput) String() string { return "Dummy Output" } ================================================ FILE: output_file.go ================================================ package goreplay import ( "bufio" "compress/gzip" "errors" "fmt" "github.com/buger/goreplay/internal/size" "io" "log" "math/rand" "os" "path/filepath" "runtime/debug" "sort" "strconv" "strings" "sync" "time" ) var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") var instanceID string func init() { instanceID = randSeq(8) } func randSeq(n int) string { b := make([]rune, n) for i := range b { b[i] = letters[rand.Intn(len(letters))] } return string(b) } var dateFileNameFuncs = map[string]func(*FileOutput) string{ "%Y": func(o *FileOutput) string { return time.Now().Format("2006") }, "%m": func(o *FileOutput) string { return time.Now().Format("01") }, "%d": func(o *FileOutput) string { return time.Now().Format("02") }, "%H": func(o *FileOutput) string { return time.Now().Format("15") }, "%M": func(o *FileOutput) string { return time.Now().Format("04") }, "%S": func(o *FileOutput) string { return time.Now().Format("05") }, "%NS": func(o *FileOutput) string { return fmt.Sprint(time.Now().Nanosecond()) }, "%r": func(o *FileOutput) string { return string(o.currentID) }, "%t": func(o *FileOutput) string { return string(o.payloadType) }, "%i": func(o *FileOutput) string { return instanceID }, } // FileOutputConfig ... type FileOutputConfig struct { FlushInterval time.Duration `json:"output-file-flush-interval"` SizeLimit size.Size `json:"output-file-size-limit"` OutputFileMaxSize size.Size `json:"output-file-max-size-limit"` QueueLimit int `json:"output-file-queue-limit"` Append bool `json:"output-file-append"` BufferPath string `json:"output-file-buffer"` onClose func(string) } // FileOutput output plugin type FileOutput struct { sync.RWMutex pathTemplate string currentName string file *os.File QueueLength int writer io.Writer requestPerFile bool currentID []byte payloadType []byte closed bool currentFileSize int totalFileSize size.Size config *FileOutputConfig } // NewFileOutput constructor for FileOutput, accepts path func NewFileOutput(pathTemplate string, config *FileOutputConfig) *FileOutput { o := new(FileOutput) o.pathTemplate = pathTemplate o.config = config if strings.Contains(pathTemplate, "%r") { o.requestPerFile = true } if config.FlushInterval == 0 { config.FlushInterval = 100 * time.Millisecond } go func() { for { time.Sleep(config.FlushInterval) if o.IsClosed() { break } o.flush() } }() return o } func getFileIndex(name string) int { ext := filepath.Ext(name) withoutExt := strings.TrimSuffix(name, ext) if idx := strings.LastIndex(withoutExt, "_"); idx != -1 { if i, err := strconv.Atoi(withoutExt[idx+1:]); err == nil { return i } } return -1 } func setFileIndex(name string, idx int) string { idxS := strconv.Itoa(idx) ext := filepath.Ext(name) withoutExt := strings.TrimSuffix(name, ext) if i := strings.LastIndex(withoutExt, "_"); i != -1 { if _, err := strconv.Atoi(withoutExt[i+1:]); err == nil { withoutExt = withoutExt[:i] } } return withoutExt + "_" + idxS + ext } func withoutIndex(s string) string { if i := strings.LastIndex(s, "_"); i != -1 { return s[:i] } return s } type sortByFileIndex []string func (s sortByFileIndex) Len() int { return len(s) } func (s sortByFileIndex) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s sortByFileIndex) Less(i, j int) bool { if withoutIndex(s[i]) == withoutIndex(s[j]) { return getFileIndex(s[i]) < getFileIndex(s[j]) } return s[i] < s[j] } func (o *FileOutput) filename() string { o.RLock() defer o.RUnlock() path := o.pathTemplate for name, fn := range dateFileNameFuncs { path = strings.Replace(path, name, fn(o), -1) } if !o.config.Append { nextChunk := false if o.currentName == "" || ((o.config.QueueLimit > 0 && o.QueueLength >= o.config.QueueLimit) || (o.config.SizeLimit > 0 && o.currentFileSize >= int(o.config.SizeLimit))) { nextChunk = true } ext := filepath.Ext(path) withoutExt := strings.TrimSuffix(path, ext) if matches, err := filepath.Glob(withoutExt + "*" + ext); err == nil { if len(matches) == 0 { return setFileIndex(path, 0) } sort.Sort(sortByFileIndex(matches)) last := matches[len(matches)-1] fileIndex := 0 if idx := getFileIndex(last); idx != -1 { fileIndex = idx if nextChunk { fileIndex++ } } return setFileIndex(last, fileIndex) } } return path } func (o *FileOutput) updateName() { name := filepath.Clean(o.filename()) o.Lock() o.currentName = name o.Unlock() } // PluginWrite writes message to this plugin func (o *FileOutput) PluginWrite(msg *Message) (n int, err error) { if o.requestPerFile { o.Lock() meta := payloadMeta(msg.Meta) o.currentID = meta[1] o.payloadType = meta[0] o.Unlock() } o.updateName() o.Lock() defer o.Unlock() if o.file == nil || o.currentName != o.file.Name() { o.closeLocked() o.file, err = os.OpenFile(o.currentName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660) o.file.Sync() if strings.HasSuffix(o.currentName, ".gz") { o.writer = gzip.NewWriter(o.file) } else { o.writer = bufio.NewWriter(o.file) } if err != nil { log.Fatal(o, "Cannot open file %q. Error: %s", o.currentName, err) } o.QueueLength = 0 } var nn int n, err = o.writer.Write(msg.Meta) nn, err = o.writer.Write(msg.Data) n += nn nn, err = o.writer.Write(payloadSeparatorAsBytes) n += nn o.totalFileSize += size.Size(n) o.currentFileSize += n o.QueueLength++ if Settings.OutputFileConfig.OutputFileMaxSize > 0 && o.totalFileSize >= Settings.OutputFileConfig.OutputFileMaxSize { return n, errors.New("File output reached size limit") } return n, err } func (o *FileOutput) flush() { // Don't exit on panic defer func() { if r := recover(); r != nil { Debug(0, "[OUTPUT-FILE] PANIC while file flush: ", r, o, string(debug.Stack())) } }() o.Lock() defer o.Unlock() if o.file != nil { if strings.HasSuffix(o.currentName, ".gz") { o.writer.(*gzip.Writer).Flush() } else { o.writer.(*bufio.Writer).Flush() } if stat, err := o.file.Stat(); err == nil { o.currentFileSize = int(stat.Size()) } else { Debug(0, "[OUTPUT-HTTP] error accessing file size", err) } } } func (o *FileOutput) String() string { return "File output: " + o.file.Name() } func (o *FileOutput) closeLocked() error { if o.file != nil { if strings.HasSuffix(o.currentName, ".gz") { o.writer.(*gzip.Writer).Close() } else { o.writer.(*bufio.Writer).Flush() } o.file.Close() if o.config.onClose != nil { o.config.onClose(o.file.Name()) } } o.closed = true o.currentFileSize = 0 return nil } // Close closes the output file that is being written to. func (o *FileOutput) Close() error { o.Lock() defer o.Unlock() return o.closeLocked() } // IsClosed returns if the output file is closed or not. func (o *FileOutput) IsClosed() bool { o.Lock() defer o.Unlock() return o.closed } ================================================ FILE: output_file_test.go ================================================ package goreplay import ( "fmt" "github.com/buger/goreplay/internal/size" "math/rand" "os" "reflect" "sort" "sync" "sync/atomic" "testing" "time" ) func TestFileOutput(t *testing.T) { wg := new(sync.WaitGroup) input := NewTestInput() output := NewFileOutput("/tmp/test_requests.gor", &FileOutputConfig{FlushInterval: time.Minute, Append: true}) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output}, } plugins.All = append(plugins.All, input, output) emitter := NewEmitter() go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 100; i++ { wg.Add(2) input.EmitGET() input.EmitPOST() } time.Sleep(100 * time.Millisecond) output.flush() emitter.Close() var counter int64 input2 := NewFileInput("/tmp/test_requests.gor", false, 100, 0, false) output2 := NewTestOutput(func(*Message) { atomic.AddInt64(&counter, 1) wg.Done() }) plugins2 := &InOutPlugins{ Inputs: []PluginReader{input2}, Outputs: []PluginWriter{output2}, } plugins2.All = append(plugins2.All, input2, output2) emitter2 := NewEmitter() go emitter2.Start(plugins2, Settings.Middleware) wg.Wait() emitter2.Close() } func TestFileOutputWithNameCleaning(t *testing.T) { output := &FileOutput{pathTemplate: "./test_requests.gor", config: &FileOutputConfig{FlushInterval: time.Minute, Append: false}} expectedFileName := "test_requests_0.gor" output.updateName() if expectedFileName != output.currentName { t.Errorf("Expected path %s but got %s", expectedFileName, output.currentName) } } func TestFileOutputPathTemplate(t *testing.T) { output := &FileOutput{pathTemplate: "/tmp/log-%Y-%m-%d-%S-%t", config: &FileOutputConfig{FlushInterval: time.Minute, Append: true}} now := time.Now() output.payloadType = []byte("3") expectedPath := fmt.Sprintf("/tmp/log-%s-%s-%s-%s-3", now.Format("2006"), now.Format("01"), now.Format("02"), now.Format("05")) path := output.filename() if expectedPath != path { t.Errorf("Expected path %s but got %s", expectedPath, path) } } func TestFileOutputMultipleFiles(t *testing.T) { output := NewFileOutput("/tmp/log-%Y-%m-%d-%S", &FileOutputConfig{Append: true, FlushInterval: time.Minute}) if output.file != nil { t.Error("Should not initialize file if no writes") } output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")}) name1 := output.file.Name() output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")}) name2 := output.file.Name() time.Sleep(time.Second) output.updateName() output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")}) name3 := output.file.Name() if name2 != name1 { t.Error("Fast changes should happen in same file:", name1, name2, name3) } if name3 == name1 { t.Error("File name should change:", name1, name2, name3) } os.Remove(name1) os.Remove(name3) } func TestFileOutputFilePerRequest(t *testing.T) { output := NewFileOutput("/tmp/log-%Y-%m-%d-%S-%r", &FileOutputConfig{Append: true}) if output.file != nil { t.Error("Should not initialize file if no writes") } output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")}) name1 := output.file.Name() output.PluginWrite(&Message{Meta: []byte("1 2 1\r\n"), Data: []byte("test")}) name2 := output.file.Name() time.Sleep(time.Second) output.updateName() output.PluginWrite(&Message{Meta: []byte("1 3 1\r\n"), Data: []byte("test")}) name3 := output.file.Name() if name3 == name2 || name2 == name1 || name3 == name1 { t.Error("File name should change:", name1, name2, name3) } os.Remove(name1) os.Remove(name2) os.Remove(name3) } func TestFileOutputCompression(t *testing.T) { output := NewFileOutput("/tmp/log-%Y-%m-%d-%S.gz", &FileOutputConfig{Append: true, FlushInterval: time.Minute}) if output.file != nil { t.Error("Should not initialize file if no writes") } for i := 0; i < 1000; i++ { output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")}) } name := output.file.Name() output.Close() s, _ := os.Stat(name) if s.Size() == 12*1000 { t.Error("Should be compressed file:", s.Size()) } os.Remove(name) } func TestGetFileIndex(t *testing.T) { var tests = []struct { path string index int }{ {"/tmp/logs", -1}, {"/tmp/logs_1", 1}, {"/tmp/logs_2.gz", 2}, {"/tmp/logs_0.gz", 0}, } for _, c := range tests { if getFileIndex(c.path) != c.index { t.Error(c.path, "should be", c.index, "instead", getFileIndex(c.path)) } } } func TestSetFileIndex(t *testing.T) { var tests = []struct { path string index int newPath string }{ {"/tmp/logs", 0, "/tmp/logs_0"}, {"/tmp/logs.gz", 1, "/tmp/logs_1.gz"}, {"/tmp/logs_1", 0, "/tmp/logs_0"}, {"/tmp/logs_0", 10, "/tmp/logs_10"}, {"/tmp/logs_0.gz", 10, "/tmp/logs_10.gz"}, {"/tmp/logs_underscores.gz", 10, "/tmp/logs_underscores_10.gz"}, } for _, c := range tests { if setFileIndex(c.path, c.index) != c.newPath { t.Error(c.path, "should be", c.newPath, "instead", setFileIndex(c.path, c.index)) } } } func TestFileOutputAppendQueueLimitOverflow(t *testing.T) { rnd := rand.Int63() name := fmt.Sprintf("/tmp/%d", rnd) output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 2}) output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")}) name1 := output.file.Name() output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")}) name2 := output.file.Name() output.updateName() output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")}) name3 := output.file.Name() if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) { t.Error("Fast changes should happen in same file:", name1, name2, name3) } if name3 == name1 || name3 != fmt.Sprintf("/tmp/%d_1", rnd) { t.Error("File name should change:", name1, name2, name3) } os.Remove(name1) os.Remove(name3) } func TestFileOutputAppendQueueLimitNoOverflow(t *testing.T) { rnd := rand.Int63() name := fmt.Sprintf("/tmp/%d", rnd) output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 3}) output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")}) name1 := output.file.Name() output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")}) name2 := output.file.Name() output.updateName() output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")}) name3 := output.file.Name() if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) { t.Error("Fast changes should happen in same file:", name1, name2, name3) } if name3 != name1 || name3 != fmt.Sprintf("/tmp/%d_0", rnd) { t.Error("File name should not change:", name1, name2, name3) } os.Remove(name1) os.Remove(name3) } func TestFileOutputAppendQueueLimitGzips(t *testing.T) { rnd := rand.Int63() name := fmt.Sprintf("/tmp/%d.gz", rnd) output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, QueueLimit: 2}) output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")}) name1 := output.file.Name() output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")}) name2 := output.file.Name() output.updateName() output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")}) name3 := output.file.Name() if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0.gz", rnd) { t.Error("Fast changes should happen in same file:", name1, name2, name3) } if name3 == name1 || name3 != fmt.Sprintf("/tmp/%d_1.gz", rnd) { t.Error("File name should change:", name1, name2, name3) } os.Remove(name1) os.Remove(name3) } func TestFileOutputSort(t *testing.T) { var files = []string{"2016_0", "2014_10", "2015_0", "2015_10", "2015_2"} var expected = []string{"2014_10", "2015_0", "2015_2", "2015_10", "2016_0"} sort.Sort(sortByFileIndex(files)) if !reflect.DeepEqual(files, expected) { t.Error("Should properly sort file names using indexes", files, expected) } } func TestFileOutputAppendSizeLimitOverflow(t *testing.T) { rnd := rand.Int63() name := fmt.Sprintf("/tmp/%d", rnd) message := []byte("1 1 1\r\ntest") messageSize := len(message) + len(payloadSeparator) output := NewFileOutput(name, &FileOutputConfig{Append: false, FlushInterval: time.Minute, SizeLimit: size.Size(2 * messageSize)}) output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")}) name1 := output.file.Name() output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")}) name2 := output.file.Name() output.flush() output.PluginWrite(&Message{Meta: []byte("1 1 1\r\n"), Data: []byte("test")}) name3 := output.file.Name() if name2 != name1 || name1 != fmt.Sprintf("/tmp/%d_0", rnd) { t.Error("Fast changes should happen in same file:", name1, name2, name3) } if name3 == name1 || name3 != fmt.Sprintf("/tmp/%d_1", rnd) { t.Error("File name should change:", name1, name2, name3) } os.Remove(name1) os.Remove(name3) } ================================================ FILE: output_http.go ================================================ package goreplay import ( "bufio" "bytes" "crypto/tls" "fmt" "log" "math" "net/http" "net/http/httputil" "net/url" "sync/atomic" "time" "github.com/buger/goreplay/internal/size" ) const ( initialDynamicWorkers = 10 readChunkSize = 64 * 1024 maxResponseSize = 1073741824 ) type response struct { payload []byte uuid []byte startedAt int64 roundTripTime int64 } // HTTPOutputConfig struct for holding http output configuration type HTTPOutputConfig struct { TrackResponses bool `json:"output-http-track-response"` Stats bool `json:"output-http-stats"` OriginalHost bool `json:"output-http-original-host"` RedirectLimit int `json:"output-http-redirect-limit"` WorkersMin int `json:"output-http-workers-min"` WorkersMax int `json:"output-http-workers"` StatsMs int `json:"output-http-stats-ms"` QueueLen int `json:"output-http-queue-len"` ElasticSearch string `json:"output-http-elasticsearch"` Timeout time.Duration `json:"output-http-timeout"` WorkerTimeout time.Duration `json:"output-http-worker-timeout"` BufferSize size.Size `json:"output-http-response-buffer"` SkipVerify bool `json:"output-http-skip-verify"` CompatibilityMode bool `json:"output-http-compatibility-mode"` RequestGroup string `json:"output-http-request-group"` Debug bool `json:"output-http-debug"` rawURL string url *url.URL } func (hoc *HTTPOutputConfig) Copy() *HTTPOutputConfig { return &HTTPOutputConfig{ TrackResponses: hoc.TrackResponses, Stats: hoc.Stats, OriginalHost: hoc.OriginalHost, RedirectLimit: hoc.RedirectLimit, WorkersMin: hoc.WorkersMin, WorkersMax: hoc.WorkersMax, StatsMs: hoc.StatsMs, QueueLen: hoc.QueueLen, ElasticSearch: hoc.ElasticSearch, Timeout: hoc.Timeout, WorkerTimeout: hoc.WorkerTimeout, BufferSize: hoc.BufferSize, SkipVerify: hoc.SkipVerify, CompatibilityMode: hoc.CompatibilityMode, RequestGroup: hoc.RequestGroup, Debug: hoc.Debug, } } // HTTPOutput plugin manage pool of workers which send request to replayed server // By default workers pool is dynamic and starts with 1 worker or workerMin workers // You can specify maximum number of workers using `--output-http-workers` type HTTPOutput struct { activeWorkers int64 config *HTTPOutputConfig queueStats *GorStat elasticSearch *ESPlugin client *HTTPClient stopWorker chan struct{} queue chan *Message responses chan *response stop chan bool // Channel used only to indicate goroutine should shutdown workerSessions map[string]*httpWorker } type httpWorker struct { output *HTTPOutput client *HTTPClient lastActivity time.Time queue chan *Message stop chan bool } func newHTTPWorker(output *HTTPOutput, queue chan *Message) *httpWorker { client := NewHTTPClient(output.config) w := &httpWorker{client: client, output: output} if queue == nil { w.queue = make(chan *Message, 100) } else { w.queue = queue } w.stop = make(chan bool) go func() { for { select { case msg := <-w.queue: output.sendRequest(client, msg) case <-w.stop: return } } }() return w } // NewHTTPOutput constructor for HTTPOutput // Initialize workers func NewHTTPOutput(address string, config *HTTPOutputConfig) PluginReadWriter { o := new(HTTPOutput) var err error newConfig := config.Copy() newConfig.url, err = url.Parse(address) if err != nil { log.Fatal(fmt.Sprintf("[OUTPUT-HTTP] parse HTTP output URL error[%q]", err)) } if newConfig.url.Scheme == "" { newConfig.url.Scheme = "http" } newConfig.rawURL = newConfig.url.String() if newConfig.Timeout < time.Millisecond*100 { newConfig.Timeout = time.Second } if newConfig.BufferSize <= 0 { newConfig.BufferSize = 100 * 1024 // 100kb } if newConfig.WorkersMin <= 0 { newConfig.WorkersMin = 1 } if newConfig.WorkersMin > 1000 { newConfig.WorkersMin = 1000 } if newConfig.WorkersMax <= 0 { newConfig.WorkersMax = math.MaxInt32 // ideally so large } if newConfig.WorkersMax < newConfig.WorkersMin { newConfig.WorkersMax = newConfig.WorkersMin } if newConfig.QueueLen <= 0 { newConfig.QueueLen = 1000 } if newConfig.RedirectLimit < 0 { newConfig.RedirectLimit = 0 } if newConfig.WorkerTimeout <= 0 { newConfig.WorkerTimeout = time.Second * 2 } o.config = newConfig o.stop = make(chan bool) if o.config.Stats { o.queueStats = NewGorStat("output_http", o.config.StatsMs) } o.queue = make(chan *Message, o.config.QueueLen) if o.config.TrackResponses { o.responses = make(chan *response, o.config.QueueLen) } // it should not be buffered to avoid races o.stopWorker = make(chan struct{}) if o.config.ElasticSearch != "" { o.elasticSearch = new(ESPlugin) o.elasticSearch.Init(o.config.ElasticSearch) } o.client = NewHTTPClient(o.config) if Settings.RecognizeTCPSessions { o.workerSessions = make(map[string]*httpWorker, 100) go o.sessionWorkerMaster() } else { o.activeWorkers += int64(o.config.WorkersMin) for i := 0; i < o.config.WorkersMin; i++ { go o.startWorker() } go o.workerMaster() } return o } func (o *HTTPOutput) workerMaster() { var timer = time.NewTimer(o.config.WorkerTimeout) defer func() { // recover from panics caused by trying to send in // a closed chan(o.stopWorker) recover() }() defer timer.Stop() for { select { case <-o.stop: return default: <-timer.C } // rollback workers rollback: if atomic.LoadInt64(&o.activeWorkers) > int64(o.config.WorkersMin) && len(o.queue) < 1 { // close one worker o.stopWorker <- struct{}{} atomic.AddInt64(&o.activeWorkers, -1) goto rollback } timer.Reset(o.config.WorkerTimeout) } } func (o *HTTPOutput) sessionWorkerMaster() { gc := time.Tick(time.Second) for { select { case msg := <-o.queue: id := payloadID(msg.Meta) sessionID := string(id[0:20]) worker, ok := o.workerSessions[sessionID] if !ok { atomic.AddInt64(&o.activeWorkers, 1) worker = newHTTPWorker(o, nil) o.workerSessions[sessionID] = worker } worker.queue <- msg worker.lastActivity = time.Now() case <-gc: now := time.Now() for id, w := range o.workerSessions { if !w.lastActivity.IsZero() && now.Sub(w.lastActivity) >= 120*time.Second { w.stop <- true delete(o.workerSessions, id) atomic.AddInt64(&o.activeWorkers, -1) } } } } } func (o *HTTPOutput) startWorker() { for { select { case <-o.stopWorker: return case msg := <-o.queue: o.sendRequest(o.client, msg) } } } // PluginWrite writes message to this plugin func (o *HTTPOutput) PluginWrite(msg *Message) (n int, err error) { if !isRequestPayload(msg.Meta) { return len(msg.Data), nil } select { case <-o.stop: return 0, ErrorStopped case o.queue <- msg: } if o.config.Stats { o.queueStats.Write(len(o.queue)) } if !Settings.RecognizeTCPSessions && o.config.WorkersMax != o.config.WorkersMin { workersCount := int(atomic.LoadInt64(&o.activeWorkers)) if len(o.queue) > workersCount { extraWorkersReq := len(o.queue) - workersCount + 1 maxWorkersAvailable := o.config.WorkersMax - workersCount if extraWorkersReq > maxWorkersAvailable { extraWorkersReq = maxWorkersAvailable } if extraWorkersReq > 0 { for i := 0; i < extraWorkersReq; i++ { go o.startWorker() atomic.AddInt64(&o.activeWorkers, 1) } } } } return len(msg.Data) + len(msg.Meta), nil } // PluginRead reads message from this plugin func (o *HTTPOutput) PluginRead() (*Message, error) { if !o.config.TrackResponses { return nil, ErrorStopped } var resp *response var msg Message select { case <-o.stop: return nil, ErrorStopped case resp = <-o.responses: msg.Data = resp.payload } msg.Meta = payloadHeader(ReplayedResponsePayload, resp.uuid, resp.startedAt, resp.roundTripTime) return &msg, nil } func (o *HTTPOutput) sendRequest(client *HTTPClient, msg *Message) { if !isRequestPayload(msg.Meta) { return } uuid := payloadID(msg.Meta) start := time.Now() resp, err := client.Send(msg.Data) stop := time.Now() if err != nil { Debug(1, fmt.Sprintf("[HTTP-OUTPUT] error when sending: %q", err)) return } if resp == nil { return } if o.config.TrackResponses { o.responses <- &response{resp, uuid, start.UnixNano(), stop.UnixNano() - start.UnixNano()} } if o.elasticSearch != nil { o.elasticSearch.ResponseAnalyze(msg.Data, resp, start, stop) } } func (o *HTTPOutput) String() string { return "HTTP output: " + o.config.rawURL } // Close closes the data channel so that data func (o *HTTPOutput) Close() error { close(o.stop) close(o.stopWorker) return nil } // HTTPClient holds configurations for a single HTTP client type HTTPClient struct { config *HTTPOutputConfig Client *http.Client } // NewHTTPClient returns new http client with check redirects policy func NewHTTPClient(config *HTTPOutputConfig) *HTTPClient { client := new(HTTPClient) client.config = config var transport *http.Transport client.Client = &http.Client{ Timeout: client.config.Timeout, CheckRedirect: func(req *http.Request, via []*http.Request) error { if len(via) >= client.config.RedirectLimit { Debug(1, fmt.Sprintf("[HTTPCLIENT] maximum output-http-redirects[%d] reached!", client.config.RedirectLimit)) return http.ErrUseLastResponse } lastReq := via[len(via)-1] resp := req.Response Debug(2, fmt.Sprintf("[HTTPCLIENT] HTTP redirects from %q to %q with %q", lastReq.Host, req.Host, resp.Status)) return nil }, } if config.SkipVerify { // clone to avoid modifying global default RoundTripper transport = http.DefaultTransport.(*http.Transport).Clone() transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} client.Client.Transport = transport } return client } // Send sends an http request using client created by NewHTTPClient func (c *HTTPClient) Send(data []byte) ([]byte, error) { var req *http.Request var resp *http.Response var err error req, err = http.ReadRequest(bufio.NewReader(bytes.NewReader(data))) if err != nil { return nil, err } // we don't send CONNECT or OPTIONS request if req.Method == http.MethodConnect { return nil, nil } if !c.config.OriginalHost { req.Host = c.config.url.Host } // fix #862 if c.config.url.Path == "" && c.config.url.RawQuery == "" { req.URL.Scheme = c.config.url.Scheme req.URL.Host = c.config.url.Host } else { req.URL = c.config.url } // force connection to not be closed, which can affect the global client req.Close = false // it's an error if this is not equal to empty string req.RequestURI = "" resp, err = c.Client.Do(req) if err != nil { return nil, err } if c.config.TrackResponses { return httputil.DumpResponse(resp, true) } _ = resp.Body.Close() return nil, nil } ================================================ FILE: output_http_test.go ================================================ package goreplay import ( "io/ioutil" "net/http" "net/http/httptest" _ "net/http/httputil" "sync" "testing" ) func TestHTTPOutput(t *testing.T) { wg := new(sync.WaitGroup) input := NewTestInput() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { if req.Header.Get("User-Agent") != "Gor" { t.Error("Wrong header") } if req.Method == "OPTIONS" { t.Error("Wrong method") } if req.Method == "POST" { defer req.Body.Close() body, _ := ioutil.ReadAll(req.Body) if string(body) != "a=1&b=2" { t.Error("Wrong POST body:", string(body)) } } wg.Done() })) defer server.Close() headers := HTTPHeaders{httpHeader{"User-Agent", "Gor"}} methods := HTTPMethods{[]byte("GET"), []byte("PUT"), []byte("POST")} Settings.ModifierConfig = HTTPModifierConfig{Headers: headers, Methods: methods} httpOutput := NewHTTPOutput(server.URL, &HTTPOutputConfig{TrackResponses: false}) output := NewTestOutput(func(*Message) { wg.Done() }) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{httpOutput, output}, } plugins.All = append(plugins.All, input, output, httpOutput) emitter := NewEmitter() go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 10; i++ { // 2 http-output, 2 - test output request wg.Add(4) // OPTIONS should be ignored input.EmitPOST() input.EmitOPTIONS() input.EmitGET() } wg.Wait() emitter.Close() Settings.ModifierConfig = HTTPModifierConfig{} } func TestHTTPOutputKeepOriginalHost(t *testing.T) { wg := new(sync.WaitGroup) input := NewTestInput() server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { if req.Host != "custom-host.com" { t.Error("Wrong header", req.Host) } wg.Done() })) defer server.Close() headers := HTTPHeaders{httpHeader{"Host", "custom-host.com"}} Settings.ModifierConfig = HTTPModifierConfig{Headers: headers} output := NewHTTPOutput(server.URL, &HTTPOutputConfig{OriginalHost: true, SkipVerify: true}) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output}, } plugins.All = append(plugins.All, input, output) emitter := NewEmitter() go emitter.Start(plugins, Settings.Middleware) wg.Add(1) input.EmitGET() wg.Wait() emitter.Close() Settings.ModifierConfig = HTTPModifierConfig{} } func TestHTTPOutputSSL(t *testing.T) { wg := new(sync.WaitGroup) // Origing and Replay server initialization server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { wg.Done() })) input := NewTestInput() output := NewHTTPOutput(server.URL, &HTTPOutputConfig{SkipVerify: true}) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output}, } plugins.All = append(plugins.All, input, output) emitter := NewEmitter() go emitter.Start(plugins, Settings.Middleware) wg.Add(2) input.EmitPOST() input.EmitGET() wg.Wait() emitter.Close() } func TestHTTPOutputSessions(t *testing.T) { wg := new(sync.WaitGroup) input := NewTestInput() input.skipHeader = true server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { wg.Done() })) defer server.Close() Settings.RecognizeTCPSessions = true Settings.SplitOutput = true output := NewHTTPOutput(server.URL, &HTTPOutputConfig{}) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output}, } plugins.All = append(plugins.All, input, output) emitter := NewEmitter() go emitter.Start(plugins, Settings.Middleware) uuid1 := []byte("1234567890123456789a0000") uuid2 := []byte("1234567890123456789d0000") for i := 0; i < 10; i++ { wg.Add(1) // OPTIONS should be ignored copy(uuid1[20:], randByte(4)) input.EmitBytes([]byte("1 " + string(uuid1) + " 1\n" + "GET / HTTP/1.1\r\n\r\n")) } for i := 0; i < 10; i++ { wg.Add(1) // OPTIONS should be ignored copy(uuid2[20:], randByte(4)) input.EmitBytes([]byte("1 " + string(uuid2) + " 1\n" + "GET / HTTP/1.1\r\n\r\n")) } wg.Wait() emitter.Close() Settings.RecognizeTCPSessions = false Settings.SplitOutput = false } func BenchmarkHTTPOutput(b *testing.B) { wg := new(sync.WaitGroup) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { wg.Done() })) defer server.Close() input := NewTestInput() output := NewHTTPOutput(server.URL, &HTTPOutputConfig{WorkersMax: 1}) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output}, } plugins.All = append(plugins.All, input, output) emitter := NewEmitter() go emitter.Start(plugins, Settings.Middleware) for i := 0; i < b.N; i++ { wg.Add(1) input.EmitPOST() } wg.Wait() emitter.Close() } func BenchmarkHTTPOutputTLS(b *testing.B) { wg := new(sync.WaitGroup) server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { wg.Done() })) defer server.Close() input := NewTestInput() output := NewHTTPOutput(server.URL, &HTTPOutputConfig{SkipVerify: true, WorkersMax: 1}) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output}, } plugins.All = append(plugins.All, input, output) emitter := NewEmitter() go emitter.Start(plugins, Settings.Middleware) for i := 0; i < b.N; i++ { wg.Add(1) input.EmitPOST() } wg.Wait() emitter.Close() } ================================================ FILE: output_kafka.go ================================================ package goreplay import ( "encoding/json" "github.com/buger/goreplay/internal/byteutils" "github.com/buger/goreplay/proto" "log" "strings" "time" "github.com/Shopify/sarama" "github.com/Shopify/sarama/mocks" ) // KafkaOutput is used for sending payloads to kafka in JSON format. type KafkaOutput struct { config *OutputKafkaConfig producer sarama.AsyncProducer } // KafkaOutputFrequency in milliseconds const KafkaOutputFrequency = 500 // NewKafkaOutput creates instance of kafka producer client with TLS config func NewKafkaOutput(_ string, config *OutputKafkaConfig, tlsConfig *KafkaTLSConfig) PluginWriter { c := NewKafkaConfig(&config.SASLConfig, tlsConfig) var producer sarama.AsyncProducer if mock, ok := config.producer.(*mocks.AsyncProducer); ok && mock != nil { producer = config.producer } else { c.Producer.RequiredAcks = sarama.WaitForLocal c.Producer.Compression = sarama.CompressionSnappy c.Producer.Flush.Frequency = KafkaOutputFrequency * time.Millisecond brokerList := strings.Split(config.Host, ",") var err error producer, err = sarama.NewAsyncProducer(brokerList, c) if err != nil { log.Fatalln("Failed to start Sarama(Kafka) producer:", err) } } o := &KafkaOutput{ config: config, producer: producer, } // Start infinite loop for tracking errors for kafka producer. go o.ErrorHandler() return o } // ErrorHandler should receive errors func (o *KafkaOutput) ErrorHandler() { for err := range o.producer.Errors() { Debug(1, "Failed to write access log entry:", err) } } // PluginWrite writes a message to this plugin func (o *KafkaOutput) PluginWrite(msg *Message) (n int, err error) { var message sarama.StringEncoder if !o.config.UseJSON { message = sarama.StringEncoder(byteutils.SliceToString(msg.Meta) + byteutils.SliceToString(msg.Data)) } else { mimeHeader := proto.ParseHeaders(msg.Data) header := make(map[string]string) for k, v := range mimeHeader { header[k] = strings.Join(v, ", ") } meta := payloadMeta(msg.Meta) req := msg.Data kafkaMessage := KafkaMessage{ ReqURL: byteutils.SliceToString(proto.Path(req)), ReqType: byteutils.SliceToString(meta[0]), ReqID: byteutils.SliceToString(meta[1]), ReqTs: byteutils.SliceToString(meta[2]), ReqMethod: byteutils.SliceToString(proto.Method(req)), ReqBody: byteutils.SliceToString(proto.Body(req)), ReqHeaders: header, } jsonMessage, _ := json.Marshal(&kafkaMessage) message = sarama.StringEncoder(byteutils.SliceToString(jsonMessage)) } o.producer.Input() <- &sarama.ProducerMessage{ Topic: o.config.Topic, Value: message, } return len(message), nil } ================================================ FILE: output_kafka_test.go ================================================ package goreplay import ( "testing" "github.com/Shopify/sarama" "github.com/Shopify/sarama/mocks" ) func TestOutputKafkaRAW(t *testing.T) { config := sarama.NewConfig() config.Producer.Return.Successes = true producer := mocks.NewAsyncProducer(t, config) producer.ExpectInputAndSucceed() output := NewKafkaOutput("", &OutputKafkaConfig{ producer: producer, Topic: "test", UseJSON: false, }, nil) output.PluginWrite(&Message{Meta: []byte("1 2 3\n"), Data: []byte("GET / HTTP1.1\r\nHeader: 1\r\n\r\n")}) resp := <-producer.Successes() data, _ := resp.Value.Encode() if string(data) != "1 2 3\nGET / HTTP1.1\r\nHeader: 1\r\n\r\n" { t.Errorf("Message not properly encoded: %q", data) } } func TestOutputKafkaJSON(t *testing.T) { config := sarama.NewConfig() config.Producer.Return.Successes = true producer := mocks.NewAsyncProducer(t, config) producer.ExpectInputAndSucceed() output := NewKafkaOutput("", &OutputKafkaConfig{ producer: producer, Topic: "test", UseJSON: true, }, nil) output.PluginWrite(&Message{Meta: []byte("1 2 3\n"), Data: []byte("GET / HTTP1.1\r\nHeader: 1\r\n\r\n")}) resp := <-producer.Successes() data, _ := resp.Value.Encode() if string(data) != `{"Req_URL":"","Req_Type":"1","Req_ID":"2","Req_Ts":"3","Req_Method":"GET"}` { t.Error("Message not properly encoded: ", string(data)) } } ================================================ FILE: output_null.go ================================================ package goreplay // NullOutput used for debugging, prints nothing type NullOutput struct { } // NewNullOutput constructor for NullOutput func NewNullOutput() (o *NullOutput) { return new(NullOutput) } // PluginWrite writes message to this plugin func (o *NullOutput) PluginWrite(msg *Message) (int, error) { return len(msg.Data) + len(msg.Meta), nil } func (o *NullOutput) String() string { return "Null Output" } ================================================ FILE: output_s3.go ================================================ //go:build !pro package goreplay import ( "errors" "fmt" ) // S3Output output plugin type S3Output struct{} // NewS3Output constructor for FileOutput, accepts path func NewS3Output(pathTemplate string, config *FileOutputConfig) *S3Output { fmt.Println("S3 output is only available in the pro version") return &S3Output{} } func (o *S3Output) PluginWrite(msg *Message) (n int, err error) { return 0, errors.New("S3 output is only available in the pro version") } func (o *S3Output) String() string { return "S3 output (pro version only)" } func (o *S3Output) Close() error { return errors.New("S3 output is only available in the pro version") } ================================================ FILE: output_s3_pro.go ================================================ //go:build pro package goreplay import ( _ "bufio" "fmt" _ "io" "log" "math/rand" "os" "path/filepath" "strings" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" _ "github.com/aws/aws-sdk-go/service/s3/s3manager" ) var _ PluginWriter = (*S3Output)(nil) // S3Output output plugin type S3Output struct { pathTemplate string buffer *FileOutput session *session.Session config *FileOutputConfig closeC chan struct{} } // NewS3Output constructor for FileOutput, accepts path func NewS3Output(pathTemplate string, config *FileOutputConfig) *S3Output { o := new(S3Output) o.pathTemplate = pathTemplate o.config = config o.config.onClose = o.onBufferUpdate if config.BufferPath == "" { config.BufferPath = "/tmp" } rnd := rand.Int63() buffer_name := fmt.Sprintf("gor_output_s3_%d_buf_", rnd) pathParts := strings.Split(pathTemplate, "/") buffer_name += pathParts[len(pathParts)-1] if strings.HasSuffix(o.pathTemplate, ".gz") { buffer_name += ".gz" } buffer_path := filepath.Join(config.BufferPath, buffer_name) o.buffer = NewFileOutput(buffer_path, config) o.connect() return o } func (o *S3Output) connect() { if o.session == nil { o.session = session.Must(session.NewSession(awsConfig())) log.Println("[S3 Output] S3 connection succesfully initialized") } } func (o *S3Output) PluginWrite(msg *Message) (n int, err error) { return o.buffer.PluginWrite(msg) } func (o *S3Output) String() string { return "S3 output: " + o.pathTemplate } func (o *S3Output) Close() error { return o.buffer.Close() } func (o *S3Output) keyPath(idx int) (bucket, key string) { bucket, key = parseS3Url(o.pathTemplate) for name, fn := range dateFileNameFuncs { key = strings.Replace(key, name, fn(o.buffer), -1) } key = setFileIndex(key, idx) return } func (o *S3Output) onBufferUpdate(path string) { svc := s3.New(o.session) idx := getFileIndex(path) bucket, key := o.keyPath(idx) file, _ := os.Open(path) // reader := bufio.NewReader(file) _, err := svc.PutObject(&s3.PutObjectInput{ Body: file, Bucket: aws.String(bucket), Key: aws.String(key), }) if err != nil { log.Printf("[S3 Output] Failed to upload data to %s/%s, %s\n", bucket, key, err) os.Remove(path) return } os.Remove(path) if o.closeC != nil { o.closeC <- struct{}{} } } ================================================ FILE: output_tcp.go ================================================ package goreplay import ( "context" "crypto/tls" "fmt" "hash/fnv" "net" "time" ) // TCPOutput used for sending raw tcp payloads // Currently used for internal communication between listener and replay server // Can be used for transferring binary payloads like protocol buffers type TCPOutput struct { address string limit int buf []chan *Message bufStats *GorStat config *TCPOutputConfig workerIndex uint32 close bool } // TCPOutputConfig tcp output configuration type TCPOutputConfig struct { Secure bool `json:"output-tcp-secure"` Sticky bool `json:"output-tcp-sticky"` SkipVerify bool `json:"output-tcp-skip-verify"` Workers int `json:"output-tcp-workers"` GetInitMessage func() *Message `json:"-"` WriteBeforeMessage func(conn net.Conn, msg *Message) error `json:"-"` } // NewTCPOutput constructor for TCPOutput // Initialize X workers which hold keep-alive connection func NewTCPOutput(address string, config *TCPOutputConfig) PluginWriter { o := new(TCPOutput) o.address = address o.config = config if Settings.OutputTCPStats { o.bufStats = NewGorStat("output_tcp", 5000) } // create X buffers and send the buffer index to the worker o.buf = make([]chan *Message, o.config.Workers) for i := 0; i < o.config.Workers; i++ { o.buf[i] = make(chan *Message, 100) go o.worker(i) } return o } func (o *TCPOutput) worker(bufferIndex int) { retries := 0 conn, err := o.connect(o.address) for { if o.close { return } if err == nil { break } Debug(1, fmt.Sprintf("Can't connect to aggregator instance, reconnecting in 1 second. Retries:%d", retries)) time.Sleep(1 * time.Second) conn, err = o.connect(o.address) retries++ } if retries > 0 { Debug(2, fmt.Sprintf("Connected to aggregator instance after %d retries", retries)) } defer conn.Close() if o.config.GetInitMessage != nil { msg := o.config.GetInitMessage() _ = o.writeToConnection(conn, msg) } for { msg := <-o.buf[bufferIndex] err = o.writeToConnection(conn, msg) if err != nil { Debug(2, "INFO: TCP output connection closed, reconnecting") go o.worker(bufferIndex) o.buf[bufferIndex] <- msg break } } } func (o *TCPOutput) writeToConnection(conn net.Conn, msg *Message) (err error) { if o.config.WriteBeforeMessage != nil { err = o.config.WriteBeforeMessage(conn, msg) } if err == nil { if _, err = conn.Write(msg.Meta); err == nil { if _, err = conn.Write(msg.Data); err == nil { _, err = conn.Write(payloadSeparatorAsBytes) } } } return err } func (o *TCPOutput) getBufferIndex(msg *Message) int { if !o.config.Sticky { o.workerIndex++ return int(o.workerIndex) % o.config.Workers } hasher := fnv.New32a() hasher.Write(payloadID(msg.Meta)) return int(hasher.Sum32()) % o.config.Workers } // PluginWrite writes message to this plugin func (o *TCPOutput) PluginWrite(msg *Message) (n int, err error) { if !isOriginPayload(msg.Meta) { return len(msg.Data), nil } bufferIndex := o.getBufferIndex(msg) o.buf[bufferIndex] <- msg if Settings.OutputTCPStats { o.bufStats.Write(len(o.buf[bufferIndex])) } return len(msg.Data) + len(msg.Meta), nil } func (o *TCPOutput) connect(address string) (conn net.Conn, err error) { if o.config.Secure { var d tls.Dialer d.Config = &tls.Config{InsecureSkipVerify: o.config.SkipVerify} ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() conn, err = d.DialContext(ctx, "tcp", address) } else { var d net.Dialer ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() conn, err = d.DialContext(ctx, "tcp", address) } return } func (o *TCPOutput) String() string { return fmt.Sprintf("TCP output %s, limit: %d", o.address, o.limit) } func (o *TCPOutput) Close() { o.close = true } ================================================ FILE: output_tcp_test.go ================================================ package goreplay import ( "bufio" "log" "net" "strings" "sync" "testing" "time" "github.com/stretchr/testify/assert" ) func TestTCPOutput(t *testing.T) { wg := new(sync.WaitGroup) listener := startTCP(func(data []byte) { wg.Done() }) output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{Workers: 10}) runTCPOutput(wg, output, 10, false) } func startTCP(cb func([]byte)) net.Listener { listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { log.Fatal("Can't start:", err) } go func() { for { conn, _ := listener.Accept() go func(conn net.Conn) { defer conn.Close() reader := bufio.NewReader(conn) scanner := bufio.NewScanner(reader) scanner.Split(payloadScanner) for scanner.Scan() { cb(scanner.Bytes()) } }(conn) } }() return listener } func BenchmarkTCPOutput(b *testing.B) { wg := new(sync.WaitGroup) listener := startTCP(func(data []byte) { wg.Done() }) input := NewTestInput() input.data = make(chan []byte, b.N) for i := 0; i < b.N; i++ { input.EmitGET() } wg.Add(b.N) output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{Workers: 10}) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output}, } emitter := NewEmitter() // avoid counting above initialization b.ResetTimer() go emitter.Start(plugins, Settings.Middleware) wg.Wait() emitter.Close() } func TestStickyDisable(t *testing.T) { tcpOutput := TCPOutput{config: &TCPOutputConfig{Sticky: false, Workers: 10}} for i := 0; i < 10; i++ { index := tcpOutput.getBufferIndex(getTestBytes()) if index != (i+1)%10 { t.Errorf("Sticky is disable. Got: %d want %d", index, (i+1)%10) } } } func TestBufferDistribution(t *testing.T) { numberOfWorkers := 10 numberOfMessages := 10000 percentDistributionErrorRange := 20 buffer := make([]int, numberOfWorkers) tcpOutput := TCPOutput{config: &TCPOutputConfig{Sticky: true, Workers: 10}} for i := 0; i < numberOfMessages; i++ { buffer[tcpOutput.getBufferIndex(getTestBytes())]++ } expectedDistribution := numberOfMessages / numberOfWorkers lowerDistribution := expectedDistribution - (expectedDistribution * percentDistributionErrorRange / 100) upperDistribution := expectedDistribution + (expectedDistribution * percentDistributionErrorRange / 100) for i := 0; i < numberOfWorkers; i++ { if buffer[i] < lowerDistribution { t.Errorf("Under expected distribution. Got %d expected lower distribution %d", buffer[i], lowerDistribution) } if buffer[i] > upperDistribution { t.Errorf("Under expected distribution. Got %d expected upper distribution %d", buffer[i], upperDistribution) } } } func getTestBytes() *Message { return &Message{ Meta: payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1), Data: []byte("GET / HTTP/1.1\r\nHost: www.w3.org\r\nUser-Agent: Go 1.1 package http\r\nAccept-Encoding: gzip\r\n\r\n"), } } func TestTCPOutputGetInitMessage(t *testing.T) { wg := new(sync.WaitGroup) var dataList [][]byte listener := startTCP(func(data []byte) { dataList = append(dataList, data) wg.Done() }) getInitMessage := func() *Message { return &Message{ Meta: []byte{}, Data: []byte("test1"), } } output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{Workers: 1, GetInitMessage: getInitMessage}) runTCPOutput(wg, output, 1, true) if assert.Equal(t, 2, len(dataList)) { assert.Equal(t, "test1", string(dataList[0])) } } func TestTCPOutputGetInitMessageAndWriteBeforeMessage(t *testing.T) { wg := new(sync.WaitGroup) var dataList [][]byte listener := startTCP(func(data []byte) { dataList = append(dataList, data) wg.Done() }) getInitMessage := func() *Message { return &Message{ Meta: []byte{}, Data: []byte("test2"), } } writeBeforeMessage := func(conn net.Conn, _ *Message) error { _, err := conn.Write([]byte("before")) return err } output := NewTCPOutput(listener.Addr().String(), &TCPOutputConfig{Workers: 1, GetInitMessage: getInitMessage, WriteBeforeMessage: writeBeforeMessage}) runTCPOutput(wg, output, 1, true) if assert.Equal(t, 2, len(dataList)) { assert.Equal(t, "beforetest2", string(dataList[0])) assert.True(t, strings.HasPrefix(string(dataList[1]), "before")) } } func runTCPOutput(wg *sync.WaitGroup, output PluginWriter, repeat int, initMessage bool) { input := NewTestInput() plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output}, } emitter := NewEmitter() go emitter.Start(plugins, Settings.Middleware) if initMessage { wg.Add(1) } for i := 0; i < repeat; i++ { wg.Add(1) input.EmitGET() } wg.Wait() emitter.Close() } ================================================ FILE: output_ws.go ================================================ package goreplay import ( "context" "crypto/tls" "encoding/base64" "fmt" "hash/fnv" "log" "net/http" "net/url" "strings" "time" "github.com/gorilla/websocket" ) // WebSocketOutput used for sending raw tcp payloads // Can be used for transferring binary payloads like protocol buffers type WebSocketOutput struct { address string limit int buf []chan *Message bufStats *GorStat config *WebSocketOutputConfig workerIndex uint32 headers http.Header close bool } // WebSocketOutputConfig WebSocket output configuration type WebSocketOutputConfig struct { Sticky bool `json:"output-ws-sticky"` SkipVerify bool `json:"output-ws-skip-verify"` Workers int `json:"output-ws-workers"` Headers map[string][]string `json:"output-ws-headers"` } // NewWebSocketOutput constructor for WebSocketOutput // Initialize X workers which hold keep-alive connection func NewWebSocketOutput(address string, config *WebSocketOutputConfig) PluginWriter { o := new(WebSocketOutput) u, err := url.Parse(address) if err != nil { log.Fatal(fmt.Sprintf("[OUTPUT-WS] parse WS output URL error[%q]", err)) } o.config = config o.headers = http.Header{ "Authorization": []string{"Basic " + base64.StdEncoding.EncodeToString([]byte(u.User.String()))}, } for k, values := range config.Headers { for _, v := range values { o.headers.Add(k, v) } } u.User = nil // must be after creating the headers o.address = u.String() if Settings.OutputWebSocketStats { o.bufStats = NewGorStat("output_ws", 5000) } // create X buffers and send the buffer index to the worker o.buf = make([]chan *Message, o.config.Workers) for i := 0; i < o.config.Workers; i++ { o.buf[i] = make(chan *Message, 100) go o.worker(i) } return o } func (o *WebSocketOutput) worker(bufferIndex int) { retries := 0 conn, err := o.connect(o.address) for { if o.close { return } if err == nil { break } Debug(1, fmt.Sprintf("Can't connect to aggregator instance, reconnecting in 1 second. Retries:%d", retries)) time.Sleep(1 * time.Second) conn, err = o.connect(o.address) retries++ } if retries > 0 { Debug(2, fmt.Sprintf("Connected to aggregator instance after %d retries", retries)) } defer conn.Close() for { msg := <-o.buf[bufferIndex] err = conn.WriteMessage(websocket.BinaryMessage, append(msg.Meta, msg.Data...)) if err != nil { Debug(2, "INFO: WebSocket output connection closed, reconnecting "+err.Error()) go o.worker(bufferIndex) o.buf[bufferIndex] <- msg break } } } func (o *WebSocketOutput) getBufferIndex(msg *Message) int { if !o.config.Sticky { o.workerIndex++ return int(o.workerIndex) % o.config.Workers } hasher := fnv.New32a() hasher.Write(payloadID(msg.Meta)) return int(hasher.Sum32()) % o.config.Workers } // PluginWrite writes message to this plugin func (o *WebSocketOutput) PluginWrite(msg *Message) (n int, err error) { if !isOriginPayload(msg.Meta) { return len(msg.Data), nil } bufferIndex := o.getBufferIndex(msg) o.buf[bufferIndex] <- msg if Settings.OutputTCPStats { o.bufStats.Write(len(o.buf[bufferIndex])) } return len(msg.Data) + len(msg.Meta), nil } func (o *WebSocketOutput) connect(address string) (conn *websocket.Conn, err error) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() d := websocket.DefaultDialer if strings.HasPrefix(address, "wss://") { d.TLSClientConfig = &tls.Config{InsecureSkipVerify: o.config.SkipVerify} } conn, _, err = d.DialContext(ctx, address, o.headers) return } func (o *WebSocketOutput) String() string { return fmt.Sprintf("WebSocket output %s, limit: %d", o.address, o.limit) } // Close closes the output func (o *WebSocketOutput) Close() { o.close = true } ================================================ FILE: output_ws_test.go ================================================ package goreplay import ( "log" "net/http" "sync" "testing" "github.com/gorilla/websocket" "github.com/stretchr/testify/assert" ) func TestWebSocketOutput(t *testing.T) { wg := new(sync.WaitGroup) var gotHeader http.Header wsAddr := startWebsocket(func(data []byte) { wg.Done() }, func(header http.Header) { gotHeader = header }) input := NewTestInput() headers := map[string][]string{ "key1": {"value1"}, "key2": {"value2"}, } output := NewWebSocketOutput(wsAddr, &WebSocketOutputConfig{Workers: 1, Headers: headers}) plugins := &InOutPlugins{ Inputs: []PluginReader{input}, Outputs: []PluginWriter{output}, } emitter := NewEmitter() go emitter.Start(plugins, Settings.Middleware) for i := 0; i < 10; i++ { wg.Add(1) input.EmitGET() } wg.Wait() emitter.Close() if assert.NotNil(t, gotHeader) { assert.Equal(t, "Basic dXNlcjE=", gotHeader.Get("Authorization")) for k, values := range headers { assert.Equal(t, 1, len(values)) assert.Equal(t, values[0], gotHeader.Get(k)) } } } func startWebsocket(cb func([]byte), headercb func(http.Header)) string { upgrader := websocket.Upgrader{} http.HandleFunc("/test", func(w http.ResponseWriter, r *http.Request) { headercb(r.Header) c, err := upgrader.Upgrade(w, r, nil) if err != nil { log.Print("upgrade:", err) return } go func(conn *websocket.Conn) { defer conn.Close() for { _, msg, _ := conn.ReadMessage() cb(msg) } }(c) }) go func() { err := http.ListenAndServe("localhost:8081", nil) if err != nil { log.Fatal("Can't start:", err) } }() return "ws://user1@localhost:8081/test" } ================================================ FILE: plugins.go ================================================ package goreplay import ( "reflect" "strings" ) // Message represents data across plugins type Message struct { Meta []byte // metadata Data []byte // actual data } // PluginReader is an interface for input plugins type PluginReader interface { PluginRead() (msg *Message, err error) } // PluginWriter is an interface for output plugins type PluginWriter interface { PluginWrite(msg *Message) (n int, err error) } // PluginReadWriter is an interface for plugins that support reading and writing type PluginReadWriter interface { PluginReader PluginWriter } // InOutPlugins struct for holding references to plugins type InOutPlugins struct { Inputs []PluginReader Outputs []PluginWriter All []interface{} } // extractLimitOptions detects if plugin get called with limiter support // Returns address and limit func extractLimitOptions(options string) (string, string) { split := strings.Split(options, "|") if len(split) > 1 { return split[0], split[1] } return split[0], "" } // Automatically detects type of plugin and initialize it // // See this article if curious about reflect stuff below: http://blog.burntsushi.net/type-parametric-functions-golang func (plugins *InOutPlugins) registerPlugin(constructor interface{}, options ...interface{}) { var path, limit string vc := reflect.ValueOf(constructor) // Pre-processing options to make it work with reflect vo := []reflect.Value{} for _, oi := range options { vo = append(vo, reflect.ValueOf(oi)) } if len(vo) > 0 { // Removing limit options from path path, limit = extractLimitOptions(vo[0].String()) // Writing value back without limiter "|" options vo[0] = reflect.ValueOf(path) } // Calling our constructor with list of given options plugin := vc.Call(vo)[0].Interface() if limit != "" { plugin = NewLimiter(plugin, limit) } // Some of the output can be Readers as well because return responses if r, ok := plugin.(PluginReader); ok { plugins.Inputs = append(plugins.Inputs, r) } if w, ok := plugin.(PluginWriter); ok { plugins.Outputs = append(plugins.Outputs, w) } plugins.All = append(plugins.All, plugin) } // NewPlugins specify and initialize all available plugins func NewPlugins() *InOutPlugins { plugins := new(InOutPlugins) for _, options := range Settings.InputDummy { plugins.registerPlugin(NewDummyInput, options) } for range Settings.OutputDummy { plugins.registerPlugin(NewDummyOutput) } if Settings.OutputStdout { plugins.registerPlugin(NewDummyOutput) } if Settings.OutputNull { plugins.registerPlugin(NewNullOutput) } for _, options := range Settings.InputRAW { plugins.registerPlugin(NewRAWInput, options, Settings.InputRAWConfig) } for _, options := range Settings.InputTCP { plugins.registerPlugin(NewTCPInput, options, &Settings.InputTCPConfig) } for _, options := range Settings.OutputTCP { plugins.registerPlugin(NewTCPOutput, options, &Settings.OutputTCPConfig) } for _, options := range Settings.OutputWebSocket { plugins.registerPlugin(NewWebSocketOutput, options, &Settings.OutputWebSocketConfig) } for _, options := range Settings.InputFile { plugins.registerPlugin(NewFileInput, options, Settings.InputFileLoop, Settings.InputFileReadDepth, Settings.InputFileMaxWait, Settings.InputFileDryRun) } for _, path := range Settings.OutputFile { if strings.HasPrefix(path, "s3://") { plugins.registerPlugin(NewS3Output, path, &Settings.OutputFileConfig) } else { plugins.registerPlugin(NewFileOutput, path, &Settings.OutputFileConfig) } } for _, options := range Settings.InputHTTP { plugins.registerPlugin(NewHTTPInput, options) } // If we explicitly set Host header http output should not rewrite it // Fix: https://github.com/buger/gor/issues/174 for _, header := range Settings.ModifierConfig.Headers { if header.Name == "Host" { Settings.OutputHTTPConfig.OriginalHost = true break } } for _, options := range Settings.OutputHTTP { plugins.registerPlugin(NewHTTPOutput, options, &Settings.OutputHTTPConfig) } for _, options := range Settings.OutputBinary { plugins.registerPlugin(NewBinaryOutput, options, &Settings.OutputBinaryConfig) } if Settings.OutputKafkaConfig.Host != "" && Settings.OutputKafkaConfig.Topic != "" { plugins.registerPlugin(NewKafkaOutput, "", &Settings.OutputKafkaConfig, &Settings.KafkaTLSConfig) } if Settings.InputKafkaConfig.Host != "" && Settings.InputKafkaConfig.Topic != "" { plugins.registerPlugin(NewKafkaInput, Settings.InputKafkaConfig.Offset, &Settings.InputKafkaConfig, &Settings.KafkaTLSConfig) } return plugins } ================================================ FILE: plugins_test.go ================================================ package goreplay import ( "testing" ) func TestPluginsRegistration(t *testing.T) { Settings.InputDummy = []string{"[]"} Settings.OutputDummy = []string{"[]"} Settings.OutputHTTP = []string{"www.example.com|10"} Settings.InputFile = []string{"/dev/null"} plugins := NewPlugins() if len(plugins.Inputs) != 3 { t.Errorf("Should be 3 inputs got %d", len(plugins.Inputs)) } if _, ok := plugins.Inputs[0].(*DummyInput); !ok { t.Errorf("First input should be DummyInput") } if _, ok := plugins.Inputs[1].(*FileInput); !ok { t.Errorf("Second input should be FileInput") } if len(plugins.Outputs) != 2 { t.Errorf("Should be 2 output %d", len(plugins.Outputs)) } if _, ok := plugins.Outputs[0].(*DummyOutput); !ok { t.Errorf("First output should be DummyOutput") } if l, ok := plugins.Outputs[1].(*Limiter); ok { if _, ok := l.plugin.(*HTTPOutput); !ok { t.Errorf("HTTPOutput should be wrapped in limiter") } } else { t.Errorf("Second output should be Limiter") } } ================================================ FILE: pro.go ================================================ //go:build pro package goreplay // PRO this value indicates if goreplay is running in PRO mode.. // it must not be modified explicitly in production var PRO = true // SettingsHook is intentionally left as a no-op var SettingsHook = func(*AppSettings) {} ================================================ FILE: proto/fuzz.go ================================================ //go:build gofuzz package proto func Fuzz(data []byte) int { ParseHeaders(data) return 1 } ================================================ FILE: proto/proto.go ================================================ /* Package proto provides byte-level interaction with HTTP request payload. Example of HTTP payload for future references, new line symbols escaped: POST /upload HTTP/1.1\r\n User-Agent: Gor\r\n Content-Length: 11\r\n \r\n Hello world GET /index.html HTTP/1.1\r\n User-Agent: Gor\r\n \r\n \r\n */ package proto import ( "bufio" "bytes" "github.com/buger/goreplay/internal/byteutils" _ "fmt" "net/http" "net/textproto" "strings" ) // CRLF In HTTP newline defined by 2 bytes (for both windows and *nix support) var CRLF = []byte("\r\n") // EmptyLine acts as separator: end of Headers or Body (in some cases) var EmptyLine = []byte("\r\n\r\n") // HeaderDelim Separator for Header line. Header looks like: `HeaderName: value` var HeaderDelim = []byte(": ") // MIMEHeadersEndPos finds end of the Headers section, which should end with empty line. func MIMEHeadersEndPos(payload []byte) int { pos := bytes.Index(payload, EmptyLine) if pos < 0 { return -1 } return pos + 4 } // MIMEHeadersStartPos finds start of Headers section // It just finds position of second line (first contains location and method). func MIMEHeadersStartPos(payload []byte) int { pos := bytes.Index(payload, CRLF) if pos < 0 { return -1 } return pos + 2 // Find first line end } // header return value and positions of header/value start/end. // If not found, value will be blank, and headerStart will be -1 // Do not support multi-line headers. func header(payload []byte, name []byte) (value []byte, headerStart, headerEnd, valueStart, valueEnd int) { if HasTitle(payload) { headerStart = MIMEHeadersStartPos(payload) if headerStart < 0 { return } } else { headerStart = 0 } var colonIndex int for headerStart < len(payload) { headerEnd = bytes.IndexByte(payload[headerStart:], '\n') if headerEnd == -1 { break } headerEnd += headerStart colonIndex = bytes.IndexByte(payload[headerStart:headerEnd], ':') if colonIndex == -1 { // Malformed header, skip, most likely packet with partial headers headerStart = headerEnd + 1 continue } colonIndex += headerStart if bytes.EqualFold(payload[headerStart:colonIndex], name) { valueStart = colonIndex + 1 valueEnd = headerEnd - 2 break } headerStart = headerEnd + 1 // move to the next header } if valueStart == 0 { headerStart = -1 headerEnd = -1 valueEnd = -1 valueStart = -1 return } // ignore empty space after ':' for valueStart < valueEnd { if payload[valueStart] < 0x21 { valueStart++ } else { break } } // ignore empty space at end of header value for valueEnd > valueStart { if payload[valueEnd] < 0x21 { valueEnd-- } else { break } } value = payload[valueStart : valueEnd+1] return } // ParseHeaders Parsing headers from the payload func ParseHeaders(p []byte) textproto.MIMEHeader { // trimming off the title of the request if HasTitle(p) { headerStart := MIMEHeadersStartPos(p) if headerStart > len(p)-1 { return nil } p = p[headerStart:] } headerEnd := MIMEHeadersEndPos(p) if headerEnd > 1 { p = p[:headerEnd] } return GetHeaders(p) } // GetHeaders returns mime headers from the payload func GetHeaders(p []byte) textproto.MIMEHeader { reader := textproto.NewReader(bufio.NewReader(bytes.NewReader(p))) mime, err := reader.ReadMIMEHeader() if err != nil { return nil } return mime } // Header returns header value, if header not found, value will be blank func Header(payload, name []byte) []byte { val, _, _, _, _ := header(payload, name) return val } // SetHeader sets header value. If header not found it creates new one. // Returns modified request payload func SetHeader(payload, name, value []byte) []byte { _, hs, _, vs, ve := header(payload, name) if hs != -1 { // If header found we just replace its value return byteutils.Replace(payload, vs, ve+1, value) } return AddHeader(payload, name, value) } // AddHeader takes http payload and appends new header to the start of headers section // Returns modified request payload func AddHeader(payload, name, value []byte) []byte { mimeStart := MIMEHeadersStartPos(payload) if mimeStart < 1 { return payload } header := make([]byte, len(name)+2+len(value)+2) copy(header[0:], name) copy(header[len(name):], HeaderDelim) copy(header[len(name)+2:], value) copy(header[len(header)-2:], CRLF) return byteutils.Insert(payload, mimeStart, header) } // DeleteHeader takes http payload and removes header name from headers section // Returns modified request payload func DeleteHeader(payload, name []byte) []byte { _, hs, he, _, _ := header(payload, name) if hs != -1 { return byteutils.Cut(payload, hs, he+1) } return payload } // Body returns request/response body func Body(payload []byte) []byte { pos := MIMEHeadersEndPos(payload) if pos == -1 || len(payload) <= pos { return nil } return payload[pos:] } // Path takes payload and returns request path: Split(firstLine, ' ')[1] func Path(payload []byte) []byte { if !HasRequestTitle(payload) { return nil } start := bytes.IndexByte(payload, ' ') + 1 end := bytes.IndexByte(payload[start:], ' ') return payload[start : start+end] } // SetPath takes payload, sets new path and returns modified payload func SetPath(payload, path []byte) []byte { if !HasTitle(payload) { return nil } start := bytes.IndexByte(payload, ' ') + 1 end := bytes.IndexByte(payload[start:], ' ') return byteutils.Replace(payload, start, start+end, path) } // PathParam returns URL query attribute by given name, if no found: valueStart will be -1 func PathParam(payload, name []byte) (value []byte, valueStart, valueEnd int) { path := Path(payload) paramStart := -1 if paramStart = bytes.Index(path, append([]byte{'&'}, append(name, '=')...)); paramStart == -1 { if paramStart = bytes.Index(path, append([]byte{'?'}, append(name, '=')...)); paramStart == -1 { return []byte(""), -1, -1 } } valueStart = paramStart + len(name) + 2 paramEnd := bytes.IndexByte(path[valueStart:], '&') // Param can end with '&' (another param), or end of line if paramEnd == -1 { // It is final param paramEnd = len(path) } else { paramEnd += valueStart } return path[valueStart:paramEnd], valueStart, paramEnd } // SetPathParam takes payload and updates path Query attribute // If query param not found, it will append new // Returns modified payload func SetPathParam(payload, name, value []byte) []byte { path := Path(payload) _, vs, ve := PathParam(payload, name) if vs != -1 { // If param found, replace its value and set new Path newPath := make([]byte, len(path)) copy(newPath, path) newPath = byteutils.Replace(newPath, vs, ve, value) return SetPath(payload, newPath) } // if param not found append to end of url // Adding 2 because of '?' or '&' at start, and '=' in middle newParam := make([]byte, len(name)+len(value)+2) if bytes.IndexByte(path, '?') == -1 { newParam[0] = '?' } else { newParam[0] = '&' } // Copy "param=value" into buffer, after it looks like "?param=value" copy(newParam[1:], name) newParam[1+len(name)] = '=' copy(newParam[2+len(name):], value) // Append param to the end of path newPath := make([]byte, len(path)+len(newParam)) copy(newPath, path) copy(newPath[len(path):], newParam) return SetPath(payload, newPath) } // SetHost updates Host header for HTTP/1.1 or updates host in path for HTTP/1.0 or Proxy requests // Returns modified payload func SetHost(payload, url, host []byte) []byte { // If this is HTTP 1.0 traffic or proxy traffic it may include host right into path variable, so instead of setting Host header we rewrite Path // Fix for https://github.com/buger/gor/issues/156 if path := Path(payload); bytes.HasPrefix(path, []byte("http")) { hostStart := bytes.IndexByte(path, ':') // : position "https?:" hostStart += 3 // Skip 1 ':' and 2 '\' hostEnd := hostStart + bytes.IndexByte(path[hostStart:], '/') newPath := make([]byte, len(path)) copy(newPath, path) newPath = byteutils.Replace(newPath, 0, hostEnd, url) return SetPath(payload, newPath) } return SetHeader(payload, []byte("Host"), host) } // Method returns HTTP method func Method(payload []byte) []byte { end := bytes.IndexByte(payload, ' ') if end == -1 { return nil } return payload[:end] } // Status returns response status. // It happens to be in same position as request payload path func Status(payload []byte) []byte { if !HasResponseTitle(payload) { return nil } start := bytes.IndexByte(payload, ' ') + 1 // status code are in range 100-600 return payload[start : start+3] } // Methods holds the http methods ordered in ascending order var Methods = [...]string{ http.MethodConnect, http.MethodDelete, http.MethodGet, http.MethodHead, http.MethodOptions, http.MethodPatch, http.MethodPost, http.MethodPut, http.MethodTrace, } const ( //MinRequestCount GET / HTTP/1.1\r\n MinRequestCount = 16 // MinResponseCount HTTP/1.1 200\r\n MinResponseCount = 14 // VersionLen HTTP/1.1 VersionLen = 8 ) // HasResponseTitle reports whether this payload has an HTTP/1 response title func HasResponseTitle(payload []byte) bool { s := byteutils.SliceToString(payload) if len(s) < MinResponseCount { return false } titleLen := bytes.Index(payload, CRLF) if titleLen == -1 { return false } major, minor, ok := http.ParseHTTPVersion(s[0:VersionLen]) if !(ok && major == 1 && (minor == 0 || minor == 1)) { return false } if s[VersionLen] != ' ' { return false } status, ok := atoI(payload[VersionLen+1:VersionLen+4], 10) if !ok { return false } // only validate status codes mentioned in rfc2616. if http.StatusText(status) == "" { return false } // handle cases from #875 return payload[VersionLen+4] == ' ' || payload[VersionLen+4] == '\r' } // HasRequestTitle reports whether this payload has an HTTP/1 request title func HasRequestTitle(payload []byte) bool { s := byteutils.SliceToString(payload) if len(s) < MinRequestCount { return false } titleLen := bytes.Index(payload, CRLF) if titleLen == -1 { return false } if strings.Count(s[:titleLen], " ") != 2 { return false } method := string(Method(payload)) var methodFound bool for _, m := range Methods { if methodFound = method == m; methodFound { break } } if !methodFound { return false } path := strings.Index(s[len(method)+1:], " ") if path == -1 { return false } major, minor, ok := http.ParseHTTPVersion(s[path+len(method)+2 : titleLen]) return ok && major == 1 && (minor == 0 || minor == 1) } // HasTitle reports if this payload has an http/1 title func HasTitle(payload []byte) bool { return HasRequestTitle(payload) || HasResponseTitle(payload) } // CheckChunked checks HTTP/1 chunked data integrity(https://tools.ietf.org/html/rfc7230#section-4.1) // and returns the length of total valid scanned chunks(including chunk size, extensions and CRLFs) and // full is true if all chunks was scanned. func CheckChunked(bufs ...[]byte) (chunkEnd int, full bool) { var buf []byte if len(bufs) > 0 { buf = bufs[0] } for chunkEnd < len(buf) { sz := bytes.IndexByte(buf[chunkEnd:], '\r') if sz < 1 { break } // don't parse chunk extensions https://github.com/golang/go/issues/13135. // chunks extensions are no longer a thing, but we do check if the byte // following the parsed hex number is ';' sz += chunkEnd chkLen, ok := atoI(buf[chunkEnd:sz], 16) if !ok && bytes.IndexByte(buf[chunkEnd:sz], ';') < 1 { break } sz++ // + '\n' // total length = SIZE + CRLF + OCTETS + CRLF allChunk := sz + chkLen + 2 if allChunk >= len(buf) || buf[sz]&buf[allChunk] != '\n' || buf[allChunk-1] != '\r' { break } chunkEnd = allChunk + 1 if chkLen == 0 { full = true break } } return } // ProtocolStateSetter is an interface used to provide protocol state for future use type ProtocolStateSetter interface { SetProtocolState(interface{}) ProtocolState() interface{} } type HTTPState struct { Body int // body index HeaderStart int HeaderEnd int HeaderParsed bool // we checked necessary headers HasFullPayload bool // all chunks has been parsed IsChunked bool // Transfer-Encoding: chunked BodyLen int // Content-Length's value HasTrailer bool // Trailer header? Continue100 bool } // HasFullPayload checks if this message has full or valid payloads and returns true. // Message param is optional but recommended on cases where 'data' is storing // partial-to-full stream of bytes(packets). func HasFullPayload(m ProtocolStateSetter, payloads ...[]byte) bool { var state *HTTPState if m != nil { state, _ = m.ProtocolState().(*HTTPState) } if state == nil { state = new(HTTPState) if m != nil { m.SetProtocolState(state) } } // Http Packets can only start with a few things, check if this is one of them if len(payloads) == 0 { return false } if !HasRequestTitle(payloads[0]) && !HasResponseTitle(payloads[0]) { return false } if state.HeaderStart < 1 { for _, data := range payloads { state.HeaderStart = MIMEHeadersStartPos(data) if state.HeaderStart < 0 { return false } else { break } } } if state.Body < 1 || state.HeaderEnd < 1 { var pos int for _, data := range payloads { endPos := MIMEHeadersEndPos(data) if endPos < 0 { pos += len(data) } else { pos += endPos state.HeaderEnd = pos } if endPos > 0 { state.Body = pos break } } } if state.HeaderEnd < 1 { return false } if !state.HeaderParsed { var pos int for _, data := range payloads { chunked := Header(data, []byte("Transfer-Encoding")) if len(chunked) > 0 && bytes.Index(data, []byte("chunked")) > 0 { state.IsChunked = true // trailers are generally not allowed in non-chunks body state.HasTrailer = len(Header(data, []byte("Trailer"))) > 0 } else { contentLen := Header(data, []byte("Content-Length")) state.BodyLen, _ = atoI(contentLen, 10) } pos += len(data) if string(Header(data, []byte("Expect"))) == "100-continue" { state.Continue100 = true } if state.BodyLen > 0 || pos >= state.Body { state.HeaderParsed = true break } } } bodyLen := 0 for _, data := range payloads { bodyLen += len(data) } bodyLen -= state.Body if state.IsChunked { // check chunks if bodyLen < 1 { return false } // check trailer headers if state.HasTrailer { if bytes.HasSuffix(payloads[len(payloads)-1], []byte("\r\n\r\n")) { return true } } else { if bytes.HasSuffix(payloads[len(payloads)-1], []byte("0\r\n\r\n")) { state.HasFullPayload = true return true } } return false } // check for content-length header return state.BodyLen == bodyLen } // this works with positive integers func atoI(s []byte, base int) (num int, ok bool) { var v int ok = true for i := 0; i < len(s); i++ { if s[i] > 127 { ok = false break } v = int(hexTable[s[i]]) if v >= base || (v == 0 && s[i] != '0') { ok = false break } num = (num * base) + v } return } var hexTable = [128]byte{ '0': 0, '1': 1, '2': 2, '3': 3, '4': 4, '5': 5, '6': 6, '7': 7, '8': 8, '9': 9, 'A': 10, 'a': 10, 'B': 11, 'b': 11, 'C': 12, 'c': 12, 'D': 13, 'd': 13, 'E': 14, 'e': 14, 'F': 15, 'f': 15, } ================================================ FILE: proto/proto_test.go ================================================ package proto import ( "bytes" "net/textproto" "reflect" "testing" ) func TestHeader(t *testing.T) { var payload, val []byte var headerStart int // Value with space at start payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") if val = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) { t.Error("Should find header value") } // Value with space at end payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7 \r\nHost: www.w3.org\r\n\r\na=1&b=2") if val = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) { t.Error("Should find header value without space after 7") } // Value without space at start payload = []byte("POST /post HTTP/1.1\r\nContent-Length:7\r\nHost: www.w3.org\r\n\r\na=1&b=2") if val = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) { t.Error("Should find header value without space after :") } // Value is empty payload = []byte("GET /p HTTP/1.1\r\nCookie:\r\nHost: www.w3.org\r\n\r\n") if val = Header(payload, []byte("Cookie")); len(val) > 0 { t.Error("Should return empty value") } // Header not found if _, headerStart, _, _, _ = header(payload, []byte("Not-Found")); headerStart != -1 { t.Error("Should not found header") } // Lower case headers payload = []byte("POST /post HTTP/1.1\r\ncontent-length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") if val = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("7")) { t.Error("Should find lower case 2 word header") } payload = []byte("POST /post HTTP/1.1\r\ncontent-length: 7\r\nhost: www.w3.org\r\n\r\na=1&b=2") if val = Header(payload, []byte("host")); !bytes.Equal(val, []byte("www.w3.org")) { t.Error("Should find lower case 1 word header") } payload = []byte("GT\r\nContent-Length: 10\r\n\r\n") if val = Header(payload, []byte("Content-Length")); !bytes.Equal(val, []byte("10")) { t.Error("Should find in partial payload") } } func TestMIMEHeadersEndPos(t *testing.T) { head := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\n") payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") end := MIMEHeadersEndPos(payload) if !bytes.Equal(payload[:end], head) { t.Error("Wrong headers end position:", end, head, payload[:end]) } } func TestMIMEHeadersStartPos(t *testing.T) { headers := []byte("Content-Length: 7\r\nHost: www.w3.org") payload := []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") start := MIMEHeadersStartPos(payload) end := MIMEHeadersEndPos(payload) - 4 if !bytes.Equal(payload[start:end], headers) { t.Error("Wrong headers end position:", start, end, payload[start:end]) } } func TestSetHeader(t *testing.T) { var payload, payloadAfter []byte payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") payloadAfter = []byte("POST /post HTTP/1.1\r\nContent-Length: 14\r\nHost: www.w3.org\r\n\r\na=1&b=2") if payload = SetHeader(payload, []byte("Content-Length"), []byte("14")); !bytes.Equal(payload, payloadAfter) { t.Error("Should update header if it exists", string(payload)) } payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") payloadAfter = []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") if payload = SetHeader(payload, []byte("User-Agent"), []byte("Gor")); !bytes.Equal(payload, payloadAfter) { t.Error("Should add header if not found", string(payload)) } invalidPayload := []byte("POST /post HTTP/1.1") if invalidPayload = SetHeader(invalidPayload, []byte("User-Agent"), []byte("Gor")); !bytes.Equal(invalidPayload, []byte("POST /post HTTP/1.1")) { t.Error("Should not modify payload if request is invalid", string(payload)) } } func TestDeleteHeader(t *testing.T) { var payload, payloadAfter []byte payload = []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") payloadAfter = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") if payload = DeleteHeader(payload, []byte("User-Agent")); !bytes.Equal(payload, payloadAfter) { t.Error("Should delete header if found", string(payload), string(payloadAfter)) } //Whitespace at end of User-Agent payload = []byte("POST /post HTTP/1.1\r\nUser-Agent: Gor \r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") payloadAfter = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") if payload = DeleteHeader(payload, []byte("User-Agent")); !bytes.Equal(payload, payloadAfter) { t.Error("Should delete header if found", string(payload), string(payloadAfter)) } } func TestParseHeaders(t *testing.T) { payload := [][]byte{[]byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.or"), []byte("g\r\nUser-Ag"), []byte("ent:Chrome\r\n\r\n"), []byte("Fake-Header: asda")} headers := ParseHeaders(bytes.Join(payload, nil)) expected := textproto.MIMEHeader{ "Content-Length": []string{"7"}, "Host": []string{"www.w3.org"}, "User-Agent": []string{"Chrome"}, } if !reflect.DeepEqual(headers, expected) { t.Error("Headers do not properly parsed", headers) } // Response with Reason phrase payload = [][]byte{[]byte("HTTP/1.1 200 OK\r\nContent-Length: 7\r\nHost: www.w3.org\r\nUser-Agent:Chrome\r\n\r\nbody")} headers = ParseHeaders(bytes.Join(payload, nil)) if !reflect.DeepEqual(headers, expected) { t.Error("Headers do not properly parsed", headers) } // Response without Reason phrase payload = [][]byte{[]byte("HTTP/1.1 200\r\nContent-Length: 7\r\nHost: www.w3.org\r\nUser-Agent:Chrome\r\n\r\nbody")} headers = ParseHeaders(bytes.Join(payload, nil)) if !reflect.DeepEqual(headers, expected) { t.Error("Headers do not properly parsed", headers) } } // See https://github.com/dvyukov/go-fuzz and fuzz.go func TestFuzzCrashers(t *testing.T) { var crashers = []string{ "\n:00\n", } for _, f := range crashers { ParseHeaders([]byte(f)) } } func TestParseHeadersWithComplexUserAgent(t *testing.T) { // User-Agent could contain inside ':' // Parser should wait for \r\n payload := [][]byte{[]byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.or"), []byte("g\r\nUser-Ag"), []byte("ent:Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko\r\n\r\n"), []byte("Fake-Header: asda")} headers := ParseHeaders(bytes.Join(payload, nil)) expected := map[string]string{ "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko", } if expected["User-Agent"] != headers["User-Agent"][0] { t.Errorf("Header 'User-Agent' expected '%s' and parsed: '%s'", expected["User-Agent"], headers["User-Agent"]) } } func TestParseHeadersWithOrigin(t *testing.T) { // User-Agent could contain inside ':' // Parser should wait for \r\n payload := [][]byte{[]byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.or"), []byte("g\r\nReferrer: http://127.0.0.1:3000\r\nOrigi"), []byte("n: https://www.example.com\r\nUser-Ag"), []byte("ent:Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko\r\n\r\n"), []byte("in:https://www.example.com\r\n\r\n"), []byte("Fake-Header: asda")} headers := ParseHeaders(bytes.Join(payload, nil)) expected := map[string]string{ "Origin": "https://www.example.com", "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko", "Referrer": "http://127.0.0.1:3000", } if expected["Referrer"] != headers["Referrer"][0] { t.Errorf("Header 'Referrer' expected '%s' and parsed: '%s'", expected["Referrer"], headers["Referrer"]) } if expected["Origin"] != headers["Origin"][0] { t.Errorf("Header 'Origin' expected '%s' and parsed: '%s'", expected["Origin"], headers["Origin"]) } if expected["User-Agent"] != headers["User-Agent"][0] { t.Errorf("Header 'User-Agent' expected '%s' and parsed: '%s'", expected["User-Agent"], headers["User-Agent"]) } } func TestPath(t *testing.T) { var path, payload []byte payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") if path = Path(payload); !bytes.Equal(path, []byte("/post")) { t.Error("Should find path", string(path)) } payload = []byte("GET /get\r\n\r\nHost: www.w3.org\r\n\r\n") if path = Path(payload); !bytes.Equal(path, nil) { t.Error("1Should not find path", string(path)) } payload = []byte("GET /get\n") if path = Path(payload); !bytes.Equal(path, nil) { t.Error("2Should not find path", string(path)) } payload = []byte("GET /get") if path = Path(payload); !bytes.Equal(path, nil) { t.Error("3Should not find path", string(path)) } } func TestStatus(t *testing.T) { var status, payload []byte payload = []byte("HTTP/1.1 200 OK\r\n") if status = Status(payload); !bytes.Equal(status, []byte("200")) { t.Error("Should find status 200 but:", string(status)) } payload = []byte("HTTP/1.1 200\r\n") if status = Status(payload); !bytes.Equal(status, []byte("200")) { t.Error("1Should find status 200 but:", string(status)) } payload = []byte("HTTP/1.1 404 Not Found\r\n") if status = Status(payload); !bytes.Equal(status, []byte("404")) { t.Error("2Should find status 404 but:", string(status)) } } func TestSetPath(t *testing.T) { var payload, payloadAfter []byte payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") payloadAfter = []byte("POST /new_path HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") if payload = SetPath(payload, []byte("/new_path")); !bytes.Equal(payload, payloadAfter) { t.Error("Should replace path", string(payload)) } } func TestPathParam(t *testing.T) { var payload []byte payload = []byte("POST /post?param=test&user_id=1&d_type=1&type=2&d_type=3 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") if val, _, _ := PathParam(payload, []byte("param")); !bytes.Equal(val, []byte("test")) { t.Error("Should detect attribute", string(val)) } if val, _, _ := PathParam(payload, []byte("user_id")); !bytes.Equal(val, []byte("1")) { t.Error("Should detect attribute", string(val)) } if val, _, _ := PathParam(payload, []byte("type")); !bytes.Equal(val, []byte("2")) { t.Error("Should detect attribute", string(val)) } if val, _, _ := PathParam(payload, []byte("d_type")); !bytes.Equal(val, []byte("1")) { // this function is not designed for cases with duplicate param keys t.Error("Should detect attribute", string(val)) } } func TestSetPathParam(t *testing.T) { var payload, payloadAfter []byte payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") payloadAfter = []byte("POST /post?param=new&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") if payload = SetPathParam(payload, []byte("param"), []byte("new")); !bytes.Equal(payload, payloadAfter) { t.Error("Should replace existing value", string(payload)) } payload = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") payloadAfter = []byte("POST /post?param=test&user_id=2 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") if payload = SetPathParam(payload, []byte("user_id"), []byte("2")); !bytes.Equal(payload, payloadAfter) { t.Error("Should replace existing value", string(payload)) } payload = []byte("POST /post HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") payloadAfter = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") if payload = SetPathParam(payload, []byte("param"), []byte("test")); !bytes.Equal(payload, payloadAfter) { t.Error("Should set param if url have no params", string(payload)) } payload = []byte("POST /post?param=test HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") payloadAfter = []byte("POST /post?param=test&user_id=1 HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") if payload = SetPathParam(payload, []byte("user_id"), []byte("1")); !bytes.Equal(payload, payloadAfter) { t.Error("Should set param at the end if url params", string(payload)) } } func TestSetHostHTTP10(t *testing.T) { var payload, payloadAfter []byte payload = []byte("POST http://example.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") payloadAfter = []byte("POST http://new.com/post HTTP/1.0\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") if payload = SetHost(payload, []byte("http://new.com"), []byte("new.com")); !bytes.Equal(payload, payloadAfter) { t.Error("Should replace host", string(payload)) } payload = []byte("POST /post HTTP/1.0\r\nContent-Length: 7\r\nHost: example.com\r\n\r\na=1&b=2") payloadAfter = []byte("POST /post HTTP/1.0\r\nContent-Length: 7\r\nHost: new.com\r\n\r\na=1&b=2") if payload = SetHost(payload, nil, []byte("new.com")); !bytes.Equal(payload, payloadAfter) { t.Error("Should replace host", string(payload)) } payload = []byte("POST /post HTTP/1.0\r\nContent-Length: 7\r\n\r\na=1&b=2") if payload = SetHost(payload, nil, []byte("new.com")); !bytes.Equal(payload, payload) { t.Error("Should replace host", string(payload)) } } func TestHasResponseTitle(t *testing.T) { var m = map[string]bool{ "HTTP": false, "": false, "HTTP/1.1 100 Continue": false, "HTTP/1.1 100 Continue\r\n": true, "HTTP/1.1 \r\n": false, "HTTP/4.0 100Continue\r\n": false, "HTTP/1.0 100Continue\r\n": false, "HTTP/1.0 10r Continue\r\n": false, "HTTP/1.1 200\r\n": true, "HTTP/1.1 200\r\nServer: Tengine\r\nContent-Length: 0\r\nConnection: close\r\n\r\n": true, } for k, v := range m { if HasResponseTitle([]byte(k)) != v { t.Errorf("%q should yield %v", k, v) break } } } func TestHasRequestTitle(t *testing.T) { var m = map[string]bool{ "POST /post HTTP/1.0\r\n": true, "": false, "POST /post HTTP/1.\r\n": false, "POS /post HTTP/1.1\r\n": false, "GET / HTTP/1.1\r\n": true, "GET / HTTP/1.1\r": false, "GET / HTTP/1.400\r\n": false, } for k, v := range m { if HasRequestTitle([]byte(k)) != v { t.Errorf("%q should yield %v", k, v) break } } } func TestCheckChunks(t *testing.T) { var m = "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n" chunkEnd, _ := CheckChunked([]byte(m)) expected := len(m) if chunkEnd != expected { t.Errorf("expected %d to equal %d", chunkEnd, expected) } m = "7\r\nMozia\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\n" chunkEnd, _ = CheckChunked([]byte(m)) if chunkEnd != 0 { t.Errorf("expected %d to equal %d", chunkEnd, 0) } // with trailers m = "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n0\r\n\r\nEXpires" chunkEnd, _ = CheckChunked([]byte(m)) expected = len(m) - 7 if chunkEnd != expected { t.Errorf("expected %d to equal %d", chunkEnd, expected) } // last chunk inside the body // with trailers m = "4\r\nWiki\r\n5\r\npedia\r\nE\r\n in\r\n\r\nchunks.\r\n3\r\n0\r\n\r\n0\r\n\r\nEXpires" chunkEnd, _ = CheckChunked([]byte(m)) expected = len(m) - 7 if chunkEnd != expected { t.Errorf("expected %d to equal %d", chunkEnd, expected) } // checks with chucks-extensions m = "4\r\nWiki\r\n5\r\npedia\r\nE; name='quoted string'\r\n in\r\n\r\nchunks.\r\n3\r\n0\r\n\r\n0\r\n\r\nEXpires" chunkEnd, _ = CheckChunked([]byte(m)) expected = len(m) - 7 if chunkEnd != expected { t.Errorf("expected %d to equal %d", chunkEnd, expected) } } func TestHasFullPayload(t *testing.T) { var m string var got, expected bool got = HasFullPayload(nil, []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n"), []byte("Transfer-Encoding: chunked\r\n\r\n"), []byte("7\r\nMozilla\r\n9\r\nDeveloper\r\n"), []byte("7\r\nNetwork\r\n0\r\n\r\n")) expected = true if got != expected { t.Errorf("expected %v to equal %v", got, expected) } // check chunks with trailers m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\nTrailer: Expires\r\n\r\n7\r\nMozilla\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\nExpires: Wed, 21 Oct 2015 07:28:00 GMT\r\n\r\n" got = HasFullPayload(nil, []byte(m)) expected = true if got != expected { t.Errorf("expected %v to equal %v", got, expected) } // check with missing trailers m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\nTrailer: Expires\r\n\r\n7\r\nMozilla\r\n9\r\nDeveloper\r\n7\r\nNetwork\r\n0\r\n\r\nExpires: Wed, 21 Oct 2015 07:28:00" got = HasFullPayload(nil, []byte(m)) expected = false if got != expected { t.Errorf("expected %v to equal %v", got, expected) } // check with content-length m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\nMozillaDeveloperNetwork" got = HasFullPayload(nil, []byte(m)) expected = true if got != expected { t.Errorf("expected %v to equal %v", got, expected) } // check missing total length m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 23\r\n\r\nMozillaDeveloperNet" got = HasFullPayload(nil, []byte(m)) expected = false if got != expected { t.Errorf("expected %v to equal %v", got, expected) } // check with no body m = "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n\r\n" got = HasFullPayload(nil, []byte(m)) expected = true if got != expected { t.Errorf("expected %v to equal %v", got, expected) } // check with trailer and no header m = "Content-Type: text/plain\r\nContent-Length: 23\r\n\r\nMozillaDeveloperNetwork" got = HasFullPayload(nil, []byte(m)) expected = false if got != expected { t.Errorf("expected %v to equal %v", got, expected) } } func BenchmarkHasFullPayload(b *testing.B) { data := []byte("HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nTransfer-Encoding: chunked\r\n\r\n1e\r\n111111111111111111111111111111\r\n0\r\n\r\n") for i := 0; i < b.N; i++ { if !HasFullPayload(nil, data) { b.Fail() } } } ================================================ FILE: protocol.go ================================================ package goreplay import ( "bytes" "crypto/rand" "encoding/hex" "fmt" ) // These constants help to indicate the type of payload const ( RequestPayload = '1' ResponsePayload = '2' ReplayedResponsePayload = '3' ) func randByte(len int) []byte { b := make([]byte, len/2) rand.Read(b) h := make([]byte, len) hex.Encode(h, b) return h } func uuid() []byte { return randByte(24) } var payloadSeparator = "\n🐵🙈🙉\n" func payloadScanner(data []byte, atEOF bool) (advance int, token []byte, err error) { if atEOF && len(data) == 0 { return 0, nil, nil } if i := bytes.Index(data, []byte(payloadSeparator)); i >= 0 { // We have a full newline-terminated line. return i + len([]byte(payloadSeparator)), data[0:i], nil } if atEOF { return len(data), data, nil } return 0, nil, nil } // Timing is request start or round-trip time, depending on payloadType func payloadHeader(payloadType byte, uuid []byte, timing int64, latency int64) (header []byte) { //Example: // 3 f45590522cd1838b4a0d5c5aab80b77929dea3b3 13923489726487326 1231\n return []byte(fmt.Sprintf("%c %s %d %d\n", payloadType, uuid, timing, latency)) } func payloadBody(payload []byte) []byte { headerSize := bytes.IndexByte(payload, '\n') return payload[headerSize+1:] } func payloadMeta(payload []byte) [][]byte { headerSize := bytes.IndexByte(payload, '\n') if headerSize < 0 { return nil } return bytes.Split(payload[:headerSize], []byte{' '}) } func payloadMetaWithBody(payload []byte) (meta, body []byte) { if i := bytes.IndexByte(payload, '\n'); i > 0 && len(payload) > i+1 { meta = payload[:i+1] body = payload[i+1:] return } // we assume the message did not have meta data return nil, payload } func payloadID(payload []byte) (id []byte) { meta := payloadMeta(payload) if len(meta) < 2 { return } return meta[1] } func isOriginPayload(payload []byte) bool { return payload[0] == RequestPayload || payload[0] == ResponsePayload } func isRequestPayload(payload []byte) bool { return payload[0] == RequestPayload } ================================================ FILE: s3/index.html ================================================ Gor PRO

Gor PRO releases

See releases page on GitHub for changelog

v0.16.2

v0.16.1

v0.16.0

v0.15.1

v0.15.0

v0.14.1

================================================ FILE: s3_reader.go ================================================ package goreplay import ( "bytes" "log" "os" "strconv" "strings" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" ) // S3ReadCloser ... type S3ReadCloser struct { bucket string key string offset int totalSize int readBytes int sess *session.Session buf *bytes.Buffer } func awsConfig() *aws.Config { region := os.Getenv("AWS_DEFAULT_REGION") if region == "" { region = os.Getenv("AWS_REGION") if region == "" { region = "us-east-1" } } config := &aws.Config{Region: aws.String(region)} if endpoint := os.Getenv("AWS_ENDPOINT_URL"); endpoint != "" { config.Endpoint = aws.String(endpoint) log.Println("Custom endpoint:", endpoint) } log.Println("Connecting to S3. Region: " + region) config.CredentialsChainVerboseErrors = aws.Bool(true) if os.Getenv("AWS_DEBUG") != "" { config.LogLevel = aws.LogLevel(aws.LogDebugWithHTTPBody) } return config } // NewS3ReadCloser returns new instance of S3 read closer func NewS3ReadCloser(path string) *S3ReadCloser { if !PRO { log.Fatal("Using S3 input and output require PRO license") return nil } bucket, key := parseS3Url(path) sess := session.Must(session.NewSession(awsConfig())) log.Println("[S3 Input] S3 connection successfully initialized", path) return &S3ReadCloser{ bucket: bucket, key: key, sess: sess, buf: &bytes.Buffer{}, } } // Read reads buffer from s3 session func (s *S3ReadCloser) Read(b []byte) (n int, e error) { if s.readBytes == 0 || s.readBytes+len(b) > s.offset { svc := s3.New(s.sess) objectRange := "bytes=" + strconv.Itoa(s.offset) s.offset += 1000000 // Reading in chunks of 1 mb objectRange += "-" + strconv.Itoa(s.offset-1) params := &s3.GetObjectInput{ Bucket: aws.String(s.bucket), Key: aws.String(s.key), Range: aws.String(objectRange), } resp, err := svc.GetObject(params) if err != nil { log.Println("[S3 Input] Error during getting file", s.bucket, s.key, err) } else { s.totalSize, _ = strconv.Atoi(strings.Split(*resp.ContentRange, "/")[1]) s.buf.ReadFrom(resp.Body) } } s.readBytes += len(b) return s.buf.Read(b) } // Close is here to make S3ReadCloser satisfy ReadCloser interface func (s *S3ReadCloser) Close() error { return nil } ================================================ FILE: s3_test.go ================================================ //go:build pro package goreplay import ( "fmt" "math/rand" "os" "path/filepath" "testing" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" ) func TestS3Output(t *testing.T) { bucket := aws.String("test-gor") rnd := rand.Int63() path := fmt.Sprintf("s3://test-gor/%d/requests.gz", rnd) output := NewS3Output(path, &FileOutputConfig{queueLimit: 2}) svc := s3.New(output.session) output.Write([]byte("1 1 1\ntest")) output.Write([]byte("1 1 1\ntest")) output.buffer.updateName() output.Write([]byte("1 1 1\ntest")) output.Write([]byte("1 1 1\ntest")) output.buffer.updateName() output.Write([]byte("1 1 1\ntest")) time.Sleep(time.Second) params := &s3.ListObjectsInput{ Bucket: bucket, Prefix: aws.String(fmt.Sprintf("%d", rnd)), } resp, _ := svc.ListObjects(params) if len(resp.Contents) != 2 { t.Error("Should create 2 objects", len(resp.Contents)) } else { if *resp.Contents[0].Key != fmt.Sprintf("%d/requests_0.gz", rnd) || *resp.Contents[1].Key != fmt.Sprintf("%d/requests_1.gz", rnd) { t.Error("Should assign proper names", resp.Contents) } } for _, c := range resp.Contents { svc.DeleteObject(&s3.DeleteObjectInput{Bucket: bucket, Key: c.Key}) } matches, _ := filepath.Glob(fmt.Sprintf("/tmp/gor_output_s3_*")) for _, m := range matches { os.Remove(m) } } func TestS3OutputQueueLimit(t *testing.T) { bucket := aws.String("test-gor") rnd := rand.Int63() path := fmt.Sprintf("s3://test-gor/%d/requests.gz", rnd) output := NewS3Output(path, &FileOutputConfig{queueLimit: 100}) output.closeCh = make(chan struct{}, 3) svc := s3.New(output.session) for i := 0; i < 3; i++ { for i := 0; i < 100; i++ { output.Write([]byte("1 1 1\ntest")) } output.buffer.updateName() } output.buffer.updateName() output.Write([]byte("1 1 1\ntest")) for i := 0; i < 3; i++ { <-output.closeCh } params := &s3.ListObjectsInput{ Bucket: bucket, Prefix: aws.String(fmt.Sprintf("%d", rnd)), } resp, _ := svc.ListObjects(params) if len(resp.Contents) != 3 { t.Error("Should create 3 object", len(resp.Contents)) } else { if *resp.Contents[0].Key != fmt.Sprintf("%d/requests_0.gz", rnd) || *resp.Contents[1].Key != fmt.Sprintf("%d/requests_1.gz", rnd) { t.Error("Should assign proper names", resp.Contents) } } for _, c := range resp.Contents { svc.DeleteObject(&s3.DeleteObjectInput{Bucket: bucket, Key: c.Key}) } matches, _ := filepath.Glob(fmt.Sprintf("/tmp/gor_output_s3_*")) for _, m := range matches { os.Remove(m) } } func TestInputFileFromS3(t *testing.T) { rnd := rand.Int63() path := fmt.Sprintf("s3://test-gor-eu/%d/requests.gz", rnd) output := NewS3Output(path, &FileOutputConfig{queueLimit: 5000}) output.closeCh = make(chan struct{}, 10) for i := 0; i <= 20000; i++ { output.Write([]byte("1 1 1\ntest")) if i%5000 == 0 { output.buffer.updateName() } } output.Write([]byte("1 1 1\ntest")) for i := 0; i < 2; i++ { <-output.closeCh } input := NewFileInput(fmt.Sprintf("s3://test-gor-eu/%d", rnd), false, 100, 0, false) buf := make([]byte, 1000) for i := 0; i <= 19999; i++ { input.Read(buf) } // Cleanup artifacts bucket := aws.String("test-gor") svc := s3.New(output.session) params := &s3.ListObjectsInput{ Bucket: bucket, Prefix: aws.String(fmt.Sprintf("%d", rnd)), } resp, _ := svc.ListObjects(params) for _, c := range resp.Contents { svc.DeleteObject(&s3.DeleteObjectInput{Bucket: bucket, Key: c.Key}) } } ================================================ FILE: settings.go ================================================ package goreplay import ( "flag" "fmt" "os" "strconv" "sync" "time" "github.com/buger/goreplay/internal/size" ) // DEMO indicates that goreplay is running in demo mode var DEMO string // MultiOption allows to specify multiple flags with same name and collects all values into array type MultiOption struct { a *[]string } func (h *MultiOption) String() string { if h.a == nil { return "" } return fmt.Sprint(*h.a) } // Set gets called multiple times for each flag with same name func (h *MultiOption) Set(value string) error { if h.a == nil { return nil } *h.a = append(*h.a, value) return nil } // MultiOption allows to specify multiple flags with same name and collects all values into array type MultiIntOption struct { a *[]int } func (h *MultiIntOption) String() string { if h.a == nil { return "" } return fmt.Sprint(*h.a) } // Set gets called multiple times for each flag with same name func (h *MultiIntOption) Set(value string) error { if h.a == nil { return nil } val, _ := strconv.Atoi(value) *h.a = append(*h.a, val) return nil } // AppSettings is the struct of main configuration type AppSettings struct { Verbose int `json:"verbose"` Stats bool `json:"stats"` ExitAfter time.Duration `json:"exit-after"` SplitOutput bool `json:"split-output"` RecognizeTCPSessions bool `json:"recognize-tcp-sessions"` Pprof string `json:"http-pprof"` CopyBufferSize size.Size `json:"copy-buffer-size"` InputDummy []string `json:"input-dummy"` OutputDummy []string OutputStdout bool `json:"output-stdout"` OutputNull bool `json:"output-null"` InputTCP []string `json:"input-tcp"` InputTCPConfig TCPInputConfig OutputTCP []string `json:"output-tcp"` OutputTCPConfig TCPOutputConfig OutputTCPStats bool `json:"output-tcp-stats"` OutputWebSocket []string `json:"output-ws"` OutputWebSocketConfig WebSocketOutputConfig OutputWebSocketStats bool `json:"output-ws-stats"` InputFile []string `json:"input-file"` InputFileLoop bool `json:"input-file-loop"` InputFileReadDepth int `json:"input-file-read-depth"` InputFileDryRun bool `json:"input-file-dry-run"` InputFileMaxWait time.Duration `json:"input-file-max-wait"` OutputFile []string `json:"output-file"` OutputFileConfig FileOutputConfig InputRAW []string `json:"input_raw"` InputRAWConfig RAWInputConfig Middleware string `json:"middleware"` InputHTTP []string OutputHTTP []string `json:"output-http"` PrettifyHTTP bool `json:"prettify-http"` OutputHTTPConfig HTTPOutputConfig OutputBinary []string `json:"output-binary"` OutputBinaryConfig BinaryOutputConfig ModifierConfig HTTPModifierConfig InputKafkaConfig InputKafkaConfig OutputKafkaConfig OutputKafkaConfig KafkaTLSConfig KafkaTLSConfig } // Settings holds Gor configuration var Settings AppSettings func usage() { fmt.Printf("Gor is a simple http traffic replication tool written in Go. Its main goal is to replay traffic from production servers to staging and dev environments.\nProject page: https://github.com/buger/gor\nAuthor: leonsbox@gmail.com\nCurrent Version: v%s\n\n", VERSION) flag.PrintDefaults() os.Exit(2) } func init() { flag.Usage = usage flag.StringVar(&Settings.Pprof, "http-pprof", "", "Enable profiling. Starts http server on specified port, exposing special /debug/pprof endpoint. Example: `:8181`") flag.IntVar(&Settings.Verbose, "verbose", 0, "set the level of verbosity, if greater than zero then it will turn on debug output") flag.BoolVar(&Settings.Stats, "stats", false, "Turn on queue stats output") if DEMO == "" { flag.DurationVar(&Settings.ExitAfter, "exit-after", 0, "exit after specified duration") } else { Settings.ExitAfter = 5 * time.Minute } flag.BoolVar(&Settings.SplitOutput, "split-output", false, "By default each output gets same traffic. If set to `true` it splits traffic equally among all outputs.") flag.BoolVar(&Settings.RecognizeTCPSessions, "recognize-tcp-sessions", false, "[PRO] If turned on http output will create separate worker for each TCP session. Splitting output will session based as well.") flag.Var(&MultiOption{&Settings.InputDummy}, "input-dummy", "Used for testing outputs. Emits 'Get /' request every 1s") flag.BoolVar(&Settings.OutputStdout, "output-stdout", false, "Used for testing inputs. Just prints to console data coming from inputs.") flag.BoolVar(&Settings.OutputNull, "output-null", false, "Used for testing inputs. Drops all requests.") flag.Var(&MultiOption{&Settings.InputTCP}, "input-tcp", "Used for internal communication between Gor instances. Example: \n\t# Receive requests from other Gor instances on 28020 port, and redirect output to staging\n\tgor --input-tcp :28020 --output-http staging.com") flag.BoolVar(&Settings.InputTCPConfig.Secure, "input-tcp-secure", false, "Turn on TLS security. Do not forget to specify certificate and key files.") flag.StringVar(&Settings.InputTCPConfig.CertificatePath, "input-tcp-certificate", "", "Path to PEM encoded certificate file. Used when TLS turned on.") flag.StringVar(&Settings.InputTCPConfig.KeyPath, "input-tcp-certificate-key", "", "Path to PEM encoded certificate key file. Used when TLS turned on.") flag.Var(&MultiOption{&Settings.OutputTCP}, "output-tcp", "Used for internal communication between Gor instances. Example: \n\t# Listen for requests on 80 port and forward them to other Gor instance on 28020 port\n\tgor --input-raw :80 --output-tcp replay.local:28020") flag.BoolVar(&Settings.OutputTCPConfig.Secure, "output-tcp-secure", false, "Use TLS secure connection. --input-file on another end should have TLS turned on as well.") flag.BoolVar(&Settings.OutputTCPConfig.SkipVerify, "output-tcp-skip-verify", false, "Don't verify hostname on TLS secure connection.") flag.BoolVar(&Settings.OutputTCPConfig.Sticky, "output-tcp-sticky", false, "Use Sticky connection. Request/Response with same ID will be sent to the same connection.") flag.IntVar(&Settings.OutputTCPConfig.Workers, "output-tcp-workers", 10, "Number of parallel tcp connections, default is 10") flag.BoolVar(&Settings.OutputTCPStats, "output-tcp-stats", false, "Report TCP output queue stats to console every 5 seconds.") flag.Var(&MultiOption{&Settings.OutputWebSocket}, "output-ws", "Just like output tcp, just with WebSocket. Example: \n\t# Listen for requests on 80 port and forward them to other Gor instance on 28020 port\n\tgor --input-raw :80 --output-ws wss://replay.local:28020/endpoint") flag.BoolVar(&Settings.OutputWebSocketConfig.SkipVerify, "output-ws-skip-verify", false, "Don't verify hostname on TLS secure connection.") flag.BoolVar(&Settings.OutputWebSocketConfig.Sticky, "output-ws-sticky", false, "Use Sticky connection. Request/Response with same ID will be sent to the same connection.") flag.IntVar(&Settings.OutputWebSocketConfig.Workers, "output-ws-workers", 10, "Number of parallel ws connections, default is 10") flag.BoolVar(&Settings.OutputWebSocketStats, "output-ws-stats", false, "Report WebSocket output queue stats to console every 5 seconds.") flag.Var(&MultiOption{&Settings.InputFile}, "input-file", "Read requests from file: \n\tgor --input-file ./requests.gor --output-http staging.com") flag.BoolVar(&Settings.InputFileLoop, "input-file-loop", false, "Loop input files, useful for performance testing.") flag.IntVar(&Settings.InputFileReadDepth, "input-file-read-depth", 100, "GoReplay tries to read and cache multiple records, in advance. In parallel it also perform sorting of requests, if they came out of order. Since it needs hold this buffer in memory, bigger values can cause worse performance") flag.BoolVar(&Settings.InputFileDryRun, "input-file-dry-run", false, "Simulate reading from the data source without replaying it. You will get information about expected replay time, number of found records etc.") flag.DurationVar(&Settings.InputFileMaxWait, "input-file-max-wait", 0, "Set the maximum time between requests. Can help in situations when you have too long periods between request, and you want to skip them. Example: --input-raw-max-wait 1s") flag.Var(&MultiOption{&Settings.OutputFile}, "output-file", "Write incoming requests to file: \n\tgor --input-raw :80 --output-file ./requests.gor") flag.DurationVar(&Settings.OutputFileConfig.FlushInterval, "output-file-flush-interval", time.Second, "Interval for forcing buffer flush to the file, default: 1s.") flag.BoolVar(&Settings.OutputFileConfig.Append, "output-file-append", false, "The flushed chunk is appended to existence file or not. ") flag.Var(&Settings.OutputFileConfig.SizeLimit, "output-file-size-limit", "Size of each chunk. Default: 32mb") flag.IntVar(&Settings.OutputFileConfig.QueueLimit, "output-file-queue-limit", 256, "The length of the chunk queue. Default: 256") flag.Var(&Settings.OutputFileConfig.OutputFileMaxSize, "output-file-max-size-limit", "Max size of output file, Default: 1TB") flag.StringVar(&Settings.OutputFileConfig.BufferPath, "output-file-buffer", "/tmp", "The path for temporary storing current buffer: \n\tgor --input-raw :80 --output-file s3://mybucket/logs/%Y-%m-%d.gz --output-file-buffer /mnt/logs") flag.BoolVar(&Settings.PrettifyHTTP, "prettify-http", false, "If enabled, will automatically decode requests and responses with: Content-Encoding: gzip and Transfer-Encoding: chunked. Useful for debugging, in conjunction with --output-stdout") flag.Var(&Settings.CopyBufferSize, "copy-buffer-size", "Set the buffer size for an individual request (default 5MB)") // input raw flags flag.Var(&MultiOption{&Settings.InputRAW}, "input-raw", "Capture traffic from given port (use RAW sockets and require *sudo* access):\n\t# Capture traffic from 8080 port\n\tgor --input-raw :8080 --output-http staging.com") flag.BoolVar(&Settings.InputRAWConfig.TrackResponse, "input-raw-track-response", false, "If turned on Gor will track responses in addition to requests, and they will be available to middleware and file output.") flag.IntVar(&Settings.InputRAWConfig.VXLANPort, "input-raw-vxlan-port", 4789, "VXLAN port. Can be used only when engine set to `vxlan`. Default: 4789") flag.Var(&MultiIntOption{&Settings.InputRAWConfig.VXLANVNIs}, "input-raw-vxlan-vni", "VXLAN VNI to capture. By default capture all VNIs. Ignore VNI by setting them with minus sign, example: `--input-raw-vxlan-vni -2`") flag.BoolVar(&Settings.InputRAWConfig.VLAN, "input-raw-vlan", false, "Enable VLAN (802.1Q) support") flag.Var(&MultiIntOption{&Settings.InputRAWConfig.VLANVIDs}, "input-raw-vlan-vid", "VLAN VID to capture. By default capture all VIDs") flag.Var(&Settings.InputRAWConfig.Engine, "input-raw-engine", "Intercept traffic using `libpcap` (default), `raw_socket`, `pcap_file`, `vxlan`") flag.Var(&Settings.InputRAWConfig.Protocol, "input-raw-protocol", "Specify application protocol of intercepted traffic. Possible values: http, binary") flag.StringVar(&Settings.InputRAWConfig.RealIPHeader, "input-raw-realip-header", "", "If not blank, injects header with given name and real IP value to the request payload. Usually this header should be named: X-Real-IP") flag.DurationVar(&Settings.InputRAWConfig.Expire, "input-raw-expire", time.Second*2, "How much it should wait for the last TCP packet, till consider that TCP message complete.") flag.StringVar(&Settings.InputRAWConfig.BPFFilter, "input-raw-bpf-filter", "", "BPF filter to write custom expressions. Can be useful in case of non standard network interfaces like tunneling or SPAN port. Example: --input-raw-bpf-filter 'dst port 80'") flag.StringVar(&Settings.InputRAWConfig.TimestampType, "input-raw-timestamp-type", "", "Possible values: PCAP_TSTAMP_HOST, PCAP_TSTAMP_HOST_LOWPREC, PCAP_TSTAMP_HOST_HIPREC, PCAP_TSTAMP_ADAPTER, PCAP_TSTAMP_ADAPTER_UNSYNCED. This values not supported on all systems, GoReplay will tell you available values of you put wrong one.") flag.BoolVar(&Settings.InputRAWConfig.Snaplen, "input-raw-override-snaplen", false, "Override the capture snaplen to be 64k. Required for some Virtualized environments") flag.DurationVar(&Settings.InputRAWConfig.BufferTimeout, "input-raw-buffer-timeout", 0, "set the pcap timeout. for immediate mode don't set this flag") flag.Var(&Settings.InputRAWConfig.BufferSize, "input-raw-buffer-size", "Controls size of the OS buffer which holds packets until they dispatched. Default value depends by system: in Linux around 2MB. If you see big package drop, increase this value.") flag.BoolVar(&Settings.InputRAWConfig.Promiscuous, "input-raw-promisc", false, "enable promiscuous mode") flag.BoolVar(&Settings.InputRAWConfig.Monitor, "input-raw-monitor", false, "enable RF monitor mode") flag.BoolVar(&Settings.InputRAWConfig.Stats, "input-raw-stats", false, "enable stats generator on raw TCP messages") flag.BoolVar(&Settings.InputRAWConfig.AllowIncomplete, "input-raw-allow-incomplete", false, "If turned on Gor will record HTTP messages with missing packets") flag.Var(&MultiOption{&Settings.InputRAWConfig.IgnoreInterface}, "input-raw-ignore-interface", "In case if you want listen for all interfaces except a few ones. Can be used in k8s environment. Example: --input-raw-ignore-interface cbr0 --input-raw-ignore-interface eth0 --input-raw-ignore-interface localhost") flag.StringVar(&Settings.Middleware, "middleware", "", "Used for modifying traffic using external command") flag.Var(&MultiOption{&Settings.OutputHTTP}, "output-http", "Forwards incoming requests to given http address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --output-http http://staging.com") /* outputHTTPConfig */ flag.Var(&Settings.OutputHTTPConfig.BufferSize, "output-http-response-buffer", "HTTP response buffer size, all data after this size will be discarded.") flag.IntVar(&Settings.OutputHTTPConfig.WorkersMin, "output-http-workers-min", 0, "Gor uses dynamic worker scaling. Enter a number to set a minimum number of workers. default = 1.") flag.IntVar(&Settings.OutputHTTPConfig.WorkersMax, "output-http-workers", 0, "Gor uses dynamic worker scaling. Enter a number to set a maximum number of workers. default = 0 = unlimited.") flag.IntVar(&Settings.OutputHTTPConfig.QueueLen, "output-http-queue-len", 1000, "Number of requests that can be queued for output, if all workers are busy. default = 1000") flag.BoolVar(&Settings.OutputHTTPConfig.SkipVerify, "output-http-skip-verify", false, "Don't verify hostname on TLS secure connection.") flag.DurationVar(&Settings.OutputHTTPConfig.WorkerTimeout, "output-http-worker-timeout", 2*time.Second, "Duration to rollback idle workers.") flag.IntVar(&Settings.OutputHTTPConfig.RedirectLimit, "output-http-redirects", 0, "Enable how often redirects should be followed.") flag.DurationVar(&Settings.OutputHTTPConfig.Timeout, "output-http-timeout", 5*time.Second, "Specify HTTP request/response timeout. By default 5s. Example: --output-http-timeout 30s") flag.BoolVar(&Settings.OutputHTTPConfig.TrackResponses, "output-http-track-response", false, "If turned on, HTTP output responses will be set to all outputs like stdout, file and etc.") flag.BoolVar(&Settings.OutputHTTPConfig.Stats, "output-http-stats", false, "Report http output queue stats to console every N milliseconds. See output-http-stats-ms") flag.IntVar(&Settings.OutputHTTPConfig.StatsMs, "output-http-stats-ms", 5000, "Report http output queue stats to console every N milliseconds. default: 5000") flag.BoolVar(&Settings.OutputHTTPConfig.OriginalHost, "http-original-host", false, "Normally gor replaces the Host http header with the host supplied with --output-http. This option disables that behavior, preserving the original Host header.") flag.StringVar(&Settings.OutputHTTPConfig.ElasticSearch, "output-http-elasticsearch", "", "Send request and response stats to ElasticSearch:\n\tgor --input-raw :8080 --output-http staging.com --output-http-elasticsearch 'es_host:api_port/index_name'") /* outputHTTPConfig */ flag.Var(&MultiOption{&Settings.OutputBinary}, "output-binary", "Forwards incoming binary payloads to given address.\n\t# Redirect all incoming requests to staging.com address \n\tgor --input-raw :80 --input-raw-protocol binary --output-binary staging.com:80") /* outputBinaryConfig */ flag.Var(&Settings.OutputBinaryConfig.BufferSize, "output-tcp-response-buffer", "TCP response buffer size, all data after this size will be discarded.") flag.IntVar(&Settings.OutputBinaryConfig.Workers, "output-binary-workers", 0, "Gor uses dynamic worker scaling by default. Enter a number to run a set number of workers.") flag.DurationVar(&Settings.OutputBinaryConfig.Timeout, "output-binary-timeout", 0, "Specify HTTP request/response timeout. By default 5s. Example: --output-binary-timeout 30s") flag.BoolVar(&Settings.OutputBinaryConfig.TrackResponses, "output-binary-track-response", false, "If turned on, Binary output responses will be set to all outputs like stdout, file and etc.") flag.BoolVar(&Settings.OutputBinaryConfig.Debug, "output-binary-debug", false, "Enables binary debug output.") /* outputBinaryConfig */ flag.StringVar(&Settings.OutputKafkaConfig.Host, "output-kafka-host", "", "Read request and response stats from Kafka:\n\tgor --input-raw :8080 --output-kafka-host '192.168.0.1:9092,192.168.0.2:9092'") flag.StringVar(&Settings.OutputKafkaConfig.Topic, "output-kafka-topic", "", "Read request and response stats from Kafka:\n\tgor --input-raw :8080 --output-kafka-topic 'kafka-log'") flag.BoolVar(&Settings.OutputKafkaConfig.UseJSON, "output-kafka-json-format", false, "If turned on, it will serialize messages from GoReplay text format to JSON.") flag.BoolVar(&Settings.OutputKafkaConfig.SASLConfig.UseSASL, "output-kafka-use-sasl", false, "--output-kafka-use-sasl true") flag.StringVar(&Settings.OutputKafkaConfig.SASLConfig.Mechanism, "output-kafka-mechanism", "", "mechanism\n\tgor --input-raw :8080 --output-kafka-mechanism 'SCRAM-SHA-512'") flag.StringVar(&Settings.OutputKafkaConfig.SASLConfig.Username, "output-kafka-username", "", "username\n\tgor --input-raw :8080 --output-kafka-username 'username'") flag.StringVar(&Settings.OutputKafkaConfig.SASLConfig.Password, "output-kafka-password", "", "password\n\tgor --input-raw :8080 --output-kafka-password 'password'") flag.StringVar(&Settings.InputKafkaConfig.Host, "input-kafka-host", "", "Send request and response stats to Kafka:\n\tgor --output-stdout --input-kafka-host '192.168.0.1:9092,192.168.0.2:9092'") flag.StringVar(&Settings.InputKafkaConfig.Topic, "input-kafka-topic", "", "Send request and response stats to Kafka:\n\tgor --output-stdout --input-kafka-topic 'kafka-log'") flag.BoolVar(&Settings.InputKafkaConfig.UseJSON, "input-kafka-json-format", false, "If turned on, it will assume that messages coming in JSON format rather than GoReplay text format.") flag.BoolVar(&Settings.InputKafkaConfig.SASLConfig.UseSASL, "input-kafka-use-sasl", false, "use-sasl\n\t--use-sasl true") flag.StringVar(&Settings.InputKafkaConfig.SASLConfig.Mechanism, "input-kafka-mechanism", "", "mechanism\n\tgor --input-raw :8080 --output-kafka-mechanism 'SCRAM-SHA-512'") flag.StringVar(&Settings.InputKafkaConfig.SASLConfig.Username, "input-kafka-username", "", "username\n\tgor --input-raw :8080 --output-kafka-username 'username'") flag.StringVar(&Settings.InputKafkaConfig.SASLConfig.Password, "input-kafka-password", "", "password\n\tgor --input-raw :8080 --output-kafka-password 'password'") flag.StringVar(&Settings.InputKafkaConfig.Offset, "input-kafka-offset", "-1", "Specify offset in Kafka partitions start to consume\n\t-1: Starts from newest, -2: Starts from oldest\nAnd supported for showdown or speedup for emitting!\n\tgor --input-kafka-offset \"-2|200%\"") flag.StringVar(&Settings.KafkaTLSConfig.CACert, "kafka-tls-ca-cert", "", "CA certificate for Kafka TLS Config:\n\tgor --input-raw :3000 --output-kafka-host '192.168.0.1:9092' --output-kafka-topic 'topic' --kafka-tls-ca-cert cacert.cer.pem --kafka-tls-client-cert client.cer.pem --kafka-tls-client-key client.key.pem") flag.StringVar(&Settings.KafkaTLSConfig.ClientCert, "kafka-tls-client-cert", "", "Client certificate for Kafka TLS Config (mandatory with to kafka-tls-ca-cert and kafka-tls-client-key)") flag.StringVar(&Settings.KafkaTLSConfig.ClientKey, "kafka-tls-client-key", "", "Client Key for Kafka TLS Config (mandatory with to kafka-tls-client-cert and kafka-tls-client-key)") flag.Var(&Settings.ModifierConfig.Headers, "http-set-header", "Inject additional headers to http request:\n\tgor --input-raw :8080 --output-http staging.com --http-set-header 'User-Agent: Gor'") flag.Var(&Settings.ModifierConfig.HeaderRewrite, "http-rewrite-header", "Rewrite the request header based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-header Host: (.*).example.com,$1.beta.example.com") flag.Var(&Settings.ModifierConfig.Params, "http-set-param", "Set request url param, if param already exists it will be overwritten:\n\tgor --input-raw :8080 --output-http staging.com --http-set-param api_key=1") flag.Var(&Settings.ModifierConfig.Methods, "http-allow-method", "Whitelist of HTTP methods to replay. Anything else will be dropped:\n\tgor --input-raw :8080 --output-http staging.com --http-allow-method GET --http-allow-method OPTIONS") flag.Var(&Settings.ModifierConfig.URLRegexp, "http-allow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-url ^www.") flag.Var(&Settings.ModifierConfig.URLNegativeRegexp, "http-disallow-url", "A regexp to match requests against. Filter get matched against full url with domain. Anything else will be forwarded:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-url ^www.") flag.Var(&Settings.ModifierConfig.URLRewrite, "http-rewrite-url", "Rewrite the request url based on a mapping:\n\tgor --input-raw :8080 --output-http staging.com --http-rewrite-url /v1/user/([^\\/]+)/ping:/v2/user/$1/ping") flag.Var(&Settings.ModifierConfig.HeaderFilters, "http-allow-header", "A regexp to match a specific header against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-allow-header api-version:^v1") flag.Var(&Settings.ModifierConfig.HeaderNegativeFilters, "http-disallow-header", "A regexp to match a specific header against. Requests with matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-disallow-header \"User-Agent: Replayed by Gor\"") flag.Var(&Settings.ModifierConfig.HeaderBasicAuthFilters, "http-basic-auth-filter", "A regexp to match the decoded basic auth string against. Requests with non-matching headers will be dropped:\n\t gor --input-raw :8080 --output-http staging.com --http-basic-auth-filter \"^customer[0-9].*\"") flag.Var(&Settings.ModifierConfig.HeaderHashFilters, "http-header-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific header:\n\t gor --input-raw :8080 --output-http staging.com --http-header-limiter user-id:25%") flag.Var(&Settings.ModifierConfig.ParamHashFilters, "http-param-limiter", "Takes a fraction of requests, consistently taking or rejecting a request based on the FNV32-1A hash of a specific GET param:\n\t gor --input-raw :8080 --output-http staging.com --http-param-limiter user_id:25%") // default values, using for tests Settings.OutputFileConfig.SizeLimit = 33554432 Settings.OutputFileConfig.OutputFileMaxSize = 1099511627776 Settings.CopyBufferSize = 5242880 } func CheckSettings() { SettingsHook(&Settings) if Settings.OutputFileConfig.SizeLimit < 1 { Settings.OutputFileConfig.SizeLimit.Set("32mb") } if Settings.OutputFileConfig.OutputFileMaxSize < 1 { Settings.OutputFileConfig.OutputFileMaxSize.Set("1tb") } if Settings.CopyBufferSize < 1 { Settings.CopyBufferSize.Set("5mb") } } var previousDebugTime = time.Now() var debugMutex sync.Mutex // Debug take an effect only if --verbose greater than 0 is specified func Debug(level int, args ...interface{}) { if Settings.Verbose >= level { debugMutex.Lock() defer debugMutex.Unlock() now := time.Now() diff := now.Sub(previousDebugTime) previousDebugTime = now fmt.Fprintf(os.Stderr, "[DEBUG][elapsed %s]: ", diff) fmt.Fprintln(os.Stderr, args...) } } ================================================ FILE: settings_test.go ================================================ package goreplay import ( "encoding/json" "testing" ) func TestAppSettings(t *testing.T) { a := AppSettings{} _, err := json.Marshal(&a) if err != nil { t.Error(err) } } ================================================ FILE: sidenav.css ================================================ .wy-affix { position: fixed; top: 1.618em; } .wy-menu a:hover { text-decoration: none; } .wy-menu-vertical header, .wy-menu-vertical p.caption { height: 32px; display: inline-block; line-height: 32px; padding: 0 1.618em; margin-bottom: 0; margin-top: 14px; display: block; font-weight: 700; text-transform: uppercase; font-size: 85%; color: #ccc; white-space: nowrap; } .wy-menu-vertical span { color: #666; } .wy-menu-vertical ul { margin-bottom: 0; } .wy-menu-vertical li.divide-top { border-top: solid 1px #404040; } .wy-menu-vertical li.divide-bottom { border-bottom: solid 1px #404040; } .wy-menu-vertical li.current { background-color: #e5e5e5; } .wy-menu-vertical li.current a { color: rgba(0, 93, 255, 0.7); border-right: none; } .wy-menu-vertical li.current a:hover { color: rgba(0, 93, 255, 0.9); } .wy-menu-vertical li code, .wy-menu-vertical li .rst-content tt, .rst-content .wy-menu-vertical li tt { border: none; background: inherit; color: inherit; padding-left: 0; padding-right: 0 } .wy-menu-vertical li span.toctree-expand { display: block; float: left; margin-left: -1.2em; font-size: .8em; line-height: 1.6em; color: #999; } .wy-menu-vertical li.on a, .wy-menu-vertical li.current>a { color: rgba(0, 93, 255, 0.9); font-weight: 700; position: relative; background: #fafafa; border: none; } /*.wy-menu-vertical li.on a:hover span.toctree-expand, .wy-menu-vertical li.current>a:hover span.toctree-expand { color: gray; }*/ .wy-menu-vertical li.on a span.toctree-expand, .wy-menu-vertical li.current>a span.toctree-expand { display: block; font-size: .8em; line-height: 1.6em; color: #333; } .wy-menu-vertical li.toctree-l1.current li.toctree-l2>ul, .wy-menu-vertical li.toctree-l2.current li.toctree-l3>ul { display: none; } .wy-menu-vertical li.toctree-l1.current li.toctree-l2.current>ul, .wy-menu-vertical li.toctree-l2.current li.toctree-l3.current>ul { display: block; } .wy-menu-vertical li.toctree-l2.current li.toctree-l3>a { display: block; padding: .4045em 4.045em; } .wy-menu-vertical li.toctree-l2 a:hover span.toctree-expand { color: #999; } .wy-menu-vertical li.toctree-l2 span.toctree-expand { color: #999; } .wy-menu-vertical li.toctree-l3 { background-color: #eee; font-size: .9em; } .wy-menu-vertical li.toctree-l3.current>a { padding: .4045em 4.045em; } .wy-menu-vertical li.toctree-l3.current li.toctree-l4>a { display: block; padding: .4045em 5.663em; border-top: none; border-bottom: none; } .wy-menu-vertical li.toctree-l3 a:hover span.toctree-expand { color: rgba(0, 93, 255, 0.9); } .wy-menu-vertical li.toctree-l3 span.toctree-expand { color: #999; } .wy-menu-vertical li.toctree-l4 { font-size: .9em; } .wy-menu-vertical li.current ul { display: block; } .wy-menu-vertical .local-toc li ul { display: block; } .wy-menu-vertical li ul li a { margin-bottom: 0; color: rgba(0, 93, 255, 0.7); font-weight: 400; } .wy-menu-vertical a { display: inline-block; line-height: 18px; padding: .4045em 1.618em; display: block; position: relative; font-size: 90%; color: rgba(0, 93, 255, 0.7); } .wy-menu-vertical li.on a:hover, .wy-menu-vertical li.current>a:hover { background-color: #fafafa; } .wy-menu-vertical a:hover { color: rgba(0, 93, 255, 0.9); cursor: pointer; background-color: #fafafa; } .wy-menu-vertical a:hover span.toctree-expand { color: rgba(0, 93, 255, 0.5); } .wy-menu-vertical a:active span.toctree-expand { color: rgba(0, 93, 255, 0.7); } /* Search */ .wy-side-nav-search { z-index: 200; background-color: #fafafa; border-bottom: #333; text-align: center; padding: .809em; display: block; color: #333; margin-bottom: .809em } .wy-side-nav-search input[type=text] { width: 100%; color: #333; border-radius: 3px; outline: 0; padding: 10px; background-color: #fff; border: solid 1px #6d6d6d; box-shadow: none } .wy-side-nav-search img { display: block; margin: auto auto .809em; height: 45px; width: 45px; background-color: #2980B9; padding: 5px; border-radius: 100% } .wy-side-nav-search>a, .wy-side-nav-search .wy-dropdown>a { color: #333; font-size: 100%; font-weight: 700; display: inline-block; padding: 4px 6px; margin-bottom: .809em } .wy-side-nav-search>a:hover, .wy-side-nav-search .wy-dropdown>a:hover { background: rgba(255,255,255,0.1) } .wy-side-nav-search>a img.logo, .wy-side-nav-search .wy-dropdown>a img.logo { display: block; margin: 0 auto; height: auto; width: auto; border-radius: 0; max-width: 100%; background: transparent } .wy-side-nav-search>a.icon img.logo, .wy-side-nav-search .wy-dropdown>a.icon img.logo { margin-top: .85em } .wy-nav .wy-menu-vertical header { color: #2980B9 } .wy-nav .wy-menu-vertical a { color: #b3b3b3 } .wy-nav .wy-menu-vertical a:hover { background-color: #2980B9; color: #fff } [data-menu-wrap] { -webkit-transition: all .2s ease-in; -moz-transition: all .2s ease-in; transition: all .2s ease-in; position: absolute; opacity: 1; width: 100%; opacity: 0 } [data-menu-wrap].move-center { left: 0; right: auto; opacity: 1 } [data-menu-wrap].move-left { right: auto; left: -100%; opacity: 0 } [data-menu-wrap].move-right { right: -100%; left: auto; opacity: 0 } .wy-body-for-nav { background: left repeat-y #fcfcfc; background-image: url(data:image/png; base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyRpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMy1jMDExIDY2LjE0NTY2MSwgMjAxMi8wMi8wNi0xNDo1NjoyNyAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENTNiAoTWFjaW50b3NoKSIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDoxOERBMTRGRDBFMUUxMUUzODUwMkJCOThDMEVFNURFMCIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDoxOERBMTRGRTBFMUUxMUUzODUwMkJCOThDMEVFNURFMCI+IDx4bXBNTTpEZXJpdmVkRnJvbSBzdFJlZjppbnN0YW5jZUlEPSJ4bXAuaWlkOjE4REExNEZCMEUxRTExRTM4NTAyQkI5OEMwRUU1REUwIiBzdFJlZjpkb2N1bWVudElEPSJ4bXAuZGlkOjE4REExNEZDMEUxRTExRTM4NTAyQkI5OEMwRUU1REUwIi8+IDwvcmRmOkRlc2NyaXB0aW9uPiA8L3JkZjpSREY+IDwveDp4bXBtZXRhPiA8P3hwYWNrZXQgZW5kPSJyIj8+EwrlwAAAAA5JREFUeNpiMDU0BAgwAAE2AJgB9BnaAAAAAElFTkSuQmCC); background-size: 300px 1px } .wy-grid-for-nav { position: absolute; width: 100%; height: 100% } .wy-nav-side { position: fixed; top: 0; bottom: 0; left: 0; padding-bottom: 2em; width: 300px; overflow-x: hidden; overflow-y: scroll; min-height: 100%; background-color: #fafafa; z-index: 200; border-right: 2px solid #eee; } .wy-nav-top { display: none; background-color: #333; color: #fff; padding: .4045em .809em; position: relative; line-height: 50px; text-align: center; font-size: 100%; *zoom: 1 } .wy-nav-top:before, .wy-nav-top:after { display: table; content: "" } .wy-nav-top:after { clear: both } .wy-nav-top a { color: #fff; font-weight: 700 } .wy-nav-top img { margin-right: 12px; height: 45px; width: 45px; background-color: #2980B9; padding: 5px; border-radius: 100% } .wy-nav-top i { font-size: 30px; line-height: 50px; float: left; cursor: pointer } .wy-nav-content-wrap { margin-left: 300px; background: #fcfcfc; min-height: 100% } .wy-nav-content { padding: 1.618em 3.236em; height: 100%; max-width: 1100px; margin: auto } .wy-body-mask { position: fixed; width: 100%; height: 100%; background: rgba(0,0,0,0.2); display: none; z-index: 499 } .wy-body-mask.on { display: block } @media screen and (max-width: 768px) { .wy-body-for-nav { background: #fcfcfc } .wy-nav-top { display: block } .wy-nav-side { left: -300px } .wy-nav-side.shift { width: 85%; left: 0 } .wy-nav-content-wrap { margin-left: 0 } .wy-nav-content-wrap .wy-nav-content { padding: 1.618em } .wy-nav-content-wrap.shift { position: fixed; min-width: 100%; left: 85%; top: 0; height: 100%; overflow: hidden } } @media screen and (min-width: 1400px) { .wy-nav-content { margin: 0; background: #fcfcfc } } ================================================ FILE: site/.gitignore ================================================ _site .sass-cache .jekyll-metadata ================================================ FILE: site/Gemfile ================================================ source "https://rubygems.org" ruby RUBY_VERSION # Hello! This is where you manage which Jekyll version is used to run. # When you want to use a different version, change it below, save the # file and run `bundle install`. Run Jekyll with `bundle exec`, like so: # # bundle exec jekyll serve # # This will help ensure the proper Jekyll version is running. # Happy Jekylling! gem "jekyll", "3.3.1" # This is the default theme for new Jekyll sites. You may change this to anything you like. gem "minima", "~> 2.0" # If you want to use GitHub Pages, remove the "gem "jekyll"" above and # uncomment the line below. To upgrade, run `bundle update github-pages`. # gem "github-pages", group: :jekyll_plugins # If you have any plugins, put them here! group :jekyll_plugins do gem "jekyll-feed", "~> 0.6" end ================================================ FILE: site/_config.yml ================================================ # Welcome to Jekyll! # # This config file is meant for settings that affect your whole blog, values # which you are expected to set up once and rarely edit after that. If you find # yourself editing this file very often, consider using Jekyll's data files # feature for the data you need to update frequently. # # For technical reasons, this file is *NOT* reloaded automatically when you use # 'bundle exec jekyll serve'. If you change this file, please restart the server process. # Site settings # These are used to personalize your new site. If you look in the HTML files, # you will see them accessed via {{ site.title }}, {{ site.email }}, and so on. # You can create any custom variable you would like, and they will be accessible # in the templates via {{ site.myvariable }}. title: Your awesome title email: your-email@domain.com description: > # this means to ignore newlines until "baseurl:" Write an awesome description for your new site here. You can edit this line in _config.yml. It will appear in your document head meta (for Google search results) and in your feed.xml site description. baseurl: "" # the subpath of your site, e.g. /blog url: "" # the base hostname & protocol for your site, e.g. http://example.com twitter_username: jekyllrb github_username: jekyll # Build settings markdown: kramdown theme: minima gems: - jekyll-feed exclude: - Gemfile - Gemfile.lock ================================================ FILE: site/_posts/2017-01-06-welcome-to-jekyll.markdown ================================================ --- layout: post title: "Welcome to Jekyll!" date: 2017-01-06 11:19:34 +0300 categories: jekyll update --- You’ll find this post in your `_posts` directory. Go ahead and edit it and re-build the site to see your changes. You can rebuild the site in many different ways, but the most common way is to run `jekyll serve`, which launches a web server and auto-regenerates your site when a file is updated. To add new posts, simply add a file in the `_posts` directory that follows the convention `YYYY-MM-DD-name-of-post.ext` and includes the necessary front matter. Take a look at the source for this post to get an idea about how it works. Jekyll also offers powerful support for code snippets: {% highlight ruby %} def print_hi(name) puts "Hi, #{name}" end print_hi('Tom') #=> prints 'Hi, Tom' to STDOUT. {% endhighlight %} Check out the [Jekyll docs][jekyll-docs] for more info on how to get the most out of Jekyll. File all bugs/feature requests at [Jekyll’s GitHub repo][jekyll-gh]. If you have questions, you can ask them on [Jekyll Talk][jekyll-talk]. [jekyll-docs]: http://jekyllrb.com/docs/home [jekyll-gh]: https://github.com/jekyll/jekyll [jekyll-talk]: https://talk.jekyllrb.com/ ================================================ FILE: site/about.md ================================================ --- layout: page title: About permalink: /about/ --- This is the base Jekyll theme. You can find out more info about customizing your Jekyll theme, as well as basic Jekyll usage documentation at [jekyllrb.com](http://jekyllrb.com/) You can find the source code for the Jekyll new theme at: {% include icon-github.html username="jekyll" %} / [minima](https://github.com/jekyll/minima) You can find the source code for Jekyll at {% include icon-github.html username="jekyll" %} / [jekyll](https://github.com/jekyll/jekyll) ================================================ FILE: site/index.md ================================================ --- # You don't need to edit this file, it's empty on purpose. # Edit theme's home layout instead if you wanna make some changes # See: https://jekyllrb.com/docs/themes/#overriding-theme-defaults layout: home --- ================================================ FILE: snapcraft.yaml ================================================ name: goreplay version: '1.0' summary: GoReplay is an open-source tool for capturing and replaying live HTTP traffic description: | GoReplay is an open-source tool for capturing and replaying live HTTP traffic into a test environment in order to continuously test your system with real data. It can be used to increase confidence in code deployments, configuration changes and infrastructure changes. grade: stable confinement: strict base: core18 parts: goreplay: plugin: go source: https://github.com/buger/goreplay.git go-importpath: github.com/buger/goreplay build-packages: - build-essential - libpcap-dev stage-packages: - libpcap0.8 apps: goreplay: command: bin/goreplay daemon: simple restart-condition: on-abnormal plugs: - home - network - network-bind - network-control - network-observe - netlink-connector - netlink-audit - bluetooth-control - firewall-control - x11 ================================================ FILE: tcp_client.go ================================================ package goreplay import ( "crypto/tls" "io" "net" "runtime/debug" "syscall" "time" ) // TCPClientConfig client configuration type TCPClientConfig struct { Debug bool ConnectionTimeout time.Duration Timeout time.Duration ResponseBufferSize int Secure bool } // TCPClient client connection properties type TCPClient struct { baseURL string addr string conn net.Conn respBuf []byte config *TCPClientConfig redirectsCount int } // NewTCPClient returns new TCPClient func NewTCPClient(addr string, config *TCPClientConfig) *TCPClient { if config.Timeout.Nanoseconds() == 0 { config.Timeout = 5 * time.Second } config.ConnectionTimeout = config.Timeout if config.ResponseBufferSize == 0 { config.ResponseBufferSize = 100 * 1024 // 100kb } client := &TCPClient{config: config, addr: addr} client.respBuf = make([]byte, config.ResponseBufferSize) return client } // Connect creates a tcp connection of the client func (c *TCPClient) Connect() (err error) { c.Disconnect() c.conn, err = net.DialTimeout("tcp", c.addr, c.config.ConnectionTimeout) if c.config.Secure { tlsConn := tls.Client(c.conn, &tls.Config{InsecureSkipVerify: true}) if err = tlsConn.Handshake(); err != nil { return } c.conn = tlsConn } return } // Disconnect closes the client connection func (c *TCPClient) Disconnect() { if c.conn != nil { c.conn.Close() c.conn = nil Debug(1, "[TCPClient] Disconnected: ", c.baseURL) } } func (c *TCPClient) isAlive() bool { one := make([]byte, 1) // Ready 1 byte from socket without timeout to check if it not closed c.conn.SetReadDeadline(time.Now().Add(time.Millisecond)) _, err := c.conn.Read(one) if err == nil { return true } else if err == io.EOF { Debug(1, "[TCPClient] connection closed, reconnecting") return false } else if err == syscall.EPIPE { Debug(1, "Detected broken pipe.", err) return false } return true } // Send sends data over created tcp connection func (c *TCPClient) Send(data []byte) (response []byte, err error) { // Don't exit on panic defer func() { if r := recover(); r != nil { Debug(1, "[TCPClient]", r, string(data)) if _, ok := r.(error); !ok { Debug(1, "[TCPClient] Failed to send request: ", string(data)) Debug(1, "PANIC: pkg:", r, debug.Stack()) } } }() if c.conn == nil || !c.isAlive() { Debug(1, "[TCPClient] Connecting:", c.baseURL) if err = c.Connect(); err != nil { Debug(1, "[TCPClient] Connection error:", err) return } } timeout := time.Now().Add(c.config.Timeout) c.conn.SetWriteDeadline(timeout) if c.config.Debug { Debug(1, "[TCPClient] Sending:", string(data)) } if _, err = c.conn.Write(data); err != nil { Debug(1, "[TCPClient] Write error:", err, c.baseURL) return } var readBytes, n int var currentChunk []byte timeout = time.Now().Add(c.config.Timeout) for { c.conn.SetReadDeadline(timeout) if readBytes < len(c.respBuf) { n, err = c.conn.Read(c.respBuf[readBytes:]) readBytes += n if err != nil { if err == io.EOF { err = nil } break } } else { if currentChunk == nil { currentChunk = make([]byte, readChunkSize) } n, err = c.conn.Read(currentChunk) if err == io.EOF { break } else if err != nil { Debug(1, "[TCPClient] Read the whole body error:", err, c.baseURL) break } readBytes += int(n) } if readBytes >= maxResponseSize { Debug(1, "[TCPClient] Body is more than the max size", maxResponseSize, c.baseURL) break } // For following chunks expect less timeout timeout = time.Now().Add(c.config.Timeout / 5) } if err != nil { Debug(1, "[TCPClient] Response read error", err, c.conn, readBytes) return } if readBytes > len(c.respBuf) { readBytes = len(c.respBuf) } payload := make([]byte, readBytes) copy(payload, c.respBuf[:readBytes]) if c.config.Debug { Debug(1, "[TCPClient] Received:", string(payload)) } return payload, err } ================================================ FILE: test_input.go ================================================ package goreplay import ( "encoding/base64" "errors" "math/rand" "time" ) // ErrorStopped is the error returned when the go routines reading the input is stopped. var ErrorStopped = errors.New("reading stopped") // TestInput used for testing purpose, it allows emitting requests on demand type TestInput struct { data chan []byte skipHeader bool stop chan bool // Channel used only to indicate goroutine should shutdown } // NewTestInput constructor for TestInput func NewTestInput() (i *TestInput) { i = new(TestInput) i.data = make(chan []byte, 100) i.stop = make(chan bool) return } // PluginRead reads message from this plugin func (i *TestInput) PluginRead() (*Message, error) { var msg Message select { case buf := <-i.data: msg.Data = buf if !i.skipHeader { msg.Meta = payloadHeader(RequestPayload, uuid(), time.Now().UnixNano(), -1) } else { msg.Meta, msg.Data = payloadMetaWithBody(msg.Data) } return &msg, nil case <-i.stop: return nil, ErrorStopped } } // Close closes this plugin func (i *TestInput) Close() error { close(i.stop) return nil } // EmitBytes sends data func (i *TestInput) EmitBytes(data []byte) { i.data <- data } // EmitGET emits GET request without headers func (i *TestInput) EmitGET() { i.data <- []byte("GET / HTTP/1.1\r\n\r\n") } // EmitPOST emits POST request with Content-Length func (i *TestInput) EmitPOST() { i.data <- []byte("POST /pub/WWW/ HTTP/1.1\r\nContent-Length: 7\r\nHost: www.w3.org\r\n\r\na=1&b=2") } // EmitChunkedPOST emits POST request with `Transfer-Encoding: chunked` and chunked body func (i *TestInput) EmitChunkedPOST() { i.data <- []byte("POST /pub/WWW/ HTTP/1.1\r\nHost: www.w3.org\r\nTransfer-Encoding: chunked\r\n\r\n4\r\nWiki\r\n5\r\npedia\r\ne\r\n in\r\n\r\nchunks.\r\n0\r\n\r\n") } // EmitLargePOST emits POST request with large payload (5mb) func (i *TestInput) EmitLargePOST() { size := 5 * 1024 * 1024 // 5 MB rb := make([]byte, size) rand.Read(rb) rs := base64.URLEncoding.EncodeToString(rb) i.data <- []byte("POST / HTTP/1.1\r\nHost: www.w3.org\nContent-Length:5242880\r\n\r\n" + rs) } // EmitSizedPOST emit a POST with a payload set to a supplied size func (i *TestInput) EmitSizedPOST(payloadSize int) { rb := make([]byte, payloadSize) rand.Read(rb) rs := base64.URLEncoding.EncodeToString(rb) i.data <- []byte("POST / HTTP/1.1\r\nHost: www.w3.org\nContent-Length:5242880\r\n\r\n" + rs) } // EmitOPTIONS emits OPTIONS request, similar to GET func (i *TestInput) EmitOPTIONS() { i.data <- []byte("OPTIONS / HTTP/1.1\r\nHost: www.w3.org\r\n\r\n") } func (i *TestInput) String() string { return "Test Input" } ================================================ FILE: test_output.go ================================================ package goreplay type writeCallback func(*Message) // TestOutput used in testing to intercept any output into callback type TestOutput struct { cb writeCallback } // NewTestOutput constructor for TestOutput, accepts callback which get called on each incoming Write func NewTestOutput(cb writeCallback) PluginWriter { i := new(TestOutput) i.cb = cb return i } // PluginWrite write message to this plugin func (i *TestOutput) PluginWrite(msg *Message) (int, error) { i.cb(msg) return len(msg.Data) + len(msg.Meta), nil } func (i *TestOutput) String() string { return "Test Output" } ================================================ FILE: version.go ================================================ package goreplay // VERSION the current version of goreplay var VERSION = "2.0.0"