Repository: Shopify/toxiproxy Branch: main Commit: c4e60d7e07b2 Files: 87 Total size: 277.7 KB Directory structure: gitextract_8n0j6szh/ ├── .editorconfig ├── .github/ │ ├── dependabot.yml │ └── workflows/ │ ├── analysis.yml │ ├── cla.yml │ ├── release.yml │ └── test.yml ├── .gitignore ├── .golangci.yml ├── .goreleaser.yml ├── .yamllint.yml ├── CHANGELOG.md ├── CREATING_TOXICS.md ├── Dockerfile ├── LICENSE ├── METRICS.md ├── Makefile ├── README.md ├── RELEASE.md ├── _examples/ │ ├── tests/ │ │ ├── README.md │ │ ├── cluster.yml │ │ ├── db.go │ │ ├── db_test.go │ │ ├── main.go │ │ ├── models.go │ │ └── resources.yml │ └── toxics/ │ ├── README.md │ ├── debug_toxic.go │ └── http_toxic.go ├── api.go ├── api_test.go ├── client/ │ ├── README.md │ ├── api_error.go │ ├── client.go │ ├── client_test.go │ ├── proxy.go │ └── toxic.go ├── cmd/ │ ├── cli/ │ │ └── cli.go │ └── server/ │ └── server.go ├── collectors/ │ ├── common.go │ ├── proxy.go │ └── runtime.go ├── dev.yml ├── go.mod ├── go.sum ├── link.go ├── link_test.go ├── metrics.go ├── metrics_test.go ├── proxy.go ├── proxy_collection.go ├── proxy_collection_test.go ├── proxy_test.go ├── scripts/ │ ├── hazelcast.xml │ ├── test-e2e │ ├── test-e2e-hazelcast │ └── test-release ├── share/ │ └── toxiproxy.conf ├── stream/ │ ├── direction.go │ ├── direction_test.go │ ├── io_chan.go │ └── io_chan_test.go ├── test/ │ └── e2e/ │ ├── benchmark_test.go │ └── endpoint.go ├── testhelper/ │ ├── tcp_server.go │ ├── tcp_server_test.go │ ├── timeout_after.go │ ├── timeout_after_test.go │ └── upstream.go ├── toxic_collection.go ├── toxics/ │ ├── bandwidth.go │ ├── bandwidth_test.go │ ├── latency.go │ ├── latency_test.go │ ├── limit_data.go │ ├── limit_data_test.go │ ├── noop.go │ ├── reset_peer.go │ ├── reset_peer_test.go │ ├── slicer.go │ ├── slicer_test.go │ ├── slow_close.go │ ├── timeout.go │ ├── timeout_test.go │ ├── toxic.go │ └── toxic_test.go ├── toxiproxy_test.go └── version.go ================================================ FILE CONTENTS ================================================ ================================================ FILE: .editorconfig ================================================ # EditorConfig is awesome: https://EditorConfig.org # Unix-style newlines with a newline ending every file [*] charset = utf-8 end_of_line = lf insert_final_newline = true trim_trailing_whitespace = true max_line_length = 100 [Makefile] indent_style = tab [{scripts/*, *.sh}] max_line_length = 80 indent_style = space indent_size = 2 shell_variant = bash binary_next_line = true switch_case_indent = true space_redirects = true keep_padding = true [*.{yaml,yml}] indent_style = space indent_size = 2 [*.go] indent_style = tab ================================================ FILE: .github/dependabot.yml ================================================ version: 2 updates: - package-ecosystem: gomod directory: / schedule: interval: weekly registries: "*" - package-ecosystem: github-actions directory: / schedule: interval: weekly ================================================ FILE: .github/workflows/analysis.yml ================================================ --- name: Analysis on: push: branches: [ main ] pull_request: branches: [ main ] schedule: - cron: 13 7 * * 6 jobs: linting: runs-on: ubuntu-latest steps: - name: checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - name: golangci-lint uses: golangci/golangci-lint-action@1e7e51e771db61008b38414a730f564565cf7c20 with: version: v2.1.6 - name: shellcheck uses: azohra/shell-linter@30a9cf3f6cf25c08fc98f10d7dc4167f7b5c0c00 with: path: scripts/test-* severity: error - name: yaml-lint uses: ibiqlik/action-yamllint@2576378a8e339169678f9939646ee3ee325e845c # v3.1.1 with: config_file: .yamllint.yml vulnerabilities: runs-on: ubuntu-latest permissions: security-events: write steps: - name: checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - name: initialize uses: github/codeql-action/init@0d579ffd059c29b07949a3cce3983f0780820c98 # v3.29.5 with: languages: go - name: codeql analyze uses: github/codeql-action/analyze@0d579ffd059c29b07949a3cce3983f0780820c98 # v3.29.5 ================================================ FILE: .github/workflows/cla.yml ================================================ --- name: Contributor License Agreement (CLA) on: pull_request_target: types: [opened, synchronize] issue_comment: types: [created] jobs: cla: permissions: actions: write pull-requests: write runs-on: ubuntu-latest if: | (github.event.issue.pull_request && !github.event.issue.pull_request.merged_at && contains(github.event.comment.body, 'signed') ) || (github.event.pull_request && !github.event.pull_request.merged) steps: - uses: Shopify/shopify-cla-action@v1 with: github-token: ${{ secrets.GITHUB_TOKEN }} cla-token: ${{ secrets.CLA_TOKEN }} ================================================ FILE: .github/workflows/release.yml ================================================ --- name: Release on: push: tags: [ v*.*.* ] env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} jobs: release: runs-on: ubuntu-latest permissions: contents: write packages: write steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - name: GPG config run: | mkdir -p ~/.gnupg cat << EOF >> ~/.gnupg/gpg.conf keyserver hkps://keys.openpgp.org auto-key-import auto-key-retrieve EOF - name: Verify tag signature run: | # NOTE: Solve the problem with Github action checkout # https://github.com/actions/checkout/issues/290 git fetch --tags --force version=${GITHUB_REF#refs/tags/*} git show $version git tag -v $version - name: Log into registry ${{ env.REGISTRY }} uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Go uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 with: go-version: 1.22 check-latest: true cache: true - name: Build release changelog run: | version=${GITHUB_REF#refs/tags/v*} mkdir -p tmp sed '/^# \['$version'\]/,/^# \[/!d;//d;/^\s*$/d' CHANGELOG.md > tmp/release_changelog.md - name: Release uses: goreleaser/goreleaser-action@ec59f474b9834571250b370d4735c50f8e2d1e29 with: distribution: goreleaser version: v1.10.3 args: release --rm-dist --release-notes=tmp/release_changelog.md env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} ================================================ FILE: .github/workflows/test.yml ================================================ --- name: Test on: push: branches: [main] pull_request: branches: [main] jobs: test: runs-on: ubuntu-latest strategy: fail-fast: true matrix: go: [1.22.8] name: go ${{ matrix.go }} steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - name: Setup go uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 with: go-version: ${{ matrix.go }} check-latest: true cache: true - name: Tests run: make test - name: benchmarks run: make bench build: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 - name: Setup go uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0 with: go-version: 1.22 check-latest: true cache: true - name: E2E tests run: make test-e2e - name: Build uses: goreleaser/goreleaser-action@ec59f474b9834571250b370d4735c50f8e2d1e29 with: distribution: goreleaser version: v1.10.3 args: build --snapshot --rm-dist --skip-post-hooks --skip-validate --single-target env: GOOS: linux ================================================ FILE: .gitignore ================================================ # goreleaser output directory dist/ # go mod dependencies vendor/ tmp/ ================================================ FILE: .golangci.yml ================================================ --- version: "2" run: go: "1.22" linters: default: none enable: - bodyclose - dogsled - exhaustive - funlen - gocritic - gocyclo - godot - gosec - govet - ineffassign - lll - misspell - staticcheck - unused - whitespace settings: funlen: lines: 80 statements: 30 gosec: excludes: - G107 lll: line-length: 100 tab-width: 2 misspell: locale: US exclusions: generated: lax presets: - comments - common-false-positives - legacy - std-error-handling paths: - third_party$ - builtin$ - examples$ formatters: enable: - gofmt - goimports exclusions: generated: lax paths: - third_party$ - builtin$ - examples$ ================================================ FILE: .goreleaser.yml ================================================ --- project_name: toxiproxy env: - GO111MODULE=on before: hooks: - go mod download - go mod tidy builds: - &build_default id: server main: ./cmd/server binary: toxiproxy-server-{{.Os}}-{{.Arch}} no_unique_dist_dir: true env: - CGO_ENABLED=0 goos: - darwin - freebsd - linux - netbsd - openbsd - solaris - windows goarch: - amd64 - arm64 ignore: - goos: windows goarch: arm64 - goarch: arm ldflags: - -s -w -X github.com/Shopify/toxiproxy/v2.Version={{.Version}} - <<: *build_default id: server-arm goarch: - arm goarm: - "6" - "7" ignore: - goos: windows goarch: arm binary: toxiproxy-server-{{.Os}}-{{.Arch}}v{{.Arm}} - &build_client <<: *build_default id: client main: ./cmd/cli binary: toxiproxy-cli-{{.Os}}-{{.Arch}} - <<: *build_client id: client-arm goarch: - arm goarm: - "6" - "7" ignore: - goos: windows goarch: arm binary: toxiproxy-cli-{{.Os}}-{{.Arch}}v{{.Arm}} - <<: *build_default id: pkg-server no_unique_dist_dir: false binary: toxiproxy-server - <<: *build_default id: pkg-client no_unique_dist_dir: false main: ./cmd/cli binary: toxiproxy-cli checksum: name_template: checksums.txt snapshot: name_template: "{{ incpatch .Version }}-next" nfpms: - id: packages package_name: toxiproxy homepage: https://github.com/Shopify/toxiproxy maintainer: Shopify Opensource description: TCP proxy to simulate network and system conditions. license: MIT bindir: /usr/bin builds: - pkg-server - pkg-client formats: - apk - deb - rpm # NOTE: Starting with Ubuntu 15.04, Upstart will be deprecated in favor of Systemd. # contents: # - src: share/toxiproxy.conf # dst: /etc/init/toxiproxy.conf dockers: - &docker use: buildx dockerfile: Dockerfile ids: - server - client goos: linux goarch: amd64 image_templates: - ghcr.io/shopify/toxiproxy:{{ .Version }}-amd64 - ghcr.io/shopify/toxiproxy:v{{ .Major }}-amd64 - ghcr.io/shopify/toxiproxy:v{{ .Major }}.{{ .Minor }}-amd64 build_flag_templates: - --platform=linux/amd64 - --no-cache - --label=org.opencontainers.image.title={{ .ProjectName }} - --label=org.opencontainers.image.description={{ .ProjectName }} - --label=org.opencontainers.image.url=https://github.com/Shopify/{{ .ProjectName }} - --label=org.opencontainers.image.source=https://github.com/Shopify/{{ .ProjectName }} - --label=org.opencontainers.image.version={{ .Version }} - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} - --label=org.opencontainers.image.revision={{ .FullCommit }} - --label=org.opencontainers.image.licenses=MIT - <<: *docker goarch: arm64 image_templates: - ghcr.io/shopify/toxiproxy:{{ .Version }}-arm64 - ghcr.io/shopify/toxiproxy:v{{ .Major }}-arm64 - ghcr.io/shopify/toxiproxy:v{{ .Major }}.{{ .Minor }}-arm64 build_flag_templates: - --platform=linux/arm64/v8 - --no-cache - --label=org.opencontainers.image.title={{ .ProjectName }} - --label=org.opencontainers.image.description={{ .ProjectName }} - --label=org.opencontainers.image.url=https://github.com/Shopify/{{ .ProjectName }} - --label=org.opencontainers.image.source=https://github.com/Shopify/{{ .ProjectName }} - --label=org.opencontainers.image.version={{ .Version }} - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} - --label=org.opencontainers.image.revision={{ .FullCommit }} - --label=org.opencontainers.image.licenses=MIT - <<: *docker goarch: arm goarm: "7" image_templates: - ghcr.io/shopify/toxiproxy:{{ .Version }}-armv7 - ghcr.io/shopify/toxiproxy:v{{ .Major }}-armv7 - ghcr.io/shopify/toxiproxy:v{{ .Major }}.{{ .Minor }}-armv7 build_flag_templates: - --platform=linux/arm/v6 - --no-cache - --label=org.opencontainers.image.title={{ .ProjectName }} - --label=org.opencontainers.image.description={{ .ProjectName }} - --label=org.opencontainers.image.url=https://github.com/Shopify/{{ .ProjectName }} - --label=org.opencontainers.image.source=https://github.com/Shopify/{{ .ProjectName }} - --label=org.opencontainers.image.version={{ .Version }} - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} - --label=org.opencontainers.image.revision={{ .FullCommit }} - --label=org.opencontainers.image.licenses=MIT - <<: *docker goarch: arm goarm: "6" image_templates: - ghcr.io/shopify/toxiproxy:{{ .Version }}-armv6 - ghcr.io/shopify/toxiproxy:v{{ .Major }}-armv6 - ghcr.io/shopify/toxiproxy:v{{ .Major }}.{{ .Minor }}-armv6 build_flag_templates: - --platform=linux/arm/v6 - --no-cache - --label=org.opencontainers.image.title={{ .ProjectName }} - --label=org.opencontainers.image.description={{ .ProjectName }} - --label=org.opencontainers.image.url=https://github.com/Shopify/{{ .ProjectName }} - --label=org.opencontainers.image.source=https://github.com/Shopify/{{ .ProjectName }} - --label=org.opencontainers.image.version={{ .Version }} - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }} - --label=org.opencontainers.image.revision={{ .FullCommit }} - --label=org.opencontainers.image.licenses=MIT docker_manifests: - name_template: ghcr.io/shopify/{{ .ProjectName }}:{{ .Version }} image_templates: - ghcr.io/shopify/{{ .ProjectName }}:{{ .Version }}-amd64 - ghcr.io/shopify/{{ .ProjectName }}:{{ .Version }}-arm64 - ghcr.io/shopify/{{ .ProjectName }}:{{ .Version }}-armv6 - ghcr.io/shopify/{{ .ProjectName }}:{{ .Version }}-armv7 - name_template: ghcr.io/shopify/{{ .ProjectName }}:latest image_templates: - ghcr.io/shopify/{{ .ProjectName }}:{{ .Version }}-amd64 - ghcr.io/shopify/{{ .ProjectName }}:{{ .Version }}-arm64 - ghcr.io/shopify/{{ .ProjectName }}:{{ .Version }}-armv6 - ghcr.io/shopify/{{ .ProjectName }}:{{ .Version }}-armv7 changelog: sort: asc filters: exclude: - "^docs:" - "^test:" - ^Merge archives: - id: archive_binaries format: binary name_template: "{{ .Binary }}" builds: - server - client - id: archive_default format: tar.gz builds: - pkg-server - pkg-client ================================================ FILE: .yamllint.yml ================================================ --- yaml-files: - "*.yaml" - "*.yml" ignore: | vendor/**/* dist/*.yaml .github/**/* rules: comments: require-starting-space: true comments-indentation: enable document-start: present: true indentation: spaces: 2 indent-sequences: true check-multi-line-strings: true line-length: max: 100 level: warning allow-non-breakable-words: true allow-non-breakable-inline-mappings: false key-duplicates: enable new-lines: type: unix trailing-spaces: enable quoted-strings: quote-type: double required: only-when-needed ================================================ FILE: CHANGELOG.md ================================================ # [Unreleased] # [2.12.0] - Update go version to 1.23.0 (#628) - Do not restart proxies when using hostnames to specify listen address when updating a proxy and populating a collection (#631, @robinbrandt) - Update various go packages # [2.11.0] - 2024-10-16 - Do not pin minimum patch version in module (#595, @jaimem88) # [2.10.0] - 2024-10-08 - Update various go packages - Update go version to 1.22.8 (#594, @abecevello) # [2.9.0] - 2024-03-12 - Updated go version to 1.22.1 to fix 3 CVEs (#559, @dianadevasia) - Updated the version of golangci to 1.56.2 and disabled depguard rule in golangci (#559, @dianadevasia) # [2.8.0] - 2024-02-27 - toxiproxy-cli - sortedAttributes sort by attribute.key instead attribute.value (#543, @jesseward) # [2.7.0] - 2023-10-25 - Fix invalid JSON in /version endpoint response (#538, @whatyouhide) - Update minimum supported Go version 1.19. (@abecevello) # [2.6.0] - 2023-08-22 - Gracefull shutdown of HTTP server. (#439, @miry) - Support PATCH HTTP method for Proxy update(`PATCH /proxies/{proxy}`) and Toxic update(`PATCH /proxies/{proxy}/toxics/{toxic}`) endpoints. Deprecat POST HTTP method for those endpoints. (@miry) - Client does not parse response body in case of errors for Populate. Requires to get current proxies with new command. (#441, @miry) - Client specifies `User-Agent` HTTP header for all requests as "toxiproxy-cli/ /". Specifies client request content type as `application/json`. (#441, @miry) - Replace Api.Listen parameters `host` and `port` with single `addr`. (#445, @miry) # [2.5.0] - 2022-09-10 - Update Release steps. (#369, @neufeldtech) - Migrate off probot-CLA to new GitHub Action. (#405, @cursedcoder) - Support go 1.18, 1.19. (#415, @miry) - `toxiproxy.NewProxy` now accepts `name`, `listen addr` and `upstream addr`. (#418, @miry) - Replace logrus with zerolog. (#413, @miry) - Log HTTP requests to API server. (#413, #421, @miry) - Add TimeoutHandler for the HTTP API server. (#420, @miry) - Set Write and Read timeouts for HTTP API server connections. (#423, @miry) - Show unique request id in API HTTP response. (#425, @miry) - Add method to parse `stream.Direction` from a string. Allow converting `stream.Direction` to string. (#430, @miry) - Add the possibility to write to Output with a deadline. On interrupting Bandwidth toxic, use non-blocking writes. (#436, @miry) - Update minimum supported Go version 1.17. (#438, @miry) # [2.4.0] - 2022-03-07 - Verify git tag on release (#347, @miry) - Fix MacOS 12 tests for go17 with -race flag (#351, @strech) - Rename `testing/` and `bin/` folders (#354, @strech) - Added verbose error on proxy upstream dialing (#355, @f-dg) - Improve server startup message (#358, @areveny) - Introduce yaml linter. (#362, @miry) - Handle slicer toxic with zero `SizeVariation` and fix slicing randomization (#359, @areveny) - Added /metrics endpoint for exposing Prometheus-compatible internal metrics (#366, @neufeldtech) # [2.3.0] - 2021-12-23 - Store all the executable `main` packages in `cmd` folder. (#335, @miry) - Extract common test helpers to own files. (#336, @miry) - Client: Allow HTTPS endpoints. (#338, @chen-anders) - client.Populate assign client to proxy. (#291, @hellodudu) - fix: The release-test task is always success. add: Allow to run release-test on arm machines. (#340, @miry) - Upgrade `goreleaser`. Support `armv7` and `armv6` oses. (#339, @mitchellrj) - Allow to change log level for server. (#346, @miry) # [2.2.0] - 2021-10-17 - Update linux packages to use `/usr/bin` folder as binary destination and change the executable names to exclude ARCH and OS names. New pathes: ``` /usr/bin/toxiproxy-cli /usr/bin/toxiproxy-server ``` (#331, @miry) - A new toxic to simulate TCP RESET (Connection reset by peer) on the connections by closing the stub Input immediately or after a timeout. (#247 and #333, @chaosbox) # [2.1.7] - 2021-09-23 - Set the valid version during the build process. Verify the correct verion of the built binaries with `make release-dry` (#328, @miry) # [2.1.6] - 2021-09-23 - Use CHANGELOG.md for release description (#306, @miry) - Dependency updates in #294 introduced a breaking change in CLI argument parsing. Now [flags must be specified before arguments](https://github.com/urfave/cli/blob/master/docs/migrate-v1-to-v2.md#flags-before-args). Previously, arguments could be specified prior to flags. Update usage help text and documentation. (#308, @miry) - Run e2e tests to validate the command line and basic features of server, client and application (#309, @miry) - Add /v2 suffix to module import path (#311, @dnwe) - Setup automated checking source code for security vulnerabilities (#312, @miry) - Setup code linter (#314, @miry) - Max line length is 100 characters (#316, @miry) - Linter to check whether HTTP response body is closed successfully (#317, @miry) - Make sure the function are not big (#318, @miry) - Extract client flags specs to seprate methods. Introduce a new way to manage toxics with `ToxicOptions` structure (#321, @miry) - Split `Proxy.server` to multiple small (#322, @miry) - Extract initializetion of fake upstream server to test helper (#323, @miry) - Support a list of well knonwn linters (#326, @miry) - `--host` flag uses `TOXIPROXY_URL` if it is set (#319, @maaslalani) - Run benchmarks in CI/CD (#320, @miry) - Use scratch docker base image instead of alpine (#325, @miry) # [2.1.5] - 2021-09-01 - Move to Go Modules from godeps (#253, @epk) - Update the example in `client/README.md` (#251, @nothinux) - Update TOC in `README.md` (4ca1eddddfcd0c50c8f6dfb97089bb68e6310fd9, @dwradcliffe) - Add an example of `config.json` file to `README.md` (#260, @JesseEstum) - Add Link to Elixir Client (#287, @Jcambass) - Add Rust client link (#293, @itarato) - Renovations: formatting code, update dependicies, make govet/staticcheck pass (#294, @dnwe) - Remove `openssl` from `dev.yml` to use `dev` tool (#298, @pedro-stanaka) - Update `go` versions in development (#299, @miry) - Mention `MacPorts` in `README.md` (#290, @amake) - Fix some typos in `README.md` and `CHANGELOG.md` (#222, @jwilk) - Replace TravisCI with Github Actions to run tests (#303, @miry) - Build and release binaries with `goreleaser`. Support `arm64` and BSD oses. (#301, @miry) - Automate release with Github actions (#304, @miry) # [2.1.4] - 2019-01-11 - Bug fix: Fix OOM in toxic. #232 - Documentation updates. - CI and test updates. # [2.1.3] - 2018-03-05 - Update `/version` endpoint to also return a charset of utf-8. #204 - Bug fix: Double http concatenation. #191 - Update cli examples to be more accurate. #187 # [2.1.2] - 2017-07-10 - go 1.8, make Sirupsen lower case, update godeps (issue #179) - Handle SIGTERM to exit cleanly (issue #180) - Address security issue by disallowing browsers from accessing API # [2.1.1] - 2017-05-16 - Fix timeout toxic causing hang (issue #159) # [2.1.0] - 2016-12-07 - Add -config server option to populate on startup #154 - Updated CLI for scriptability #133 - Add `/populate` endpoint to server #111 - Change error responses from `title` to `error` - Allow hostname to be specified in CLI #129 - Add support for stateful toxics #127 - Add limit_data toxic # [2.0.0] - 2016-04-25 - Add CLI (`toxiproxy-cli`) and rename server binary to `toxiproxy-server` #93 - Fix removing a timeout toxic causing API to hang #89 - API and client return toxics as array rather than a map of name to toxic #92 - Fix multiple latency toxics not accumulating #94 - Change default toxic name to `_` #96 - Nest toxic attributes rather than having a flat structure #98 - 2.0 RFC: #54 and PR #62 - Change toxic API endpoints to an Add/Update/Remove structure - Remove `enabled` field, and add `name` and `type` fields to toxics - Add global toxic fields to a wrapper struct - Chain toxics together dynamically instead of in a fixed length chain - Register toxics in `init()` functions instead of a hard-coded list - Clean up API error codes to make them more consistent - Move toxics to their own package to allow 3rd party toxics - Remove stream direction from API urls #73 - Add `toxicity` field for toxics #75 - Refactor Go client to make usage easier with 2.0 #76 - Make `ChanReader` in the `stream` package interruptible #77 - Define proxy buffer sizes per-toxic (Fixes #72) - Fix slicer toxic testing race condition #71 # [1.2.1] - 2015-07-24 - Fix proxy name conflicts leaking an open port #69 # [1.2.0] - 2015-07-23 - Add a Toxic and Toxics type for the Go client - Add `Dockerfile` - Fix latency toxic limiting bandwidth #67 - Add Slicer toxic # [1.1.0] - 2015-05-05 - Remove /toxics endpoint in favour of /proxies - Add bandwidth toxic # [1.0.3] - 2015-04-29 - Rename Go library package to Toxiproxy from Client - Fix latency toxic send to closed channel panic #46 - Fix latency toxic accumulating delay #47 # [1.0.2] - 2015-04-12 - Added Toxic support to Go client # [1.0.1] - 2015-03-31 - Various improvements to the documentation - Initial version of Go client - Fix toxic disabling bug #42 # [1.0.0] - 2015-01-07 Initial public release. [Unreleased]: https://github.com/Shopify/toxiproxy/compare/v2.12.0...HEAD [2.12.0]: https://github.com/Shopify/toxiproxy/compare/v2.11.0...v2.12.0 [2.11.0]: https://github.com/Shopify/toxiproxy/compare/v2.10.0...v2.11.0 [2.10.0]: https://github.com/Shopify/toxiproxy/compare/v2.9.0...v2.10.0 [2.9.0]: https://github.com/Shopify/toxiproxy/compare/v2.8.0...v2.9.0 [2.8.0]: https://github.com/Shopify/toxiproxy/compare/v2.7.0...v2.8.0 [2.7.0]: https://github.com/Shopify/toxiproxy/compare/v2.6.0...v2.7.0 [2.6.0]: https://github.com/Shopify/toxiproxy/compare/v2.5.0...v2.6.0 [2.5.0]: https://github.com/Shopify/toxiproxy/compare/v2.4.0...v2.5.0 [2.4.0]: https://github.com/Shopify/toxiproxy/compare/v2.3.0...v2.4.0 [2.3.0]: https://github.com/Shopify/toxiproxy/compare/v2.2.0...v2.3.0 [2.2.0]: https://github.com/Shopify/toxiproxy/compare/v2.1.7...v2.2.0 [2.1.7]: https://github.com/Shopify/toxiproxy/compare/v2.1.6...v2.1.7 [2.1.6]: https://github.com/Shopify/toxiproxy/compare/v2.1.5...v2.1.6 [2.1.5]: https://github.com/Shopify/toxiproxy/compare/v2.1.4...v2.1.5 [2.1.4]: https://github.com/Shopify/toxiproxy/compare/v2.1.3...v2.1.4 [2.1.3]: https://github.com/Shopify/toxiproxy/compare/v2.1.2...v2.1.3 [2.1.2]: https://github.com/Shopify/toxiproxy/compare/v2.1.1...v2.1.2 [2.1.1]: https://github.com/Shopify/toxiproxy/compare/v2.1.0...v2.1.1 [2.1.0]: https://github.com/Shopify/toxiproxy/compare/v2.0.0...v2.1.0 [2.0.0]: https://github.com/Shopify/toxiproxy/compare/v1.2.1...v2.0.0 [1.2.1]: https://github.com/Shopify/toxiproxy/compare/v1.2.0...v1.2.1 [1.2.0]: https://github.com/Shopify/toxiproxy/compare/v1.1.0...v1.2.0 [1.1.0]: https://github.com/Shopify/toxiproxy/compare/v1.0.3...v1.1.0 [1.0.3]: https://github.com/Shopify/toxiproxy/compare/v1.0.2...v1.0.3 [1.0.2]: https://github.com/Shopify/toxiproxy/compare/v1.0.1...v1.0.2 [1.0.1]: https://github.com/Shopify/toxiproxy/compare/v1.0.0...v1.0.1 [1.0.0]: https://github.com/Shopify/toxiproxy/releases/tag/v1.0.0 ================================================ FILE: CREATING_TOXICS.md ================================================ # Creating custom toxics Creating a toxic is done by implementing the `Toxic` interface: ```go type Toxic interface { Pipe(*toxics.ToxicStub) } ``` The `Pipe()` function defines how data flows through the toxic, and is passed a `ToxicStub` to operate on. A `ToxicStub` stores the input and output channels for the toxic, as well as an interrupt channel that is used to pause operation of the toxic. The input and output channels in a `ToxicStub` send and receive `StreamChunk` structs, which are similar to network packets. A `StreamChunk` contains a `byte[]` of stream data, and a timestamp of when Toxiproxy received the data from the client or server. This is used instead of just a plain `byte[]` so that toxics like latency can find out how long a chunk of data has been waiting in the proxy. Toxics are registered in an `init()` function so that they can be used by the server: ```go func init() { toxics.Register("toxic_name", new(ExampleToxic)) } ``` In order to use your own toxics, you will need to compile your own binary. This can be done by copying [server](./cmd/server/server.go) into a new project and registering your toxic with the server. This will allow you to add toxics without having to make a full fork of the project. If you think your toxics will be useful to others, contribute them back with a Pull Request. An example project for building a separate binary can be found here: [examples](./_examples/toxics/) ## A basic toxic The most basic implementation of a toxic is the [noop toxic](./toxics/noop.go), which just passes data through without any modifications. ```go type NoopToxic struct{} func (t *NoopToxic) Pipe(stub *toxics.ToxicStub) { for { select { case <-stub.Interrupt: return case c := <-stub.Input: if c == nil { stub.Close() return } stub.Output <- c } } } ``` The above code reads from `stub.Input` in a loop, and passes the `StreamChunk` along to `stub.Output`. Since reading from `stub.Input` will block until a chunk is available, we need to check for interrupts as the same time. Toxics will be interrupted whenever they are being updated, or possibly removed. This can happen at any point within the `Pipe()` function, so all blocking operations (including sleep), should be interruptible. When an interrupt is received, the toxic should return from the `Pipe()` function after it has written any "in-flight" data back to `stub.Output`. It is important that all data read from `stub.Input` is passed along to `stub.Output`, otherwise the stream will be missing bytes and become corrupted. When an `end of stream` is reached, `stub.Input` will return a `nil` chunk. Whenever a nil chunk is returned, the toxic should call `Close()` on the stub, and return from `Pipe()`. ## Toxic configuration Toxic configuration information can be stored in the toxic struct. The toxic will be json encoded and decoded by the api, so all public fields will be api accessible. An example of a toxic that uses configuration values is the [latency toxic](./toxics/latency.go) ```go type LatencyToxic struct { Latency int64 `json:"latency"` Jitter int64 `json:"jitter"` } ``` These fields can be used inside the `Pipe()` function, but generally should not be written to from the toxic. A separate instance of the toxic exists for each connection through the proxy, and may be replaced when updated by the api. If state is required in your toxic, it is better to use a local variable at the top of `Pipe()`, since struct fields are not guaranteed to be persisted across interrupts. ## Toxic buffering By default, toxics are not buffered. This means that writes to `stub.Output` will block until either the endpoint or another toxic reads it. Since toxics are chained together, this means not reading from `stub.Input` will block other toxics (and endpoint writes) from operating. If this is not behavior you want your toxic to have, you can specify a buffer size for your toxic's input. The [latency toxic](./toxics/latency.go) uses this in order to prevent added latency from limiting the proxy bandwidth. Specifying a buffer size is done by implementing the `BufferedToxic` interface, which adds the `GetBufferSize()` function: ```go func (t *LatencyToxic) GetBufferSize() int { return 1024 } ``` The unit used by `GetBufferSize()` is `StreamChunk`s. Chunks are generally anywhere from 1 byte, up to 32KB, so keep this in mind when thinking about how much buffering you need, and how much memory you are comfortable with using. ## Stateful toxics If a toxic needs to store extra information for a connection such as the number of bytes transferred (See `limit_data` toxic), a state object can be created by implementing the `StatefulToxic` interface. This interface defines the `NewState()` function that can create a new state object with default values set. When a stateful toxic is created, the state object will be stored on the `ToxicStub` and can be accessed from `toxic.Pipe()`: ```go state := stub.State.(*ExampleToxicState) ``` If necessary, some global state can be stored in the toxic struct, which will not be instanced per-connection. These fields cannot have a custom default value set and will not be thread-safe, so proper locking or atomic operations will need to be used. ## Using `io.Reader` and `io.Writer` If your toxic involves modifying the data going through a proxy, you can use the `ChanReader` and `ChanWriter` interfaces in the [stream package](./stream). These allow reading and writing from the input and output channels as you would a normal data stream such as a TCP socket. An implementation of the noop toxic above using the stream package would look something like this: ```go func (t *NoopToxic) Pipe(stub *toxics.ToxicStub) { buf := make([]byte, 32*1024) writer := stream.NewChanWriter(stub.Output) reader := stream.NewChanReader(stub.Input) reader.SetInterrupt(stub.Interrupt) for { n, err := reader.Read(buf) if err == stream.ErrInterrupted { writer.Write(buf[:n]) return } else if err == io.EOF { stub.Close() return } writer.Write(buf[:n]) } } ``` See [examples](./_examples/toxics/) for a full example of using the stream package with Go's http package. ================================================ FILE: Dockerfile ================================================ FROM scratch EXPOSE 8474 ENTRYPOINT ["/toxiproxy"] CMD ["-host=0.0.0.0"] ENV LOG_LEVEL=info COPY toxiproxy-server-linux-* /toxiproxy COPY toxiproxy-cli-linux-* /toxiproxy-cli ================================================ FILE: LICENSE ================================================ The MIT License (MIT) Copyright (c) 2014 Shopify Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: METRICS.md ================================================ # Metrics - [Metrics](#metrics) - [Runtime Metrics](#runtime-metrics) - [Proxy Metrics](#proxy-metrics) - [toxiproxy_proxy_received_bytes_total / toxiproxy_proxy_sent_bytes_total](#toxiproxy_proxy_received_bytes_total--toxiproxy_proxy_sent_bytes_total) ### Runtime Metrics To enable runtime metrics related to the state of the go runtime, build version, process info, use the `-runtime-metrics` flag. For more details, see below: - [NewGoCollector](https://pkg.go.dev/github.com/prometheus/client_golang/prometheus/collectors#NewGoCollector) - [NewBuildInfoCollector](https://pkg.go.dev/github.com/prometheus/client_golang/prometheus/collectors#NewBuildInfoCollector) - [NewProcessCollector](https://pkg.go.dev/github.com/prometheus/client_golang/prometheus/collectors#NewProcessCollector) ### Proxy Metrics To enable metrics related to toxiproxy internals, use the `-proxy-metrics` flag. #### toxiproxy_proxy_received_bytes_total / toxiproxy_proxy_sent_bytes_total The total number of bytes received/sent on a given proxy link in a given direction ```mermaid sequenceDiagram Client->>+Toxiproxy: toxiproxy_proxy_received_bytes_total{direction="upstream"} Toxiproxy->>+Server: toxiproxy_proxy_sent_bytes_total{direction="upstream"} Server->>+Toxiproxy: toxiproxy_proxy_received_bytes_total{direction="downstream"} Toxiproxy->>+Client: toxiproxy_proxy_sent_bytes_total{direction="downstream"} ``` **Type** Counter **Labels** | Label | Description | Example | |-----------|--------------------------------|-----------------------| | direction | Direction of the link | upstream / downstream | | listener | Listener address of this proxy | 0.0.0.0:8080 | | proxy | Proxy name | my-proxy | | upstream | Upstream address of this proxy | httpbin.org:80 | ================================================ FILE: Makefile ================================================ OS := $(shell uname -s) ARCH := $(shell uname -m) GO_VERSION := $(shell go version | cut -f3 -d" ") GO_MINOR_VERSION := $(shell echo $(GO_VERSION) | cut -f2 -d.) GO_PATCH_VERSION := $(shell echo $(GO_VERSION) | cut -f3 -d. | sed "s/^\s*$$/0/") MALLOC_ENV := $(shell [ $(OS) = Darwin -a $(GO_MINOR_VERSION) -eq 17 -a $(GO_PATCH_VERSION) -lt 6 ] && echo "MallocNanoZone=0") .PHONY: all all: setup build test bench fmt lint .PHONY: test test: # NOTE: https://github.com/golang/go/issues/49138 $(MALLOC_ENV) go test -v -race -timeout 1m ./... .PHONY: test-e2e test-e2e: build container.build scripts/test-e2e timeout -v --foreground 20m scripts/test-e2e-hazelcast toxiproxy .PHONY: test-release test-release: lint fmt test bench test-e2e release-dry scripts/test-release .PHONY: bench bench: # TODO: Investigate why benchmarks require more sockets: ulimit -n 10240 go test -bench=. -v *.go go test -bench=. -v toxics/*.go .PHONY: fmt fmt: go fmt ./... goimports -w **/*.go golangci-lint run --fix shfmt -l -s -w -kp -i 2 scripts/test-* .PHONY: lint lint: golangci-lint run shellcheck scripts/test-* shfmt -l -s -d -kp -i 2 scripts/test-* yamllint . .PHONY: build build: dist clean go build -ldflags="-s -w" -o ./dist/toxiproxy-server ./cmd/server go build -ldflags="-s -w" -o ./dist/toxiproxy-cli ./cmd/cli .PHONY: container.build container.build: env GOOS=linux CGO_ENABLED=0 go build -ldflags="-s -w" -o ./dist/toxiproxy-server-linux-$(ARCH) ./cmd/server env GOOS=linux CGO_ENABLED=0 go build -ldflags="-s -w" -o ./dist/toxiproxy-cli-linux-$(ARCH) ./cmd/cli docker build -f Dockerfile -t toxiproxy dist docker run --rm toxiproxy --version .PHONY: release release: goreleaser release --rm-dist .PHONY: release-dry release-dry: goreleaser release --rm-dist --skip-publish --skip-validate .PHONY: setup setup: go mod download go mod tidy dist: mkdir -p dist .PHONY: clean clean: rm -fr dist/* ================================================ FILE: README.md ================================================ # Toxiproxy [![GitHub release](https://img.shields.io/github/release/Shopify/toxiproxy.svg)](https://github.com/Shopify/toxiproxy/releases/latest) [![Build Status](https://github.com/Shopify/toxiproxy/actions/workflows/test.yml/badge.svg)](https://github.com/Shopify/toxiproxy/actions/workflows/test.yml) ![](http://i.imgur.com/sOaNw0o.png) Toxiproxy is a framework for simulating network conditions. It's made specifically to work in testing, CI and development environments, supporting deterministic tampering with connections, but with support for randomized chaos and customization. **Toxiproxy is the tool you need to prove with tests that your application doesn't have single points of failure.** We've been successfully using it in all development and test environments at Shopify since October, 2014. See our [blog post][blog] on resiliency for more information. Toxiproxy usage consists of two parts. A TCP proxy written in Go (what this repository contains) and a client communicating with the proxy over HTTP. You configure your application to make all test connections go through Toxiproxy and can then manipulate their health via HTTP. See [Usage](#usage) below on how to set up your project. For example, to add 1000ms of latency to the response of MySQL from the [Ruby client](https://github.com/Shopify/toxiproxy-ruby): ```ruby Toxiproxy[:mysql_master].downstream(:latency, latency: 1000).apply do Shop.first # this takes at least 1s end ``` To take down all Redis instances: ```ruby Toxiproxy[/redis/].down do Shop.first # this will throw an exception end ``` While the examples in this README are currently in Ruby, there's nothing stopping you from creating a client in any other language (see [Clients](#clients)). ## Table of Contents - [Toxiproxy](#toxiproxy) - [Table of Contents](#table-of-contents) - [Why yet another chaotic TCP proxy?](#why-yet-another-chaotic-tcp-proxy) - [Clients](#clients) - [Example](#example) - [Usage](#usage) - [1. Installing Toxiproxy](#1-installing-toxiproxy) - [Upgrading from Toxiproxy 1.x](#upgrading-from-toxiproxy-1x) - [2. Populating Toxiproxy](#2-populating-toxiproxy) - [3. Using Toxiproxy](#3-using-toxiproxy) - [4. Logging](#4-logging) - [Toxics](#toxics) - [latency](#latency) - [down](#down) - [bandwidth](#bandwidth) - [slow_close](#slow_close) - [timeout](#timeout) - [reset_peer](#reset_peer) - [slicer](#slicer) - [limit_data](#limit_data) - [HTTP API](#http-api) - [Proxy fields:](#proxy-fields) - [Toxic fields:](#toxic-fields) - [Endpoints](#endpoints) - [Populating Proxies](#populating-proxies) - [CLI Example](#cli-example) - [Metrics](#metrics) - [Frequently Asked Questions](#frequently-asked-questions) - [Development](#development) - [Release](#release) ## Why yet another chaotic TCP proxy? The existing ones we found didn't provide the kind of dynamic API we needed for integration and unit testing. Linux tools like `nc` and so on are not cross-platform and require root, which makes them problematic in test, development and CI environments. ## Clients * [toxiproxy-ruby](https://github.com/Shopify/toxiproxy-ruby) * [toxiproxy-go](https://github.com/Shopify/toxiproxy/tree/main/client) * [toxiproxy-python](https://github.com/douglas/toxiproxy-python) * [toxiproxy.net](https://github.com/mdevilliers/Toxiproxy.Net) * [toxiproxy-php-client](https://github.com/ihsw/toxiproxy-php-client) * [toxiproxy-node-client](https://github.com/ihsw/toxiproxy-node-client) * [toxiproxy-java](https://github.com/trekawek/toxiproxy-java) * [toxiproxy-haskell](https://github.com/jpittis/toxiproxy-haskell) * [toxiproxy-rust](https://github.com/itarato/toxiproxy_rust) * [toxiproxy-elixir](https://github.com/Jcambass/toxiproxy_ex) ## Example Let's walk through an example with a Rails application. Note that Toxiproxy is in no way tied to Ruby, it's just been our first use case. You can see the full example at [sirupsen/toxiproxy-rails-example](https://github.com/sirupsen/toxiproxy-rails-example). To get started right away, jump down to [Usage](#usage). For our popular blog, for some reason we're storing the tags for our posts in Redis and the posts themselves in MySQL. We might have a `Post` class that includes some methods to manipulate tags in a [Redis set](http://redis.io/commands#set): ```ruby class Post < ActiveRecord::Base # Return an Array of all the tags. def tags TagRedis.smembers(tag_key) end # Add a tag to the post. def add_tag(tag) TagRedis.sadd(tag_key, tag) end # Remove a tag from the post. def remove_tag(tag) TagRedis.srem(tag_key, tag) end # Return the key in Redis for the set of tags for the post. def tag_key "post:tags:#{self.id}" end end ``` We've decided that erroring while writing to the tag data store (adding/removing) is OK. However, if the tag data store is down, we should be able to see the post with no tags. We could simply rescue the `Redis::CannotConnectError` around the `SMEMBERS` Redis call in the `tags` method. Let's use Toxiproxy to test that. Since we've already installed Toxiproxy and it's running on our machine, we can skip to step 2. This is where we need to make sure Toxiproxy has a mapping for Redis tags. To `config/boot.rb` (before any connection is made) we add: ```ruby require 'toxiproxy' Toxiproxy.populate([ { name: "toxiproxy_test_redis_tags", listen: "127.0.0.1:22222", upstream: "127.0.0.1:6379" } ]) ``` Then in `config/environments/test.rb` we set the `TagRedis` to be a Redis client that connects to Redis through Toxiproxy by adding this line: ```ruby TagRedis = Redis.new(port: 22222) ``` All calls in the test environment now go through Toxiproxy. That means we can add a unit test where we simulate a failure: ```ruby test "should return empty array when tag redis is down when listing tags" do @post.add_tag "mammals" # Take down all Redises in Toxiproxy Toxiproxy[/redis/].down do assert_equal [], @post.tags end end ``` The test fails with `Redis::CannotConnectError`. Perfect! Toxiproxy took down the Redis successfully for the duration of the closure. Let's fix the `tags` method to be resilient: ```ruby def tags TagRedis.smembers(tag_key) rescue Redis::CannotConnectError [] end ``` The tests pass! We now have a unit test that proves fetching the tags when Redis is down returns an empty array, instead of throwing an exception. For full coverage you should also write an integration test that wraps fetching the entire blog post page when Redis is down. Full example application is at [sirupsen/toxiproxy-rails-example](https://github.com/sirupsen/toxiproxy-rails-example). ## Usage Configuring a project to use Toxiproxy consists of three steps: 1. Installing Toxiproxy 2. Populating Toxiproxy 3. Using Toxiproxy ### 1. Installing Toxiproxy **Linux** See [`Releases`](https://github.com/Shopify/toxiproxy/releases) for the latest binaries and system packages for your architecture. **Ubuntu** ```bash $ wget -O toxiproxy-2.1.4.deb https://github.com/Shopify/toxiproxy/releases/download/v2.1.4/toxiproxy_2.1.4_amd64.deb $ sudo dpkg -i toxiproxy-2.1.4.deb $ sudo service toxiproxy start ``` **OS X** With [Homebrew](https://brew.sh/): ```bash $ brew tap shopify/shopify $ brew install toxiproxy ``` Or with [MacPorts](https://www.macports.org/): ```bash $ port install toxiproxy ``` **Windows** Toxiproxy for Windows is available for download at https://github.com/Shopify/toxiproxy/releases/download/v2.1.4/toxiproxy-server-windows-amd64.exe **Docker** Toxiproxy is available on [Github container registry](https://github.com/Shopify/toxiproxy/pkgs/container/toxiproxy). Old versions `<= 2.1.4` are available on on [Docker Hub](https://hub.docker.com/r/shopify/toxiproxy/). ```bash $ docker pull ghcr.io/shopify/toxiproxy $ docker run --rm -it ghcr.io/shopify/toxiproxy ``` If using Toxiproxy from the host rather than other containers, enable host networking with `--net=host`. ```shell $ docker run --rm --entrypoint="/toxiproxy-cli" -it ghcr.io/shopify/toxiproxy list ``` **Source** If you have Go installed, you can build Toxiproxy from source using the make file: ```bash $ make build $ ./toxiproxy-server ``` #### Upgrading from Toxiproxy 1.x In Toxiproxy 2.0 several changes were made to the API that make it incompatible with version 1.x. In order to use version 2.x of the Toxiproxy server, you will need to make sure your client library supports the same version. You can check which version of Toxiproxy you are running by looking at the `/version` endpoint. See the documentation for your client library for specific library changes. Detailed changes for the Toxiproxy server can been found in [CHANGELOG.md](./CHANGELOG.md). ### 2. Populating Toxiproxy When your application boots, it needs to make sure that Toxiproxy knows which endpoints to proxy where. The main parameters are: name, address for Toxiproxy to **listen** on and the address of the upstream. Some client libraries have helpers for this task, which is essentially just making sure each proxy in a list is created. Example from the Ruby client: ```ruby # Make sure `shopify_test_redis_master` and `shopify_test_mysql_master` are # present in Toxiproxy Toxiproxy.populate([ { name: "shopify_test_redis_master", listen: "127.0.0.1:22220", upstream: "127.0.0.1:6379" }, { name: "shopify_test_mysql_master", listen: "127.0.0.1:24220", upstream: "127.0.0.1:3306" } ]) ``` This code needs to run as early in boot as possible, before any code establishes a connection through Toxiproxy. Please check your client library for documentation on the population helpers. Alternatively use the CLI to create proxies, e.g.: ```bash toxiproxy-cli create -l localhost:26379 -u localhost:6379 shopify_test_redis_master ``` We recommend a naming such as the above: `___`. This makes sure there are no clashes between applications using the same Toxiproxy. For large application we recommend storing the Toxiproxy configurations in a separate configuration file. We use `config/toxiproxy.json`. This file can be passed to the server using the `-config` option, or loaded by the application to use with the `populate` function. An example `config/toxiproxy.json`: ```json [ { "name": "web_dev_frontend_1", "listen": "[::]:18080", "upstream": "webapp.domain:8080", "enabled": true }, { "name": "web_dev_mysql_1", "listen": "[::]:13306", "upstream": "database.domain:3306", "enabled": true } ] ``` Use ports outside the ephemeral port range to avoid random port conflicts. It's `32,768` to `61,000` on Linux by default, see `/proc/sys/net/ipv4/ip_local_port_range`. ### 3. Using Toxiproxy To use Toxiproxy, you now need to configure your application to connect through Toxiproxy. Continuing with our example from step two, we can configure our Redis client to connect through Toxiproxy: ```ruby # old straight to redis redis = Redis.new(port: 6380) # new through toxiproxy redis = Redis.new(port: 22220) ``` Now you can tamper with it through the Toxiproxy API. In Ruby: ```ruby redis = Redis.new(port: 22220) Toxiproxy[:shopify_test_redis_master].downstream(:latency, latency: 1000).apply do redis.get("test") # will take 1s end ``` Or via the CLI: ```bash toxiproxy-cli toxic add -t latency -a latency=1000 shopify_test_redis_master ``` Please consult your respective client library on usage. ### 4. Logging There are the following log levels: panic, fatal, error, warn or warning, info, debug and trace. The level could be updated via environment variable `LOG_LEVEL`. ### Toxics Toxics manipulate the pipe between the client and upstream. They can be added and removed from proxies using the [HTTP api](#http-api). Each toxic has its own parameters to change how it affects the proxy links. For documentation on implementing custom toxics, see [CREATING_TOXICS.md](./CREATING_TOXICS.md) #### latency Add a delay to all data going through the proxy. The delay is equal to `latency` +/- `jitter`. Attributes: - `latency`: time in milliseconds - `jitter`: time in milliseconds #### down Bringing a service down is not technically a toxic in the implementation of Toxiproxy. This is done by `POST`ing to `/proxies/{proxy}` and setting the `enabled` field to `false`. #### bandwidth Limit a connection to a maximum number of kilobytes per second. Attributes: - `rate`: rate in KB/s #### slow_close Delay the TCP socket from closing until `delay` has elapsed. Attributes: - `delay`: time in milliseconds #### timeout Stops all data from getting through, and closes the connection after `timeout`. If `timeout` is 0, the connection won't close, and data will be dropped until the toxic is removed. Attributes: - `timeout`: time in milliseconds #### reset_peer Simulate TCP RESET (Connection reset by peer) on the connections by closing the stub Input immediately or after a `timeout`. Attributes: - `timeout`: time in milliseconds #### slicer Slices TCP data up into small bits, optionally adding a delay between each sliced "packet". Attributes: - `average_size`: size in bytes of an average packet - `size_variation`: variation in bytes of an average packet (should be smaller than average_size) - `delay`: time in microseconds to delay each packet by #### limit_data Closes connection when transmitted data exceeded limit. - `bytes`: number of bytes it should transmit before connection is closed ### HTTP API All communication with the Toxiproxy daemon from the client happens through the HTTP interface, which is described here. Toxiproxy listens for HTTP on port **8474**. #### Proxy fields: - `name`: proxy name (string) - `listen`: listen address (string) - `upstream`: proxy upstream address (string) - `enabled`: true/false (defaults to true on creation) To change a proxy's name, it must be deleted and recreated. Changing the `listen` or `upstream` fields will restart the proxy and drop any active connections. If `listen` is specified with a port of 0, toxiproxy will pick an ephemeral port. The `listen` field in the response will be updated with the actual port. If you change `enabled` to `false`, it will take down the proxy. You can switch it back to `true` to reenable it. #### Toxic fields: - `name`: toxic name (string, defaults to `_`) - `type`: toxic type (string) - `stream`: link direction to affect (defaults to `downstream`) - `toxicity`: probability of the toxic being applied to a link (defaults to 1.0, 100%) - `attributes`: a map of toxic-specific attributes See [Toxics](#toxics) for toxic-specific attributes. The `stream` direction must be either `upstream` or `downstream`. `upstream` applies the toxic on the `client -> server` connection, while `downstream` applies the toxic on the `server -> client` connection. This can be used to modify requests and responses separately. #### Endpoints All endpoints are JSON. - **GET /proxies** - List existing proxies and their toxics - **POST /proxies** - Create a new proxy - **POST /populate** - Create or replace a list of proxies - **GET /proxies/{proxy}** - Show the proxy with all its active toxics - **POST /proxies/{proxy}** - Update a proxy's fields - **DELETE /proxies/{proxy}** - Delete an existing proxy - **GET /proxies/{proxy}/toxics** - List active toxics - **POST /proxies/{proxy}/toxics** - Create a new toxic - **GET /proxies/{proxy}/toxics/{toxic}** - Get an active toxic's fields - **POST /proxies/{proxy}/toxics/{toxic}** - Update an active toxic - **DELETE /proxies/{proxy}/toxics/{toxic}** - Remove an active toxic - **POST /reset** - Enable all proxies and remove all active toxics - **GET /version** - Returns the server version number - **GET /metrics** - Returns Prometheus-compatible metrics #### Populating Proxies Proxies can be added and configured in bulk using the `/populate` endpoint. This is done by passing a json array of proxies to toxiproxy. If a proxy with the same name already exists, it will be compared to the new proxy and replaced if the `upstream` and `listen` address don't match. A `/populate` call can be included for example at application start to ensure all required proxies exist. It is safe to make this call several times, since proxies will be untouched as long as their fields are consistent with the new data. ### CLI Example ```bash $ toxiproxy-cli create -l localhost:26379 -u localhost:6379 redis Created new proxy redis $ toxiproxy-cli list Listen Upstream Name Enabled Toxics ====================================================================== 127.0.0.1:26379 localhost:6379 redis true None Hint: inspect toxics with `toxiproxy-client inspect ` ``` ```bash $ redis-cli -p 26379 127.0.0.1:26379> SET omg pandas OK 127.0.0.1:26379> GET omg "pandas" ``` ```bash $ toxiproxy-cli toxic add -t latency -a latency=1000 redis Added downstream latency toxic 'latency_downstream' on proxy 'redis' ``` ```bash $ redis-cli -p 26379 127.0.0.1:26379> GET omg "pandas" (1.00s) 127.0.0.1:26379> DEL omg (integer) 1 (1.00s) ``` ```bash $ toxiproxy-cli toxic remove -n latency_downstream redis Removed toxic 'latency_downstream' on proxy 'redis' ``` ```bash $ redis-cli -p 26379 127.0.0.1:26379> GET omg (nil) ``` ```bash $ toxiproxy-cli delete redis Deleted proxy redis ``` ```bash $ redis-cli -p 26379 Could not connect to Redis at 127.0.0.1:26379: Connection refused ``` ### Metrics Toxiproxy exposes Prometheus-compatible metrics via its HTTP API at /metrics. See [METRICS.md](./METRICS.md) for full descriptions ### Frequently Asked Questions **How fast is Toxiproxy?** The speed of Toxiproxy depends largely on your hardware, but you can expect a latency of *< 100µs* when no toxics are enabled. When running with `GOMAXPROCS=4` on a Macbook Pro we achieved *~1000MB/s* throughput, and as high as *2400MB/s* on a higher end desktop. Basically, you can expect Toxiproxy to move data around at least as fast the app you're testing. **Can Toxiproxy do randomized testing?** Many of the available toxics can be configured to have randomness, such as `jitter` in the `latency` toxic. There is also a global `toxicity` parameter that specifies the percentage of connections a toxic will affect. This is most useful for things like the `timeout` toxic, which would allow X% of connections to timeout. **I am not seeing my Toxiproxy actions reflected for MySQL**. MySQL will prefer the local Unix domain socket for some clients, no matter which port you pass it if the host is set to `localhost`. Configure your MySQL server to not create a socket, and use `127.0.0.1` as the host. Remember to remove the old socket after you restart the server. **Toxiproxy causes intermittent connection failures**. Use ports outside the ephemeral port range to avoid random port conflicts. It's `32,768` to `61,000` on Linux by default, see `/proc/sys/net/ipv4/ip_local_port_range`. **Should I run a Toxiproxy for each application?** No, we recommend using the same Toxiproxy for all applications. To distinguish between services we recommend naming your proxies with the scheme: `___`. For example, `shopify_test_redis_master` or `shopify_development_mysql_1`. ### Development * `make`. Build a toxiproxy development binary for the current platform. * `make all`. Build Toxiproxy binaries and packages for all platforms. Requires to have Go compiled with cross compilation enabled on Linux and Darwin (amd64) as well as [`goreleaser`](https://goreleaser.com/) in your `$PATH` to build binaries the Linux package. * `make test`. Run the Toxiproxy tests. ### Release See [RELEASE.md](./RELEASE.md) [blog]: https://shopify.engineering/building-and-testing-resilient-ruby-on-rails-applications ================================================ FILE: RELEASE.md ================================================ # Releasing - [Releasing](#releasing) - [Before You Begin](#before-you-begin) - [Local Release Preparation](#local-release-preparation) - [Checkout latest code](#checkout-latest-code) - [Update the CHANGELOG.md](#update-the-changelogmd) - [Create Release Commit and Tag](#create-release-commit-and-tag) - [Run Pre-Release Tests](#run-pre-release-tests) - [Push Release Tag](#push-release-tag) - [Verify Github Release](#verify-github-release) - [Update Homebrew versions](#update-homebrew-versions) ## Before You Begin Ensure your local workstation is configured to be able to [Sign commits](https://docs.github.com/en/authentication/managing-commit-signature-verification/signing-commits) ## Local Release Preparation ### Checkout latest code ```shell git checkout main git pull origin main ``` ### Update the [CHANGELOG.md](CHANGELOG.md) - Add a new version header at the top of the document, just after `# [Unreleased]` - Update links at bottom of changelog ### Create Release Commit and Tag ```shell export RELEASE_VERSION=2.x.y git commit -a -S -m "Release $RELEASE_VERSION" git tag -s "v$RELEASE_VERSION" # When prompted for a commit message, enter the 'release notes' style message, just like on the releases page ``` ### Run Pre-Release Tests ```shell make test-release ``` - Push to Main Branch ```shell git push origin main --follow-tags ``` ## Push Release Tag - On your local machine again, push your tag to github ```shell git push origin "v$RELEASE_VERSION" ``` ## Verify Github Release - Github Actions should kick off a build and release after the tag is pushed. - Verify that a [Release gets created in Github](https://github.com/Shopify/toxiproxy/releases) and verify that the release notes look correct - Github Actions should also attatch the built binaries to the release (it might take a few mins) ## Update Homebrew versions - Update [homebrew-shopify toxiproxy.rb](https://github.com/Shopify/homebrew-shopify/blob/master/toxiproxy.rb#L9) manifest 1. Update `app_version` string to your released version 2. Update hashes for all platforms (find the hashes in the checksums.txt from your release notes) - Do a manual check of installing toxiproxy via brew 1. While in the homebrew-shopify directory... ```shell brew install ./toxiproxy.rb --debug ``` Note: it's normal to get some errors when homebrew attempts to load the file as a Cask instead of a formula, just make sure that it still gets installed. - PR the version update change and merge ================================================ FILE: _examples/tests/README.md ================================================ ## Tests with Toxiproxy ### Setup ```shell $ brew install shopify/shopify/toxiproxy kind $ kind create cluster --config=cluster.yml $ kubectl --context kind-kind apply -f resources.yml $ kubectl wait deploy postgres --for condition=available --timeout=5m $ psql -h 127.0.0.1 -U postgres -c "DROP DATABASE IF EXISTS sample" $ psql -h 127.0.0.1 -U postgres -c "CREATE DATABASE sample" $ psql -h 127.0.0.1 -U postgres -c "CREATE DATABASE sample_test" ``` ### Run ```shell $ go run ./ ``` ### Test ```shell $ go test -v . $ go test -v . -run TestMultipleToxics ``` ================================================ FILE: _examples/tests/cluster.yml ================================================ --- kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 networking: ipFamily: ipv4 nodes: - role: control-plane extraPortMappings: # port forward 5432 on the host to 5432 on this node - containerPort: 30950 hostPort: 5432 # optional: set the bind address on the host # 0.0.0.0 is the current default listenAddress: 127.0.0.1 # optional: set the protocol to one of TCP, UDP, SCTP. # TCP is the default protocol: TCP ... ================================================ FILE: _examples/tests/db.go ================================================ package main import ( pg "github.com/go-pg/pg/v10" "github.com/go-pg/pg/v10/orm" ) func setupDB(addr, database string) (*pg.DB, error) { db := pg.Connect(&pg.Options{ Addr: addr, User: "postgres", Database: database, }) err := createSchema(db) if err != nil { return nil, err } err = seed(db) if err != nil { return nil, err } return db, nil } func createSchema(db *pg.DB) error { models := []interface{}{ (*User)(nil), (*Story)(nil), } for _, model := range models { err := db.Model(model).CreateTable(&orm.CreateTableOptions{ Temp: true, }) if err != nil { return err } } return nil } func seed(db *pg.DB) error { user1 := &User{ Name: "admin", Emails: []string{"admin1@admin", "admin2@admin"}, } _, err := db.Model(user1).Insert() if err != nil { return err } _, err = db.Model(&User{ Name: "root", Emails: []string{"root1@root", "root2@root"}, }).Insert() if err != nil { return err } story1 := &Story{ Title: "Cool story", AuthorId: user1.Id, } _, err = db.Model(story1).Insert() if err != nil { return err } return nil } ================================================ FILE: _examples/tests/db_test.go ================================================ package main import ( "fmt" "log" "net" "testing" "time" toxiServer "github.com/Shopify/toxiproxy/v2" toxiproxy "github.com/Shopify/toxiproxy/v2/client" pg "github.com/go-pg/pg/v10" "github.com/prometheus/client_golang/prometheus" "github.com/rs/zerolog" ) var db *pg.DB var toxi *toxiproxy.Client var proxies map[string]*toxiproxy.Proxy func DB() *pg.DB { if db == nil { var err error db, err = setupDB(":35432", "sample_test") if err != nil { log.Panicf("Could not connect to DB: %+v", err) } } return db } func connectDB(addr string) *pg.DB { return pg.Connect(&pg.Options{ Addr: addr, User: "postgres", Database: "sample_test", }) } func init() { log.SetFlags(log.LstdFlags | log.Lshortfile) fmt.Println("=== SETUP") runToxiproxyServer() populateProxies() } func populateProxies() { if toxi == nil { toxi = toxiproxy.NewClient("localhost:8474") } var err error _, err = toxi.Populate([]toxiproxy.Proxy{{ Name: "postgresql", Listen: "localhost:35432", Upstream: "localhost:5432", Enabled: true, }}) if err != nil { panic(err) } proxies, err = toxi.Proxies() if err != nil { panic(err) } } func runToxiproxyServer() { var err error timeout := 5 * time.Second // Check if there is instance run conn, err := net.DialTimeout("tcp", "localhost:8474", timeout) if err == nil { conn.Close() return } go func() { metricsContainer := toxiServer.NewMetricsContainer(prometheus.NewRegistry()) server := toxiServer.NewServer(metricsContainer, zerolog.Nop()) server.Listen("localhost:8474") }() for i := 0; i < 10; i += 1 { conn, err := net.DialTimeout("tcp", "localhost:8474", timeout) if err == nil { conn.Close() return } } panic(err) } func TestSlowDBConnection(t *testing.T) { db := DB() // Add 1s latency to 100% of downstream connections proxies["postgresql"].AddToxic("latency_down", "latency", "downstream", 1.0, toxiproxy.Attributes{ "latency": 10000, }) defer proxies["postgresql"].RemoveToxic("latency_down") err := process(db) if err != nil { t.Fatalf("got error %v, wanted no errors", err) } } func TestOutageResetPeer(t *testing.T) { db := DB() // Add broken TCP connection proxies["postgresql"].AddToxic("reset_peer_down", "reset_peer", "downstream", 1.0, toxiproxy.Attributes{ "timeout": 10, }) defer proxies["postgresql"].RemoveToxic("reset_peer_down") err := process(db) if err == nil { t.Fatalf("expect error") } } ================================================ FILE: _examples/tests/main.go ================================================ package main import ( "context" "log" "os" pg "github.com/go-pg/pg/v10" ) func main() { err := run() if err != nil { log.Printf("ERROR: %v", err) os.Exit(1) } } func run() error { db, err := setupDB(":5432", "sample") if err != nil { return err } defer db.Close() ctx := context.Background() if err := db.Ping(ctx); err != nil { return err } process(db) return nil } func process(db *pg.DB) error { var users []User err := db.Model(&users).Select() if err != nil { return err } for _, user := range users { log.Printf("user: %v", user) } // Select story and associated author in one query. story := new(Story) err = db.Model(story). Relation("Author"). Where("story.id = ?", 1). Select() if err != nil { return err } log.Printf("story: %v", story) return nil } ================================================ FILE: _examples/tests/models.go ================================================ package main import "fmt" type User struct { Id int64 Name string Emails []string } func (u User) String() string { return fmt.Sprintf("User<%d %s %v>", u.Id, u.Name, u.Emails) } type Story struct { Id int64 Title string AuthorId int64 Author *User `pg:"rel:has-one"` } func (s Story) String() string { return fmt.Sprintf("Story<%d %s %s>", s.Id, s.Title, s.Author) } ================================================ FILE: _examples/tests/resources.yml ================================================ --- apiVersion: v1 kind: ConfigMap metadata: name: postgres labels: app: postgres data: POSTGRES_DB: postgres POSTGRES_USER: postgres POSTGRES_PASSWORD: Welcome POSTGRES_HOST_AUTH_METHOD: trust ... --- apiVersion: apps/v1 kind: Deployment metadata: name: postgres labels: app: postgres spec: replicas: 1 selector: matchLabels: app: postgres template: metadata: labels: app: postgres spec: containers: - name: postgres image: postgres imagePullPolicy: IfNotPresent ports: - containerPort: 5432 envFrom: - configMapRef: name: postgres ... --- apiVersion: v1 kind: Service metadata: name: postgres labels: app: postgres spec: type: NodePort ports: - port: 5432 nodePort: 30950 selector: app: postgres ... ================================================ FILE: _examples/toxics/README.md ================================================ ## Create Toxic Example how to start building own toxics. ### Debug toxic Run custom toxiserver with DebugToxic. ```shell $ go run debug_toxic.go ``` Run redis-server in separate terminal: ```shell $ redis-server ``` Test toxic with: ```shell $ toxiproxy-cli --host "http://localhost:8484" create -l :16379 -u localhost:6379 redis $ toxiproxy-cli --host "http://localhost:8484" toxic add --type debug redis $ redis-cli -p 16379 "keys" "*" ``` Custom Toxiproxy should print bytes in hex format. ### HTTP toxic Run custom toxiserver with DebugToxic. ```shell $ go run http_toxic.go ``` Test toxic with command and verify output: ```shell $ toxiproxy-cli --host "http://localhost:8484" create -l :18080 -u example.com:80 example $ toxiproxy-cli --host "http://localhost:8484" toxic add --type http example $ curl -v localhost:18080/hello ... < HTTP/1.1 404 Not Found < Location: https://github.com/Shopify/toxiproxy ``` ================================================ FILE: _examples/toxics/debug_toxic.go ================================================ // Ported from https://github.com/xthexder/toxic-example/blob/master/noop.go package main import ( "os" "log" "fmt" "io" "github.com/prometheus/client_golang/prometheus" "github.com/rs/zerolog" "github.com/Shopify/toxiproxy/v2" "github.com/Shopify/toxiproxy/v2/toxics" "github.com/Shopify/toxiproxy/v2/stream" ) // DebugToxic prints bytes processed through pipe. type DebugToxic struct{} func (t *DebugToxic) PrintHex(data []byte) { for i := 0; i < len(data); { for j := 0; j < 4; j +=1 { x := i + 8 if x >= len(data) { x = len(data) - 1 fmt.Printf("% x\n", data[i:x]) return } fmt.Printf("% x\t\t", data[i:x]) i = x } fmt.Println() } } func (t *DebugToxic) Pipe(stub *toxics.ToxicStub) { buf := make([]byte, 32*1024) writer := stream.NewChanWriter(stub.Output) reader := stream.NewChanReader(stub.Input) reader.SetInterrupt(stub.Interrupt) for { n, err := reader.Read(buf) log.Printf("-- [DebugToxic] Processed %d bytes\n", n) if err == stream.ErrInterrupted { writer.Write(buf[:n]) return } else if err == io.EOF { stub.Close() return } t.PrintHex(buf[:n]) writer.Write(buf[:n]) } } func main() { toxics.Register("debug", new(DebugToxic)) logger := zerolog.New(os.Stderr).With().Caller().Timestamp().Logger() metrics := toxiproxy.NewMetricsContainer(prometheus.NewRegistry()) server := toxiproxy.NewServer(metrics, logger) server.Listen("0.0.0.0:8484") } ================================================ FILE: _examples/toxics/http_toxic.go ================================================ // Ported from https://github.com/xthexder/toxic-example/blob/master/http.go package main import ( "bufio" "bytes" "io" "net/http" "os" "github.com/prometheus/client_golang/prometheus" "github.com/rs/zerolog" "github.com/Shopify/toxiproxy/v2" "github.com/Shopify/toxiproxy/v2/stream" "github.com/Shopify/toxiproxy/v2/toxics" ) type HttpToxic struct{} func (t *HttpToxic) ModifyResponse(resp *http.Response) { resp.Header.Set("Location", "https://github.com/Shopify/toxiproxy") } func (t *HttpToxic) Pipe(stub *toxics.ToxicStub) { buffer := bytes.NewBuffer(make([]byte, 0, 32*1024)) writer := stream.NewChanWriter(stub.Output) reader := stream.NewChanReader(stub.Input) reader.SetInterrupt(stub.Interrupt) for { tee := io.TeeReader(reader, buffer) resp, err := http.ReadResponse(bufio.NewReader(tee), nil) if err == stream.ErrInterrupted { buffer.WriteTo(writer) return } else if err == io.EOF { stub.Close() return } if err != nil { buffer.WriteTo(writer) } else { t.ModifyResponse(resp) resp.Write(writer) } buffer.Reset() } } func main() { toxics.Register("http", new(HttpToxic)) logger := zerolog.New(os.Stderr).With().Caller().Timestamp().Logger() metrics := toxiproxy.NewMetricsContainer(prometheus.NewRegistry()) server := toxiproxy.NewServer(metrics, logger) server.Listen("0.0.0.0:8484") } ================================================ FILE: api.go ================================================ package toxiproxy import ( "context" "encoding/json" "fmt" "net/http" "os" "strings" "time" "github.com/gorilla/mux" "github.com/rs/zerolog" "github.com/rs/zerolog/hlog" "github.com/Shopify/toxiproxy/v2/toxics" ) func stopBrowsersMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if strings.HasPrefix(r.UserAgent(), "Mozilla/") { http.Error(w, "User agent not allowed", http.StatusForbidden) } else { next.ServeHTTP(w, r) } }) } func timeoutMiddleware(next http.Handler) http.Handler { return http.TimeoutHandler(next, 25*time.Second, "") } type ApiServer struct { Collection *ProxyCollection Metrics *metricsContainer Logger *zerolog.Logger http *http.Server } const ( wait_timeout = 30 * time.Second read_timeout = 15 * time.Second ) func NewServer(m *metricsContainer, logger zerolog.Logger) *ApiServer { return &ApiServer{ Collection: NewProxyCollection(), Metrics: m, Logger: &logger, } } func (server *ApiServer) Listen(addr string) error { server.Logger. Info(). Str("address", addr). Msg("Starting Toxiproxy HTTP server") server.http = &http.Server{ Addr: addr, Handler: server.Routes(), WriteTimeout: wait_timeout, ReadTimeout: read_timeout, IdleTimeout: 60 * time.Second, } err := server.http.ListenAndServe() if err == http.ErrServerClosed { err = nil } return err } func (server *ApiServer) Shutdown() error { if server.http == nil { return nil } ctx, cancel := context.WithTimeout(context.Background(), wait_timeout) defer cancel() err := server.http.Shutdown(ctx) if err != nil { return err } return nil } func (server *ApiServer) Routes() *mux.Router { r := mux.NewRouter() r.Use(hlog.NewHandler(*server.Logger)) r.Use(hlog.RequestIDHandler("request_id", "X-Toxiproxy-Request-Id")) r.Use(hlog.AccessHandler(func(r *http.Request, status, size int, duration time.Duration) { handler := mux.CurrentRoute(r).GetName() zerolog.Ctx(r.Context()). Debug(). Str("client", r.RemoteAddr). Str("method", r.Method). Stringer("url", r.URL). Str("user_agent", r.Header.Get("User-Agent")). Int("status", status). Int("size", size). Dur("duration", duration). Str("handler", handler). Msg("") })) r.Use(stopBrowsersMiddleware) r.Use(timeoutMiddleware) r.HandleFunc("/reset", server.ResetState).Methods("POST"). Name("ResetState") r.HandleFunc("/proxies", server.ProxyIndex).Methods("GET"). Name("ProxyIndex") r.HandleFunc("/proxies", server.ProxyCreate).Methods("POST"). Name("ProxyCreate") r.HandleFunc("/populate", server.Populate).Methods("POST"). Name("Populate") r.HandleFunc("/proxies/{proxy}", server.ProxyShow).Methods("GET"). Name("ProxyShow") r.HandleFunc("/proxies/{proxy}", server.ProxyUpdate).Methods("POST", "PATCH"). Name("ProxyUpdate") r.HandleFunc("/proxies/{proxy}", server.ProxyDelete).Methods("DELETE"). Name("ProxyDelete") r.HandleFunc("/proxies/{proxy}/toxics", server.ToxicIndex).Methods("GET"). Name("ToxicIndex") r.HandleFunc("/proxies/{proxy}/toxics", server.ToxicCreate).Methods("POST"). Name("ToxicCreate") r.HandleFunc("/proxies/{proxy}/toxics/{toxic}", server.ToxicShow).Methods("GET"). Name("ToxicShow") r.HandleFunc("/proxies/{proxy}/toxics/{toxic}", server.ToxicUpdate).Methods("POST", "PATCH"). Name("ToxicUpdate") r.HandleFunc("/proxies/{proxy}/toxics/{toxic}", server.ToxicDelete).Methods("DELETE"). Name("ToxicDelete") r.HandleFunc("/version", server.Version).Methods("GET").Name("Version") if server.Metrics.anyMetricsEnabled() { r.Handle("/metrics", server.Metrics.handler()).Name("Metrics") } return r } func (server *ApiServer) PopulateConfig(filename string) { file, err := os.Open(filename) logger := server.Logger if err != nil { logger.Err(err).Str("config", filename).Msg("Error reading config file") return } proxies, err := server.Collection.PopulateJson(server, file) if err != nil { logger.Err(err).Msg("Failed to populate proxies from file") } else { logger.Info().Int("proxies", len(proxies)).Msg("Populated proxies from file") } } func (server *ApiServer) ProxyIndex(response http.ResponseWriter, request *http.Request) { proxies := server.Collection.Proxies() marshalData := make(map[string]interface{}, len(proxies)) for name, proxy := range proxies { marshalData[name] = proxyWithToxics(proxy) } data, err := json.Marshal(marshalData) if server.apiError(response, err) { return } response.Header().Set("Content-Type", "application/json") _, err = response.Write(data) if err != nil { log := zerolog.Ctx(request.Context()) log.Warn().Err(err).Msg("ProxyIndex: Failed to write response to client") } } func (server *ApiServer) ResetState(response http.ResponseWriter, request *http.Request) { ctx := request.Context() proxies := server.Collection.Proxies() for _, proxy := range proxies { err := proxy.Start() if err != ErrProxyAlreadyStarted && server.apiError(response, err) { return } proxy.Toxics.ResetToxics(ctx) } response.WriteHeader(http.StatusNoContent) _, err := response.Write(nil) if err != nil { log := zerolog.Ctx(ctx) log.Warn().Err(err).Msg("ResetState: Failed to write headers to client") } } func (server *ApiServer) ProxyCreate(response http.ResponseWriter, request *http.Request) { // Default fields to enable the proxy right away input := Proxy{Enabled: true} err := json.NewDecoder(request.Body).Decode(&input) if server.apiError(response, joinError(err, ErrBadRequestBody)) { return } if len(input.Name) < 1 { server.apiError(response, joinError(fmt.Errorf("name"), ErrMissingField)) return } if len(input.Upstream) < 1 { server.apiError(response, joinError(fmt.Errorf("upstream"), ErrMissingField)) return } proxy := NewProxy(server, input.Name, input.Listen, input.Upstream) err = server.Collection.Add(proxy, input.Enabled) if server.apiError(response, err) { return } data, err := json.Marshal(proxyWithToxics(proxy)) if server.apiError(response, err) { return } response.Header().Set("Content-Type", "application/json") response.WriteHeader(http.StatusCreated) _, err = response.Write(data) if err != nil { log := zerolog.Ctx(request.Context()) log.Warn().Err(err).Msg("ProxyCreate: Failed to write response to client") } } func (server *ApiServer) Populate(response http.ResponseWriter, request *http.Request) { proxies, err := server.Collection.PopulateJson(server, request.Body) log := zerolog.Ctx(request.Context()) if err != nil { log.Warn().Err(err).Msg("Populate errors") } apiErr, ok := err.(*ApiError) if !ok && err != nil { log.Warn().Err(err).Msg("Error did not include status code") apiErr = &ApiError{err.Error(), http.StatusInternalServerError} } data, err := json.Marshal(struct { *ApiError `json:",omitempty"` Proxies []proxyToxics `json:"proxies"` }{apiErr, proxiesWithToxics(proxies)}) if server.apiError(response, err) { return } responseCode := http.StatusCreated if apiErr != nil { responseCode = apiErr.StatusCode } response.Header().Set("Content-Type", "application/json") response.WriteHeader(responseCode) _, err = response.Write(data) if err != nil { log.Warn().Err(err).Msg("Populate: Failed to write response to client") } } func (server *ApiServer) ProxyShow(response http.ResponseWriter, request *http.Request) { vars := mux.Vars(request) proxy, err := server.Collection.Get(vars["proxy"]) if server.apiError(response, err) { return } data, err := json.Marshal(proxyWithToxics(proxy)) if server.apiError(response, err) { return } response.Header().Set("Content-Type", "application/json") _, err = response.Write(data) if err != nil { server.Logger.Warn().Err(err).Msg("ProxyShow: Failed to write response to client") } } func (server *ApiServer) ProxyUpdate(response http.ResponseWriter, request *http.Request) { log := zerolog.Ctx(request.Context()) if request.Method == "POST" { log.Warn().Msg("ProxyUpdate: HTTP method POST is deprecated. Use HTTP PATCH instead.") } vars := mux.Vars(request) proxy, err := server.Collection.Get(vars["proxy"]) if server.apiError(response, err) { return } // Default fields are the same as existing proxy input := Proxy{Listen: proxy.Listen, Upstream: proxy.Upstream, Enabled: proxy.Enabled} err = json.NewDecoder(request.Body).Decode(&input) if server.apiError(response, joinError(err, ErrBadRequestBody)) { return } err = proxy.Update(&input) if server.apiError(response, err) { return } data, err := json.Marshal(proxyWithToxics(proxy)) if server.apiError(response, err) { return } response.Header().Set("Content-Type", "application/json") _, err = response.Write(data) if err != nil { log.Warn().Err(err).Msg("ProxyUpdate: Failed to write response to client") } } func (server *ApiServer) ProxyDelete(response http.ResponseWriter, request *http.Request) { vars := mux.Vars(request) err := server.Collection.Remove(vars["proxy"]) if server.apiError(response, err) { return } response.WriteHeader(http.StatusNoContent) _, err = response.Write(nil) if err != nil { log := zerolog.Ctx(request.Context()) log.Warn().Err(err).Msg("ProxyDelete: Failed to write headers to client") } } func (server *ApiServer) ToxicIndex(response http.ResponseWriter, request *http.Request) { vars := mux.Vars(request) proxy, err := server.Collection.Get(vars["proxy"]) if server.apiError(response, err) { return } toxics := proxy.Toxics.GetToxicArray() data, err := json.Marshal(toxics) if server.apiError(response, err) { return } response.Header().Set("Content-Type", "application/json") _, err = response.Write(data) if err != nil { log := zerolog.Ctx(request.Context()) log.Warn().Err(err).Msg("ToxicIndex: Failed to write response to client") } } func (server *ApiServer) ToxicCreate(response http.ResponseWriter, request *http.Request) { vars := mux.Vars(request) proxy, err := server.Collection.Get(vars["proxy"]) if server.apiError(response, err) { return } toxic, err := proxy.Toxics.AddToxicJson(request.Body) if server.apiError(response, err) { return } data, err := json.Marshal(toxic) if server.apiError(response, err) { return } response.Header().Set("Content-Type", "application/json") _, err = response.Write(data) if err != nil { log := zerolog.Ctx(request.Context()) log.Warn().Err(err).Msg("ToxicCreate: Failed to write response to client") } } func (server *ApiServer) ToxicShow(response http.ResponseWriter, request *http.Request) { vars := mux.Vars(request) proxy, err := server.Collection.Get(vars["proxy"]) if server.apiError(response, err) { return } toxic := proxy.Toxics.GetToxic(vars["toxic"]) if toxic == nil { server.apiError(response, ErrToxicNotFound) return } data, err := json.Marshal(toxic) if server.apiError(response, err) { return } response.Header().Set("Content-Type", "application/json") _, err = response.Write(data) if err != nil { log := zerolog.Ctx(request.Context()) log.Warn().Err(err).Msg("ToxicShow: Failed to write response to client") } } func (server *ApiServer) ToxicUpdate(response http.ResponseWriter, request *http.Request) { log := zerolog.Ctx(request.Context()) if request.Method == "POST" { log.Warn().Msg("ToxicUpdate: HTTP method POST is deprecated. Use HTTP PATCH instead.") } vars := mux.Vars(request) proxy, err := server.Collection.Get(vars["proxy"]) if server.apiError(response, err) { return } toxic, err := proxy.Toxics.UpdateToxicJson(vars["toxic"], request.Body) if server.apiError(response, err) { return } data, err := json.Marshal(toxic) if server.apiError(response, err) { return } response.Header().Set("Content-Type", "application/json") _, err = response.Write(data) if err != nil { log.Warn().Err(err).Msg("ToxicUpdate: Failed to write response to client") } } func (server *ApiServer) ToxicDelete(response http.ResponseWriter, request *http.Request) { vars := mux.Vars(request) ctx := request.Context() log := zerolog.Ctx(ctx) proxy, err := server.Collection.Get(vars["proxy"]) if server.apiError(response, err) { return } err = proxy.Toxics.RemoveToxic(ctx, vars["toxic"]) if server.apiError(response, err) { return } response.WriteHeader(http.StatusNoContent) _, err = response.Write(nil) if err != nil { log.Warn().Err(err).Msg("ToxicDelete: Failed to write headers to client") } } func (server *ApiServer) Version(response http.ResponseWriter, request *http.Request) { log := zerolog.Ctx(request.Context()) response.Header().Set("Content-Type", "application/json;charset=utf-8") version := fmt.Sprintf("{\"version\": \"%s\"}\n", Version) _, err := response.Write([]byte(version)) if err != nil { log.Warn().Err(err).Msg("Version: Failed to write response to client") } } type ApiError struct { Message string `json:"error"` StatusCode int `json:"status"` } func (e *ApiError) Error() string { return e.Message } func newError(msg string, status int) *ApiError { return &ApiError{msg, status} } func joinError(err error, wrapper *ApiError) *ApiError { if err != nil { return &ApiError{wrapper.Message + ": " + err.Error(), wrapper.StatusCode} } return nil } var ( ErrBadRequestBody = newError("bad request body", http.StatusBadRequest) ErrMissingField = newError("missing required field", http.StatusBadRequest) ErrProxyNotFound = newError("proxy not found", http.StatusNotFound) ErrProxyAlreadyExists = newError("proxy already exists", http.StatusConflict) ErrInvalidStream = newError( "stream was invalid, can be either upstream or downstream", http.StatusBadRequest, ) ErrInvalidToxicType = newError("invalid toxic type", http.StatusBadRequest) ErrToxicAlreadyExists = newError("toxic already exists", http.StatusConflict) ErrToxicNotFound = newError("toxic not found", http.StatusNotFound) ) func (server *ApiServer) apiError(resp http.ResponseWriter, err error) bool { obj, ok := err.(*ApiError) if !ok && err != nil { server.Logger.Warn().Err(err).Msg("Error did not include status code") obj = &ApiError{err.Error(), http.StatusInternalServerError} } if obj == nil { return false } data, err2 := json.Marshal(obj) if err2 != nil { server.Logger.Warn().Err(err2).Msg("Error json encoding error (╯°□°）╯︵ ┻━┻ ") } resp.Header().Set("Content-Type", "application/json") http.Error(resp, string(data), obj.StatusCode) return true } type proxyToxics struct { *Proxy Toxics []toxics.Toxic `json:"toxics"` } func proxyWithToxics(proxy *Proxy) (result proxyToxics) { result.Proxy = proxy result.Toxics = proxy.Toxics.GetToxicArray() return } func proxiesWithToxics(proxies []*Proxy) (result []proxyToxics) { for _, proxy := range proxies { result = append(result, proxyWithToxics(proxy)) } return } ================================================ FILE: api_test.go ================================================ package toxiproxy_test import ( "bytes" "flag" "io" "net/http" "os" "testing" "time" "github.com/prometheus/client_golang/prometheus" "github.com/rs/zerolog" "github.com/Shopify/toxiproxy/v2" tclient "github.com/Shopify/toxiproxy/v2/client" ) var testServer *toxiproxy.ApiServer var client = tclient.NewClient("http://127.0.0.1:8475") func WithServer(t *testing.T, f func(string)) { log := zerolog.Nop() if flag.Lookup("test.v").DefValue == "true" { log = zerolog.New(os.Stdout).With().Caller().Timestamp().Logger() } // Make sure only one server is running at a time. Apparently there's no clean // way to shut it down between each test run. if testServer == nil { testServer = toxiproxy.NewServer( toxiproxy.NewMetricsContainer(prometheus.NewRegistry()), log, ) go testServer.Listen("localhost:8475") // Allow server to start. There's no clean way to know when it listens. time.Sleep(50 * time.Millisecond) } defer func() { err := testServer.Collection.Clear() if err != nil { t.Error("Failed to clear collection", err) } }() f("http://localhost:8475") } func TestRequestId(t *testing.T) { WithServer(t, func(addr string) { client := http.Client{} req, _ := http.NewRequest("GET", "http://localhost:8475/version", nil) req.Header.Add("User-Agent", "curl") resp, err := client.Do(req) if err != nil { t.Fatalf("Does not expect errors from client: %+v", err) } defer resp.Body.Close() if _, ok := resp.Header["X-Toxiproxy-Request-Id"]; !ok { t.Fatalf("Expect http response with header X-Toxiproxy-Request-Id, got %+v", resp.Header) } }) } func TestBrowserGets403(t *testing.T) { WithServer(t, func(addr string) { client := http.Client{} req, _ := http.NewRequest("GET", "http://localhost:8475/proxies", nil) req.Header.Add( "User-Agent", "Mozilla/5.0 (Linux; Android 4.4.2); Nexus 5 Build/KOT49H) AppleWebKit/537.36"+ "(KHTML, like Gecko) Chrome/33.0.1750.117 Mobile Safari/537.36 OPR/20.0.1396.72047", ) resp, err := client.Do(req) if err != nil { t.Fatalf("Does not expect errors from client: %v", err) } defer resp.Body.Close() if resp.StatusCode != 403 { t.Fatal("Browser-like UserAgent was not denied access to Toxiproxy") } }) } func TestNonBrowserGets200(t *testing.T) { WithServer(t, func(addr string) { client := http.Client{} req, _ := http.NewRequest("GET", "http://localhost:8475/proxies", nil) req.Header.Add("User-Agent", "Wget/2.1") resp, err := client.Do(req) if err != nil { t.Fatalf("Does not expect errors from client: %v", err) } defer resp.Body.Close() if resp.StatusCode == 403 { t.Fatal("Non-Browser-like UserAgent was denied access to Toxiproxy") } }) } func TestIndexWithNoProxies(t *testing.T) { WithServer(t, func(addr string) { client := tclient.NewClient(addr) proxies, err := client.Proxies() if err != nil { t.Fatal("Failed getting proxies:", err) } if len(proxies) > 0 { t.Fatal("Expected no proxies, got:", proxies) } }) } func TestCreateProxyBlankName(t *testing.T) { WithServer(t, func(addr string) { _, err := client.CreateProxy("", "", "") expected := "create: HTTP 400: missing required field: name" if err == nil { t.Error("Expected error creating proxy, got nil") } else if err.Error() != expected { t.Errorf("Expected error `%s',\n\tgot: `%s'", expected, err) } }) } func TestCreateProxyBlankUpstream(t *testing.T) { WithServer(t, func(addr string) { _, err := client.CreateProxy("test", "", "") if err == nil { t.Error("Expected error creating proxy, got nil") } else if err.Error() != "create: HTTP 400: missing required field: upstream" { t.Error("Expected different error creating proxy:", err) } }) } func TestPopulateProxy(t *testing.T) { WithServer(t, func(addr string) { testProxies, err := client.Populate([]tclient.Proxy{ { Name: "one", Listen: "localhost:7070", Upstream: "localhost:7171", Enabled: true, }, { Name: "two", Listen: "localhost:7373", Upstream: "localhost:7474", Enabled: true, }, }) if err != nil { t.Fatal("Unable to populate:", err) } if len(testProxies) != 2 { t.Fatalf("Wrong number of proxies returned: %d != 2", len(testProxies)) } if testProxies[0].Name != "one" || testProxies[1].Name != "two" { t.Fatalf("Wrong proxy names returned: %s, %s", testProxies[0].Name, testProxies[1].Name) } for _, p := range testProxies { AssertProxyUp(t, p.Listen, true) } }) } func TestPopulateDefaultEnabled(t *testing.T) { WithServer(t, func(addr string) { request := []byte( `[{"name": "test", "listen": "localhost:7070", "upstream": "localhost:7171"}]`, ) resp, err := http.Post(addr+"/populate", "application/json", bytes.NewReader(request)) if err != nil { t.Fatal("Failed to send POST to /populate:", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusCreated { message, _ := io.ReadAll(resp.Body) t.Fatalf("Failed to populate proxy list: HTTP %s\n%s", resp.Status, string(message)) } proxies, err := client.Proxies() if err != nil { t.Fatal(err) } else if len(proxies) != 1 { t.Fatalf("Wrong number of proxies created: %d != 1", len(proxies)) } else if _, ok := proxies["test"]; !ok { t.Fatalf("Wrong proxy name returned") } for _, p := range proxies { AssertProxyUp(t, p.Listen, true) } }) } func TestPopulateDisabledProxy(t *testing.T) { WithServer(t, func(addr string) { testProxies, err := client.Populate([]tclient.Proxy{ { Name: "one", Listen: "localhost:7070", Upstream: "localhost:7171", Enabled: false, }, { Name: "two", Listen: "localhost:7373", Upstream: "localhost:7474", Enabled: true, }, }) if err != nil { t.Fatal("Unable to populate:", err) } if len(testProxies) != 2 { t.Fatalf("Wrong number of proxies returned: %d != 2", len(testProxies)) } if testProxies[0].Name != "one" || testProxies[1].Name != "two" { t.Fatalf("Wrong proxy names returned: %s, %s", testProxies[0].Name, testProxies[1].Name) } AssertProxyUp(t, "localhost:7070", false) AssertProxyUp(t, "localhost:7373", true) }) } func TestPopulateExistingProxy(t *testing.T) { WithServer(t, func(addr string) { testProxy, err := client.CreateProxy("one", "localhost:7070", "localhost:7171") if err != nil { t.Fatal("Unable to create proxy:", err) } _, err = client.CreateProxy("two", "localhost:7373", "localhost:7474") if err != nil { t.Fatal("Unable to create proxy:", err) } // Create a toxic so we can make sure the proxy wasn't replaced _, err = testProxy.AddToxic("", "latency", "downstream", 1, nil) if err != nil { t.Fatal("Unable to create toxic:", err) } testProxies, err := client.Populate([]tclient.Proxy{ { Name: "one", Listen: "localhost:7070", // intentional: this should be resolved to 127.0.0.1:7070 Upstream: "localhost:7171", Enabled: true, }, { Name: "two", Listen: "127.0.0.1:7575", Upstream: "localhost:7676", Enabled: true, }, }) if err != nil { t.Fatal("Unable to populate:", err) } if len(testProxies) != 2 { t.Fatalf("Wrong number of proxies returned: %d != 2", len(testProxies)) } if testProxies[0].Name != "one" || testProxies[1].Name != "two" { t.Fatalf("Wrong proxy names returned: %s, %s", testProxies[0].Name, testProxies[1].Name) } if testProxies[0].Listen != "127.0.0.1:7070" || testProxies[1].Listen != "127.0.0.1:7575" { t.Fatalf("Wrong proxy listen addresses returned: %s, %s", testProxies[0].Listen, testProxies[1].Listen, ) } toxics, err := testProxy.Toxics() if err != nil { t.Fatal("Unable to get toxics:", err) } if len(toxics) != 1 || toxics[0].Type != "latency" { t.Fatalf("Populate did not preseve existing proxy. (%d toxics)", len(toxics)) } for _, p := range testProxies { AssertProxyUp(t, p.Listen, true) } }) } func TestPopulateWithBadName(t *testing.T) { WithServer(t, func(addr string) { testProxies, err := client.Populate([]tclient.Proxy{ { Name: "one", Listen: "localhost:7070", Upstream: "localhost:7171", Enabled: true, }, { Name: "", Listen: "", Enabled: true, }, }) if err == nil { t.Fatal("Expected Populate to fail.") } expected := "Populate: HTTP 400: missing required field: name at proxy 2" if err.Error() != expected { t.Fatalf("Expected error `%s',\n\tgot: `%s'", expected, err) } if len(testProxies) != 0 { t.Fatalf("Wrong number of proxies returned: %d != 0", len(testProxies)) } proxies, err := client.Proxies() if err != nil { t.Fatal(err) } else if len(proxies) != 0 { t.Fatalf("Expected no proxies to be created: %d != 0", len(proxies)) } }) } func TestPopulateProxyWithBadDataShouldReturnError(t *testing.T) { WithServer(t, func(addr string) { testProxies, err := client.Populate([]tclient.Proxy{ { Name: "one", Listen: "localhost:7070", Upstream: "localhost:7171", Enabled: true, }, { Name: "two", Listen: "local373", Upstream: "localhost:7474", Enabled: true, }, { Name: "three", Listen: "localhost:7575", Upstream: "localhost:7676", Enabled: true, }, }) if err == nil { t.Fatal("Expected Populate to fail.") } if len(testProxies) != 0 { t.Fatalf("Expected Proxies to be empty, got %v", testProxies) } proxies, err := client.Proxies() if err != nil { t.Fatalf("Expected no error, got: %v", err) } if len(proxies) != 1 { t.Fatalf("Wrong number of proxies returned: %d != %d", len(proxies), 1) } if _, ok := proxies["one"]; !ok { t.Fatal("Proxy `one' was not created!") } for _, p := range testProxies { AssertProxyUp(t, p.Listen, true) } for _, p := range proxies { if p.Name == "two" || p.Name == "three" { t.Fatalf("Proxy %s exists, populate did not fail correctly.", p.Name) } } }) } func TestPopulateAddToxic(t *testing.T) { WithServer(t, func(addr string) { testProxies, err := client.Populate([]tclient.Proxy{ { Name: "one", Listen: "localhost:7070", Upstream: "localhost:7171", Enabled: true, }, }) if err != nil { t.Fatal("Unable to populate:", err) } if len(testProxies) != 1 { t.Fatalf("Wrong number of proxies returned: %d != %d", len(testProxies), 1) } if testProxies[0].Name != "one" { t.Fatalf("Wrong proxy name returned: %s != one", testProxies[0].Name) } _, err = testProxies[0].AddToxic("", "latency", "downstream", 1, nil) if err != nil { t.Fatal("Failed to AddToxic.") } }) } func TestListingProxies(t *testing.T) { WithServer(t, func(addr string) { _, err := client.CreateProxy("mysql_master", "localhost:3310", "localhost:20001") if err != nil { t.Fatal("Unable to create proxy:", err) } proxies, err := client.Proxies() if err != nil { t.Fatal("Error listing proxies:", err) } if len(proxies) == 0 { t.Fatal("Expected new proxy in list") } proxy, ok := proxies["mysql_master"] if !ok { t.Fatal("Expected to see mysql_master proxy in list") } if proxy.Name != "mysql_master" || proxy.Listen != "127.0.0.1:3310" || proxy.Upstream != "localhost:20001" { t.Fatalf( "Unexpected proxy metadata: %s, %s, %s", proxy.Name, proxy.Listen, proxy.Upstream, ) } AssertToxicExists(t, proxy.ActiveToxics, "latency", "", "", false) }) } func TestCreateAndGetProxy(t *testing.T) { WithServer(t, func(addr string) { _, err := client.CreateProxy("mysql_master", "localhost:3310", "localhost:20001") if err != nil { t.Fatal("Unable to create proxy:", err) } proxy, err := client.Proxy("mysql_master") if err != nil { t.Fatal("Unable to retriecve proxy:", err) } if proxy.Name != "mysql_master" || proxy.Listen != "127.0.0.1:3310" || proxy.Upstream != "localhost:20001" || !proxy.Enabled { t.Fatalf( "Unexpected proxy metadata: %s, %s, %s, %v", proxy.Name, proxy.Listen, proxy.Upstream, proxy.Enabled, ) } AssertToxicExists(t, proxy.ActiveToxics, "latency", "", "", false) }) } func TestCreateProxyWithSave(t *testing.T) { WithServer(t, func(addr string) { testProxy := client.NewProxy() testProxy.Name = "mysql_master" testProxy.Listen = "localhost:3310" testProxy.Upstream = "localhost:20001" testProxy.Enabled = true err := testProxy.Save() if err != nil { t.Fatal("Unable to create proxy:", err) } proxy, err := client.Proxy("mysql_master") if err != nil { t.Fatal("Unable to retriecve proxy:", err) } if proxy.Name != "mysql_master" || proxy.Listen != "127.0.0.1:3310" || proxy.Upstream != "localhost:20001" || !proxy.Enabled { t.Fatalf( "Unexpected proxy metadata: %s, %s, %s, %v", proxy.Name, proxy.Listen, proxy.Upstream, proxy.Enabled, ) } AssertProxyUp(t, proxy.Listen, true) }) } func TestCreateDisabledProxy(t *testing.T) { WithServer(t, func(addr string) { disabledProxy := client.NewProxy() disabledProxy.Name = "mysql_master" disabledProxy.Listen = "localhost:3310" disabledProxy.Upstream = "localhost:20001" err := disabledProxy.Save() if err != nil { t.Fatal("Unable to create proxy:", err) } proxy, err := client.Proxy("mysql_master") if err != nil { t.Fatal("Unable to retriecve proxy:", err) } if proxy.Name != "mysql_master" || proxy.Listen != "localhost:3310" || proxy.Upstream != "localhost:20001" || proxy.Enabled { t.Fatalf( "Unexpected proxy metadata: %s, %s, %s, %v", proxy.Name, proxy.Listen, proxy.Upstream, proxy.Enabled, ) } AssertProxyUp(t, proxy.Listen, false) }) } func TestCreateDisabledProxyAndEnable(t *testing.T) { WithServer(t, func(addr string) { disabledProxy := client.NewProxy() disabledProxy.Name = "mysql_master" disabledProxy.Listen = "localhost:3310" disabledProxy.Upstream = "localhost:20001" err := disabledProxy.Save() if err != nil { t.Fatal("Unable to create proxy:", err) } proxy, err := client.Proxy("mysql_master") if err != nil { t.Fatal("Unable to retriecve proxy:", err) } if proxy.Name != "mysql_master" || proxy.Listen != "localhost:3310" || proxy.Upstream != "localhost:20001" || proxy.Enabled { t.Fatalf( "Unexpected proxy metadata: %s, %s, %s, %v", proxy.Name, proxy.Listen, proxy.Upstream, proxy.Enabled, ) } proxy.Enabled = true err = proxy.Save() if err != nil { t.Fatal("Failed to update proxy:", err) } AssertProxyUp(t, proxy.Listen, true) proxy.Enabled = false err = proxy.Save() if err != nil { t.Fatal("Failed to update proxy:", err) } AssertProxyUp(t, proxy.Listen, false) }) } func TestDeleteProxy(t *testing.T) { WithServer(t, func(addr string) { testProxy, err := client.CreateProxy("mysql_master", "localhost:3310", "localhost:20001") if err != nil { t.Fatal("Unable to create proxy:", err) } proxies, err := client.Proxies() if err != nil { t.Fatal("Error listing proxies:", err) } if len(proxies) == 0 { t.Fatal("Expected new proxy in list") } AssertProxyUp(t, testProxy.Listen, true) err = testProxy.Delete() if err != nil { t.Fatal("Failed deleting proxy:", err) } AssertProxyUp(t, testProxy.Listen, false) proxies, err = client.Proxies() if err != nil { t.Fatal("Error listing proxies:", err) } if len(proxies) > 0 { t.Fatal("Expected proxy to be deleted from list") } expected := "Delete: HTTP 404: proxy not found" err = testProxy.Delete() if err == nil { t.Error("Proxy did not result in not found.") } else if err.Error() != expected { t.Errorf("Expected error `%s',\n\tgot: `%s'", expected, err) } }) } func TestCreateProxyPortConflict(t *testing.T) { WithServer(t, func(addr string) { testProxy, err := client.CreateProxy("mysql_master", "localhost:3310", "localhost:20001") if err != nil { t.Fatal("Unable to create proxy:", err) } expected := "create: HTTP 500: listen tcp 127.0.0.1:3310: bind: address already in use" _, err = client.CreateProxy("test", "localhost:3310", "localhost:20001") if err == nil { t.Error("Proxy did not result in conflict.") } else if err.Error() != expected { t.Errorf("Expected error `%s',\n\tgot: `%s'", expected, err) } err = testProxy.Delete() if err != nil { t.Fatal("Unable to delete proxy:", err) } _, err = client.CreateProxy("test", "localhost:3310", "localhost:20001") if err != nil { t.Fatal("Unable to create proxy:", err) } }) } func TestCreateProxyNameConflict(t *testing.T) { WithServer(t, func(addr string) { testProxy, err := client.CreateProxy("mysql_master", "localhost:3310", "localhost:20001") if err != nil { t.Fatal("Unable to create proxy:", err) } expected := "create: HTTP 409: proxy already exists" _, err = client.CreateProxy("mysql_master", "localhost:3311", "localhost:20001") if err == nil { t.Fatal("Proxy did not result in conflict.") } else if err.Error() != expected { t.Fatalf("Expected error `%s',\n\tgot: `%s'", expected, err) } err = testProxy.Delete() if err != nil { t.Fatal("Unable to delete proxy:", err) } _, err = client.CreateProxy("mysql_master", "localhost:3311", "localhost:20001") if err != nil { t.Fatal("Unable to create proxy:", err) } }) } func TestResetState(t *testing.T) { WithServer(t, func(addr string) { testProxy, err := client.CreateProxy("mysql_master", "localhost:3310", "localhost:20001") if err != nil { t.Fatal("Unable to create proxy:", err) } latency, err := testProxy.AddToxic("", "latency", "downstream", 1, tclient.Attributes{ "latency": 100, "jitter": 10, }) if err != nil { t.Fatal("Error setting toxic:", err) } if latency.Attributes["latency"] != 100.0 || latency.Attributes["jitter"] != 10.0 { t.Fatal("Latency toxic did not start up with correct settings") } err = client.ResetState() if err != nil { t.Fatal("unable to reset state:", err) } proxies, err := client.Proxies() if err != nil { t.Fatal("Error listing proxies:", err) } proxy, ok := proxies["mysql_master"] if !ok { t.Fatal("Expected proxy to still exist") } if !proxy.Enabled { t.Fatal("Expected proxy to be enabled") } toxics, err := proxy.Toxics() if err != nil { t.Fatal("Error requesting toxics:", err) } AssertToxicExists(t, toxics, "latency", "", "", false) AssertProxyUp(t, proxy.Listen, true) }) } func TestListingToxics(t *testing.T) { WithServer(t, func(addr string) { testProxy, err := client.CreateProxy("mysql_master", "localhost:3310", "localhost:20001") if err != nil { t.Fatal("Unable to create proxy:", err) } toxics, err := testProxy.Toxics() if err != nil { t.Fatal("Error returning toxics:", err) } AssertToxicExists(t, toxics, "latency", "", "", false) }) } func TestAddToxic(t *testing.T) { WithServer(t, func(addr string) { testProxy, err := client.CreateProxy("mysql_master", "localhost:3310", "localhost:20001") if err != nil { t.Fatal("Unable to create proxy:", err) } latency, err := testProxy.AddToxic("foobar", "latency", "downstream", 1, tclient.Attributes{ "latency": 100, "jitter": 10, }) if err != nil { t.Fatal("Error setting toxic:", err) } if latency.Attributes["latency"] != 100.0 || latency.Attributes["jitter"] != 10.0 { t.Fatal("Latency toxic did not start up with correct settings") } toxics, err := testProxy.Toxics() if err != nil { t.Fatal("Error returning toxics:", err) } toxic := AssertToxicExists(t, toxics, "foobar", "latency", "downstream", true) if toxic.Toxicity != 1.0 || toxic.Attributes["latency"] != 100.0 || toxic.Attributes["jitter"] != 10.0 { t.Fatal("Toxic was not read back correctly:", toxic) } }) } func TestAddMultipleToxics(t *testing.T) { WithServer(t, func(addr string) { testProxy, err := client.CreateProxy("mysql_master", "localhost:3310", "localhost:20001") if err != nil { t.Fatal("Unable to create proxy:", err) } _, err = testProxy.AddToxic("latency1", "latency", "downstream", 1, nil) if err != nil { t.Fatal("Error setting toxic:", err) } _, err = testProxy.AddToxic("latency2", "latency", "downstream", 1, nil) if err != nil { t.Fatal("Error setting toxic:", err) } toxics, err := testProxy.Toxics() if err != nil { t.Fatal("Error returning toxics:", err) } AssertToxicExists(t, toxics, "latency1", "latency", "downstream", true) toxic := AssertToxicExists(t, toxics, "latency2", "latency", "downstream", true) if toxic.Toxicity != 1.0 || toxic.Attributes["latency"] != 0.0 || toxic.Attributes["jitter"] != 0.0 { t.Fatal("Toxic was not read back correctly:", toxic) } AssertToxicExists(t, toxics, "latency1", "", "upstream", false) AssertToxicExists(t, toxics, "latency2", "", "upstream", false) }) } func TestAddConflictingToxic(t *testing.T) { WithServer(t, func(addr string) { testProxy, err := client.CreateProxy("mysql_master", "localhost:3310", "localhost:20001") if err != nil { t.Fatal("Unable to create proxy:", err) } _, err = testProxy.AddToxic("foobar", "latency", "downstream", 1, nil) if err != nil { t.Fatal("Error setting toxic:", err) } _, err = testProxy.AddToxic("foobar", "slow_close", "downstream", 1, nil) if err == nil { t.Fatal("Toxic did not result in conflict.") } else if err.Error() != "AddToxic: HTTP 409: toxic already exists" { t.Fatal("Incorrect error setting toxic:", err) } toxics, err := testProxy.Toxics() if err != nil { t.Fatal("Error returning toxics:", err) } toxic := AssertToxicExists(t, toxics, "foobar", "latency", "downstream", true) if toxic.Toxicity != 1.0 || toxic.Attributes["latency"] != 0.0 || toxic.Attributes["jitter"] != 0.0 { t.Fatal("Toxic was not read back correctly:", toxic) } AssertToxicExists(t, toxics, "foobar", "", "upstream", false) }) } func TestAddConflictingToxicsMultistream(t *testing.T) { WithServer(t, func(addr string) { testProxy, err := client.CreateProxy("mysql_master", "localhost:3310", "localhost:20001") if err != nil { t.Fatal("Unable to create proxy:", err) } _, err = testProxy.AddToxic("foobar", "latency", "upstream", 1, nil) if err != nil { t.Fatal("Error setting toxic:", err) } _, err = testProxy.AddToxic("foobar", "latency", "downstream", 1, nil) if err == nil { t.Fatal("Toxic did not result in conflict.") } else if err.Error() != "AddToxic: HTTP 409: toxic already exists" { t.Fatal("Incorrect error setting toxic:", err) } toxics, err := testProxy.Toxics() if err != nil { t.Fatal("Error returning toxics:", err) } toxic := AssertToxicExists(t, toxics, "foobar", "latency", "upstream", true) if toxic.Toxicity != 1.0 || toxic.Attributes["latency"] != 0.0 || toxic.Attributes["jitter"] != 0.0 { t.Fatal("Toxic was not read back correctly:", toxic) } AssertToxicExists(t, toxics, "foobar", "", "downstream", false) }) } func TestAddConflictingToxicsMultistreamDefaults(t *testing.T) { WithServer(t, func(addr string) { testProxy, err := client.CreateProxy("mysql_master", "localhost:3310", "localhost:20001") if err != nil { t.Fatal("Unable to create proxy:", err) } _, err = testProxy.AddToxic("", "latency", "upstream", 1, nil) if err != nil { t.Fatal("Error setting toxic:", err) } _, err = testProxy.AddToxic("", "latency", "downstream", 1, nil) if err != nil { t.Fatal("Error setting toxic:", err) } toxics, err := testProxy.Toxics() if err != nil { t.Fatal("Error returning toxics:", err) } toxic := AssertToxicExists(t, toxics, "latency_upstream", "latency", "upstream", true) if toxic.Toxicity != 1.0 || toxic.Attributes["latency"] != 0.0 || toxic.Attributes["jitter"] != 0.0 { t.Fatal("Toxic was not read back correctly:", toxic) } toxic = AssertToxicExists(t, toxics, "latency_downstream", "latency", "downstream", true) if toxic.Toxicity != 1.0 || toxic.Attributes["latency"] != 0.0 || toxic.Attributes["jitter"] != 0.0 { t.Fatal("Toxic was not read back correctly:", toxic) } }) } func TestAddToxicWithToxicity(t *testing.T) { WithServer(t, func(addr string) { testProxy, err := client.CreateProxy("mysql_master", "localhost:3310", "localhost:20001") if err != nil { t.Fatal("Unable to create proxy:", err) } latency, err := testProxy.AddToxic("", "latency", "downstream", 0.2, tclient.Attributes{ "latency": 100, "jitter": 10, }) if err != nil { t.Fatal("Error setting toxic:", err) } if latency.Toxicity != 0.2 || latency.Attributes["latency"] != 100.0 || latency.Attributes["jitter"] != 10.0 { t.Fatal("Latency toxic did not start up with correct settings:", latency) } toxics, err := testProxy.Toxics() if err != nil { t.Fatal("Error returning toxics:", err) } toxic := AssertToxicExists(t, toxics, "latency_downstream", "latency", "downstream", true) if toxic.Toxicity != 0.2 || toxic.Attributes["latency"] != 100.0 || toxic.Attributes["jitter"] != 10.0 { t.Fatal("Toxic was not read back correctly:", toxic) } }) } func TestAddNoop(t *testing.T) { WithServer(t, func(addr string) { testProxy, err := client.CreateProxy("mysql_master", "localhost:3310", "localhost:20001") if err != nil { t.Fatal("Unable to create proxy:", err) } noop, err := testProxy.AddToxic("foobar", "noop", "", 1, nil) if err != nil { t.Fatal("Error setting toxic:", err) } if noop.Toxicity != 1.0 || noop.Name != "foobar" || noop.Type != "noop" || noop.Stream != "downstream" { t.Fatal("Noop toxic did not start up with correct settings:", noop) } toxics, err := testProxy.Toxics() if err != nil { t.Fatal("Error returning toxics:", err) } toxic := AssertToxicExists(t, toxics, "foobar", "noop", "downstream", true) if toxic.Toxicity != 1.0 { t.Fatal("Toxic was not read back correctly:", toxic) } }) } func TestUpdateToxics(t *testing.T) { WithServer(t, func(addr string) { testProxy, err := client.CreateProxy("mysql_master", "localhost:3310", "localhost:20001") if err != nil { t.Fatal("Unable to create proxy:", err) } latency, err := testProxy.AddToxic("", "latency", "downstream", -1, tclient.Attributes{ "latency": 100, "jitter": 10, }) if err != nil { t.Fatal("Error setting toxic:", err) } if latency.Toxicity != 1.0 || latency.Attributes["latency"] != 100.0 || latency.Attributes["jitter"] != 10.0 { t.Fatal("Latency toxic did not start up with correct settings:", latency) } latency, err = testProxy.UpdateToxic("latency_downstream", 0.5, tclient.Attributes{ "latency": 1000, }) if err != nil { t.Fatal("Error setting toxic:", err) } if latency.Toxicity != 0.5 || latency.Attributes["latency"] != 1000.0 || latency.Attributes["jitter"] != 10.0 { t.Fatal("Latency toxic did not get updated with the correct settings:", latency) } latency, err = testProxy.UpdateToxic("latency_downstream", -1, tclient.Attributes{ "latency": 500, }) if err != nil { t.Fatal("Error setting toxic:", err) } if latency.Toxicity != 0.5 || latency.Attributes["latency"] != 500.0 || latency.Attributes["jitter"] != 10.0 { t.Fatal("Latency toxic did not get updated with the correct settings:", latency) } toxics, err := testProxy.Toxics() if err != nil { t.Fatal("Error returning toxics:", err) } toxic := AssertToxicExists(t, toxics, "latency_downstream", "latency", "downstream", true) if toxic.Toxicity != 0.5 || toxic.Attributes["latency"] != 500.0 || toxic.Attributes["jitter"] != 10.0 { t.Fatal("Toxic was not read back correctly:", toxic) } }) } func TestRemoveToxic(t *testing.T) { WithServer(t, func(addr string) { testProxy, err := client.CreateProxy("mysql_master", "localhost:3310", "localhost:20001") if err != nil { t.Fatal("Unable to create proxy:", err) } _, err = testProxy.AddToxic("", "latency", "downstream", 1, nil) if err != nil { t.Fatal("Error setting toxic:", err) } toxics, err := testProxy.Toxics() if err != nil { t.Fatal("Error returning toxics:", err) } toxic := AssertToxicExists(t, toxics, "latency_downstream", "latency", "downstream", true) if toxic.Toxicity != 1.0 || toxic.Attributes["latency"] != 0.0 || toxic.Attributes["jitter"] != 0.0 { t.Fatal("Toxic was not read back correctly:", toxic) } err = testProxy.RemoveToxic("latency_downstream") if err != nil { t.Fatal("Error removing toxic:", err) } toxics, err = testProxy.Toxics() if err != nil { t.Fatal("Error returning toxics:", err) } AssertToxicExists(t, toxics, "latency_downstream", "", "", false) }) } func TestVersionEndpointReturnsVersion(t *testing.T) { WithServer(t, func(addr string) { resp, err := http.Get(addr + "/version") if err != nil { t.Fatal("Failed to get index", err) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { t.Fatal("Unable to read body from response") } if string(body) != "{\"version\": \"git\"}\n" { t.Fatal("Expected to return Version from /version, got:", string(body)) } }) } func TestInvalidStream(t *testing.T) { WithServer(t, func(addr string) { testProxy, err := client.CreateProxy("mysql_master", "localhost:3310", "localhost:20001") if err != nil { t.Fatal("Unable to create proxy:", err) } _, err = testProxy.AddToxic("", "latency", "walrustream", 1, nil) if err == nil { t.Fatal("Error setting toxic:", err) } }) } func AssertToxicExists( t *testing.T, toxics tclient.Toxics, name, typeName, stream string, exists bool, ) *tclient.Toxic { var toxic *tclient.Toxic var actualType, actualStream string for i, tox := range toxics { if name == tox.Name { toxic = &toxics[i] actualType = tox.Type actualStream = tox.Stream } } if exists { if toxic == nil { t.Fatalf("Expected to see %s toxic in list", name) } if actualType != typeName { t.Fatalf("Expected %s to be of type %s, found %s", name, typeName, actualType) } if actualStream != stream { t.Fatalf("Expected %s to be in stream %s, found %s", name, stream, actualStream) } } else if toxic != nil && actualStream == stream { t.Fatalf("Expected %s toxic to be missing from list, found type %s", name, actualType) } return toxic } ================================================ FILE: client/README.md ================================================ # toxiproxy-go This is the Go client library for the [Toxiproxy](https://github.com/shopify/toxiproxy) API. Please read the [usage section in the Toxiproxy README](https://github.com/shopify/toxiproxy#usage) before attempting to use the client. This client is compatible with Toxiproxy 2.x, for the latest 1.x client see [v1.2.1](https://github.com/Shopify/toxiproxy/tree/v1.2.1/client). ## Changes in Toxiproxy-go Client 2.x In order to make use of the 2.0 api, and to make usage a little easier, the client api has changed: - `client.NewProxy()` no longer accepts a proxy as an argument. - `proxy.Create()` is removed in favour of using `proxy.Save()`. - Proxies can be created in a single call using `client.CreateProxy()`. - `proxy.Disable()` and `proxy.Enable()` have been added to simplify taking down a proxy. - `proxy.ToxicsUpstream` and `proxy.ToxicsDownstream` have been merged into a single `ActiveToxics` list. - `proxy.Toxics()` no longer requires a direction to be specified, and will return toxics for both directions. - `proxy.SetToxic()` has been replaced by `proxy.AddToxic()`, `proxy.UpdateToxic()`, and `proxy.RemoveToxic()`. ## Usage For detailed API docs please [see the Godoc documentation](http://godoc.org/github.com/Shopify/toxiproxy/client). First import toxiproxy and create a new client: ```go import toxiproxy "github.com/Shopify/toxiproxy/v2/client" client := toxiproxy.NewClient("localhost:8474") ``` You can then create a new proxy using the client: ```go proxy, err := client.CreateProxy("redis", "localhost:26379", "localhost:6379") if err != nil { panic(err) } ``` For large amounts of proxies, they can also be created using a configuration file: ```go var config []toxiproxy.Proxy data, _ := ioutil.ReadFile("config.json") json.Unmarshal(data, &config) proxies, err = client.Populate(config) ``` ```json [{ "name": "redis", "listen": "localhost:26379", "upstream": "localhost:6379" }] ``` Toxics can be added as follows: ```go // Add 1s latency to 100% of downstream connections proxy.AddToxic("latency_down", "latency", "downstream", 1.0, toxiproxy.Attributes{ "latency": 1000, }) // Change downstream latency to add 100ms of jitter proxy.UpdateToxic("latency_down", 1.0, toxiproxy.Attributes{ "jitter": 100, }) // Remove the latency toxic proxy.RemoveToxic("latency_down") ``` The proxy can be taken down using `Disable()`: ```go proxy.Disable() ``` When a proxy is no longer needed, it can be cleaned up with `Delete()`: ```go proxy.Delete() ``` ## Full Example ```go import ( "testing" "time" toxiproxy "github.com/Shopify/toxiproxy/v2/client" "github.com/gomodule/redigo/redis" ) var toxiClient *toxiproxy.Client func init() { var err error toxiClient = toxiproxy.NewClient("localhost:8474") _, err = toxiClient.Populate([]toxiproxy.Proxy{{ Name: "redis", Listen: "localhost:26379", Upstream: "localhost:6379", // note: you cannot set toxics here via ActiveToxics }}) if err != nil { panic(err) } // Alternatively, create the proxies manually with // toxiClient.CreateProxy("redis", "localhost:26379", "localhost:6379") } func TestRedisBackendDown(t *testing.T) { var proxy, _ = toxiClient.Proxy("redis") proxy.Disable() defer proxy.Enable() // Test that redis is down _, err := redis.Dial("tcp", ":26379") if err == nil { t.Fatal("Connection to redis did not fail") } } func TestRedisBackendSlow(t *testing.T) { var proxy, _ = toxiClient.Proxy("redis") proxy.AddToxic("", "latency", "", 1, toxiproxy.Attributes{ "latency": 1000, }) proxy.Save() defer removeToxic(proxy, "latency_downstream") // Test that redis is slow start := time.Now() conn, err := redis.Dial("tcp", ":26379") if err != nil { t.Fatal("Connection to redis failed", err) } _, err = conn.Do("GET", "test") if err != nil { t.Fatal("Redis command failed", err) } else if time.Since(start) < 900*time.Millisecond { t.Fatal("Redis command did not take long enough:", time.Since(start)) } } func removeToxic(p *toxiproxy.Proxy, n string) { p.RemoveToxic(n) p.Save() } ``` ================================================ FILE: client/api_error.go ================================================ // Package Toxiproxy provides a client wrapper around the Toxiproxy HTTP API for // testing the resiliency of Go applications. // // For use with Toxiproxy 2.x package toxiproxy import ( "fmt" ) type ApiError struct { Message string `json:"error"` Status int `json:"status"` } func (err *ApiError) Error() string { return fmt.Sprintf("HTTP %d: %s", err.Status, err.Message) } ================================================ FILE: client/client.go ================================================ // Package Toxiproxy provides a client wrapper around the Toxiproxy HTTP API for // testing the resiliency of Go applications. // // For use with Toxiproxy 2.x package toxiproxy import ( "bytes" "encoding/json" "fmt" "io" "net/http" "strings" "time" ) // Client holds information about where to connect to Toxiproxy. type Client struct { UserAgent string endpoint string http *http.Client } // NewClient creates a new client which provides the base of all communication // with Toxiproxy. Endpoint is the address to the proxy (e.g. localhost:8474 if // not overridden). func NewClient(endpoint string) *Client { if !strings.HasPrefix(endpoint, "https://") && !strings.HasPrefix(endpoint, "http://") { endpoint = "http://" + endpoint } http := &http.Client{ Timeout: 30 * time.Second, } return &Client{ UserAgent: "toxiproxy-cli", endpoint: endpoint, http: http, } } // Version returns a Toxiproxy running version. func (client *Client) Version() ([]byte, error) { return client.get("/version") } // Proxies returns a map with all the proxies and their toxics. func (client *Client) Proxies() (map[string]*Proxy, error) { resp, err := client.get("/proxies") if err != nil { return nil, err } proxies := make(map[string]*Proxy) err = json.Unmarshal(resp, &proxies) if err != nil { return nil, err } for _, proxy := range proxies { proxy.client = client proxy.created = true } return proxies, nil } // Generates a new uncommitted proxy instance. In order to use the result, the // proxy fields will need to be set and have `Save()` called. func (client *Client) NewProxy() *Proxy { return &Proxy{ client: client, } } // CreateProxy instantiates a new proxy and starts listening on the specified address. // This is an alias for `NewProxy()` + `proxy.Save()`. func (client *Client) CreateProxy(name, listen, upstream string) (*Proxy, error) { proxy := &Proxy{ Name: name, Listen: listen, Upstream: upstream, Enabled: true, client: client, } err := proxy.Save() if err != nil { return nil, fmt.Errorf("create: %w", err) } return proxy, nil } // Proxy returns a proxy by name. func (client *Client) Proxy(name string) (*Proxy, error) { resp, err := client.get("/proxies/" + name) if err != nil { return nil, err } proxy := new(Proxy) err = json.Unmarshal(resp, &proxy) if err != nil { return nil, err } proxy.client = client proxy.created = true return proxy, nil } // Create a list of proxies using a configuration list. If a proxy already exists, // it will be replaced with the specified configuration. // For large amounts of proxies, `config` can be loaded from a file. // Returns a list of the successfully created proxies. func (client *Client) Populate(config []Proxy) ([]*Proxy, error) { proxies := struct { Proxies []*Proxy `json:"proxies"` }{} request, err := json.Marshal(config) if err != nil { return nil, err } resp, err := client.post("/populate", bytes.NewReader(request)) if err != nil { return nil, fmt.Errorf("Populate: %w", err) } err = json.Unmarshal(resp, &proxies) if err != nil { return nil, err } for _, proxy := range proxies.Proxies { proxy.client = client } return proxies.Proxies, err } // AddToxic creates a toxic to proxy. func (client *Client) AddToxic(options *ToxicOptions) (*Toxic, error) { proxy, err := client.Proxy(options.ProxyName) if err != nil { return nil, fmt.Errorf("failed to retrieve proxy with name `%s`: %v", options.ProxyName, err) } toxic, err := proxy.AddToxic( options.ToxicName, options.ToxicType, options.Stream, options.Toxicity, options.Attributes, ) if err != nil { return nil, fmt.Errorf("failed to add toxic to proxy %s: %v", options.ProxyName, err) } return toxic, nil } // UpdateToxic update a toxic in proxy. func (client *Client) UpdateToxic(options *ToxicOptions) (*Toxic, error) { proxy, err := client.Proxy(options.ProxyName) if err != nil { return nil, fmt.Errorf("failed to retrieve proxy with name `%s`: %v", options.ProxyName, err) } toxic, err := proxy.UpdateToxic( options.ToxicName, options.Toxicity, options.Attributes, ) if err != nil { return nil, fmt.Errorf( "failed to update toxic '%s' of proxy '%s': %v", options.ToxicName, options.ProxyName, err, ) } return toxic, nil } // RemoveToxic removes toxic from proxy. func (client *Client) RemoveToxic(options *ToxicOptions) error { proxy, err := client.Proxy(options.ProxyName) if err != nil { return fmt.Errorf("failed to retrieve proxy with name `%s`: %v", options.ProxyName, err) } err = proxy.RemoveToxic(options.ToxicName) if err != nil { return fmt.Errorf( "failed to remove toxic '%s' from proxy '%s': %v", options.ToxicName, options.ProxyName, err, ) } return nil } // ResetState resets the state of all proxies and toxics in Toxiproxy. func (client *Client) ResetState() error { _, err := client.post("/reset", bytes.NewReader([]byte{})) return err } func (c *Client) get(path string) ([]byte, error) { return c.send("GET", path, nil) } func (c *Client) post(path string, body io.Reader) ([]byte, error) { return c.send("POST", path, body) } func (c *Client) patch(path string, body io.Reader) ([]byte, error) { return c.send("PATCH", path, body) } func (c *Client) delete(path string) error { _, err := c.send("DELETE", path, nil) return err } func (c *Client) send(verb, path string, body io.Reader) ([]byte, error) { req, err := http.NewRequest(verb, c.endpoint+path, body) if err != nil { return nil, err } req.Header.Set("User-Agent", c.UserAgent) req.Header.Set("Content-Type", "application/json") resp, err := c.http.Do(req) if err != nil { return nil, fmt.Errorf("fail to request: %w", err) } err = c.validateResponse(resp) if err != nil { return nil, err } defer resp.Body.Close() result, err := io.ReadAll(resp.Body) if err != nil { return nil, err } return result, nil } func (c *Client) validateResponse(resp *http.Response) error { if resp.StatusCode < 300 && resp.StatusCode >= 200 { return nil } apiError := new(ApiError) err := json.NewDecoder(resp.Body).Decode(&apiError) if err != nil { return err } resp.Body.Close() if err != nil { apiError.Message = fmt.Sprintf( "Unexpected response code %d", resp.StatusCode, ) apiError.Status = resp.StatusCode } return apiError } ================================================ FILE: client/client_test.go ================================================ package toxiproxy_test import ( "net/http" "net/http/httptest" "testing" toxiproxy "github.com/Shopify/toxiproxy/v2/client" ) func TestClient_Headers(t *testing.T) { t.Parallel() expected := "toxiproxy-cli/v1.25.0 (darwin/arm64)" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) ua := r.Header.Get("User-Agent") if ua != expected { t.Errorf("User-Agent for %s %s is expected `%s', got: `%s'", r.Method, r.URL, expected, ua) } contentType := r.Header.Get("Content-Type") if contentType != "application/json" { t.Errorf("Content-Type for %s %s is expected `application/json', got: `%s'", r.Method, r.URL, contentType) } w.Write([]byte(`foo`)) })) defer server.Close() client := toxiproxy.NewClient(server.URL) client.UserAgent = expected cases := []struct { name string fn func(c *toxiproxy.Client) }{ {"get version", func(c *toxiproxy.Client) { c.Version() }}, {"get proxies", func(c *toxiproxy.Client) { c.Proxies() }}, {"create proxy", func(c *toxiproxy.Client) { c.CreateProxy("foo", "example.com:0", "example.com:0") }}, {"get proxy", func(c *toxiproxy.Client) { c.Proxy("foo") }}, {"post populate", func(c *toxiproxy.Client) { c.Populate([]toxiproxy.Proxy{{}}) }}, {"create toxic", func(c *toxiproxy.Client) { c.AddToxic(&toxiproxy.ToxicOptions{}) }}, {"update toxic", func(c *toxiproxy.Client) { c.UpdateToxic(&toxiproxy.ToxicOptions{}) }}, {"delete toxic", func(c *toxiproxy.Client) { c.RemoveToxic(&toxiproxy.ToxicOptions{}) }}, {"reset state", func(c *toxiproxy.Client) { c.ResetState() }}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { tc.fn(client) }) } } ================================================ FILE: client/proxy.go ================================================ // Package Toxiproxy provides a client wrapper around the Toxiproxy HTTP API for // testing the resiliency of Go applications. // // For use with Toxiproxy 2.x package toxiproxy import ( "bytes" "encoding/json" "fmt" ) type Proxy struct { Name string `json:"name"` // The name of the proxy Listen string `json:"listen"` // The address the proxy listens on Upstream string `json:"upstream"` // The upstream address to proxy to Enabled bool `json:"enabled"` // Whether the proxy is enabled // The toxics active on this proxy. Note: you cannot set this // when passing Proxy into Populate() ActiveToxics Toxics `json:"toxics"` client *Client created bool // True if this proxy exists on the server } // Save saves changes to a proxy such as its enabled status or upstream port. func (proxy *Proxy) Save() error { request, err := json.Marshal(proxy) if err != nil { return err } data := bytes.NewReader(request) var resp []byte if proxy.created { // TODO: Release PATCH only for v3.0 // resp, err = proxy.client.patch("/proxies/"+proxy.Name, data) resp, err = proxy.client.post("/proxies/"+proxy.Name, data) } else { resp, err = proxy.client.post("/proxies", data) } if err != nil { return err } err = json.Unmarshal(resp, proxy) if err != nil { return err } proxy.created = true return nil } // Enable a proxy again after it has been disabled. func (proxy *Proxy) Enable() error { proxy.Enabled = true return proxy.Save() } // Disable a proxy so that no connections can pass through. This will drop all active connections. func (proxy *Proxy) Disable() error { proxy.Enabled = false return proxy.Save() } // Delete a proxy complete and close all existing connections through it. All information about // the proxy such as listen port and active toxics will be deleted as well. If you just wish to // stop and later enable a proxy, use `Enable()` and `Disable()`. func (proxy *Proxy) Delete() error { err := proxy.client.delete("/proxies/" + proxy.Name) if err != nil { return fmt.Errorf("Delete: %w", err) } return nil } // Toxics returns a map of all the active toxics and their attributes. func (proxy *Proxy) Toxics() (Toxics, error) { resp, err := proxy.client.get("/proxies/" + proxy.Name + "/toxics") if err != nil { return nil, err } toxics := make(Toxics, 0) err = json.Unmarshal(resp, &toxics) if err != nil { return nil, err } return toxics, nil } // AddToxic adds a toxic to the given stream direction. // If a name is not specified, it will default to _. // If a stream is not specified, it will default to downstream. // See https://github.com/Shopify/toxiproxy#toxics for a list of all Toxic types. func (proxy *Proxy) AddToxic( name, typeName, stream string, toxicity float32, attrs Attributes, ) (*Toxic, error) { toxic := Toxic{name, typeName, stream, toxicity, attrs} if toxic.Toxicity == -1 { toxic.Toxicity = 1 // Just to be consistent with a toxicity of -1 using the default } request, err := json.Marshal(&toxic) if err != nil { return nil, err } resp, err := proxy.client.post( "/proxies/"+proxy.Name+"/toxics", bytes.NewReader(request), ) if err != nil { return nil, fmt.Errorf("AddToxic: %w", err) } result := &Toxic{} err = json.Unmarshal(resp, result) if err != nil { return nil, err } return result, nil } // UpdateToxic sets the parameters for an existing toxic with the given name. // If toxicity is set to -1, the current value will be used. func (proxy *Proxy) UpdateToxic(name string, toxicity float32, attrs Attributes) (*Toxic, error) { toxic := map[string]interface{}{ "attributes": attrs, } if toxicity != -1 { toxic["toxicity"] = toxicity } request, err := json.Marshal(&toxic) if err != nil { return nil, err } resp, err := proxy.client.patch( "/proxies/"+proxy.Name+"/toxics/"+name, bytes.NewReader(request), ) if err != nil { return nil, err } result := &Toxic{} err = json.Unmarshal(resp, result) if err != nil { return nil, err } return result, nil } // RemoveToxic renives the toxic with the given name. func (proxy *Proxy) RemoveToxic(name string) error { return proxy.client.delete("/proxies/" + proxy.Name + "/toxics/" + name) } ================================================ FILE: client/toxic.go ================================================ // Package Toxiproxy provides a client wrapper around the Toxiproxy HTTP API for // testing the resiliency of Go applications. // // For use with Toxiproxy 2.x package toxiproxy type Attributes map[string]interface{} type Toxic struct { Name string `json:"name"` Type string `json:"type"` Stream string `json:"stream,omitempty"` Toxicity float32 `json:"toxicity"` Attributes Attributes `json:"attributes"` } type Toxics []Toxic type ToxicOptions struct { ProxyName, ToxicName, ToxicType, Stream string Toxicity float32 Attributes Attributes } ================================================ FILE: cmd/cli/cli.go ================================================ package main import ( "fmt" "os" "runtime" "sort" "strconv" "strings" "github.com/urfave/cli/v2" terminal "golang.org/x/term" toxiproxyServer "github.com/Shopify/toxiproxy/v2" toxiproxy "github.com/Shopify/toxiproxy/v2/client" ) const ( RED = "\x1b[31m" GREEN = "\x1b[32m" YELLOW = "\x1b[33m" BLUE = "\x1b[34m" PURPLE = "\x1b[35m" NONE = "\x1b[0m" ) func color(color string) string { if isTTY { return color } else { return "" } } var toxicDescription = ` Default Toxics: latency: delay all data +/- jitter latency=,jitter= bandwidth: limit to max kb/s rate= slow_close: delay from closing delay= timeout: stop all data and close after timeout timeout= reset_peer: simulate TCP RESET (Connection reset by peer) on the connections by closing the stub Input immediately or after a timeout timeout= slicer: slice data into bits with optional delay average_size=,size_variation=,delay= toxic add: usage: toxiproxy-cli toxic add --type [--downstream|--upstream] \ --toxicName [--toxicity ] \ --attribute [--attribute ] example: toxiproxy-cli toxic add -t latency -n myToxic -a latency=100 -a jitter=50 myProxy toxic update: usage: toxiproxy-cli toxic update --toxicName [--toxicity ] \ --attribute [--attribute ] example: toxiproxy-cli toxic update -n myToxic -a jitter=25 myProxy toxic delete: usage: toxiproxy-cli toxic delete --toxicName example: toxiproxy-cli toxic delete -n myToxic myProxy ` var ( hostname string isTTY bool ) func main() { app := cli.NewApp() app.Name = "toxiproxy-cli" app.Version = toxiproxyServer.Version app.Usage = "Simulate network and system conditions" app.Commands = cliCommands() cli.HelpFlag = &cli.BoolFlag{ Name: "help", Usage: "show help", } app.Flags = []cli.Flag{ &cli.StringFlag{ Name: "host", Aliases: []string{"h"}, Value: "http://localhost:8474", Usage: "toxiproxy host to connect to", Destination: &hostname, EnvVars: []string{"TOXIPROXY_URL"}, }, } isTTY = terminal.IsTerminal(int(os.Stdout.Fd())) app.Run(os.Args) } func cliCommands() []*cli.Command { return []*cli.Command{ { Name: "list", Usage: "list all proxies\n\tusage: 'toxiproxy-cli list'\n", Aliases: []string{"l", "li", "ls"}, Action: withToxi(list), }, { Name: "inspect", Aliases: []string{"i", "ins"}, Usage: "inspect a single proxy\n\tusage: 'toxiproxy-cli inspect '\n", Action: withToxi(inspectProxy), }, { Name: "create", Usage: "create a new proxy\n\t" + "usage: 'toxiproxy-cli create --listen --upstream '\n", Aliases: []string{"c", "new"}, Flags: []cli.Flag{ &cli.StringFlag{ Name: "listen", Aliases: []string{"l"}, Usage: "proxy will listen on this address", }, &cli.StringFlag{ Name: "upstream", Aliases: []string{"u"}, Usage: "proxy will forward to this address", }, }, Action: withToxi(createProxy), }, { Name: "toggle", Usage: "\ttoggle enabled status on a proxy\n" + "\t\tusage: 'toxiproxy-cli toggle '\n", Aliases: []string{"tog"}, Action: withToxi(toggleProxy), }, { Name: "delete", Usage: "\tdelete a proxy\n\t\tusage: 'toxiproxy-cli delete '\n", Aliases: []string{"d"}, Action: withToxi(deleteProxy), }, { Name: "toxic", Aliases: []string{"t"}, Usage: "\tadd, remove or update a toxic\n\t\tusage: see 'toxiproxy-cli toxic'\n", Description: toxicDescription, Subcommands: cliToxiSubCommands(), }, } } func cliToxiSubCommands() []*cli.Command { return []*cli.Command{ cliToxiAddSubCommand(), cliToxiUpdateSubCommand(), cliToxiRemoveSubCommand(), } } func cliToxiAddSubCommand() *cli.Command { return &cli.Command{ Name: "add", Aliases: []string{"a"}, Usage: "add a new toxic", ArgsUsage: "", Flags: []cli.Flag{ &cli.StringFlag{ Name: "toxicName", Aliases: []string{"n"}, Usage: "name of the toxic", }, &cli.StringFlag{ Name: "type", Aliases: []string{"t"}, Usage: "type of toxic", }, &cli.StringFlag{ Name: "toxicity", Aliases: []string{"tox"}, Usage: "toxicity of toxic should be a float between 0 and 1", DefaultText: "1.0", }, &cli.StringSliceFlag{ Name: "attribute", Aliases: []string{"a"}, Usage: "toxic attribute in key=value format", }, &cli.BoolFlag{ Name: "upstream", Aliases: []string{"u"}, Usage: "add toxic to upstream", DefaultText: "false", }, &cli.BoolFlag{ Name: "downstream", Aliases: []string{"d"}, Usage: "add toxic to downstream", DefaultText: "true", }, }, Action: withToxi(addToxic), } } func cliToxiUpdateSubCommand() *cli.Command { return &cli.Command{ Name: "update", Aliases: []string{"u"}, Usage: "update an enabled toxic", ArgsUsage: "", Flags: []cli.Flag{ &cli.StringFlag{ Name: "toxicName", Aliases: []string{"n"}, Usage: "name of the toxic", }, &cli.StringFlag{ Name: "toxicity", Aliases: []string{"tox"}, Usage: "toxicity of toxic should be a float between 0 and 1", DefaultText: "1.0", }, &cli.StringSliceFlag{ Name: "attribute", Aliases: []string{"a"}, Usage: "toxic attribute in key=value format", }, }, Action: withToxi(updateToxic), } } func cliToxiRemoveSubCommand() *cli.Command { return &cli.Command{ Name: "remove", Aliases: []string{"r", "delete", "d"}, Usage: "remove an enabled toxic", ArgsUsage: "", Flags: []cli.Flag{ &cli.StringFlag{ Name: "toxicName", Aliases: []string{"n"}, Usage: "name of the toxic", }, }, Action: withToxi(removeToxic), } } type toxiAction func(*cli.Context, *toxiproxy.Client) error func withToxi(f toxiAction) func(*cli.Context) error { return func(c *cli.Context) error { toxiproxyClient := toxiproxy.NewClient(hostname) toxiproxyClient.UserAgent = fmt.Sprintf( "toxiproxy-cli/%s (%s/%s)", c.App.Version, runtime.GOOS, runtime.GOARCH, ) return f(c, toxiproxyClient) } } func list(c *cli.Context, t *toxiproxy.Client) error { proxies, err := t.Proxies() if err != nil { return errorf("Failed to retrieve proxies: %s", err) } var proxyNames []string for proxyName := range proxies { proxyNames = append(proxyNames, proxyName) } sort.Strings(proxyNames) if isTTY { fmt.Printf( "%sName\t\t\t%sListen\t\t%sUpstream\t\t%sEnabled\t\t%sToxics\n%s", color(GREEN), color(BLUE), color(YELLOW), color(PURPLE), color(RED), color(NONE), ) fmt.Printf( "%s======================================================================================\n", color(NONE), ) if len(proxyNames) == 0 { fmt.Printf("%sno proxies\n%s", color(RED), color(NONE)) hint("create a proxy with `toxiproxy-cli create`") return nil } } for _, proxyName := range proxyNames { proxy := proxies[proxyName] numToxics := strconv.Itoa(len(proxy.ActiveToxics)) if numToxics == "0" && isTTY { numToxics = "None" } printWidth(color(colorEnabled(proxy.Enabled)), proxy.Name, 3) printWidth(BLUE, proxy.Listen, 2) printWidth(YELLOW, proxy.Upstream, 3) printWidth(PURPLE, enabledText(proxy.Enabled), 2) fmt.Printf("%s%s%s\n", color(RED), numToxics, color(NONE)) } hint("inspect toxics with `toxiproxy-cli inspect `") return nil } func inspectProxy(c *cli.Context, t *toxiproxy.Client) error { proxyName := c.Args().First() if proxyName == "" { cli.ShowSubcommandHelp(c) return errorf("Proxy name is required as the first argument.\n") } proxy, err := t.Proxy(proxyName) if err != nil { return errorf("Failed to retrieve proxy %s: %s\n", proxyName, err.Error()) } if isTTY { fmt.Printf("%sName: %s%s\t", color(PURPLE), color(NONE), proxy.Name) fmt.Printf("%sListen: %s%s\t", color(BLUE), color(NONE), proxy.Listen) fmt.Printf("%sUpstream: %s%s\n", color(YELLOW), color(NONE), proxy.Upstream) fmt.Printf( "%s======================================================================\n", color(NONE), ) splitToxics := func(toxics toxiproxy.Toxics) (toxiproxy.Toxics, toxiproxy.Toxics) { upstream := make(toxiproxy.Toxics, 0) downstream := make(toxiproxy.Toxics, 0) for _, toxic := range toxics { if toxic.Stream == "upstream" { upstream = append(upstream, toxic) } else { downstream = append(downstream, toxic) } } return upstream, downstream } if len(proxy.ActiveToxics) == 0 { fmt.Printf("%sProxy has no toxics enabled.\n%s", color(RED), color(NONE)) } else { up, down := splitToxics(proxy.ActiveToxics) listToxics(up, "Upstream") fmt.Println() listToxics(down, "Downstream") } hint("add a toxic with `toxiproxy-cli toxic add`") } else { listToxics(proxy.ActiveToxics, "") } return nil } func toggleProxy(c *cli.Context, t *toxiproxy.Client) error { proxyName := c.Args().First() if proxyName == "" { cli.ShowSubcommandHelp(c) return errorf("Proxy name is required as the first argument.\n") } proxy, err := t.Proxy(proxyName) if err != nil { return errorf("Failed to retrieve proxy %s: %s\n", proxyName, err.Error()) } proxy.Enabled = !proxy.Enabled err = proxy.Save() if err != nil { return errorf("Failed to toggle proxy %s: %s\n", proxyName, err.Error()) } fmt.Printf( "Proxy %s%s%s is now %s%s%s\n", colorEnabled(proxy.Enabled), proxyName, color(NONE), colorEnabled(proxy.Enabled), enabledText(proxy.Enabled), color(NONE), ) return nil } func createProxy(c *cli.Context, t *toxiproxy.Client) error { proxyName := c.Args().First() if proxyName == "" { cli.ShowSubcommandHelp(c) return errorf("Proxy name is required as the first argument.\n") } listen, err := getArgOrFail(c, "listen") if err != nil { return err } upstream, err := getArgOrFail(c, "upstream") if err != nil { return err } _, err = t.CreateProxy(proxyName, listen, upstream) if err != nil { return errorf("Failed to create proxy: %s\n", err.Error()) } fmt.Printf("Created new proxy %s\n", proxyName) return nil } func deleteProxy(c *cli.Context, t *toxiproxy.Client) error { proxyName := c.Args().First() if proxyName == "" { cli.ShowSubcommandHelp(c) return errorf("Proxy name is required as the first argument.\n") } p, err := t.Proxy(proxyName) if err != nil { return errorf("Failed to retrieve proxy %s: %s\n", proxyName, err.Error()) } err = p.Delete() if err != nil { return errorf("Failed to delete proxy: %s\n", err.Error()) } fmt.Printf("Deleted proxy %s\n", proxyName) return nil } func parseToxicity(c *cli.Context, defaultToxicity float32) (float32, error) { toxicity := defaultToxicity toxicityString := c.String("toxicity") if toxicityString != "" { tox, err := strconv.ParseFloat(toxicityString, 32) if err != nil || tox > 1 || tox < 0 { return 0, errorf("toxicity should be a float between 0 and 1.\n") } toxicity = float32(tox) } return toxicity, nil } func addToxic(c *cli.Context, t *toxiproxy.Client) error { toxicParams, err := parseAddToxicParams(c) if err != nil { return err } toxic, err := t.AddToxic(toxicParams) if err != nil { return errorf("Failed to add toxic: %v\n", err) } fmt.Printf( "Added %s %s toxic '%s' on proxy '%s'\n", toxic.Stream, toxic.Type, toxic.Name, toxicParams.ProxyName, ) return nil } func updateToxic(c *cli.Context, t *toxiproxy.Client) error { toxicParams, err := parseUpdateToxicParams(c) if err != nil { return err } toxic, err := t.UpdateToxic(toxicParams) if err != nil { return errorf("Failed to update toxic: %v\n", err) } fmt.Printf( "Updated toxic '%s' on proxy '%s'\n", toxic.Name, toxicParams.ProxyName, ) return nil } func removeToxic(c *cli.Context, t *toxiproxy.Client) error { toxicParams, err := parseToxicCommonParams(c) if err != nil { return err } err = t.RemoveToxic(toxicParams) if err != nil { return errorf("Failed to remove toxic: %v\n", err) } fmt.Printf("Removed toxic '%s' on proxy '%s'\n", toxicParams.ToxicName, toxicParams.ProxyName) return nil } func parseToxicCommonParams(context *cli.Context) (*toxiproxy.ToxicOptions, error) { proxyName := context.Args().First() if proxyName == "" { cli.ShowSubcommandHelp(context) return nil, errorf("Proxy name is missing.\n") } toxicName := context.String("toxicName") return &toxiproxy.ToxicOptions{ ProxyName: proxyName, ToxicName: toxicName, }, nil } func parseUpdateToxicParams(c *cli.Context) (*toxiproxy.ToxicOptions, error) { result, err := parseToxicCommonParams(c) if err != nil { return nil, err } result.Toxicity, err = parseToxicity(c, 1.0) if err != nil { return nil, err } result.Attributes = parseAttributes(c, "attribute") return result, nil } func parseAddToxicParams(c *cli.Context) (*toxiproxy.ToxicOptions, error) { result, err := parseToxicCommonParams(c) if err != nil { return nil, err } result.ToxicType, err = getArgOrFail(c, "type") if err != nil { return nil, err } upstream := c.Bool("upstream") downstream := c.Bool("downstream") if upstream && downstream { return nil, errorf("Only one should be specified: upstream or downstream.\n") } stream := "downstream" if upstream { stream = "upstream" } result.Stream = stream result.Toxicity, err = parseToxicity(c, 1.0) if err != nil { return nil, err } result.Attributes = parseAttributes(c, "attribute") return result, nil } func parseAttributes(c *cli.Context, name string) toxiproxy.Attributes { parsed := map[string]interface{}{} args := c.StringSlice(name) for _, raw := range args { kv := strings.SplitN(raw, "=", 2) if len(kv) < 2 { continue } if float, err := strconv.ParseFloat(kv[1], 64); err == nil { parsed[kv[0]] = float } else { parsed[kv[0]] = kv[1] } } return parsed } func colorEnabled(enabled bool) string { if enabled { return color(GREEN) } return color(RED) } func enabledText(enabled bool) string { if enabled { return "enabled" } return "disabled" } type attribute struct { key string value interface{} } type attributeList []attribute func sortedAttributes(attrs toxiproxy.Attributes) attributeList { li := make(attributeList, 0, len(attrs)) for k, v := range attrs { li = append(li, attribute{k, v}) } sort.Slice(li, func(i, j int) bool { return li[i].key < li[j].key }) return li } func listToxics(toxics toxiproxy.Toxics, stream string) { if isTTY { fmt.Printf("%s%s toxics:\n%s", color(GREEN), stream, color(NONE)) if len(toxics) == 0 { fmt.Printf("%sProxy has no %s toxics enabled.\n%s", color(RED), stream, color(NONE)) return } } for _, t := range toxics { if isTTY { fmt.Printf("%s%s:%s\t", color(BLUE), t.Name, color(NONE)) } else { fmt.Printf("%s\t", t.Name) } fmt.Printf("type=%s\t", t.Type) fmt.Printf("stream=%s\t", t.Stream) fmt.Printf("toxicity=%.2f\t", t.Toxicity) fmt.Printf("attributes=[") sorted := sortedAttributes(t.Attributes) for _, a := range sorted { fmt.Printf("\t%s=", a.key) fmt.Print(a.value) } fmt.Printf("\t]\n") } } func getArgOrFail(c *cli.Context, name string) (string, error) { arg := c.String(name) if arg == "" { cli.ShowSubcommandHelp(c) return "", errorf("Required argument '%s' was empty.\n", name) } return arg, nil } func hint(m string) { if isTTY { fmt.Printf("\n%sHint: %s\n", color(NONE), m) } } func errorf(m string, args ...interface{}) error { return cli.Exit(fmt.Sprintf(m, args...), 1) } func printWidth(col string, m string, numTabs int) { if isTTY { numTabs -= len(m)/8 + 1 if numTabs < 0 { numTabs = 0 } } else { numTabs = 0 } fmt.Printf("%s%s%s\t%s", color(col), m, color(NONE), strings.Repeat("\t", numTabs)) } ================================================ FILE: cmd/server/server.go ================================================ package main import ( "flag" "fmt" "math/rand" "net" "os" "os/signal" "strconv" "syscall" "time" "github.com/prometheus/client_golang/prometheus" "github.com/rs/zerolog" "github.com/rs/zerolog/log" "github.com/Shopify/toxiproxy/v2" "github.com/Shopify/toxiproxy/v2/collectors" ) type cliArguments struct { host string port string config string seed int64 printVersion bool proxyMetrics bool runtimeMetrics bool } func parseArguments() cliArguments { result := cliArguments{} flag.StringVar(&result.host, "host", "localhost", "Host for toxiproxy's API to listen on") flag.StringVar(&result.port, "port", "8474", "Port for toxiproxy's API to listen on") flag.StringVar(&result.config, "config", "", "JSON file containing proxies to create on startup") flag.Int64Var(&result.seed, "seed", time.Now().UTC().UnixNano(), "Seed for randomizing toxics with") flag.BoolVar(&result.runtimeMetrics, "runtime-metrics", false, `enable runtime-related prometheus metrics (default "false")`) flag.BoolVar(&result.proxyMetrics, "proxy-metrics", false, `enable toxiproxy-specific prometheus metrics (default "false")`) flag.BoolVar(&result.printVersion, "version", false, `print the version (default "false")`) flag.Parse() return result } func main() { err := run() if err != nil { fmt.Printf("error: %v", err) os.Exit(1) } os.Exit(0) } func run() error { cli := parseArguments() if cli.printVersion { fmt.Printf("toxiproxy-server version %s\n", toxiproxy.Version) return nil } rand.New(rand.NewSource(cli.seed)) // #nosec G404 -- ignoring this rule logger := setupLogger() log.Logger = logger logger. Info(). Str("version", toxiproxy.Version). Msg("Starting Toxiproxy") metrics := toxiproxy.NewMetricsContainer(prometheus.NewRegistry()) server := toxiproxy.NewServer(metrics, logger) if cli.proxyMetrics { server.Metrics.ProxyMetrics = collectors.NewProxyMetricCollectors() } if cli.runtimeMetrics { server.Metrics.RuntimeMetrics = collectors.NewRuntimeMetricCollectors() } if len(cli.config) > 0 { server.PopulateConfig(cli.config) } addr := net.JoinHostPort(cli.host, cli.port) go func(server *toxiproxy.ApiServer, addr string) { err := server.Listen(addr) if err != nil { server.Logger.Err(err).Msg("Server finished with error") } }(server, addr) signals := make(chan os.Signal, 1) signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) <-signals server.Logger.Info().Msg("Shutdown started") err := server.Shutdown() if err != nil { logger.Err(err).Msg("Shutdown finished with error") } return nil } func setupLogger() zerolog.Logger { zerolog.TimestampFunc = func() time.Time { return time.Now().UTC() } zerolog.CallerMarshalFunc = func(pc uintptr, file string, line int) string { short := file for i := len(file) - 1; i > 0; i-- { if file[i] == '/' { short = file[i+1:] break } } file = short return file + ":" + strconv.Itoa(line) } logger := zerolog.New(os.Stdout).With().Caller().Timestamp().Logger() val, ok := os.LookupEnv("LOG_LEVEL") if !ok { return logger } lvl, err := zerolog.ParseLevel(val) if err == nil { logger = logger.Level(lvl) } else { l := &logger l.Err(err).Msgf("unknown LOG_LEVEL value: \"%s\"", val) } return logger } ================================================ FILE: collectors/common.go ================================================ package collectors const ( namespace string = "toxiproxy" ) ================================================ FILE: collectors/proxy.go ================================================ package collectors import ( "github.com/prometheus/client_golang/prometheus" ) type ProxyMetricCollectors struct { collectors []prometheus.Collector proxyLabels []string ReceivedBytesTotal *prometheus.CounterVec SentBytesTotal *prometheus.CounterVec } func (c *ProxyMetricCollectors) Collectors() []prometheus.Collector { return c.collectors } func NewProxyMetricCollectors() *ProxyMetricCollectors { var m ProxyMetricCollectors m.proxyLabels = []string{ "direction", "proxy", "listener", "upstream", } m.ReceivedBytesTotal = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, Subsystem: "proxy", Name: "received_bytes_total", }, m.proxyLabels) m.collectors = append(m.collectors, m.ReceivedBytesTotal) m.SentBytesTotal = prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, Subsystem: "proxy", Name: "sent_bytes_total", }, m.proxyLabels) m.collectors = append(m.collectors, m.SentBytesTotal) return &m } ================================================ FILE: collectors/runtime.go ================================================ package collectors import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/collectors" ) type RuntimeMetricCollectors struct { collectors []prometheus.Collector } func (c *RuntimeMetricCollectors) Collectors() []prometheus.Collector { return c.collectors } func NewRuntimeMetricCollectors() *RuntimeMetricCollectors { var m RuntimeMetricCollectors m.collectors = append(m.collectors, collectors.NewGoCollector()) m.collectors = append(m.collectors, collectors.NewBuildInfoCollector()) m.collectors = append(m.collectors, collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) return &m } ================================================ FILE: dev.yml ================================================ --- name: toxiproxy type: go up: - packages: - gnu-tar - golangci-lint - goreleaser - shellcheck - shfmt - yamllint - go: version: 1.22.8 modules: true ================================================ FILE: go.mod ================================================ module github.com/Shopify/toxiproxy/v2 go 1.23.0 require ( github.com/gorilla/mux v1.8.1 github.com/prometheus/client_golang v1.23.2 github.com/rs/zerolog v1.34.0 github.com/urfave/cli/v2 v2.27.7 golang.org/x/term v0.34.0 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 ) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.7 // indirect github.com/kr/text v0.2.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.66.1 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/rs/xid v1.6.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect golang.org/x/sys v0.35.0 // indirect google.golang.org/protobuf v1.36.8 // indirect ) ================================================ FILE: go.sum ================================================ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.7 h1:zbFlGlXEAKlwXpmvle3d8Oe3YnkKIK4xSRTd3sHPnBo= github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs= github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/urfave/cli/v2 v2.27.7 h1:bH59vdhbjLv3LAvIu6gd0usJHgoTTPhCFib8qqOwXYU= github.com/urfave/cli/v2 v2.27.7/go.mod h1:CyNAG/xg+iAOg0N4MPGZqVmv2rCoP267496AOXUZjA4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= ================================================ FILE: link.go ================================================ package toxiproxy import ( "context" "fmt" "io" "net" "time" "github.com/rs/zerolog" "github.com/Shopify/toxiproxy/v2/stream" "github.com/Shopify/toxiproxy/v2/toxics" ) // ToxicLinks are single direction pipelines that connects an input and output via // a chain of toxics. The chain always starts with a NoopToxic, and toxics are added // and removed as they are enabled/disabled. New toxics are always added to the end // of the chain. // // | NoopToxic LatencyToxic // | v v // | Input > ToxicStub > ToxicStub > Output. type ToxicLink struct { stubs []*toxics.ToxicStub proxy *Proxy toxics *ToxicCollection input *stream.ChanWriter output *stream.ChanReader direction stream.Direction Logger *zerolog.Logger } func NewToxicLink( proxy *Proxy, collection *ToxicCollection, direction stream.Direction, logger zerolog.Logger, ) *ToxicLink { link := &ToxicLink{ stubs: make( []*toxics.ToxicStub, len(collection.chain[direction]), cap(collection.chain[direction]), ), proxy: proxy, toxics: collection, direction: direction, Logger: &logger, } // Initialize the link with ToxicStubs last := make(chan *stream.StreamChunk) // The first toxic is always a noop link.input = stream.NewChanWriter(last) for i := 0; i < len(link.stubs); i++ { var next chan *stream.StreamChunk if i+1 < len(link.stubs) { next = make(chan *stream.StreamChunk, link.toxics.chain[direction][i+1].BufferSize) } else { next = make(chan *stream.StreamChunk) } link.stubs[i] = toxics.NewToxicStub(last, next) last = next } link.output = stream.NewChanReader(last) return link } // Start the link with the specified toxics. func (link *ToxicLink) Start( server *ApiServer, name string, source io.Reader, dest io.WriteCloser, ) { logger := link.Logger logger. Debug(). Str("direction", link.Direction()). Msg("Setup connection") labels := []string{ link.Direction(), link.proxy.Name, link.proxy.Listen, link.proxy.Upstream} go link.read(labels, server, source) for i, toxic := range link.toxics.chain[link.direction] { if stateful, ok := toxic.Toxic.(toxics.StatefulToxic); ok { link.stubs[i].State = stateful.NewState() } if _, ok := toxic.Toxic.(*toxics.ResetToxic); ok { if err := source.(*net.TCPConn).SetLinger(0); err != nil { logger.Err(err). Str("toxic", toxic.Type). Msg("source: Unable to setLinger(ms)") } if err := dest.(*net.TCPConn).SetLinger(0); err != nil { logger.Err(err). Str("toxic", toxic.Type). Msg("dest: Unable to setLinger(ms)") } } go link.stubs[i].Run(toxic) } go link.write(labels, name, server, dest) } // read copies bytes from a source to the link's input channel. func (link *ToxicLink) read( metricLabels []string, server *ApiServer, source io.Reader, ) { logger := link.Logger bytes, err := io.Copy(link.input, source) if err != nil { logger.Warn(). Int64("bytes", bytes). Err(err). Msg("Source terminated") } if server.Metrics.proxyMetricsEnabled() { server.Metrics.ProxyMetrics.ReceivedBytesTotal. WithLabelValues(metricLabels...).Add(float64(bytes)) } link.input.Close() } // write copies bytes from the link's output channel to a destination. func (link *ToxicLink) write( metricLabels []string, name string, server *ApiServer, // TODO: Replace with AppConfig for Metrics and Logger dest io.WriteCloser, ) { logger := link.Logger. With(). Str("component", "ToxicLink"). Str("method", "write"). Str("link", name). Str("proxy", link.proxy.Name). Str("link_addr", fmt.Sprintf("%p", link)). Logger() bytes, err := io.Copy(dest, link.output) if err != nil { logger.Warn(). Int64("bytes", bytes). Err(err). Msg("Could not write to destination") } else if server.Metrics.proxyMetricsEnabled() { server.Metrics.ProxyMetrics.SentBytesTotal. WithLabelValues(metricLabels...).Add(float64(bytes)) } dest.Close() logger.Trace().Msgf("Remove link %s from ToxicCollection", name) link.toxics.RemoveLink(name) logger.Trace().Msgf("RemoveConnection %s from Proxy %s", name, link.proxy.Name) link.proxy.RemoveConnection(name) } // Add a toxic to the end of the chain. func (link *ToxicLink) AddToxic(toxic *toxics.ToxicWrapper) { i := len(link.stubs) newin := make(chan *stream.StreamChunk, toxic.BufferSize) link.stubs = append(link.stubs, toxics.NewToxicStub(newin, link.stubs[i-1].Output)) // Interrupt the last toxic so that we don't have a race when moving channels if link.stubs[i-1].InterruptToxic() { link.stubs[i-1].Output = newin if stateful, ok := toxic.Toxic.(toxics.StatefulToxic); ok { link.stubs[i].State = stateful.NewState() } go link.stubs[i].Run(toxic) go link.stubs[i-1].Run(link.toxics.chain[link.direction][i-1]) } else { // This link is already closed, make sure the new toxic matches link.stubs[i].Output = newin // The real output is already closed, close this instead link.stubs[i].Close() } } // Update an existing toxic in the chain. func (link *ToxicLink) UpdateToxic(toxic *toxics.ToxicWrapper) { if link.stubs[toxic.Index].InterruptToxic() { go link.stubs[toxic.Index].Run(toxic) } } // Remove an existing toxic from the chain. func (link *ToxicLink) RemoveToxic(ctx context.Context, toxic *toxics.ToxicWrapper) { toxic_index := toxic.Index log := zerolog.Ctx(ctx). With(). Str("component", "ToxicLink"). Str("method", "RemoveToxic"). Str("toxic", toxic.Name). Str("toxic_type", toxic.Type). Int("toxic_index", toxic.Index). Str("link_addr", fmt.Sprintf("%p", link)). Str("toxic_stub_addr", fmt.Sprintf("%p", link.stubs[toxic_index])). Str("prev_toxic_stub_addr", fmt.Sprintf("%p", link.stubs[toxic_index-1])). Logger() if link.stubs[toxic_index].InterruptToxic() { cleanup, ok := toxic.Toxic.(toxics.CleanupToxic) if ok { cleanup.Cleanup(link.stubs[toxic_index]) // Cleanup could have closed the stub. if link.stubs[toxic_index].Closed() { log.Trace().Msg("Cleanup closed toxic and removed toxic") // TODO: Check if cleanup happen would link.stubs recalculated? return } } log.Trace().Msg("Interrupting the previous toxic to update its output") stop := make(chan bool) go func(stub *toxics.ToxicStub, stop chan bool) { stop <- stub.InterruptToxic() }(link.stubs[toxic_index-1], stop) // Unblock the previous toxic if it is trying to flush // If the previous toxic is closed, continue flusing until we reach the end. interrupted := false stopped := false for !interrupted { select { case interrupted = <-stop: stopped = true case tmp := <-link.stubs[toxic_index].Input: if tmp == nil { link.stubs[toxic_index].Close() if !stopped { <-stop } return // TODO: There are some steps after this to clean buffer } err := link.stubs[toxic_index].WriteOutput(tmp, 5*time.Second) if err != nil { log.Err(err). Msg("Could not write last packets after interrupt to Output") } } } // Empty the toxic's buffer if necessary for len(link.stubs[toxic_index].Input) > 0 { tmp := <-link.stubs[toxic_index].Input if tmp == nil { link.stubs[toxic_index].Close() return } err := link.stubs[toxic_index].WriteOutput(tmp, 5*time.Second) if err != nil { log.Err(err). Msg("Could not write last packets after interrupt to Output") } } link.stubs[toxic_index-1].Output = link.stubs[toxic_index].Output link.stubs = append(link.stubs[:toxic_index], link.stubs[toxic_index+1:]...) go link.stubs[toxic_index-1].Run(link.toxics.chain[link.direction][toxic_index-1]) } } // Direction returns the direction of the link (upstream or downstream). func (link *ToxicLink) Direction() string { return link.direction.String() } ================================================ FILE: link_test.go ================================================ package toxiproxy import ( "context" "encoding/binary" "flag" "io" "os" "testing" "time" "github.com/rs/zerolog" "github.com/Shopify/toxiproxy/v2/stream" "github.com/Shopify/toxiproxy/v2/testhelper" "github.com/Shopify/toxiproxy/v2/toxics" ) func TestToxicsAreLoaded(t *testing.T) { if toxics.Count() < 1 { t.Fatal("No toxics loaded!") } } func TestStubInitializaation(t *testing.T) { collection := NewToxicCollection(nil) link := NewToxicLink(nil, collection, stream.Downstream, zerolog.Nop()) if len(link.stubs) != 1 { t.Fatalf("Link created with wrong number of stubs: %d != 1", len(link.stubs)) } if cap(link.stubs) != toxics.Count()+1 { t.Fatalf("Link created with wrong capacity: %d != %d", cap(link.stubs), toxics.Count()+1) } if cap(link.stubs[0].Input) != 0 { t.Fatalf("Noop buffer was not initialized as 0: %d", cap(link.stubs[0].Input)) } if cap(link.stubs[0].Output) != 0 { t.Fatalf("Link output buffer was not initialized as 0: %d", cap(link.stubs[0].Output)) } } func TestStubInitializaationWithToxics(t *testing.T) { collection := NewToxicCollection(nil) collection.chainAddToxic(&toxics.ToxicWrapper{ Toxic: new(toxics.LatencyToxic), Type: "latency", Direction: stream.Downstream, BufferSize: 1024, Toxicity: 1, }) collection.chainAddToxic(&toxics.ToxicWrapper{ Toxic: new(toxics.BandwidthToxic), Type: "bandwidth", Direction: stream.Downstream, Toxicity: 1, }) link := NewToxicLink(nil, collection, stream.Downstream, zerolog.Nop()) if len(link.stubs) != 3 { t.Fatalf("Link created with wrong number of stubs: %d != 3", len(link.stubs)) } if cap(link.stubs) != toxics.Count()+1 { t.Fatalf("Link created with wrong capacity: %d != %d", cap(link.stubs), toxics.Count()+1) } if cap(link.stubs[len(link.stubs)-1].Output) != 0 { t.Fatalf("Link output buffer was not initialized as 0: %d", cap(link.stubs[0].Output)) } for i, toxic := range collection.chain[stream.Downstream] { if cap(link.stubs[i].Input) != toxic.BufferSize { t.Fatalf( "%s buffer was not initialized as %d: %d", toxic.Type, toxic.BufferSize, cap(link.stubs[i].Input), ) } } } func TestAddRemoveStubs(t *testing.T) { ctx := context.Background() collection := NewToxicCollection(nil) link := NewToxicLink(nil, collection, stream.Downstream, zerolog.Nop()) go link.stubs[0].Run(collection.chain[stream.Downstream][0]) collection.links["test"] = link // Add stubs collection.chainAddToxic(&toxics.ToxicWrapper{ Toxic: new(toxics.LatencyToxic), Type: "latency", Direction: stream.Downstream, BufferSize: 1024, Toxicity: 1, }) toxic := &toxics.ToxicWrapper{ Toxic: new(toxics.BandwidthToxic), Type: "bandwidth", Direction: stream.Downstream, BufferSize: 2048, Toxicity: 1, } collection.chainAddToxic(toxic) if cap(link.stubs[len(link.stubs)-1].Output) != 0 { t.Fatalf("Link output buffer was not initialized as 0: %d", cap(link.stubs[0].Output)) } for i, toxic := range collection.chain[stream.Downstream] { if cap(link.stubs[i].Input) != toxic.BufferSize { t.Fatalf( "%s buffer was not initialized as %d: %d", toxic.Type, toxic.BufferSize, cap(link.stubs[i].Input), ) } } // Remove stubs collection.chainRemoveToxic(ctx, toxic) if cap(link.stubs[len(link.stubs)-1].Output) != 0 { t.Fatalf("Link output buffer was not initialized as 0: %d", cap(link.stubs[0].Output)) } for i, toxic := range collection.chain[stream.Downstream] { if cap(link.stubs[i].Input) != toxic.BufferSize { t.Fatalf( "%s buffer was not initialized as %d: %d", toxic.Type, toxic.BufferSize, cap(link.stubs[i].Input), ) } } } func TestNoDataDropped(t *testing.T) { ctx := context.Background() collection := NewToxicCollection(nil) link := NewToxicLink(nil, collection, stream.Downstream, zerolog.Nop()) go link.stubs[0].Run(collection.chain[stream.Downstream][0]) collection.links["test"] = link toxic := &toxics.ToxicWrapper{ Toxic: &toxics.LatencyToxic{ Latency: 1000, }, Type: "latency", Direction: stream.Downstream, BufferSize: 1024, Toxicity: 1, } done := make(chan struct{}) defer close(done) go func() { for i := uint16(0); i < 65535; i++ { buf := make([]byte, 2) binary.BigEndian.PutUint16(buf, i) link.input.Write(buf) } link.input.Close() }() go func(ctx context.Context) { for { select { case <-done: return default: collection.chainAddToxic(toxic) collection.chainRemoveToxic(ctx, toxic) } } }(ctx) buf := make([]byte, 2) for i := uint16(0); i < 65535; i++ { n, err := link.output.Read(buf) if n != 2 || err != nil { t.Fatalf("Read failed: %d %v", n, err) } else { val := binary.BigEndian.Uint16(buf) if val != i { t.Fatalf("Read incorrect bytes: %v != %d", val, i) } } } n, err := link.output.Read(buf) if n != 0 || err != io.EOF { t.Fatalf("Expected EOF: %d %v", n, err) } } func TestToxicity(t *testing.T) { collection := NewToxicCollection(nil) link := NewToxicLink(nil, collection, stream.Downstream, zerolog.Nop()) go link.stubs[0].Run(collection.chain[stream.Downstream][0]) collection.links["test"] = link toxic := &toxics.ToxicWrapper{ Toxic: new(toxics.TimeoutToxic), Name: "timeout1", Type: "timeout", Direction: stream.Downstream, Toxicity: 0, } collection.chainAddToxic(toxic) // Toxic should be a Noop because of toxicity n, err := link.input.Write([]byte{42}) if n != 1 || err != nil { t.Fatalf("Write failed: %d %v", n, err) } buf := make([]byte, 2) n, err = link.output.Read(buf) if n != 1 || err != nil { t.Fatalf("Read failed: %d %v", n, err) } else if buf[0] != 42 { t.Fatalf("Read wrong byte: %x", buf[0]) } toxic.Toxicity = 1 toxic.Toxic.(*toxics.TimeoutToxic).Timeout = 100 collection.chainUpdateToxic(toxic) err = testhelper.TimeoutAfter(150*time.Millisecond, func() { n, err = link.input.Write([]byte{42}) if n != 1 || err != nil { t.Fatalf("Write failed: %d %v", n, err) } n, err = link.output.Read(buf) if n != 0 || err != io.EOF { t.Fatalf("Read did not get EOF: %d %v", n, err) } }) if err != nil { t.Fatal(err) } } func TestStateCreated(t *testing.T) { collection := NewToxicCollection(nil) log := zerolog.Nop() if flag.Lookup("test.v").DefValue == "true" { log = zerolog.New(os.Stdout).With().Caller().Timestamp().Logger() } link := NewToxicLink(nil, collection, stream.Downstream, log) go link.stubs[0].Run(collection.chain[stream.Downstream][0]) collection.links["test"] = link collection.chainAddToxic(&toxics.ToxicWrapper{ Toxic: new(toxics.LimitDataToxic), Type: "limit_data", Direction: stream.Downstream, Toxicity: 1, }) if link.stubs[len(link.stubs)-1].State == nil { t.Fatalf("New toxic did not have state object created.") } } func TestRemoveToxicWithBrokenConnection(t *testing.T) { ctx := context.Background() log := zerolog.Nop() if flag.Lookup("test.v").DefValue == "true" { log = zerolog.New(os.Stdout).With().Caller().Timestamp().Logger() } ctx = log.WithContext(ctx) collection := NewToxicCollection(nil) link := NewToxicLink(nil, collection, stream.Downstream, log) go link.stubs[0].Run(collection.chain[stream.Downstream][0]) collection.links["test"] = link toxics := [2]*toxics.ToxicWrapper{ { Toxic: &toxics.BandwidthToxic{ Rate: 0, }, Type: "bandwidth", Direction: stream.Downstream, Toxicity: 1, }, { Toxic: &toxics.BandwidthToxic{ Rate: 0, }, Type: "bandwidth", Direction: stream.Upstream, Toxicity: 1, }, } collection.chainAddToxic(toxics[0]) collection.chainAddToxic(toxics[1]) done := make(chan struct{}) defer close(done) var data uint16 = 42 go func(log zerolog.Logger) { for { select { case <-done: link.input.Close() return case <-time.After(10 * time.Second): log.Print("Finish load") return default: buf := make([]byte, 2) binary.BigEndian.PutUint16(buf, data) link.input.Write(buf) } } }(log) collection.chainRemoveToxic(ctx, toxics[0]) collection.chainRemoveToxic(ctx, toxics[1]) } ================================================ FILE: metrics.go ================================================ package toxiproxy import ( "net/http" "github.com/Shopify/toxiproxy/v2/collectors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" ) // NewMetricsContainer initializes a container for storing all prometheus metrics. func NewMetricsContainer(registry *prometheus.Registry) *metricsContainer { if registry == nil { registry = prometheus.NewRegistry() } return &metricsContainer{ registry: registry, } } type metricsContainer struct { RuntimeMetrics *collectors.RuntimeMetricCollectors ProxyMetrics *collectors.ProxyMetricCollectors registry *prometheus.Registry } func (m *metricsContainer) runtimeMetricsEnabled() bool { return m.RuntimeMetrics != nil } func (m *metricsContainer) proxyMetricsEnabled() bool { return m.ProxyMetrics != nil } // anyMetricsEnabled determines whether we have any prometheus metrics registered for exporting. func (m *metricsContainer) anyMetricsEnabled() bool { return m.runtimeMetricsEnabled() || m.proxyMetricsEnabled() } // handler returns an HTTP handler with the necessary collectors registered // via a global prometheus registry. func (m *metricsContainer) handler() http.Handler { if m.runtimeMetricsEnabled() { m.registry.MustRegister(m.RuntimeMetrics.Collectors()...) } if m.proxyMetricsEnabled() { m.registry.MustRegister(m.ProxyMetrics.Collectors()...) } return promhttp.HandlerFor( m.registry, promhttp.HandlerOpts{Registry: m.registry}) } ================================================ FILE: metrics_test.go ================================================ package toxiproxy import ( "bufio" "bytes" "net/http" "net/http/httptest" "reflect" "regexp" "strings" "testing" "github.com/prometheus/client_golang/prometheus" "github.com/rs/zerolog" "github.com/Shopify/toxiproxy/v2/collectors" "github.com/Shopify/toxiproxy/v2/stream" ) func TestProxyMetricsReceivedSentBytes(t *testing.T) { srv := NewServer(NewMetricsContainer(prometheus.NewRegistry()), zerolog.Nop()) srv.Metrics.ProxyMetrics = collectors.NewProxyMetricCollectors() proxy := NewProxy(srv, "test_proxy_metrics_received_sent_bytes", "localhost:0", "upstream") r := bufio.NewReader(bytes.NewBufferString("hello")) w := &testWriteCloser{ bufio.NewWriter(bytes.NewBuffer([]byte{})), } linkName := "testupstream" proxy.Toxics.StartLink(srv, linkName, r, w, stream.Upstream) proxy.Toxics.RemoveLink(linkName) actual := prometheusOutput(t, srv, "toxiproxy_proxy") expected := []string{ `toxiproxy_proxy_received_bytes_total{` + `direction="upstream",listener="localhost:0",` + `proxy="test_proxy_metrics_received_sent_bytes",upstream="upstream"` + `} 5`, `toxiproxy_proxy_sent_bytes_total{` + `direction="upstream",listener="localhost:0",` + `proxy="test_proxy_metrics_received_sent_bytes",upstream="upstream"` + `} 5`, } if !reflect.DeepEqual(actual, expected) { t.Fatalf( "\nexpected:\n [%v]\ngot:\n [%v]", strings.Join(expected, "\n "), strings.Join(actual, "\n "), ) } } func TestRuntimeMetricsBuildInfo(t *testing.T) { srv := NewServer(NewMetricsContainer(prometheus.NewRegistry()), zerolog.Nop()) srv.Metrics.RuntimeMetrics = collectors.NewRuntimeMetricCollectors() expected := `go_build_info{checksum="[^"]*",path="[^"]*",version="[^"]*"} 1` actual := prometheusOutput(t, srv, "go_build_info") if len(actual) != 1 { t.Fatalf( "\nexpected: 1 item\ngot: %d item(s)\nmetrics:\n %+v", len(actual), strings.Join(actual, "\n "), ) } matched, err := regexp.MatchString(expected, actual[0]) if err != nil { t.Fatalf("Unexpected error: %s", err) } if !matched { t.Fatalf("\nexpected:\n %v\nto match:\n %v", actual[0], expected) } } type testWriteCloser struct { *bufio.Writer } func (t *testWriteCloser) Close() error { return t.Flush() } func prometheusOutput(t *testing.T, apiServer *ApiServer, prefix string) []string { t.Helper() testServer := httptest.NewServer(apiServer.Metrics.handler()) defer testServer.Close() resp, err := http.Get(testServer.URL) if err != nil { t.Fatal(err) } defer resp.Body.Close() var selected []string s := bufio.NewScanner(resp.Body) for s.Scan() { if strings.HasPrefix(s.Text(), prefix) { selected = append(selected, s.Text()) } } return selected } ================================================ FILE: proxy.go ================================================ package toxiproxy import ( "errors" "net" "sync" "github.com/rs/zerolog" tomb "gopkg.in/tomb.v1" "github.com/Shopify/toxiproxy/v2/stream" ) // Proxy represents the proxy in its entirety with all its links. The main // responsibility of Proxy is to accept new client and create Links between the // client and upstream. // // Client <-> toxiproxy <-> Upstream. type Proxy struct { sync.Mutex Name string `json:"name"` Listen string `json:"listen"` Upstream string `json:"upstream"` Enabled bool `json:"enabled"` listener net.Listener started chan error tomb tomb.Tomb connections ConnectionList Toxics *ToxicCollection `json:"-"` apiServer *ApiServer Logger *zerolog.Logger } type ConnectionList struct { list map[string]net.Conn lock sync.Mutex } func (c *ConnectionList) Lock() { c.lock.Lock() } func (c *ConnectionList) Unlock() { c.lock.Unlock() } var ErrProxyAlreadyStarted = errors.New("Proxy already started") func NewProxy(server *ApiServer, name, listen, upstream string) *Proxy { l := server.Logger. With(). Str("name", name). Str("listen", listen). Str("upstream", upstream). Logger() proxy := &Proxy{ Name: name, Listen: listen, Upstream: upstream, started: make(chan error), connections: ConnectionList{list: make(map[string]net.Conn)}, apiServer: server, Logger: &l, } proxy.Toxics = NewToxicCollection(proxy) return proxy } func (proxy *Proxy) Start() error { proxy.Lock() defer proxy.Unlock() return start(proxy) } func (proxy *Proxy) Update(input *Proxy) error { proxy.Lock() defer proxy.Unlock() differs, err := proxy.Differs(input) if err != nil { return err } if differs { stop(proxy) proxy.Listen = input.Listen proxy.Upstream = input.Upstream } if input.Enabled != proxy.Enabled { if input.Enabled { return start(proxy) } stop(proxy) } return nil } func (proxy *Proxy) Stop() { proxy.Lock() defer proxy.Unlock() stop(proxy) } func (proxy *Proxy) listen() error { var err error proxy.listener, err = net.Listen("tcp", proxy.Listen) if err != nil { proxy.started <- err return err } proxy.Listen = proxy.listener.Addr().String() proxy.started <- nil proxy.Logger. Info(). Msg("Started proxy") return nil } func (proxy *Proxy) close() { // Unblock proxy.listener.Accept() err := proxy.listener.Close() if err != nil { proxy.Logger. Warn(). Err(err). Msg("Attempted to close an already closed proxy server") } } func (proxy *Proxy) Differs(other *Proxy) (bool, error) { newResolvedListen, err := net.ResolveTCPAddr("tcp", other.Listen) if err != nil { return false, err } if proxy.Listen != newResolvedListen.String() || proxy.Upstream != other.Upstream { return true, nil } return false, nil } // This channel is to kill the blocking Accept() call below by closing the // net.Listener. func (proxy *Proxy) freeBlocker(acceptTomb *tomb.Tomb) { <-proxy.tomb.Dying() // Notify ln.Accept() that the shutdown was safe acceptTomb.Killf("Shutting down from stop()") proxy.close() // Wait for the accept loop to finish processing acceptTomb.Wait() proxy.tomb.Done() } // server runs the Proxy server, accepting new clients and creating Links to // connect them to upstreams. func (proxy *Proxy) server() { err := proxy.listen() if err != nil { return } acceptTomb := &tomb.Tomb{} defer acceptTomb.Done() // This channel is to kill the blocking Accept() call below by closing the // net.Listener. go proxy.freeBlocker(acceptTomb) for { client, err := proxy.listener.Accept() if err != nil { // This is to confirm we're being shut down in a legit way. Unfortunately, // Go doesn't export the error when it's closed from Close() so we have to // sync up with a channel here. // // See http://zhen.org/blog/graceful-shutdown-of-go-net-dot-listeners/ select { case <-acceptTomb.Dying(): default: proxy.Logger. Warn(). Err(err). Msg("Error while accepting client") } return } proxy.Logger. Info(). Str("client", client.RemoteAddr().String()). Msg("Accepted client") upstream, err := net.Dial("tcp", proxy.Upstream) if err != nil { proxy.Logger. Err(err). Str("client", client.RemoteAddr().String()). Msg("Unable to open connection to upstream") client.Close() continue } name := client.RemoteAddr().String() proxy.connections.Lock() proxy.connections.list[name+"upstream"] = upstream proxy.connections.list[name+"downstream"] = client proxy.connections.Unlock() proxy.Toxics.StartLink(proxy.apiServer, name+"upstream", client, upstream, stream.Upstream) proxy.Toxics.StartLink(proxy.apiServer, name+"downstream", upstream, client, stream.Downstream) } } func (proxy *Proxy) RemoveConnection(name string) { proxy.connections.Lock() defer proxy.connections.Unlock() delete(proxy.connections.list, name) } // Starts a proxy, assumes the lock has already been taken. func start(proxy *Proxy) error { if proxy.Enabled { return ErrProxyAlreadyStarted } proxy.tomb = tomb.Tomb{} // Reset tomb, from previous starts/stops go proxy.server() err := <-proxy.started // Only enable the proxy if it successfully started proxy.Enabled = err == nil return err } // Stops a proxy, assumes the lock has already been taken. func stop(proxy *Proxy) { if !proxy.Enabled { return } proxy.Enabled = false proxy.tomb.Killf("Shutting down from stop()") proxy.tomb.Wait() // Wait until we stop accepting new connections proxy.connections.Lock() defer proxy.connections.Unlock() for _, conn := range proxy.connections.list { conn.Close() } proxy.Logger. Info(). Msg("Terminated proxy") } ================================================ FILE: proxy_collection.go ================================================ package toxiproxy import ( "encoding/json" "fmt" "io" "sync" ) // ProxyCollection is a collection of proxies. It's the interface for anything // to add and remove proxies from the toxiproxy instance. It's responsibility is // to maintain the integrity of the proxy set, by guarding for things such as // duplicate names. type ProxyCollection struct { sync.RWMutex proxies map[string]*Proxy } func NewProxyCollection() *ProxyCollection { return &ProxyCollection{ proxies: make(map[string]*Proxy), } } func (collection *ProxyCollection) Add(proxy *Proxy, start bool) error { collection.Lock() defer collection.Unlock() if _, exists := collection.proxies[proxy.Name]; exists { return ErrProxyAlreadyExists } if start { err := proxy.Start() if err != nil { return err } } collection.proxies[proxy.Name] = proxy return nil } func (collection *ProxyCollection) AddOrReplace(proxy *Proxy, start bool) (*Proxy, error) { collection.Lock() defer collection.Unlock() if existing, exists := collection.proxies[proxy.Name]; exists { differs, err := existing.Differs(proxy) if err != nil { return nil, err } if !differs { return existing, nil } existing.Stop() } if start { err := proxy.Start() if err != nil { return nil, err } } collection.proxies[proxy.Name] = proxy return proxy, nil } func (collection *ProxyCollection) PopulateJson( server *ApiServer, data io.Reader, ) ([]*Proxy, error) { input := []struct { Proxy Enabled *bool `json:"enabled"` // Overrides Proxy field to make field nullable }{} err := json.NewDecoder(data).Decode(&input) if err != nil { return nil, joinError(err, ErrBadRequestBody) } // Check for valid input before creating any proxies t := true for i := range input { if len(input[i].Name) < 1 { return nil, joinError(fmt.Errorf("name at proxy %d", i+1), ErrMissingField) } if len(input[i].Upstream) < 1 { return nil, joinError(fmt.Errorf("upstream at proxy %d", i+1), ErrMissingField) } if input[i].Enabled == nil { input[i].Enabled = &t } } proxies := make([]*Proxy, 0, len(input)) for i := range input { proxy := NewProxy(server, input[i].Name, input[i].Listen, input[i].Upstream) addedOrReplaced, err := collection.AddOrReplace(proxy, *input[i].Enabled) if err != nil { return proxies, err } proxies = append(proxies, addedOrReplaced) } return proxies, err } func (collection *ProxyCollection) Proxies() map[string]*Proxy { collection.RLock() defer collection.RUnlock() // Copy the map since using the existing one isn't thread-safe proxies := make(map[string]*Proxy, len(collection.proxies)) for k, v := range collection.proxies { proxies[k] = v } return proxies } func (collection *ProxyCollection) Get(name string) (*Proxy, error) { collection.RLock() defer collection.RUnlock() return collection.getByName(name) } func (collection *ProxyCollection) Remove(name string) error { collection.Lock() defer collection.Unlock() proxy, err := collection.getByName(name) if err != nil { return err } proxy.Stop() delete(collection.proxies, proxy.Name) return nil } func (collection *ProxyCollection) Clear() error { collection.Lock() defer collection.Unlock() for _, proxy := range collection.proxies { proxy.Stop() delete(collection.proxies, proxy.Name) } return nil } // getByName returns a proxy by its name. Its used from #remove and #get. // It assumes the lock has already been acquired. func (collection *ProxyCollection) getByName(name string) (*Proxy, error) { proxy, exists := collection.proxies[name] if !exists { return nil, ErrProxyNotFound } return proxy, nil } ================================================ FILE: proxy_collection_test.go ================================================ package toxiproxy_test import ( "bytes" "net" "testing" "github.com/Shopify/toxiproxy/v2" ) func TestAddProxyToCollection(t *testing.T) { collection := toxiproxy.NewProxyCollection() proxy := NewTestProxy("test", "localhost:20000") if _, err := collection.Get(proxy.Name); err == nil { t.Error("Expected proxies to be empty") } err := collection.Add(proxy, false) if err != nil { t.Error("Expected to be able to add first proxy to collection") } if _, err := collection.Get(proxy.Name); err != nil { t.Error("Expected proxy to be added to map") } } func TestAddTwoProxiesToCollection(t *testing.T) { collection := toxiproxy.NewProxyCollection() proxy := NewTestProxy("test", "localhost:20000") err := collection.Add(proxy, false) if err != nil { t.Error("Expected to be able to add first proxy to collection") } err = collection.Add(proxy, false) if err == nil { t.Error("Expected to not be able to add proxy with same name") } } func TestListProxies(t *testing.T) { collection := toxiproxy.NewProxyCollection() proxy := NewTestProxy("test", "localhost:20000") err := collection.Add(proxy, false) if err != nil { t.Error("Expected to be able to add first proxy to collection") } proxies := collection.Proxies() proxy, ok := proxies[proxy.Name] if !ok { t.Error("Expected to be able to see existing proxy") } else if proxy.Enabled { t.Error("Expected proxy not to be running") } } func TestAddProxyAndStart(t *testing.T) { collection := toxiproxy.NewProxyCollection() proxy := NewTestProxy("test", "localhost:20000") err := collection.Add(proxy, true) if err != nil { t.Error("Expected to be able to add proxy to collection:", err) } proxies := collection.Proxies() proxy, ok := proxies[proxy.Name] if !ok { t.Error("Expected to be able to see existing proxy") } else if !proxy.Enabled { t.Error("Expected proxy to be running") } } func TestAddAndRemoveProxyFromCollection(t *testing.T) { WithTCPProxy(t, func(conn net.Conn, response chan []byte, proxy *toxiproxy.Proxy) { collection := toxiproxy.NewProxyCollection() if _, err := collection.Get(proxy.Name); err == nil { t.Error("Expected proxies to be empty") } err := collection.Add(proxy, false) if err != nil { t.Error("Expected to be able to add first proxy to collection") } if _, err := collection.Get(proxy.Name); err != nil { t.Error("Expected proxy to be added to map") } msg := []byte("go away") _, err = conn.Write(msg) if err != nil { t.Error("Failed writing to socket to shut down server") } conn.Close() resp := <-response if !bytes.Equal(resp, msg) { t.Error("Server didn't read bytes from client") } err = collection.Remove(proxy.Name) if err != nil { t.Error("Expected to remove proxy from collection") } if _, err := collection.Get(proxy.Name); err == nil { t.Error("Expected proxies to be empty") } }) } ================================================ FILE: proxy_test.go ================================================ package toxiproxy_test import ( "bytes" "encoding/hex" "errors" "io" "net" "os" "testing" "time" "github.com/Shopify/toxiproxy/v2" "github.com/Shopify/toxiproxy/v2/testhelper" ) func TestProxySimpleMessage(t *testing.T) { WithTCPProxy(t, func(conn net.Conn, response chan []byte, proxy *toxiproxy.Proxy) { msg := []byte("hello world") _, err := conn.Write(msg) if err != nil { t.Error("Failed writing to TCP server", err) } err = conn.Close() if err != nil { t.Error("Failed to close TCP connection", err) } resp := <-response if !bytes.Equal(resp, msg) { t.Error("Server didn't read correct bytes from client", resp) } }) } func TestProxyToDownUpstream(t *testing.T) { proxy := NewTestProxy("test", "localhost:20009") proxy.Start() conn := AssertProxyUp(t, proxy.Listen, true) // Check to make sure the connection is closed conn.SetReadDeadline(time.Now().Add(500 * time.Millisecond)) _, err := conn.Read(make([]byte, 1)) if err != io.EOF { t.Error("Proxy did not close connection when upstream down", err) } proxy.Stop() } func TestProxyBigMessage(t *testing.T) { WithTCPProxy(t, func(conn net.Conn, response chan []byte, proxy *toxiproxy.Proxy) { buf := make([]byte, 32*1024) msg := make([]byte, len(buf)*2) hex.Encode(msg, buf) _, err := conn.Write(msg) if err != nil { t.Error("Failed writing to TCP server", err) } err = conn.Close() if err != nil { t.Error("Failed to close TCP connection", err) } resp := <-response if !bytes.Equal(resp, msg) { t.Error("Server didn't read correct bytes from client", resp) } }) } func TestProxyTwoPartMessage(t *testing.T) { WithTCPProxy(t, func(conn net.Conn, response chan []byte, proxy *toxiproxy.Proxy) { msg1 := []byte("hello world") msg2 := []byte("hello world") _, err := conn.Write(msg1) if err != nil { t.Error("Failed writing to TCP server", err) } _, err = conn.Write(msg2) if err != nil { t.Error("Failed writing to TCP server", err) } err = conn.Close() if err != nil { t.Error("Failed to close TCP connection", err) } msg1 = append(msg1, msg2...) resp := <-response if !bytes.Equal(resp, msg1) { t.Error("Server didn't read correct bytes from client", resp) } }) } func TestClosingProxyMultipleTimes(t *testing.T) { WithTCPProxy(t, func(conn net.Conn, response chan []byte, proxy *toxiproxy.Proxy) { proxy.Stop() proxy.Stop() proxy.Stop() }) } func TestStartTwoProxiesOnSameAddress(t *testing.T) { WithTCPProxy(t, func(conn net.Conn, response chan []byte, proxy *toxiproxy.Proxy) { proxy2 := NewTestProxy("proxy_2", "localhost:3306") proxy2.Listen = proxy.Listen if err := proxy2.Start(); err == nil { t.Fatal("Expected an err back from start") } }) } func TestStopProxyBeforeStarting(t *testing.T) { testhelper.WithTCPServer(t, func(upstream string, response chan []byte) { proxy := NewTestProxy("test", upstream) AssertProxyUp(t, proxy.Listen, false) proxy.Stop() err := proxy.Start() if err != nil { t.Error("Proxy failed to start", err) } err = proxy.Start() if err != toxiproxy.ErrProxyAlreadyStarted { t.Error("Proxy did not fail to start when already started", err) } AssertProxyUp(t, proxy.Listen, true) proxy.Stop() AssertProxyUp(t, proxy.Listen, false) }) } func TestProxyUpdate(t *testing.T) { testhelper.WithTCPServer(t, func(upstream string, response chan []byte) { proxy := NewTestProxy("test", upstream) err := proxy.Start() if err != nil { t.Error("Proxy failed to start", err) } AssertProxyUp(t, proxy.Listen, true) before := proxy.Listen input := &toxiproxy.Proxy{Listen: "localhost:0", Upstream: proxy.Upstream, Enabled: true} err = proxy.Update(input) if err != nil { t.Error("Failed to update proxy", err) } if proxy.Listen == before || proxy.Listen == input.Listen { t.Errorf("Proxy update didn't change listen address: %s to %s", before, proxy.Listen) } AssertProxyUp(t, proxy.Listen, true) input.Listen = proxy.Listen err = proxy.Update(input) if err != nil { t.Error("Failed to update proxy", err) } AssertProxyUp(t, proxy.Listen, true) input.Enabled = false err = proxy.Update(input) if err != nil { t.Error("Failed to update proxy", err) } AssertProxyUp(t, proxy.Listen, false) }) } func TestProxyUpdateWithHostname(t *testing.T) { testhelper.WithTCPServer(t, func(upstream string, response chan []byte) { proxy := NewTestProxy("test", upstream) err := proxy.Start() if err != nil { t.Error("Proxy failed to start", err) } AssertProxyUp(t, proxy.Listen, true) connectionLost := make(chan bool) // Start a goroutine to check if connection is maintained go func() { conn, err := net.Dial("tcp", proxy.Listen) if err != nil { t.Error("Failed to connect to proxy", err) } defer conn.Close() // Try to read from the connection buf := make([]byte, 1024) conn.SetReadDeadline(time.Now().Add(500 * time.Millisecond)) _, err = conn.Read(buf) if err != nil && !errors.Is(err, os.ErrDeadlineExceeded) { connectionLost <- true return } connectionLost <- false }() _, port, err := net.SplitHostPort(proxy.Listen) if err != nil { t.Error("Failed to split host and port", err) } input := &toxiproxy.Proxy{ Listen: net.JoinHostPort("localhost", port), Upstream: proxy.Upstream, Enabled: true, } err = proxy.Update(input) if err != nil { t.Error("Failed to update proxy", err) } // Check if the connection was lost during the update if lost := <-connectionLost; lost { t.Error("Connection was lost during proxy update") } // Verify proxy is still up after the update AssertProxyUp(t, proxy.Listen, true) }) } func TestRestartFailedToStartProxy(t *testing.T) { testhelper.WithTCPServer(t, func(upstream string, response chan []byte) { proxy := NewTestProxy("test", upstream) conflict := NewTestProxy("test2", upstream) err := conflict.Start() if err != nil { t.Error("Proxy failed to start", err) } AssertProxyUp(t, conflict.Listen, true) proxy.Listen = conflict.Listen err = proxy.Start() if err == nil || err == toxiproxy.ErrProxyAlreadyStarted { t.Error("Proxy started when it should have conflicted") } conflict.Stop() AssertProxyUp(t, conflict.Listen, false) err = proxy.Start() if err != nil { t.Error("Proxy failed to start after conflict went away", err) } AssertProxyUp(t, proxy.Listen, true) proxy.Stop() AssertProxyUp(t, proxy.Listen, false) }) } func TestProxyDiffers(t *testing.T) { testhelper.WithTCPServer(t, func(upstream string, response chan []byte) { proxy := NewTestProxy("test", upstream) proxy.Start() _, port, err := net.SplitHostPort(proxy.Listen) if err != nil { t.Error("Failed to split host and port", err) } otherProxy := &toxiproxy.Proxy{ Name: "other", Listen: net.JoinHostPort("localhost", port), Upstream: upstream, Enabled: true, } differs, err := proxy.Differs(otherProxy) if err != nil { t.Error("Failed to check if proxy differs", err) } if differs { t.Error("Proxy should not differ ") } }) } ================================================ FILE: scripts/hazelcast.xml ================================================ 15 20 false deadline 3 10 member-proxy:${proxyPort} 5701 member-proxy:${proxyPort0} member-proxy:${proxyPort1} member-proxy:${proxyPort2} ================================================ FILE: scripts/test-e2e ================================================ #!/usr/bin/env bash set -ueo pipefail cd "$(dirname "$0")" server="../dist/toxiproxy-server" state="started" benchmark() { go test -bench=.. ../test/e2e -v } cli() { ../dist/toxiproxy-cli "$@" 2>&1 | sed -e 's/^/[client] /' } wait_for_url() { curl -s --retry-connrefused --retry 5 --retry-delay 2 --retry-max-time 30 \ --max-time 1 -L -I -X GET "${1}" } # Stop all background jobs on exit function cleanup() { echo -e "\n\n== Teardown: state=${state}" pkill -15 -f "toxiproxy-server -proxy-metrics -runtime-metrics$" pkill -15 -f "exe/endpoint$" } trap "cleanup" EXIT SIGINT SIGTERM echo "= Toxiproxy E2E tests" echo echo "== Setup" echo echo "=== Starting Web service" pkill -15 "toxiproxy-server" || true pkill -15 -f "exe/endpoint$" || true go run ../test/e2e/endpoint.go 2>&1 | sed -e 's/^/[web] /' & echo "=== Starting Toxiproxy" LOG_LEVEL=trace $server -proxy-metrics -runtime-metrics 2>&1 | sed -e 's/^/[toxiproxy] /' & echo "=== Wait when services are available" wait_for_url http://localhost:20002/test2 wait_for_url http://localhost:8474/version echo "=== Test client to manipulate proxy" cli -h http://localhost:8474 \ create -l localhost:20000 -u localhost:20002 shopify_http cli list cli toggle shopify_http cli inspect shopify_http cli toggle shopify_http echo -e "-----------------\n" echo "== Benchmarking" echo echo "=== Without toxics" benchmark echo -e "-----------------\n" echo "=== Latency toxic downstream" cli toxic add --downstream \ --type=latency \ --toxicName="latency_downstream" \ --attribute="latency=1000" \ --attribute="jitter=50" \ --toxicity=0.99 \ shopify_http cli inspect shopify_http benchmark cli toxic update --toxicName="latency_downstream" \ --attribute="jitter=20" \ --toxicity=0.7 \ shopify_http cli inspect shopify_http cli toxic delete --toxicName="latency_downstream" shopify_http echo -e "-----------------\n" echo "=== Latency toxic upstream" cli toxic add --upstream \ --type=latency \ --toxicName="latency_upstream" \ --attribute="latency=1000" \ --attribute="jitter=50" \ --toxicity=1 \ shopify_http cli inspect shopify_http benchmark cli toxic update --toxicName="latency_upstream" \ --attribute="jitter=20" \ --toxicity=0.3 \ shopify_http cli inspect shopify_http cli toxic delete --toxicName="latency_upstream" shopify_http echo -e "-----------------\n" echo "=== Bandwidth toxic" cli toxic add --type=bandwidth \ --toxicName="bandwidth_kb_per_second" \ --attribute="rate=1" \ --toxicity=0.5 \ shopify_http cli toxic update --toxicName="bandwidth_kb_per_second" \ --attribute="rate=10" \ --toxicity=1.0 \ shopify_http benchmark cli toxic delete --toxicName="bandwidth_kb_per_second" \ shopify_http echo -e "-----------------\n" echo "=== Timeout toxic" cli toxic add --type=timeout \ --toxicName="timeout_ms" \ --attribute="timeout=10" \ --toxicity=0.1 \ shopify_http cli toxic delete --toxicName="timeout_ms" shopify_http echo -e "-----------------\n" echo "=== Slicer toxic" cli toxic add --type=slicer \ --toxicName="slicer_us" \ --attribute="average_size=64" \ --attribute="size_variation=32" \ --attribute="delay=10" \ --toxicity=1.0 \ shopify_http benchmark cli toxic delete --toxicName="slicer_us" shopify_http echo -e "-----------------\n" echo "=== Reset peer toxic" cli toxic add --type=reset_peer \ --toxicName="reset_peer" \ --attribute="timeout=2000" \ --toxicity=1.0 \ shopify_http cli inspect shopify_http cli toxic delete --toxicName="reset_peer" shopify_http echo -e "-----------------\n" echo "== Metrics test" wait_for_url http://localhost:20000/test1 curl -s http://localhost:8474/metrics | grep -E '^toxiproxy_proxy_sent_bytes_total{direction="downstream",listener="127.0.0.1:20000",proxy="shopify_http",upstream="localhost:20002"} [0-9]+' curl -s http://localhost:8474/metrics | grep -E '^go_info' curl -s http://localhost:8474/metrics | grep -E '^go_goroutines' echo -e "-----------------\n" echo -e "=================\n" echo "Succcess!" state="success" ================================================ FILE: scripts/test-e2e-hazelcast ================================================ #!/bin/bash # Usage: # test-e2e-hazelcast [docker image name for toxiproxy] set -ueo pipefail cd "$(dirname "$0")" state="started" cli() { ../dist/toxiproxy-cli "$@" 2>&1 | sed -e 's/^/[client] /' } wait_for_url() { curl -s --retry-connrefused --retry 5 --retry-delay 2 --retry-max-time 30 \ --max-time 1 -L -I -X GET "${1}" } # Stop all background jobs on exit function cleanup() { echo -e "\n\n== Teardown: state=${state}" if [[ $state != "success" ]]; then docker kill -s SIGQUIT member-proxy docker logs -t member-proxy fi docker stop member-proxy member0 member1 member2 &>/dev/null || true docker network rm toxiproxy-e2e &>/dev/null || true } trap "cleanup" EXIT SIGINT SIGTERM LATEST_TAG=$(git describe --tags --abbrev=0) IMAGE_HAZELCAST="hazelcast/hazelcast:5.1.2-slim" IMAGE_TOXIPROXY="${1:-ghcr.io/shopify/toxiproxy:${LATEST_TAG:1}}" TOXIPROXY_BASE_URL="http://localhost:8474" echo "= Toxiproxy E2E tests with Hazelcast cluster" echo echo "== Setup" echo echo "=== Starting Toxiproxy" docker rm -f member-proxy member0 member1 member2 &>/dev/null docker network rm toxiproxy-e2e &>/dev/null || true docker network create toxiproxy-e2e docker run --rm -t "${IMAGE_TOXIPROXY}" --version docker run -d \ --name member-proxy \ --network toxiproxy-e2e \ -p 8474:8474 \ -e LOG_LEVEL=trace \ "$IMAGE_TOXIPROXY" echo "=== Wait Toxiproxy API is available" wait_for_url "${TOXIPROXY_BASE_URL}/version" echo "=== Prepare proxies for Hazelcast cluster" for i in {0..2}; do echo "> Create proxy for member${i} on port 600${i}" # curl --data "{\"name\": \"member${i}\", \"upstream\": \"member${i}:5701\", \"listen\": \"0.0.0.0:600${i}\"}" "${TOXIPROXY_BASE_URL}/proxies" cli create -l "0.0.0.0:600${i}" -u "member${i}:5701" "member${i}" echo done echo echo "=== Strating Hazelcast containers" for i in {0..2}; do echo "> Start Hazelcast on host member${i}" docker run -d --rm \ --name "member${i}" \ --network toxiproxy-e2e \ --volume "${PWD}/hazelcast.xml:/opt/hazelcast/config/hazelcast-docker.xml" \ --env HZ_PHONE_HOME_ENABLED=false \ --env JAVA_OPTS="-DproxyPort=600${i} -DproxyPort0=6000 -DproxyPort1=6001 -DproxyPort2=6002" \ "$IMAGE_HAZELCAST" done echo "> Wait for cluster join (30s)..." sleep 30 echo "> Output of member0" docker logs -t -n 10 member0 echo echo "=== Initialize toxics for cluster" for i in {0..2}; do echo "> Adding toxics to member${i} proxy" # curl --data "{\"name\": \"member${i}_downstream\", \"stream\": \"downstream\", \"toxicity\": 1.0, \"type\": \"bandwidth\", \"attributes\": { \"rate\": 0 }}" "${TOXIPROXY_BASE_URL}/proxies/member${i}/toxics" cli toxic add --type=bandwidth \ --downstream \ --toxicName="member${i}_downstream" \ --attribute="rate=0" \ --toxicity=1 \ "member${i}" # curl --data "{\"name\": \"member${i}_upstream\", \"stream\": \"upstream\", \"toxicity\": 1.0, \"type\": \"bandwidth\", \"attributes\": { \"rate\": 0 }}" "${TOXIPROXY_BASE_URL}/proxies/member${i}/toxics" cli toxic add --type=bandwidth \ --upstream \ --toxicName="member${i}_upstream" \ --attribute="rate=0" \ --toxicity=1 \ "member${i}" echo cli inspect "member${i}" echo done echo "=== Wait for a the Hazelcast cluster split-brain (60s)..." sleep 60 echo "=== Validate output of Toxiproxy and single member" docker logs -t -n 10 member0 docker logs -t -n 10 member-proxy echo "=== Removing toxics from proxies" for i in {0..2}; do echo "[$(date)] > Remove downstream bandwith Toxic for member${i} proxy" # curl -v -X DELETE "${TOXIPROXY_BASE_URL}/proxies/member${i}/toxics/member${i}_downstream" cli toxic delete --toxicName="member${i}_downstream" "member${i}" echo "[$(date)] > Remove ustream bandwith Toxic for member${i} proxy" # curl -v -X DELETE "${TOXIPROXY_BASE_URL}/proxies/member${i}/toxics/member${i}_upstream" cli toxic delete --toxicName="member${i}_upstream" "member${i}" done echo "=== Validate output of Toxiproxy and single member after removing toxics" docker logs -t -n 10 member0 docker logs -t -n 10 member-proxy cli list cli inspect member0 cli inspect member1 cli inspect member2 echo -e "=================\n" echo "Succcess!" state="success" ================================================ FILE: scripts/test-release ================================================ #!/usr/bin/env bash set -ueo pipefail VERSION_FULL="$(git describe --abbrev=0 --tags)" VERSION=${VERSION_FULL:1} ARCH=$(uname -m) if [ "${ARCH}" == "x86_64" ]; then ARCH="amd64" fi goreleaser release --rm-dist --skip-publish --skip-validate docker run -v "$(PWD)"/dist:/dist --pull always --rm -it ubuntu bash -c \ "set -xe; dpkg -i /dist/toxiproxy_*_linux_${ARCH}.deb; ls -1 /usr/bin/toxiproxy-*; /usr/bin/toxiproxy-server --version; /usr/bin/toxiproxy-server --version \ | grep -o -e 'toxiproxy-server version ${VERSION}'; /usr/bin/toxiproxy-cli --version; /usr/bin/toxiproxy-cli --version \ | grep -o -e 'toxiproxy-cli version ${VERSION}'" docker run -v "$(PWD)"/dist:/dist --pull always --rm -it fedora bash -c \ "set -xe; yum localinstall -y /dist/toxiproxy_*_linux_${ARCH}.rpm; ls -1 /usr/bin/toxiproxy-*; /usr/bin/toxiproxy-server --version; /usr/bin/toxiproxy-server --version \ | grep -o -e 'toxiproxy-server version ${VERSION}'; /usr/bin/toxiproxy-cli --version; /usr/bin/toxiproxy-cli --version \ | grep -o -e 'toxiproxy-cli version ${VERSION}'" docker run -v "$(PWD)"/dist:/dist --pull always --rm -it alpine sh -c \ "set -xe; apk add --allow-untrusted --no-cache /dist/toxiproxy_*_linux_${ARCH}.apk; ls -1 /usr/bin/toxiproxy-*; /usr/bin/toxiproxy-server --version; /usr/bin/toxiproxy-server --version \ | grep -o -e 'toxiproxy-server version ${VERSION}'; /usr/bin/toxiproxy-cli --version; /usr/bin/toxiproxy-cli --version \ | grep -o -e 'toxiproxy-cli version ${VERSION}'" tar -ztvf dist/toxiproxy_*_linux_amd64.tar.gz | grep -o -e toxiproxy-server tar -ztvf dist/toxiproxy_*_linux_amd64.tar.gz | grep -o -e toxiproxy-cli goreleaser build --rm-dist --single-target --skip-validate --id server bineries=(./dist/toxiproxy-server-*) server="${bineries[0]}" $server --help 2>&1 | grep -o -e "Usage of ./dist/toxiproxy-server" $server --version | grep -o -e "toxiproxy-server version ${VERSION}" goreleaser build --rm-dist --single-target --skip-validate --id client bineries=(./dist/toxiproxy-cli-*) cli="${bineries[0]}" $cli --help 2>&1 | grep -o -e "toxiproxy-cli - Simulate network and system conditions" $cli --version | grep -o -e "toxiproxy-cli version ${VERSION}" ================================================ FILE: share/toxiproxy.conf ================================================ description "TCP proxy to simulate network and system conditions" author "Simon Eskildsen & Jacob Wirth" start on startup stop on shutdown env HOST="localhost" env PORT="8474" env BINARY="/usr/bin/toxiproxy-server" script exec $BINARY -port $PORT -host $HOST end script ================================================ FILE: stream/direction.go ================================================ package stream import ( "errors" "strings" ) type Direction uint8 var ErrInvalidDirectionParameter error = errors.New("stream: invalid direction") const ( Upstream Direction = iota Downstream NumDirections ) func (d Direction) String() string { if d >= NumDirections { return "num_directions" } return [...]string{"upstream", "downstream"}[d] } func ParseDirection(value string) (Direction, error) { switch strings.ToLower(value) { case "downstream": return Downstream, nil case "upstream": return Upstream, nil } return NumDirections, ErrInvalidDirectionParameter } ================================================ FILE: stream/direction_test.go ================================================ package stream_test import ( "testing" "github.com/Shopify/toxiproxy/v2/stream" ) func TestDirection_String(t *testing.T) { testCases := []struct { name string direction stream.Direction expected string }{ {"Downstream to string", stream.Downstream, "downstream"}, {"Upstream to string", stream.Upstream, "upstream"}, {"NumDirections to string", stream.NumDirections, "num_directions"}, {"Upstream via number direction to string", stream.Direction(0), "upstream"}, {"Downstream via number direction to string", stream.Direction(1), "downstream"}, {"High number direction to string", stream.Direction(5), "num_directions"}, } for _, tc := range testCases { tc := tc // capture range variable t.Run(tc.name, func(t *testing.T) { t.Parallel() actual := tc.direction.String() if actual != tc.expected { t.Errorf("got \"%s\"; expected \"%s\"", actual, tc.expected) } }) } } func TestParseDirection(t *testing.T) { testCases := []struct { name string input string expected stream.Direction err error }{ {"parse empty", "", stream.NumDirections, stream.ErrInvalidDirectionParameter}, {"parse upstream", "upstream", stream.Upstream, nil}, {"parse downstream", "downstream", stream.Downstream, nil}, {"parse unknown", "unknown", stream.NumDirections, stream.ErrInvalidDirectionParameter}, {"parse number", "-123", stream.NumDirections, stream.ErrInvalidDirectionParameter}, {"parse upper case", "DOWNSTREAM", stream.Downstream, nil}, {"parse camel case", "UpStream", stream.Upstream, nil}, } for _, tc := range testCases { tc := tc // capture range variable t.Run(tc.name, func(t *testing.T) { t.Parallel() actual, err := stream.ParseDirection(tc.input) if actual != tc.expected { t.Errorf("got \"%s\"; expected \"%s\"", actual, tc.expected) } if err != tc.err { t.Errorf("got \"%s\"; expected \"%s\"", err, tc.err) } }) } } ================================================ FILE: stream/io_chan.go ================================================ package stream import ( "fmt" "io" "time" ) // Stores a slice of bytes with its receive timestamp. type StreamChunk struct { Data []byte Timestamp time.Time } // Implements the io.WriteCloser interface for a chan []byte. type ChanWriter struct { output chan<- *StreamChunk } func NewChanWriter(output chan<- *StreamChunk) *ChanWriter { return &ChanWriter{output} } // Write `buf` as a StreamChunk to the channel. The full buffer is always written, and error // will always be nil. Calling `Write()` after closing the channel will panic. func (c *ChanWriter) Write(buf []byte) (int, error) { packet := &StreamChunk{make([]byte, len(buf)), time.Now()} copy(packet.Data, buf) // Make a copy before sending it to the channel c.output <- packet return len(buf), nil } // Close the output channel. func (c *ChanWriter) Close() error { close(c.output) return nil } // Implements the io.Reader interface for a chan []byte. type ChanReader struct { input <-chan *StreamChunk interrupt <-chan struct{} buffer []byte } var ErrInterrupted = fmt.Errorf("read interrupted by channel") func NewChanReader(input <-chan *StreamChunk) *ChanReader { return &ChanReader{input, make(chan struct{}), []byte{}} } // Specify a channel that can interrupt a read if it is blocking. func (c *ChanReader) SetInterrupt(interrupt <-chan struct{}) { c.interrupt = interrupt } // Read from the channel into `out`. This will block until data is available, // and can be interrupted with a channel using `SetInterrupt()`. If the read // was interrupted, `ErrInterrupted` will be returned. func (c *ChanReader) Read(out []byte) (int, error) { if c.buffer == nil { return 0, io.EOF } n := copy(out, c.buffer) c.buffer = c.buffer[n:] if len(out) <= len(c.buffer) { return n, nil } else if n > 0 { // We have some data to return, so make the channel read optional select { case p := <-c.input: if p == nil { // Stream was closed c.buffer = nil if n > 0 { return n, nil } return 0, io.EOF } n2 := copy(out[n:], p.Data) c.buffer = p.Data[n2:] return n + n2, nil default: return n, nil } } var p *StreamChunk select { case p = <-c.input: case <-c.interrupt: c.buffer = c.buffer[:0] return n, ErrInterrupted } if p == nil { // Stream was closed c.buffer = nil return 0, io.EOF } n2 := copy(out[n:], p.Data) c.buffer = p.Data[n2:] return n + n2, nil } ================================================ FILE: stream/io_chan_test.go ================================================ package stream import ( "bytes" "io" "testing" "time" ) func TestBasicReadWrite(t *testing.T) { send := []byte("hello world") c := make(chan *StreamChunk) writer := NewChanWriter(c) reader := NewChanReader(c) go writer.Write(send) buf := make([]byte, len(send)) n, err := reader.Read(buf) if n != len(send) { t.Fatalf("Read wrong number of bytes: %d expected %d", n, len(send)) } if err != nil { t.Fatal("Couldn't read from stream", err) } if !bytes.Equal(buf, send) { t.Fatal("Got wrong message from stream", string(buf)) } writer.Close() n, err = reader.Read(buf) if err != io.EOF { t.Fatal("Read returned wrong error after close:", err) } if n != 0 { t.Fatalf("Read still returned data after close: %d bytes", n) } } func TestReadMoreThanWrite(t *testing.T) { send := []byte("hello world") c := make(chan *StreamChunk) writer := NewChanWriter(c) reader := NewChanReader(c) go writer.Write(send) buf := make([]byte, len(send)+10) n, err := reader.Read(buf) if err != nil { t.Fatal("Couldn't read from stream", err) } if n != len(send) { t.Fatalf("Read wrong number of bytes: %d expected %d", n, len(send)) } if !bytes.Equal(buf[:n], send) { t.Fatal("Got wrong message from stream", string(buf[:n])) } writer.Close() n, err = reader.Read(buf) if err != io.EOF { t.Fatal("Read returned wrong error after close:", err) } if n != 0 { t.Fatalf("Read still returned data after close: %d bytes", n) } } func TestReadLessThanWrite(t *testing.T) { send := []byte("hello world") c := make(chan *StreamChunk) writer := NewChanWriter(c) reader := NewChanReader(c) go writer.Write(send) buf := make([]byte, 6) n, err := reader.Read(buf) if err != nil { t.Fatal("Couldn't read from stream", err) } if n != len(buf) { t.Fatalf("Read wrong number of bytes: %d expected %d", n, len(buf)) } if !bytes.Equal(buf, send[:len(buf)]) { t.Fatal("Got wrong message from stream", string(buf)) } writer.Close() n, err = reader.Read(buf) if err != nil { t.Fatal("Couldn't read from stream", err) } if n != len(send)-len(buf) { t.Fatalf("Read wrong number of bytes: %d expected %d", n, len(send)-len(buf)) } if !bytes.Equal(buf[:n], send[len(buf):]) { t.Fatal("Got wrong message from stream", string(buf[:n])) } n, err = reader.Read(buf) if err != io.EOF { t.Fatal("Read returned wrong error after close:", err) } if n != 0 { t.Fatalf("Read still returned data after close: %d bytes", n) } } func TestMultiReadWrite(t *testing.T) { send := []byte("hello world, this message is longer") c := make(chan *StreamChunk) writer := NewChanWriter(c) reader := NewChanReader(c) go func() { writer.Write(send[:9]) writer.Write(send[9:19]) writer.Write(send[19:]) writer.Close() }() buf := make([]byte, 10) for read := 0; read < len(send); { n, err := reader.Read(buf) if err != nil { t.Fatal("Couldn't read from stream", err, n) } if !bytes.Equal(buf[:n], send[read:read+n]) { t.Fatal("Got wrong message from stream", string(buf)) } read += n } n, err := reader.Read(buf) if err != io.EOF { t.Fatal("Read returned wrong error after close:", err, string(buf[:n])) } if !bytes.Equal(buf[:n], send[len(send)-n:]) { t.Fatal("Got wrong message from stream", string(buf[:n])) } } func TestMultiWriteWithCopy(t *testing.T) { send := []byte("hello world, this message is longer") c := make(chan *StreamChunk) writer := NewChanWriter(c) reader := NewChanReader(c) go func() { writer.Write(send[:9]) writer.Write(send[9:19]) writer.Write(send[19:]) writer.Close() }() buf := new(bytes.Buffer) n, err := io.Copy(buf, reader) if int(n) != len(send) { t.Fatalf("Read wrong number of bytes: %d expected %d", n, len(send)) } if err != nil { t.Fatal("Couldn't read from stream", err) } if !bytes.Equal(buf.Bytes(), send) { t.Fatal("Got wrong message from stream", buf.String()) } } func TestStream_ReadCorrectness(t *testing.T) { sendMsg := []byte("hello world") c := make(chan *StreamChunk) interrupt := make(chan struct{}) writer := NewChanWriter(c) reader := NewChanReader(c) reader.SetInterrupt(interrupt) go writer.Write(sendMsg) readMsg := make([]byte, len(sendMsg)) n, err := reader.Read(readMsg) if err != nil { t.Fatalf("Couldn't read from stream: %v", err) } if n != len(sendMsg) { t.Fatalf("Read wrong number of bytes: %d expected %d", n, len(sendMsg)) } if !bytes.Equal(readMsg, sendMsg) { t.Fatal("Got wrong message from stream", string(readMsg)) } } func TestStream_ReadInterrupt(t *testing.T) { sendMsg := []byte("hello world") c := make(chan *StreamChunk) interrupt := make(chan struct{}) writer := NewChanWriter(c) reader := NewChanReader(c) reader.SetInterrupt(interrupt) go writer.Write(sendMsg) readMsg := make([]byte, len(sendMsg)) reader.Read(readMsg) // Interrupting the stream mid-read go func() { time.Sleep(50 * time.Millisecond) interrupt <- struct{}{} }() n, err := reader.Read(readMsg) if err != ErrInterrupted { t.Fatalf("Read returned wrong error after interrupt: %v", err) } if n != 0 { t.Fatalf("Read still returned data after interrput: %d bytes", n) } // Try writing again after the channel was interrupted go writer.Write(sendMsg) n, err = reader.Read(readMsg) if n != len(sendMsg) { t.Fatalf("Read wrong number of bytes: %d expected %d", n, len(sendMsg)) } if err != nil { t.Fatalf("Couldn't read from stream: %v", err) } if !bytes.Equal(readMsg, sendMsg) { t.Fatal("Got wrong message from stream", string(readMsg)) } } ================================================ FILE: test/e2e/benchmark_test.go ================================================ package main import ( "io" "net/http" "testing" ) // Benchmark numbers: // // Toxiproxy 1.1 // // 1x Toxic Types: // BenchmarkDirect 3000 588148 ns/op // BenchmarkProxy 2000 999949 ns/op // BenchmarkDirectSmall 5000 291324 ns/op // BenchmarkProxySmall 3000 504501 ns/op // // 10x Toxic Types: // BenchmarkDirect 3000 599519 ns/op // BenchmarkProxy 2000 1044746 ns/op // BenchmarkDirectSmall 5000 280713 ns/op // BenchmarkProxySmall 3000 574816 ns/op // // Toxiproxy 2.0 // // No Enabled Toxics: // BenchmarkDirect 2000 597998 ns/op // BenchmarkProxy 2000 964510 ns/op // BenchmarkDirectSmall 10000 287448 ns/op // BenchmarkProxySmall 5000 560694 ns/op // Test the backend server directly, use 64k random endpoint. func BenchmarkDirect(b *testing.B) { client := http.Client{} for i := 0; i < b.N; i++ { resp, err := client.Get("http://localhost:20002/test1") if err != nil { b.Fatal(err) } _, err = io.ReadAll(resp.Body) if err != nil { b.Fatal(err) } resp.Body.Close() } client.CloseIdleConnections() } // Test the backend through toxiproxy, use 64k random endpoint. func BenchmarkProxy(b *testing.B) { client := http.Client{} for i := 0; i < b.N; i++ { resp, err := client.Get("http://localhost:20000/test1") if err != nil { b.Fatal(err) } _, err = io.ReadAll(resp.Body) if err != nil { b.Fatal(err) } resp.Body.Close() } client.CloseIdleConnections() } // Test the backend server directly, use "hello world" endpoint. func BenchmarkDirectSmall(b *testing.B) { client := http.Client{} for i := 0; i < b.N; i++ { resp, err := client.Get("http://localhost:20002/test2") if err != nil { b.Fatal(err) } _, err = io.ReadAll(resp.Body) if err != nil { b.Fatal(err) } resp.Body.Close() } client.CloseIdleConnections() } // Test the backend through toxiproxy, use "hello world" endpoint. func BenchmarkProxySmall(b *testing.B) { client := http.Client{} for i := 0; i < b.N; i++ { resp, err := client.Get("http://localhost:20000/test2") if err != nil { b.Fatal(err) } _, err = io.ReadAll(resp.Body) if err != nil { b.Fatal(err) } resp.Body.Close() } client.CloseIdleConnections() } ================================================ FILE: test/e2e/endpoint.go ================================================ package main import ( "encoding/hex" "fmt" "log" "net/http" "time" "github.com/gorilla/mux" ) var ( stuff []byte out []byte out2 []byte ) func handler1(w http.ResponseWriter, r *http.Request) { n, err := w.Write(out) if n != len(out) { fmt.Println("Short write!") } if err != nil { fmt.Println(err) } } func handler2(w http.ResponseWriter, r *http.Request) { n, err := w.Write(out2) if n != len(out2) { fmt.Println("Short write!") } if err != nil { fmt.Println(err) } } func main() { stuff = make([]byte, 32*1024) out = make([]byte, len(stuff)*2) out2 = []byte("hello world") for i := 0; i < len(stuff); i++ { stuff[i] = byte(i % 256) } hex.Encode(out, stuff) r := mux.NewRouter() r.HandleFunc("/test1", handler1) r.HandleFunc("/test2", handler2) log.Println("Listening :20002") srv := &http.Server{ Handler: r, Addr: ":20002", WriteTimeout: 3 * time.Second, ReadTimeout: 3 * time.Second, } log.Fatal(srv.ListenAndServe()) } ================================================ FILE: testhelper/tcp_server.go ================================================ package testhelper import ( "io" "net" "testing" ) func NewTCPServer() (*TCPServer, error) { result := &TCPServer{ addr: "localhost:0", response: make(chan []byte, 1), } err := result.Run() if err != nil { return nil, err } return result, nil } type TCPServer struct { addr string server net.Listener response chan []byte } func (server *TCPServer) Run() (err error) { server.server, err = net.Listen("tcp", server.addr) if err != nil { return } server.addr = server.server.Addr().String() return } func (server *TCPServer) handle_connection() (err error) { conn, err := server.server.Accept() if err != nil { return } defer conn.Close() val, err := io.ReadAll(conn) if err != nil { return } server.response <- val return } func (server *TCPServer) Close() (err error) { return server.server.Close() } func WithTCPServer(t *testing.T, block func(string, chan []byte)) { server, err := NewTCPServer() if err != nil { t.Fatal("Failed to create TCP server", err) } go func(t *testing.T, server *TCPServer) { err := server.handle_connection() if err != nil { t.Error("Failed to handle connection", err) } }(t, server) block(server.addr, server.response) } ================================================ FILE: testhelper/tcp_server_test.go ================================================ package testhelper_test import ( "bytes" "net" "testing" "github.com/Shopify/toxiproxy/v2/testhelper" ) func TestSimpleServer(t *testing.T) { testhelper.WithTCPServer(t, func(addr string, response chan []byte) { conn, err := net.Dial("tcp", addr) if err != nil { t.Error("Unable to dial TCP server", err) } msg := []byte("hello world") _, err = conn.Write(msg) if err != nil { t.Error("Failed writing to TCP server", err) } err = conn.Close() if err != nil { t.Error("Failed to close TCP connection", err) } resp := <-response if !bytes.Equal(resp, msg) { t.Error("Server didn't read bytes from client") } }) } ================================================ FILE: testhelper/timeout_after.go ================================================ package testhelper import ( "fmt" "time" ) func TimeoutAfter(after time.Duration, f func()) error { success := make(chan struct{}) go func() { f() close(success) }() select { case <-success: return nil case <-time.After(after): return fmt.Errorf("timed out after %s", after) } } ================================================ FILE: testhelper/timeout_after_test.go ================================================ package testhelper_test import ( "testing" "time" "github.com/Shopify/toxiproxy/v2/testhelper" ) func TestTimeoutAfter(t *testing.T) { err := testhelper.TimeoutAfter(5*time.Millisecond, func() {}) if err != nil { t.Fatal("Non blocking function should not timeout.") } err = testhelper.TimeoutAfter(5*time.Millisecond, func() { time.Sleep(time.Second) }) if err == nil { t.Fatal("Blocking function should timeout.") } } ================================================ FILE: testhelper/upstream.go ================================================ package testhelper import ( "net" "testing" ) type Upstream struct { listener net.Listener logger testing.TB Connections chan net.Conn } func NewUpstream(t testing.TB, ignoreData bool) *Upstream { result := &Upstream{ logger: t, } result.listen() result.accept(ignoreData) return result } func (u *Upstream) listen() { listener, err := net.Listen("tcp", "localhost:0") if err != nil { u.logger.Fatalf("Failed to create TCP server: %v", err) } u.listener = listener } func (u *Upstream) accept(ignoreData bool) { u.Connections = make(chan net.Conn) go func(u *Upstream) { conn, err := u.listener.Accept() if err != nil { u.logger.Fatalf("Unable to accept TCP connection: %v", err) } if ignoreData { buf := make([]byte, 4000) for err == nil { _, err = conn.Read(buf) } } else { u.Connections <- conn } }(u) } func (u *Upstream) Close() { u.listener.Close() } func (u *Upstream) Addr() string { return u.listener.Addr().String() } ================================================ FILE: toxic_collection.go ================================================ package toxiproxy import ( "bytes" "context" "encoding/json" "fmt" "io" "sync" "github.com/rs/zerolog" "github.com/Shopify/toxiproxy/v2/stream" "github.com/Shopify/toxiproxy/v2/toxics" ) // ToxicCollection contains a list of toxics that are chained together. Each proxy // has its own collection. A hidden noop toxic is always maintained at the beginning // of each chain so toxics have a method of pausing incoming data (by interrupting // the preceding toxic). type ToxicCollection struct { sync.Mutex noop *toxics.ToxicWrapper proxy *Proxy chain [][]*toxics.ToxicWrapper links map[string]*ToxicLink } func NewToxicCollection(proxy *Proxy) *ToxicCollection { collection := &ToxicCollection{ noop: &toxics.ToxicWrapper{ Toxic: new(toxics.NoopToxic), Type: "noop", }, proxy: proxy, chain: make([][]*toxics.ToxicWrapper, stream.NumDirections), links: make(map[string]*ToxicLink), } for dir := range collection.chain { collection.chain[dir] = make([]*toxics.ToxicWrapper, 1, toxics.Count()+1) collection.chain[dir][0] = collection.noop } return collection } func (c *ToxicCollection) ResetToxics(ctx context.Context) { c.Lock() defer c.Unlock() // Remove all but the first noop toxic for dir := range c.chain { for len(c.chain[dir]) > 1 { c.chainRemoveToxic(ctx, c.chain[dir][1]) } } } func (c *ToxicCollection) GetToxic(name string) *toxics.ToxicWrapper { c.Lock() defer c.Unlock() return c.findToxicByName(name) } func (c *ToxicCollection) GetToxicArray() []toxics.Toxic { c.Lock() defer c.Unlock() result := make([]toxics.Toxic, 0) for dir := range c.chain { for i, toxic := range c.chain[dir] { if i == 0 { // Skip the first noop toxic, it should not be visible continue } result = append(result, toxic) } } return result } func (c *ToxicCollection) AddToxicJson(data io.Reader) (*toxics.ToxicWrapper, error) { c.Lock() defer c.Unlock() var buffer bytes.Buffer // Default to a downstream toxic with a toxicity of 1. wrapper := &toxics.ToxicWrapper{ Stream: "downstream", Toxicity: 1.0, Toxic: new(toxics.NoopToxic), } err := json.NewDecoder(io.TeeReader(data, &buffer)).Decode(wrapper) if err != nil { return nil, joinError(err, ErrBadRequestBody) } wrapper.Direction, err = stream.ParseDirection(wrapper.Stream) if err != nil { return nil, ErrInvalidStream } if wrapper.Name == "" { wrapper.Name = fmt.Sprintf("%s_%s", wrapper.Type, wrapper.Stream) } if toxics.New(wrapper) == nil { return nil, ErrInvalidToxicType } found := c.findToxicByName(wrapper.Name) if found != nil { return nil, ErrToxicAlreadyExists } // Parse attributes because we now know the toxics type. attrs := &struct { Attributes interface{} `json:"attributes"` }{ wrapper.Toxic, } err = json.NewDecoder(&buffer).Decode(attrs) if err != nil { return nil, joinError(err, ErrBadRequestBody) } c.chainAddToxic(wrapper) return wrapper, nil } func (c *ToxicCollection) UpdateToxicJson( name string, data io.Reader, ) (*toxics.ToxicWrapper, error) { c.Lock() defer c.Unlock() toxic := c.findToxicByName(name) if toxic != nil { attrs := &struct { Attributes interface{} `json:"attributes"` Toxicity float32 `json:"toxicity"` }{ toxic.Toxic, toxic.Toxicity, } err := json.NewDecoder(data).Decode(attrs) if err != nil { return nil, joinError(err, ErrBadRequestBody) } toxic.Toxicity = attrs.Toxicity c.chainUpdateToxic(toxic) return toxic, nil } return nil, ErrToxicNotFound } func (c *ToxicCollection) RemoveToxic(ctx context.Context, name string) error { log := zerolog.Ctx(ctx). With(). Str("component", "ToxicCollection"). Str("method", "RemoveToxic"). Str("toxic", name). Str("proxy", c.proxy.Name). Logger() log.Trace().Msg("Acquire locking...") c.Lock() defer c.Unlock() log.Trace().Msg("Getting toxic by name...") toxic := c.findToxicByName(name) if toxic == nil { log.Trace().Msg("Could not find toxic by name") return ErrToxicNotFound } c.chainRemoveToxic(ctx, toxic) log.Trace().Msg("Finished") return nil } func (c *ToxicCollection) StartLink( server *ApiServer, name string, input io.Reader, output io.WriteCloser, direction stream.Direction, ) { c.Lock() defer c.Unlock() var logger zerolog.Logger if c.proxy.Logger != nil { logger = *c.proxy.Logger } else { logger = zerolog.Nop() } link := NewToxicLink(c.proxy, c, direction, logger) link.Start(server, name, input, output) c.links[name] = link } func (c *ToxicCollection) RemoveLink(name string) { c.Lock() defer c.Unlock() delete(c.links, name) } // All following functions assume the lock is already grabbed. func (c *ToxicCollection) findToxicByName(name string) *toxics.ToxicWrapper { for dir := range c.chain { // Skip the first noop toxic, it has no name for _, toxic := range c.chain[dir][1:] { if toxic.Name == name { return toxic } } } return nil } func (c *ToxicCollection) chainAddToxic(toxic *toxics.ToxicWrapper) { dir := toxic.Direction toxic.Index = len(c.chain[dir]) c.chain[dir] = append(c.chain[dir], toxic) // Asynchronously add the toxic to each link wg := sync.WaitGroup{} for _, link := range c.links { if link.direction == dir { wg.Add(1) go func(link *ToxicLink, wg *sync.WaitGroup) { defer wg.Done() link.AddToxic(toxic) }(link, &wg) } } wg.Wait() } func (c *ToxicCollection) chainUpdateToxic(toxic *toxics.ToxicWrapper) { c.chain[toxic.Direction][toxic.Index] = toxic // Asynchronously update the toxic in each link group := sync.WaitGroup{} for _, link := range c.links { if link.direction == toxic.Direction { group.Add(1) go func(link *ToxicLink) { defer group.Done() link.UpdateToxic(toxic) }(link) } } group.Wait() } func (c *ToxicCollection) chainRemoveToxic(ctx context.Context, toxic *toxics.ToxicWrapper) { log := zerolog.Ctx(ctx). With(). Str("component", "ToxicCollection"). Str("method", "chainRemoveToxic"). Str("toxic", toxic.Name). Str("direction", toxic.Direction.String()). Logger() dir := toxic.Direction c.chain[dir] = append(c.chain[dir][:toxic.Index], c.chain[dir][toxic.Index+1:]...) for i := toxic.Index; i < len(c.chain[dir]); i++ { c.chain[dir][i].Index = i } // Asynchronously remove the toxic from each link wg := sync.WaitGroup{} event_array := zerolog.Arr() for _, link := range c.links { if link.direction == dir { event_array = event_array.Str(fmt.Sprintf("Link[%p] %s", link, link.Direction())) wg.Add(1) go func(ctx context.Context, link *ToxicLink, log zerolog.Logger) { defer wg.Done() link.RemoveToxic(ctx, toxic) }(ctx, link, log) } } log.Trace(). Array("links", event_array). Msg("Waiting to update links") wg.Wait() toxic.Index = -1 } ================================================ FILE: toxics/bandwidth.go ================================================ package toxics import ( "fmt" "time" "github.com/rs/zerolog/log" "github.com/Shopify/toxiproxy/v2/stream" ) // The BandwidthToxic passes data through at a limited rate. type BandwidthToxic struct { // Rate in KB/s Rate int64 `json:"rate"` } func (t *BandwidthToxic) Pipe(stub *ToxicStub) { logger := log.With(). Str("component", "BandwidthToxic"). Str("method", "Pipe"). Str("toxic_type", "bandwidth"). Str("addr", fmt.Sprintf("%p", t)). Logger() var sleep time.Duration = 0 for { select { case <-stub.Interrupt: logger.Trace().Msg("BandwidthToxic was interrupted") return case p := <-stub.Input: if p == nil { stub.Close() return } if t.Rate <= 0 { sleep = 0 } else { sleep += time.Duration(len(p.Data)) * time.Millisecond / time.Duration(t.Rate) } // If the rate is low enough, split the packet up and send in 100 millisecond intervals for int64(len(p.Data)) > t.Rate*100 { select { case <-time.After(100 * time.Millisecond): stub.Output <- &stream.StreamChunk{ Data: p.Data[:t.Rate*100], Timestamp: p.Timestamp, } p.Data = p.Data[t.Rate*100:] sleep -= 100 * time.Millisecond case <-stub.Interrupt: logger.Trace().Msg("BandwidthToxic was interrupted during writing data") err := stub.WriteOutput(p, 5*time.Second) // Don't drop any data on the floor if err != nil { logger.Warn().Err(err). Msg("Could not write last packets after interrupt to Output") } return } } start := time.Now() select { case <-time.After(sleep): // time.After only seems to have ~1ms prevision, so offset the next sleep by the error sleep -= time.Since(start) stub.Output <- p case <-stub.Interrupt: logger.Trace().Msg("BandwidthToxic was interrupted during writing data") err := stub.WriteOutput(p, 5*time.Second) // Don't drop any data on the floor if err != nil { logger.Warn().Err(err). Msg("Could not write last packets after interrupt to Output") } return } } } } func init() { Register("bandwidth", new(BandwidthToxic)) } ================================================ FILE: toxics/bandwidth_test.go ================================================ package toxics_test import ( "bytes" "io" "net" "strings" "testing" "time" "github.com/Shopify/toxiproxy/v2/testhelper" "github.com/Shopify/toxiproxy/v2/toxics" ) func TestBandwidthToxic(t *testing.T) { upstream := testhelper.NewUpstream(t, false) defer upstream.Close() proxy := NewTestProxy("test", upstream.Addr()) proxy.Start() defer proxy.Stop() client, err := net.Dial("tcp", proxy.Listen) if err != nil { t.Fatalf("Unable to dial TCP server: %v", err) } upstreamConn := <-upstream.Connections rate := 1000 // 1MB/s proxy.Toxics.AddToxicJson( ToxicToJson(t, "", "bandwidth", "upstream", &toxics.BandwidthToxic{Rate: int64(rate)}), ) writtenPayload := []byte(strings.Repeat("hello world ", 40000)) // 480KB go func() { n, err := client.Write(writtenPayload) client.Close() if n != len(writtenPayload) || err != nil { t.Errorf("Failed to write buffer: (%d == %d) %v", n, len(writtenPayload), err) } }() serverRecvPayload := make([]byte, len(writtenPayload)) start := time.Now() _, err = io.ReadAtLeast(upstreamConn, serverRecvPayload, len(serverRecvPayload)) if err != nil { t.Errorf("Proxy read failed: %v", err) } else if !bytes.Equal(writtenPayload, serverRecvPayload) { t.Errorf("Server did not read correct buffer from client!") } AssertDeltaTime(t, "Bandwidth", time.Since(start), time.Duration(len(writtenPayload))*time.Second/time.Duration(rate*1000), 10*time.Millisecond, ) } func BenchmarkBandwidthToxic100MB(b *testing.B) { upstream := testhelper.NewUpstream(b, true) defer upstream.Close() proxy := NewTestProxy("test", upstream.Addr()) proxy.Start() defer proxy.Stop() client, err := net.Dial("tcp", proxy.Listen) if err != nil { b.Error("Unable to dial TCP server", err) } writtenPayload := []byte(strings.Repeat("hello world ", 1000)) proxy.Toxics.AddToxicJson( ToxicToJson(nil, "", "bandwidth", "upstream", &toxics.BandwidthToxic{Rate: 100 * 1000}), ) b.SetBytes(int64(len(writtenPayload))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { n, err := client.Write(writtenPayload) if err != nil || n != len(writtenPayload) { b.Errorf("%v, %d == %d", err, n, len(writtenPayload)) break } } err = client.Close() if err != nil { b.Error("Failed to close TCP connection", err) } } ================================================ FILE: toxics/latency.go ================================================ package toxics import ( "math/rand" "time" ) // The LatencyToxic passes data through with the a delay of latency +/- jitter added. type LatencyToxic struct { // Times in milliseconds Latency int64 `json:"latency"` Jitter int64 `json:"jitter"` } func (t *LatencyToxic) GetBufferSize() int { return 1024 } func (t *LatencyToxic) delay() time.Duration { // Delay = t.Latency +/- t.Jitter delay := t.Latency jitter := t.Jitter if jitter > 0 { // #nosec G404 -- was ignored before too delay += rand.Int63n(jitter*2) - jitter } return time.Duration(delay) * time.Millisecond } func (t *LatencyToxic) Pipe(stub *ToxicStub) { for { select { case <-stub.Interrupt: return case c := <-stub.Input: if c == nil { stub.Close() return } sleep := t.delay() - time.Since(c.Timestamp) select { case <-time.After(sleep): c.Timestamp = c.Timestamp.Add(sleep) stub.Output <- c case <-stub.Interrupt: // Exit fast without applying latency. stub.Output <- c // Don't drop any data on the floor return } } } } func init() { Register("latency", new(LatencyToxic)) } ================================================ FILE: toxics/latency_test.go ================================================ package toxics_test import ( "bufio" "bytes" "context" "io" "net" "strconv" "strings" "testing" "time" "github.com/Shopify/toxiproxy/v2" "github.com/Shopify/toxiproxy/v2/testhelper" "github.com/Shopify/toxiproxy/v2/toxics" ) func AssertDeltaTime(t *testing.T, message string, actual, expected, delta time.Duration) { diff := actual - expected if diff < 0 { diff *= -1 } if diff > delta { t.Errorf( "[%s] Time was more than %v off: got %v expected %v", message, delta, actual, expected, ) } else { t.Logf("[%s] Time was correct: %v (expected %v)", message, actual, expected) } } func DoLatencyTest(t *testing.T, upLatency, downLatency *toxics.LatencyToxic) { WithEchoProxy(t, func(conn net.Conn, response chan []byte, proxy *toxiproxy.Proxy) { if upLatency == nil { upLatency = &toxics.LatencyToxic{} } else { _, err := proxy.Toxics.AddToxicJson( ToxicToJson(t, "latency_up", "latency", "upstream", upLatency), ) if err != nil { t.Error("AddToxicJson returned error:", err) } } if downLatency == nil { downLatency = &toxics.LatencyToxic{} } else { _, err := proxy.Toxics.AddToxicJson( ToxicToJson(t, "latency_down", "latency", "downstream", downLatency), ) if err != nil { t.Error("AddToxicJson returned error:", err) } } t.Logf( "Using latency: Up: %dms +/- %dms, Down: %dms +/- %dms", upLatency.Latency, upLatency.Jitter, downLatency.Latency, downLatency.Jitter, ) msg := []byte("hello world " + strings.Repeat("a", 32*1024) + "\n") timer := time.Now() _, err := conn.Write(msg) if err != nil { t.Error("Failed writing to TCP server", err) } resp := <-response if !bytes.Equal(resp, msg) { t.Error("Server didn't read correct bytes from client:", string(resp)) } AssertDeltaTime(t, "Server read", time.Since(timer), time.Duration(upLatency.Latency)*time.Millisecond, time.Duration(upLatency.Jitter+10)*time.Millisecond, ) timer2 := time.Now() scan := bufio.NewScanner(conn) if scan.Scan() { resp = append(scan.Bytes(), '\n') if !bytes.Equal(resp, msg) { t.Error("Client didn't read correct bytes from server:", string(resp)) } } AssertDeltaTime(t, "Client read", time.Since(timer2), time.Duration(downLatency.Latency)*time.Millisecond, time.Duration(downLatency.Jitter+10)*time.Millisecond, ) AssertDeltaTime(t, "Round trip", time.Since(timer), time.Duration(upLatency.Latency+downLatency.Latency)*time.Millisecond, time.Duration(upLatency.Jitter+downLatency.Jitter+20)*time.Millisecond, ) ctx := context.Background() proxy.Toxics.RemoveToxic(ctx, "latency_up") proxy.Toxics.RemoveToxic(ctx, "latency_down") err = conn.Close() if err != nil { t.Error("Failed to close TCP connection", err) } }) } func TestUpstreamLatency(t *testing.T) { DoLatencyTest(t, &toxics.LatencyToxic{Latency: 100}, nil) } func TestDownstreamLatency(t *testing.T) { DoLatencyTest(t, nil, &toxics.LatencyToxic{Latency: 100}) } func TestFullstreamLatencyEven(t *testing.T) { DoLatencyTest(t, &toxics.LatencyToxic{Latency: 100}, &toxics.LatencyToxic{Latency: 100}) } func TestFullstreamLatencyBiasUp(t *testing.T) { DoLatencyTest(t, &toxics.LatencyToxic{Latency: 1000}, &toxics.LatencyToxic{Latency: 100}) } func TestFullstreamLatencyBiasDown(t *testing.T) { DoLatencyTest(t, &toxics.LatencyToxic{Latency: 100}, &toxics.LatencyToxic{Latency: 1000}) } func TestZeroLatency(t *testing.T) { DoLatencyTest(t, &toxics.LatencyToxic{Latency: 0}, &toxics.LatencyToxic{Latency: 0}) } func TestLatencyToxicCloseRace(t *testing.T) { ln, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatal("Failed to create TCP server", err) } defer ln.Close() proxy := NewTestProxy("test", ln.Addr().String()) proxy.Start() defer proxy.Stop() go func() { for { _, err := ln.Accept() if err != nil { return } } }() // Check for potential race conditions when interrupting toxics for i := 0; i < 1000; i++ { proxy.Toxics.AddToxicJson( ToxicToJson(t, "", "latency", "upstream", &toxics.LatencyToxic{Latency: 10}), ) conn, err := net.Dial("tcp", proxy.Listen) if err != nil { t.Error("Unable to dial TCP server", err) } conn.Write([]byte("hello")) conn.Close() proxy.Toxics.RemoveToxic(context.Background(), "latency") } } func TestTwoLatencyToxics(t *testing.T) { WithEchoProxy(t, func(conn net.Conn, response chan []byte, proxy *toxiproxy.Proxy) { toxics := []*toxics.LatencyToxic{{Latency: 500}, {Latency: 500}} for i, toxic := range toxics { _, err := proxy.Toxics.AddToxicJson( ToxicToJson(t, "latency_"+strconv.Itoa(i), "latency", "upstream", toxic), ) if err != nil { t.Error("AddToxicJson returned error:", err) } } msg := []byte("hello world " + strings.Repeat("a", 32*1024) + "\n") timer := time.Now() _, err := conn.Write(msg) if err != nil { t.Error("Failed writing to TCP server", err) } resp := <-response if !bytes.Equal(resp, msg) { t.Error("Server didn't read correct bytes from client:", string(resp)) } AssertDeltaTime(t, "Upstream two latency toxics", time.Since(timer), time.Duration(1000)*time.Millisecond, time.Duration(10)*time.Millisecond, ) for i := range toxics { proxy.Toxics.RemoveToxic(context.Background(), "latency_"+strconv.Itoa(i)) } err = conn.Close() if err != nil { t.Error("Failed to close TCP connection", err) } }) } func TestLatencyToxicBandwidth(t *testing.T) { upstream := testhelper.NewUpstream(t, false) defer upstream.Close() proxy := NewTestProxy("test", upstream.Addr()) proxy.Start() defer proxy.Stop() client, err := net.Dial("tcp", proxy.Listen) if err != nil { t.Fatalf("Unable to dial TCP server: %v", err) } writtenPayload := []byte(strings.Repeat("hello world ", 1000)) upstreamConn := <-upstream.Connections go func(conn net.Conn, payload []byte) { var err error for err == nil { _, err = conn.Write(payload) } }(upstreamConn, writtenPayload) proxy.Toxics.AddToxicJson(ToxicToJson(t, "", "latency", "", &toxics.LatencyToxic{Latency: 100})) time.Sleep(150 * time.Millisecond) // Wait for latency toxic response := make([]byte, len(writtenPayload)) start := time.Now() count := 0 for i := 0; i < 100; i++ { n, err := io.ReadFull(client, response) if err != nil { t.Fatalf("Could not read from socket: %v", err) break } count += n } // Assert the transfer was at least 100MB/s AssertDeltaTime( t, "Latency toxic bandwidth", time.Since(start), 0, time.Duration(count/100000)*time.Millisecond, ) err = client.Close() if err != nil { t.Error("Failed to close TCP connection", err) } } ================================================ FILE: toxics/limit_data.go ================================================ package toxics import "github.com/Shopify/toxiproxy/v2/stream" // LimitDataToxic has limit in bytes. type LimitDataToxic struct { Bytes int64 `json:"bytes"` } type LimitDataToxicState struct { bytesTransmitted int64 } func (t *LimitDataToxic) Pipe(stub *ToxicStub) { state := stub.State.(*LimitDataToxicState) bytesRemaining := t.Bytes - state.bytesTransmitted for { select { case <-stub.Interrupt: return case c := <-stub.Input: if c == nil { stub.Close() return } if bytesRemaining < 0 { bytesRemaining = 0 } if bytesRemaining < int64(len(c.Data)) { c = &stream.StreamChunk{ Timestamp: c.Timestamp, Data: c.Data[0:bytesRemaining], } } if len(c.Data) > 0 { stub.Output <- c state.bytesTransmitted += int64(len(c.Data)) } bytesRemaining = t.Bytes - state.bytesTransmitted if bytesRemaining <= 0 { stub.Close() return } } } } func (t *LimitDataToxic) NewState() interface{} { return new(LimitDataToxicState) } func init() { Register("limit_data", new(LimitDataToxic)) } ================================================ FILE: toxics/limit_data_test.go ================================================ package toxics_test import ( "bytes" "crypto/rand" "testing" "github.com/Shopify/toxiproxy/v2/stream" "github.com/Shopify/toxiproxy/v2/toxics" ) func buffer(size int) []byte { buf := make([]byte, size) // #nosec G404 -- used only in tests rand.Read(buf) return buf } func checkOutgoingChunk(t *testing.T, output chan *stream.StreamChunk, expected []byte) { chunk := <-output if !bytes.Equal(chunk.Data, expected) { t.Error("Data in outgoing chunk doesn't match expected values") } } func checkRemainingChunks(t *testing.T, output chan *stream.StreamChunk) { if len(output) != 0 { t.Errorf("There is %d chunks in output channel. 0 is expected.", len(output)) } } func check(t *testing.T, toxic *toxics.LimitDataToxic, chunks [][]byte, expectedChunks [][]byte) { input := make(chan *stream.StreamChunk) output := make(chan *stream.StreamChunk, 100) stub := toxics.NewToxicStub(input, output) stub.State = toxic.NewState() go toxic.Pipe(stub) for _, buf := range chunks { input <- &stream.StreamChunk{Data: buf} } for _, expected := range expectedChunks { checkOutgoingChunk(t, output, expected) } checkRemainingChunks(t, output) } func TestLimitDataToxicMayBeRestarted(t *testing.T) { toxic := &toxics.LimitDataToxic{Bytes: 100} input := make(chan *stream.StreamChunk) output := make(chan *stream.StreamChunk, 100) stub := toxics.NewToxicStub(input, output) stub.State = toxic.NewState() buf := buffer(90) buf2 := buffer(20) // Send chunk with data not exceeding limit and interrupt go func() { input <- &stream.StreamChunk{Data: buf} stub.Interrupt <- struct{}{} }() toxic.Pipe(stub) checkOutgoingChunk(t, output, buf) // Send 2nd chunk to exceed limit go func() { input <- &stream.StreamChunk{Data: buf2} }() toxic.Pipe(stub) checkOutgoingChunk(t, output, buf2[0:10]) checkRemainingChunks(t, output) } func TestLimitDataToxicMayBeInterrupted(t *testing.T) { toxic := &toxics.LimitDataToxic{Bytes: 100} input := make(chan *stream.StreamChunk) output := make(chan *stream.StreamChunk) stub := toxics.NewToxicStub(input, output) stub.State = toxic.NewState() go func() { stub.Interrupt <- struct{}{} }() toxic.Pipe(stub) } func TestLimitDataToxicNilShouldClosePipe(t *testing.T) { toxic := &toxics.LimitDataToxic{Bytes: 100} input := make(chan *stream.StreamChunk) output := make(chan *stream.StreamChunk) stub := toxics.NewToxicStub(input, output) stub.State = toxic.NewState() go func() { input <- nil }() toxic.Pipe(stub) } func TestLimitDataToxicChunkSmallerThanLimit(t *testing.T) { toxic := &toxics.LimitDataToxic{Bytes: 100} buf := buffer(50) check(t, toxic, [][]byte{buf}, [][]byte{buf}) } func TestLimitDataToxicChunkLengthMatchesLimit(t *testing.T) { toxic := &toxics.LimitDataToxic{Bytes: 100} buf := buffer(100) check(t, toxic, [][]byte{buf}, [][]byte{buf}) } func TestLimitDataToxicChunkBiggerThanLimit(t *testing.T) { toxic := &toxics.LimitDataToxic{Bytes: 100} buf := buffer(150) expected := buf[0:100] check(t, toxic, [][]byte{buf}, [][]byte{expected}) } func TestLimitDataToxicMultipleChunksMatchThanLimit(t *testing.T) { toxic := &toxics.LimitDataToxic{Bytes: 100} buf := buffer(25) check(t, toxic, [][]byte{buf, buf, buf, buf}, [][]byte{buf, buf, buf, buf}) } func TestLimitDataToxicSecondChunkWouldOverflowLimit(t *testing.T) { toxic := &toxics.LimitDataToxic{Bytes: 100} buf := buffer(90) buf2 := buffer(20) expected := buf2[0:10] check(t, toxic, [][]byte{buf, buf2}, [][]byte{buf, expected}) } func TestLimitDataToxicLimitIsSetToZero(t *testing.T) { toxic := &toxics.LimitDataToxic{Bytes: 0} buf := buffer(100) check(t, toxic, [][]byte{buf}, [][]byte{}) } ================================================ FILE: toxics/noop.go ================================================ package toxics // The NoopToxic passes all data through without any toxic effects. type NoopToxic struct{} func (t *NoopToxic) Pipe(stub *ToxicStub) { for { select { case <-stub.Interrupt: return case c := <-stub.Input: if c == nil { stub.Close() return } stub.Output <- c } } } func init() { Register("noop", new(NoopToxic)) } ================================================ FILE: toxics/reset_peer.go ================================================ package toxics import ( "time" ) /* The ResetToxic sends closes the connection abruptly after a timeout (in ms). The behavior of Close is set to discard any unsent/unacknowledged data by setting SetLinger to 0, ~= sets TCP RST flag and resets the connection. If the timeout is set to 0, then the connection will be reset immediately. Drop data since it will initiate a graceful close by sending the FIN/ACK. (io.EOF) */ type ResetToxic struct { // Timeout in milliseconds Timeout int64 `json:"timeout"` } func (t *ResetToxic) Pipe(stub *ToxicStub) { timeout := time.Duration(t.Timeout) * time.Millisecond for { select { case <-stub.Interrupt: return case <-stub.Input: <-time.After(timeout) stub.Close() return } } } func init() { Register("reset_peer", new(ResetToxic)) } ================================================ FILE: toxics/reset_peer_test.go ================================================ package toxics_test import ( "bufio" "io" "net" "os" "syscall" "testing" "time" "github.com/Shopify/toxiproxy/v2/toxics" ) const msg = "reset toxic payload\n" func TestResetToxicNoTimeout(t *testing.T) { resetTCPHelper(t, ToxicToJson(t, "resettcp", "reset_peer", "upstream", &toxics.ResetToxic{})) } func TestResetToxicWithTimeout(t *testing.T) { start := time.Now() resetToxic := toxics.ResetToxic{Timeout: 100} resetTCPHelper(t, ToxicToJson(t, "resettcp", "reset_peer", "upstream", &resetToxic)) AssertDeltaTime(t, "Reset after timeout", time.Since(start), time.Duration(resetToxic.Timeout)*time.Millisecond, time.Duration(resetToxic.Timeout+10)*time.Millisecond, ) } func TestResetToxicWithTimeoutDownstream(t *testing.T) { start := time.Now() resetToxic := toxics.ResetToxic{Timeout: 100} resetTCPHelper(t, ToxicToJson(t, "resettcp", "reset_peer", "downstream", &resetToxic)) AssertDeltaTime(t, "Reset after timeout", time.Since(start), time.Duration(resetToxic.Timeout)*time.Millisecond, time.Duration(resetToxic.Timeout+10)*time.Millisecond, ) } func checkConnectionState(t *testing.T, listenAddress string) { conn, err := net.Dial("tcp", listenAddress) if err != nil { t.Error("Unable to dial TCP server", err) } if _, err := conn.Write([]byte(msg)); err != nil { t.Error("Failed writing TCP payload", err) } tmp := make([]byte, 1000) _, err = conn.Read(tmp) defer conn.Close() if opErr, ok := err.(*net.OpError); ok { syscallErr, _ := opErr.Err.(*os.SyscallError) if syscallErr.Err != syscall.ECONNRESET { t.Error("Expected: connection reset by peer. Got:", err) } } else { t.Error( "Expected: connection reset by peer. Got:", err, "conn:", conn.RemoteAddr(), conn.LocalAddr(), ) } _, err = conn.Read(tmp) if err != io.EOF { t.Error("expected EOF from closed connection") } } func resetTCPHelper(t *testing.T, toxicJSON io.Reader) { ln, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatal("Failed to create TCP server", err) } defer ln.Close() proxy := NewTestProxy("test", ln.Addr().String()) proxy.Start() proxy.Toxics.AddToxicJson(toxicJSON) defer proxy.Stop() go func() { conn, err := ln.Accept() if err != nil { t.Error("Unable to accept TCP connection", err) } defer ln.Close() scan := bufio.NewScanner(conn) if scan.Scan() { conn.Write([]byte(msg)) } }() checkConnectionState(t, proxy.Listen) } ================================================ FILE: toxics/slicer.go ================================================ package toxics import ( "math/rand" "time" "github.com/Shopify/toxiproxy/v2/stream" ) // The SlicerToxic slices data into multiple smaller packets // to simulate real-world TCP behavior. type SlicerToxic struct { // Average number of bytes to slice at AverageSize int `json:"average_size"` // +/- bytes to vary sliced amounts. Must be less than // the average size SizeVariation int `json:"size_variation"` // Microseconds to delay each packet. May be useful since there's // usually some kind of buffering of network data Delay int `json:"delay"` } // Returns a list of chunk offsets to slice up a packet of the // given total size. For example, for a size of 100, output might be: // // | []int{0, 18, 18, 43, 43, 67, 67, 77, 77, 100} // | ^---^ ^----^ ^----^ ^----^ ^-----^ // // This tries to get fairly evenly-varying chunks (no tendency // to have a small/large chunk at the start/end). func (t *SlicerToxic) chunk(start int, end int) []int { // Base case: // If the size is within the random varation, _or already // less than the average size_, just return it. // Otherwise split the chunk in about two, and recurse. if (end-start)-t.AverageSize <= t.SizeVariation { return []int{start, end} } mid := start + (end-start)/2 if t.SizeVariation > 0 { mid += rand.Intn(t.SizeVariation*2) - t.SizeVariation // #nosec G404 -- was ignored before too } left := t.chunk(start, mid) right := t.chunk(mid, end) return append(left, right...) } func (t *SlicerToxic) Pipe(stub *ToxicStub) { for { select { case <-stub.Interrupt: return case c := <-stub.Input: if c == nil { stub.Close() return } chunks := t.chunk(0, len(c.Data)) for i := 1; i < len(chunks); i += 2 { stub.Output <- &stream.StreamChunk{ Data: c.Data[chunks[i-1]:chunks[i]], Timestamp: c.Timestamp, } select { case <-stub.Interrupt: stub.Output <- &stream.StreamChunk{ Data: c.Data[chunks[i]:], Timestamp: c.Timestamp, } return case <-time.After(time.Duration(t.Delay) * time.Microsecond): } } } } } func init() { Register("slicer", new(SlicerToxic)) } ================================================ FILE: toxics/slicer_test.go ================================================ package toxics_test import ( "bytes" "strings" "testing" "time" "github.com/Shopify/toxiproxy/v2/stream" "github.com/Shopify/toxiproxy/v2/toxics" ) func TestSlicerToxic(t *testing.T) { data := []byte(strings.Repeat("hello world ", 40000)) // 480 kb slicer := &toxics.SlicerToxic{AverageSize: 1024, SizeVariation: 512, Delay: 10} input := make(chan *stream.StreamChunk) output := make(chan *stream.StreamChunk) stub := toxics.NewToxicStub(input, output) done := make(chan bool) go func() { slicer.Pipe(stub) done <- true }() defer func() { close(input) for { select { case <-done: return case <-output: } } }() input <- &stream.StreamChunk{Data: data} buf := make([]byte, 0, len(data)) reads := 0 for timeout := false; !timeout; { select { case c := <-output: reads++ buf = append(buf, c.Data...) case <-time.After(10 * time.Millisecond): timeout = true } } if reads < 480/2 || reads > 480/2+480 { t.Errorf("Expected to read about 480 times, but read %d times.", reads) } if !bytes.Equal(buf, data) { t.Errorf("Server did not read correct buffer from client!") } } func TestSlicerToxicZeroSizeVariation(t *testing.T) { data := []byte(strings.Repeat("hello world ", 2)) // 24 bytes // SizeVariation: 0 by default slicer := &toxics.SlicerToxic{AverageSize: 1, Delay: 10} input := make(chan *stream.StreamChunk) output := make(chan *stream.StreamChunk) stub := toxics.NewToxicStub(input, output) done := make(chan bool) go func() { slicer.Pipe(stub) done <- true }() defer func() { close(input) for { select { case <-done: return case <-output: } } }() input <- &stream.StreamChunk{Data: data} buf := make([]byte, 0, len(data)) reads := 0 for timeout := false; !timeout; { select { case c := <-output: reads++ buf = append(buf, c.Data...) case <-time.After(10 * time.Millisecond): timeout = true } } if reads != 24 { t.Errorf("Expected to read 24 times, but read %d times.", reads) } if !bytes.Equal(buf, data) { t.Errorf("Server did not read correct buffer from client!") } } ================================================ FILE: toxics/slow_close.go ================================================ package toxics import "time" // The SlowCloseToxic stops the TCP connection from closing until after a delay. type SlowCloseToxic struct { // Times in milliseconds Delay int64 `json:"delay"` } func (t *SlowCloseToxic) Pipe(stub *ToxicStub) { for { select { case <-stub.Interrupt: return case c := <-stub.Input: if c == nil { delay := time.Duration(t.Delay) * time.Millisecond select { case <-time.After(delay): stub.Close() return case <-stub.Interrupt: return } } stub.Output <- c } } } func init() { Register("slow_close", new(SlowCloseToxic)) } ================================================ FILE: toxics/timeout.go ================================================ package toxics import "time" // The TimeoutToxic stops any data from flowing through, // and will close the connection after a timeout. // If the timeout is set to 0, then the connection will not be closed. type TimeoutToxic struct { // Times in milliseconds Timeout int64 `json:"timeout"` } func (t *TimeoutToxic) Pipe(stub *ToxicStub) { timeout := time.Duration(t.Timeout) * time.Millisecond if timeout > 0 { for { select { case <-time.After(timeout): stub.Close() return case <-stub.Interrupt: return case c := <-stub.Input: if c == nil { stub.Close() return } // Drop the data on the ground. } } } else { for { select { case <-stub.Interrupt: return case c := <-stub.Input: if c == nil { stub.Close() return } // Drop the data on the ground. } } } } func (t *TimeoutToxic) Cleanup(stub *ToxicStub) { stub.Close() } func init() { Register("timeout", new(TimeoutToxic)) } ================================================ FILE: toxics/timeout_test.go ================================================ package toxics_test import ( "bytes" "context" "io" "net" "testing" "time" "github.com/Shopify/toxiproxy/v2" "github.com/Shopify/toxiproxy/v2/testhelper" "github.com/Shopify/toxiproxy/v2/toxics" ) func WithEstablishedProxy(t *testing.T, f func(net.Conn, net.Conn, *toxiproxy.Proxy)) { ln, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatal("Failed to create TCP server", err) } defer ln.Close() proxy := NewTestProxy("test", ln.Addr().String()) proxy.Start() defer proxy.Stop() serverConnRecv := make(chan net.Conn) go func() { conn, err := ln.Accept() if err != nil { t.Error("Unable to accept TCP connection", err) } serverConnRecv <- conn }() conn, err := net.Dial("tcp", proxy.Listen) if err != nil { t.Fatal("Unable to dial TCP server", err) } defer conn.Close() serverConn := <-serverConnRecv defer serverConn.Close() writeAndReceive := func(from, to net.Conn) { data := []byte("foobar") _, err := from.Write(data) if err != nil { t.Fatal(err) } err = testhelper.TimeoutAfter(time.Second, func() { resp := make([]byte, len(data)) to.Read(resp) if !bytes.Equal(resp, data) { t.Fatalf("expected '%s' but got '%s'", string(data), string(resp)) } }) if err != nil { t.Fatal(err) } } // Make sure we can send and receive data before continuing. writeAndReceive(conn, serverConn) writeAndReceive(serverConn, conn) f(conn, serverConn, proxy) } func TestTimeoutToxicDoesNotCauseHang(t *testing.T) { WithEstablishedProxy(t, func(conn, _ net.Conn, proxy *toxiproxy.Proxy) { proxy.Toxics.AddToxicJson( ToxicToJson(t, "might_block", "latency", "upstream", &toxics.LatencyToxic{Latency: 10}), ) proxy.Toxics.AddToxicJson( ToxicToJson(t, "timeout", "timeout", "upstream", &toxics.TimeoutToxic{Timeout: 0}), ) for i := 0; i < 5; i++ { _, err := conn.Write([]byte("hello")) if err != nil { t.Fatal("Unable to write to proxy", err) } time.Sleep(200 * time.Millisecond) // Shitty sync waiting for latency toxi to get data. } err := testhelper.TimeoutAfter(time.Second, func() { proxy.Toxics.RemoveToxic(context.Background(), "might_block") }) if err != nil { t.Fatal(err) } }) } func TestTimeoutToxicClosesConnectionOnRemove(t *testing.T) { WithEstablishedProxy(t, func(conn, serverConn net.Conn, proxy *toxiproxy.Proxy) { proxy.Toxics.AddToxicJson( ToxicToJson(t, "to_delete", "timeout", "upstream", &toxics.TimeoutToxic{Timeout: 0}), ) proxy.Toxics.RemoveToxic(context.Background(), "to_delete") err := testhelper.TimeoutAfter(time.Second, func() { buf := make([]byte, 1) _, err := conn.Read(buf) if err != io.EOF { t.Fatal("expected EOF from closed connetion") } _, err = serverConn.Read(buf) if err != io.EOF { t.Fatal("expected EOF from closed server connetion") } }) if err != nil { t.Fatal(err) } }) } ================================================ FILE: toxics/toxic.go ================================================ package toxics import ( "fmt" "math/rand" "reflect" "sync" "time" "github.com/Shopify/toxiproxy/v2/stream" ) // A Toxic is something that can be attatched to a link to modify the way // data can be passed through (for example, by adding latency) // // Toxic // v // Client <-> ToxicStub <-> Upstream // // Toxic's work in a pipeline fashion, and can be chained together // with channels. The toxic itself only defines the settings and // Pipe() function definition, and uses the ToxicStub struct to store // per-connection information. This allows the same toxic to be used // for multiple connections. type Toxic interface { // Defines how packets flow through a ToxicStub. // Pipe() blocks until the link is closed or interrupted. Pipe(*ToxicStub) } type CleanupToxic interface { // Cleanup is called before a toxic is removed. Cleanup(*ToxicStub) } type BufferedToxic interface { // Defines the size of buffer this toxic should use GetBufferSize() int } // Stateful toxics store a per-connection state object on the ToxicStub. // The state is created once when the toxic is added and persists until the // toxic is removed or the connection is closed. type StatefulToxic interface { // Creates a new object to store toxic state in NewState() interface{} } type ToxicWrapper struct { Toxic `json:"attributes"` Name string `json:"name"` Type string `json:"type"` Stream string `json:"stream"` Toxicity float32 `json:"toxicity"` Direction stream.Direction `json:"-"` Index int `json:"-"` BufferSize int `json:"-"` } type ToxicStub struct { Input <-chan *stream.StreamChunk Output chan<- *stream.StreamChunk State interface{} Interrupt chan struct{} running chan struct{} closed chan struct{} } func NewToxicStub(input <-chan *stream.StreamChunk, output chan<- *stream.StreamChunk) *ToxicStub { return &ToxicStub{ Interrupt: make(chan struct{}), closed: make(chan struct{}), Input: input, Output: output, } } // Begin running a toxic on this stub, can be interrupted. // Runs a noop toxic randomly depending on toxicity. func (s *ToxicStub) Run(toxic *ToxicWrapper) { s.running = make(chan struct{}) defer close(s.running) randomToxicity := rand.Float32() // #nosec G404 -- was ignored before too if randomToxicity < toxic.Toxicity { toxic.Pipe(s) } else { new(NoopToxic).Pipe(s) } } // WriteOutput allows to write to Output with timeout to avoid deadlocks. // If duration is 0, then wait until other goroutines finish reading from Output. func (s *ToxicStub) WriteOutput(p *stream.StreamChunk, d time.Duration) error { if d == 0 { s.Output <- p return nil } select { case s.Output <- p: return nil case <-time.After(d): return fmt.Errorf("timeout: could not write to output in %d seconds", int(d.Seconds())) } } // Interrupt the flow of data so that the toxic controlling the stub can be replaced. // Returns true if the stream was successfully interrupted, or false if the stream is closed. func (s *ToxicStub) InterruptToxic() bool { select { case <-s.closed: return false case s.Interrupt <- struct{}{}: <-s.running // Wait for the running toxic to exit return true } } func (s *ToxicStub) Closed() bool { select { case <-s.closed: return true default: return false } } func (s *ToxicStub) Close() { if !s.Closed() { close(s.closed) close(s.Output) } } var ( ToxicRegistry map[string]Toxic registryMutex sync.RWMutex ) func Register(typeName string, toxic Toxic) { registryMutex.Lock() defer registryMutex.Unlock() if ToxicRegistry == nil { ToxicRegistry = make(map[string]Toxic) } ToxicRegistry[typeName] = toxic } func New(wrapper *ToxicWrapper) Toxic { registryMutex.RLock() defer registryMutex.RUnlock() orig, ok := ToxicRegistry[wrapper.Type] if !ok { return nil } wrapper.Toxic = reflect.New(reflect.TypeOf(orig).Elem()).Interface().(Toxic) if buffered, ok := wrapper.Toxic.(BufferedToxic); ok { wrapper.BufferSize = buffered.GetBufferSize() } else { wrapper.BufferSize = 0 } return wrapper.Toxic } func Count() int { registryMutex.RLock() defer registryMutex.RUnlock() return len(ToxicRegistry) } ================================================ FILE: toxics/toxic_test.go ================================================ package toxics_test import ( "bufio" "bytes" "context" "crypto/rand" "encoding/json" "flag" "io" "net" "os" "strings" "testing" "time" "github.com/prometheus/client_golang/prometheus" "github.com/rs/zerolog" tomb "gopkg.in/tomb.v1" "github.com/Shopify/toxiproxy/v2" "github.com/Shopify/toxiproxy/v2/collectors" "github.com/Shopify/toxiproxy/v2/stream" "github.com/Shopify/toxiproxy/v2/toxics" ) func NewTestProxy(name, upstream string) *toxiproxy.Proxy { log := zerolog.Nop() if flag.Lookup("test.v").DefValue == "true" { log = zerolog.New(os.Stdout).With().Caller().Timestamp().Logger() } srv := toxiproxy.NewServer( toxiproxy.NewMetricsContainer(prometheus.NewRegistry()), log, ) srv.Metrics.ProxyMetrics = collectors.NewProxyMetricCollectors() proxy := toxiproxy.NewProxy(srv, name, "localhost:0", upstream) return proxy } func WithEchoServer(t *testing.T, f func(string, chan []byte)) { ln, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatal("Failed to create TCP server", err) } defer ln.Close() response := make(chan []byte, 1) tomb := tomb.Tomb{} go func() { defer tomb.Done() src, err := ln.Accept() if err != nil { select { case <-tomb.Dying(): default: t.Error("Failed to accept client") return } return } ln.Close() scan := bufio.NewScanner(src) if scan.Scan() { received := append(scan.Bytes(), '\n') response <- received src.Write(received) } }() f(ln.Addr().String(), response) tomb.Killf("Function body finished") ln.Close() tomb.Wait() close(response) } func WithEchoProxy( t *testing.T, f func(proxy net.Conn, response chan []byte, proxyServer *toxiproxy.Proxy), ) { WithEchoServer(t, func(upstream string, response chan []byte) { proxy := NewTestProxy("test", upstream) proxy.Start() conn, err := net.Dial("tcp", proxy.Listen) if err != nil { t.Error("Unable to dial TCP server", err) } f(conn, response, proxy) proxy.Stop() }) } func ToxicToJson(t *testing.T, name, typeName, stream string, toxic toxics.Toxic) io.Reader { data := map[string]interface{}{ "name": name, "type": typeName, "stream": stream, "attributes": toxic, } request, err := json.Marshal(data) if err != nil { t.Errorf("Failed to marshal toxic for api (1): %v", toxic) } return bytes.NewReader(request) } func AssertEchoResponse(t *testing.T, client, server net.Conn) { msg := []byte("hello world\n") _, err := client.Write(msg) if err != nil { t.Error("Failed writing to TCP server", err) } scan := bufio.NewScanner(server) if !scan.Scan() { t.Error("Client unexpectedly closed connection") } resp := append(scan.Bytes(), '\n') if !bytes.Equal(resp, msg) { t.Error("Server didn't read correct bytes from client:", string(resp)) } _, err = server.Write(resp) if err != nil { t.Error("Failed writing to TCP client", err) } scan = bufio.NewScanner(client) if !scan.Scan() { t.Error("Server unexpectedly closed connection") } resp = append(scan.Bytes(), '\n') if !bytes.Equal(resp, msg) { t.Error("Client didn't read correct bytes from server:", string(resp)) } } func TestPersistentConnections(t *testing.T) { ctx := context.Background() ln, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatal("Failed to create TCP server", err) } defer ln.Close() proxy := NewTestProxy("test", ln.Addr().String()) proxy.Start() defer proxy.Stop() serverConnRecv := make(chan net.Conn) go func() { conn, err := ln.Accept() if err != nil { t.Error("Unable to accept TCP connection", err) } serverConnRecv <- conn }() conn, err := net.Dial("tcp", proxy.Listen) if err != nil { t.Error("Unable to dial TCP server", err) } serverConn := <-serverConnRecv proxy.Toxics.AddToxicJson(ToxicToJson(t, "noop_up", "noop", "upstream", &toxics.NoopToxic{})) proxy.Toxics.AddToxicJson( ToxicToJson(t, "noop_down", "noop", "downstream", &toxics.NoopToxic{}), ) AssertEchoResponse(t, conn, serverConn) proxy.Toxics.ResetToxics(ctx) AssertEchoResponse(t, conn, serverConn) proxy.Toxics.ResetToxics(ctx) AssertEchoResponse(t, conn, serverConn) err = conn.Close() if err != nil { t.Error("Failed to close TCP connection", err) } } func TestToxicAddRemove(t *testing.T) { ln, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatal("Failed to create TCP server", err) } defer ln.Close() proxy := NewTestProxy("test", ln.Addr().String()) proxy.Start() defer proxy.Stop() serverConnRecv := make(chan net.Conn) go func() { conn, err := ln.Accept() if err != nil { t.Error("Unable to accept TCP connection", err) } serverConnRecv <- conn }() conn, err := net.Dial("tcp", proxy.Listen) if err != nil { t.Error("Unable to dial TCP server", err) } serverConn := <-serverConnRecv running := make(chan struct{}) go func() { enabled := false for { select { case <-running: return default: if enabled { proxy.Toxics.AddToxicJson( ToxicToJson(t, "noop_up", "noop", "upstream", &toxics.NoopToxic{}), ) proxy.Toxics.RemoveToxic(context.Background(), "noop_down") } else { proxy.Toxics.RemoveToxic(context.Background(), "noop_up") proxy.Toxics.AddToxicJson( ToxicToJson(t, "noop_down", "noop", "downstream", &toxics.NoopToxic{}), ) } enabled = !enabled } } }() for i := 0; i < 100; i++ { AssertEchoResponse(t, conn, serverConn) } close(running) err = conn.Close() if err != nil { t.Error("Failed to close TCP connection", err) } } func TestProxyLatency(t *testing.T) { ln, err := net.Listen("tcp", "localhost:0") if err != nil { t.Fatal("Failed to create TCP server", err) } defer ln.Close() proxy := NewTestProxy("test", ln.Addr().String()) proxy.Start() defer proxy.Stop() serverConnRecv := make(chan net.Conn) go func() { conn, err := ln.Accept() if err != nil { t.Error("Unable to accept TCP connection", err) } serverConnRecv <- conn }() conn, err := net.Dial("tcp", proxy.Listen) if err != nil { t.Error("Unable to dial TCP server", err) } serverConn := <-serverConnRecv start := time.Now() for i := 0; i < 100; i++ { AssertEchoResponse(t, conn, serverConn) } latency := time.Since(start) / 200 if latency > 300*time.Microsecond { t.Errorf("Average proxy latency > 300µs (%v)", latency) } else { t.Logf("Average proxy latency: %v", latency) } err = conn.Close() if err != nil { t.Error("Failed to close TCP connection", err) } } func BenchmarkProxyBandwidth(b *testing.B) { ln, err := net.Listen("tcp", "localhost:0") if err != nil { b.Fatal("Failed to create TCP server", err) } defer ln.Close() proxy := NewTestProxy("test", ln.Addr().String()) proxy.Start() defer proxy.Stop() buf := []byte(strings.Repeat("hello world ", 1000)) go func() { conn, err := ln.Accept() if err != nil { b.Error("Unable to accept TCP connection", err) } buf2 := make([]byte, len(buf)) for err == nil { _, err = conn.Read(buf2) } }() conn, err := net.Dial("tcp", proxy.Listen) if err != nil { b.Error("Unable to dial TCP server", err) } b.SetBytes(int64(len(buf))) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { n, err := conn.Write(buf) if err != nil || n != len(buf) { b.Errorf("%v, %d == %d", err, n, len(buf)) break } } err = conn.Close() if err != nil { b.Error("Failed to close TCP connection", err) } } func TestToxicStub_WriteOutput(t *testing.T) { input := make(chan *stream.StreamChunk) output := make(chan *stream.StreamChunk) stub := toxics.NewToxicStub(input, output) buf := make([]byte, 42) // #nosec G404 -- used only in tests rand.Read(buf) t.Run("when no read in 1 second", func(t *testing.T) { err := stub.WriteOutput(&stream.StreamChunk{Data: buf}, time.Second) if err == nil { t.Error("Expected to have error") } expected := "timeout: could not write to output in 1 seconds" if err.Error() != expected { t.Errorf("Expected error: %s, got %s", expected, err) } }) t.Run("when read is available", func(t *testing.T) { go func(t *testing.T, stub *toxics.ToxicStub, expected []byte) { select { case <-time.After(5 * time.Second): t.Error("Timeout of running test to read from output.") case chunk := <-output: if !bytes.Equal(chunk.Data, buf) { t.Error("Data in Output different from Write") } } }(t, stub, buf) err := stub.WriteOutput(&stream.StreamChunk{Data: buf}, 5*time.Second) if err != nil { t.Errorf("Unexpected error: %+v", err) } }) } ================================================ FILE: toxiproxy_test.go ================================================ package toxiproxy_test import ( "flag" "net" "os" "testing" "github.com/prometheus/client_golang/prometheus" "github.com/rs/zerolog" "github.com/Shopify/toxiproxy/v2" "github.com/Shopify/toxiproxy/v2/collectors" "github.com/Shopify/toxiproxy/v2/testhelper" ) func NewTestProxy(name, upstream string) *toxiproxy.Proxy { log := zerolog.Nop() if flag.Lookup("test.v").DefValue == "true" { log = zerolog.New(os.Stdout).With().Caller().Timestamp().Logger() } srv := toxiproxy.NewServer( toxiproxy.NewMetricsContainer(prometheus.NewRegistry()), log, ) srv.Metrics.ProxyMetrics = collectors.NewProxyMetricCollectors() proxy := toxiproxy.NewProxy(srv, name, "localhost:0", upstream) return proxy } func WithTCPProxy( t *testing.T, f func(proxy net.Conn, response chan []byte, proxyServer *toxiproxy.Proxy), ) { testhelper.WithTCPServer(t, func(upstream string, response chan []byte) { proxy := NewTestProxy("test", upstream) proxy.Start() conn := AssertProxyUp(t, proxy.Listen, true) f(conn, response, proxy) proxy.Stop() }) } func AssertProxyUp(t *testing.T, addr string, up bool) net.Conn { conn, err := net.Dial("tcp", addr) if err != nil && up { t.Error("Expected proxy to be up:", err) } else if err == nil && !up { t.Error("Expected proxy to be down") } return conn } ================================================ FILE: version.go ================================================ package toxiproxy var Version = "git"